From 052d329c9df899d8664c202239520d5d43f32ea3 Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Fri, 11 Mar 2022 12:25:04 +0800 Subject: [PATCH] Add benchmark. --- paddleaudio/tests/benchmark/README.md | 55 ++++---- paddleaudio/tests/benchmark/features.py | 117 ++++++++++++++++++ .../tests/benchmark/features/__init__.py | 13 -- .../benchmark/features/log_mel_spectrogram.py | 50 -------- .../benchmark/features/mel_spectrogram.py | 50 -------- paddleaudio/tests/benchmark/features/mfcc.py | 50 -------- 6 files changed, 142 insertions(+), 193 deletions(-) create mode 100644 paddleaudio/tests/benchmark/features.py delete mode 100644 paddleaudio/tests/benchmark/features/__init__.py delete mode 100644 paddleaudio/tests/benchmark/features/log_mel_spectrogram.py delete mode 100644 paddleaudio/tests/benchmark/features/mel_spectrogram.py delete mode 100644 paddleaudio/tests/benchmark/features/mfcc.py diff --git a/paddleaudio/tests/benchmark/README.md b/paddleaudio/tests/benchmark/README.md index ef445449..9655632d 100644 --- a/paddleaudio/tests/benchmark/README.md +++ b/paddleaudio/tests/benchmark/README.md @@ -1,45 +1,40 @@ # 1. Prepare -First, install `line_profiler` via pip. +First, install `pytest-benchmark` via pip. ```sh -pip install line_profiler +pip install pytest-benchmark ``` # 2. Run Run the specific script for profiling. ```sh -kernprof -l features/mel_spectrogram.py -python -m line_profiler -u 1e-3 mel_spectrogram.py.lprof +pytest features.py ``` Result: ```sh -Timer unit: 0.001 s +========================================================================== test session starts ========================================================================== +platform linux -- Python 3.7.7, pytest-7.0.1, pluggy-1.0.0 +benchmark: 3.4.1 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000) +rootdir: /ssd3/chenxiaojie06/PaddleSpeech/DeepSpeech/paddleaudio +plugins: typeguard-2.12.1, benchmark-3.4.1, anyio-3.5.0 +collected 6 items -Total time: 22.1208 s -File: features/mel_spectrogram.py -Function: test_melspect_cpu at line 13 - -Line # Hits Time Per Hit % Time Line Contents -============================================================== - 13 @profile - 14 def test_melspect_cpu(input_shape, times): - 15 1 0.1 0.1 0.0 paddle.set_device('cpu') - 16 1 234.5 234.5 1.1 x = paddle.randn(input_shape) - 17 1 85.3 85.3 0.4 feature_extractor = paddleaudio.features.MelSpectrogram(**feat_conf, dtype=x.dtype) - 18 101 0.5 0.0 0.0 for i in range(times): - 19 100 21800.5 218.0 98.6 y = feature_extractor(x) +features.py ...... [100%] -Total time: 4.80543 s -File: features/mel_spectrogram.py -Function: test_melspect_gpu at line 22 -Line # Hits Time Per Hit % Time Line Contents -============================================================== - 22 @profile - 23 def test_melspect_gpu(input_shape, times): - 24 1 0.5 0.5 0.0 paddle.set_device('gpu') - 25 1 4144.8 4144.8 86.3 x = paddle.randn(input_shape) - 26 1 41.9 41.9 0.9 feature_extractor = paddleaudio.features.MelSpectrogram(**feat_conf, dtype=x.dtype) - 27 101 0.2 0.0 0.0 for i in range(times): - 28 100 618.1 6.2 12.9 y = feature_extractor(x) +------------------------------------------------------------------------------------------------- benchmark: 6 tests ------------------------------------------------------------------------------------------------ +Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +test_melspect_gpu 632.2041 (1.0) 898.7449 (1.0) 709.3824 (1.0) 109.7022 (6.91) 676.1923 (1.0) 115.2642 (22.19) 1;0 1,409.6768 (1.0) 5 1 +test_log_melspect_gpu 912.9159 (1.44) 1,222.0535 (1.36) 931.2489 (1.31) 34.4270 (2.17) 924.9896 (1.37) 5.1949 (1.0) 4;13 1,073.8268 (0.76) 82 1 +test_mfcc_gpu 1,244.8374 (1.97) 1,321.3232 (1.47) 1,262.1319 (1.78) 15.8698 (1.0) 1,258.3155 (1.86) 14.1086 (2.72) 17;9 792.3102 (0.56) 91 1 +test_melspect_cpu 19,106.5744 (30.22) 46,194.2125 (51.40) 27,458.7850 (38.71) 9,786.1071 (616.65) 23,830.0692 (35.24) 14,344.4724 (>1000.0) 3;0 36.4182 (0.03) 14 1 +test_log_melspect_cpu 19,513.7132 (30.87) 20,367.2443 (22.66) 19,765.4018 (27.86) 167.1289 (10.53) 19,750.2729 (29.21) 188.9346 (36.37) 16;1 50.5935 (0.04) 49 1 +test_mfcc_cpu 19,881.3528 (31.45) 20,427.2158 (22.73) 20,104.6574 (28.34) 129.5621 (8.16) 20,075.8977 (29.69) 150.9022 (29.05) 12;2 49.7397 (0.04) 48 1 +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +Legend: + Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile. + OPS: Operations Per Second, computed as 1 / Mean + ========================================================================== 6 passed in 20.51s =========================================================================== ``` diff --git a/paddleaudio/tests/benchmark/features.py b/paddleaudio/tests/benchmark/features.py new file mode 100644 index 00000000..67bec9e6 --- /dev/null +++ b/paddleaudio/tests/benchmark/features.py @@ -0,0 +1,117 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import librosa +import numpy as np +import paddle + +import paddleaudio + +# Feature conf +mel_conf = { + 'sr': 16000, + 'n_fft': 512, + 'hop_length': 128, + 'n_mels': 40, +} +mfcc_conf = { + 'n_mfcc': 20, + 'top_db': 80.0, +} +mfcc_conf.update(mel_conf) + +input_shape = (48000) +waveform = np.random.random(size=input_shape) +waveform_tensor = paddle.to_tensor(waveform).unsqueeze(0) + + +def enable_cpu_device(): + paddle.set_device('cpu') + + +def enable_gpu_device(): + paddle.set_device('gpu') + + +mel_extractor = paddleaudio.features.MelSpectrogram( + **mel_conf, f_min=0.0, dtype=waveform_tensor.dtype) + + +def melspectrogram(): + return mel_extractor(waveform_tensor).squeeze(0) + + +def test_melspect_cpu(benchmark): + enable_cpu_device() + feature_paddleaudio = benchmark(melspectrogram) + feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddleaudio, decimal=4) + + +def test_melspect_gpu(benchmark): + enable_gpu_device() + feature_paddleaudio = benchmark(melspectrogram) + feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddleaudio, decimal=4) + + +log_mel_extractor = paddleaudio.features.LogMelSpectrogram( + **mel_conf, f_min=0.0, dtype=waveform_tensor.dtype) + + +def log_melspectrogram(): + return log_mel_extractor(waveform_tensor).squeeze(0) + + +def test_log_melspect_cpu(benchmark): + enable_cpu_device() + feature_paddleaudio = benchmark(log_melspectrogram) + feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) + feature_librosa = librosa.power_to_db(feature_librosa, top_db=None) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddleaudio, decimal=4) + + +def test_log_melspect_gpu(benchmark): + enable_gpu_device() + feature_paddleaudio = benchmark(log_melspectrogram) + feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) + feature_librosa = librosa.power_to_db(feature_librosa, top_db=None) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddleaudio, decimal=4) + + +mfcc_extractor = paddleaudio.features.MFCC( + **mfcc_conf, f_min=0.0, dtype=waveform_tensor.dtype) + + +def mfcc(): + return mfcc_extractor(waveform_tensor).squeeze(0) + + +def test_mfcc_cpu(benchmark): + enable_cpu_device() + feature_paddleaudio = benchmark(mfcc) + feature_librosa = librosa.feature.mfcc(waveform, **mel_conf) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddleaudio, decimal=4) + + +def test_mfcc_gpu(benchmark): + enable_gpu_device() + feature_paddleaudio = benchmark(mfcc) + feature_librosa = librosa.feature.mfcc(waveform, **mel_conf) + np.testing.assert_array_almost_equal( + feature_librosa, feature_paddleaudio, decimal=4) diff --git a/paddleaudio/tests/benchmark/features/__init__.py b/paddleaudio/tests/benchmark/features/__init__.py deleted file mode 100644 index 97043fd7..00000000 --- a/paddleaudio/tests/benchmark/features/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/paddleaudio/tests/benchmark/features/log_mel_spectrogram.py b/paddleaudio/tests/benchmark/features/log_mel_spectrogram.py deleted file mode 100644 index 75c21253..00000000 --- a/paddleaudio/tests/benchmark/features/log_mel_spectrogram.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import paddle - -import paddleaudio - -feat_conf = { - 'sr': 16000, - 'n_fft': 512, - 'hop_length': 128, - 'n_mels': 40, - 'f_min': 0.0, -} - - -@profile -def test_log_melspect_cpu(input_shape, times): - paddle.set_device('cpu') - x = paddle.randn(input_shape) - feature_extractor = paddleaudio.features.LogMelSpectrogram( - **feat_conf, dtype=x.dtype) - for i in range(times): - y = feature_extractor(x) - - -@profile -def test_log_melspect_gpu(input_shape, times): - paddle.set_device('gpu') - x = paddle.randn(input_shape) - feature_extractor = paddleaudio.features.LogMelSpectrogram( - **feat_conf, dtype=x.dtype) - for i in range(times): - y = feature_extractor(x) - - -input_shape = (16, 48000) # (N, T) -times = 100 -test_log_melspect_cpu(input_shape, times) -test_log_melspect_gpu(input_shape, times) diff --git a/paddleaudio/tests/benchmark/features/mel_spectrogram.py b/paddleaudio/tests/benchmark/features/mel_spectrogram.py deleted file mode 100644 index e8e7cc20..00000000 --- a/paddleaudio/tests/benchmark/features/mel_spectrogram.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import paddle - -import paddleaudio - -feat_conf = { - 'sr': 16000, - 'n_fft': 512, - 'hop_length': 128, - 'n_mels': 40, - 'f_min': 0.0, -} - - -@profile -def test_melspect_cpu(input_shape, times): - paddle.set_device('cpu') - x = paddle.randn(input_shape) - feature_extractor = paddleaudio.features.MelSpectrogram( - **feat_conf, dtype=x.dtype) - for i in range(times): - y = feature_extractor(x) - - -@profile -def test_melspect_gpu(input_shape, times): - paddle.set_device('gpu') - x = paddle.randn(input_shape) - feature_extractor = paddleaudio.features.MelSpectrogram( - **feat_conf, dtype=x.dtype) - for i in range(times): - y = feature_extractor(x) - - -input_shape = (16, 48000) # (N, T) -times = 100 -test_melspect_cpu(input_shape, times) -test_melspect_gpu(input_shape, times) diff --git a/paddleaudio/tests/benchmark/features/mfcc.py b/paddleaudio/tests/benchmark/features/mfcc.py deleted file mode 100644 index 1d3fadd3..00000000 --- a/paddleaudio/tests/benchmark/features/mfcc.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import paddle - -import paddleaudio - -feat_conf = { - 'sr': 16000, - 'n_mfcc': 20, - 'n_fft': 512, - 'hop_length': 128, - 'n_mels': 40, - 'f_min': 0.0, - 'top_db': 80.0, -} - - -@profile -def test_mfcc_cpu(input_shape, times): - paddle.set_device('cpu') - x = paddle.randn(input_shape) - feature_extractor = paddleaudio.features.MFCC(**feat_conf, dtype=x.dtype) - for i in range(times): - y = feature_extractor(x) - - -@profile -def test_mfcc_gpu(input_shape, times): - paddle.set_device('gpu') - x = paddle.randn(input_shape) - feature_extractor = paddleaudio.features.MFCC(**feat_conf, dtype=x.dtype) - for i in range(times): - y = feature_extractor(x) - - -input_shape = (16, 48000) # (N, T) -times = 100 -test_mfcc_cpu(input_shape, times) -test_mfcc_gpu(input_shape, times)