msb-public
/
PaddleSpeech
mirror of https://github.com/PaddlePaddle/PaddleSpeech
1
0
Code Issues Projects Releases Wiki Activity

Add benchmark.

Browse Source
pull/1548/head
KP 3 years ago
parent 052d329c9d
commit c3e0a8dd7a
2 changed files with 152 additions and 26 deletions
Show Stats Download Patch File Download Diff File

33
paddleaudio/tests/benchmark/README.md
Unescape View File

@ -17,24 +17,31 @@ platform linux -- Python 3.7.7, pytest-7.0.1, pluggy-1.0.0
benchmark: 3.4.1 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000) benchmark: 3.4.1 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
rootdir: /ssd3/chenxiaojie06/PaddleSpeech/DeepSpeech/paddleaudio rootdir: /ssd3/chenxiaojie06/PaddleSpeech/DeepSpeech/paddleaudio
plugins: typeguard-2.12.1, benchmark-3.4.1, anyio-3.5.0 plugins: typeguard-2.12.1, benchmark-3.4.1, anyio-3.5.0
collected 6 items collected 12 items
features.py ...... [100%] features.py ............ [100%]
------------------------------------------------------------------------------------------------- benchmark: 6 tests ------------------------------------------------------------------------------------------------ ---------------------------------------------------------------------------------------------------- benchmark: 12 tests ----------------------------------------------------------------------------------------------------
Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations Name (time in us) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
test_melspect_gpu 632.2041 (1.0) 898.7449 (1.0) 709.3824 (1.0) 109.7022 (6.91) 676.1923 (1.0) 115.2642 (22.19) 1;0 1,409.6768 (1.0) 5 1 test_melspect_gpu_torchaudio 210.7229 (1.0) 338.5879 (1.0) 217.4949 (1.0) 11.3591 (1.02) 214.0319 (1.0) 8.3707 (1.0) 6;5 4,597.8093 (1.0) 186 1
test_log_melspect_gpu 912.9159 (1.44) 1,222.0535 (1.36) 931.2489 (1.31) 34.4270 (2.17) 924.9896 (1.37) 5.1949 (1.0) 4;13 1,073.8268 (0.76) 82 1 test_log_melspect_gpu_torchaudio 375.4422 (1.78) 1,024.8050 (3.03) 387.3589 (1.78) 18.7080 (1.69) 385.2872 (1.80) 9.4259 (1.13) 31;31 2,581.5853 (0.56) 1420 1
test_mfcc_gpu 1,244.8374 (1.97) 1,321.3232 (1.47) 1,262.1319 (1.78) 15.8698 (1.0) 1,258.3155 (1.86) 14.1086 (2.72) 17;9 792.3102 (0.56) 91 1 test_mfcc_gpu_torchaudio 422.4107 (2.00) 700.7364 (2.07) 454.9903 (2.09) 47.3926 (4.27) 436.6031 (2.04) 15.4376 (1.84) 159;193 2,197.8493 (0.48) 1078 1
test_melspect_cpu 19,106.5744 (30.22) 46,194.2125 (51.40) 27,458.7850 (38.71) 9,786.1071 (616.65) 23,830.0692 (35.24) 14,344.4724 (>1000.0) 3;0 36.4182 (0.03) 14 1 test_melspect_gpu 819.3776 (3.89) 1,161.9311 (3.43) 900.9168 (4.14) 147.0245 (13.26) 830.7453 (3.88) 115.4500 (13.79) 1;1 1,109.9805 (0.24) 5 1
test_log_melspect_cpu 19,513.7132 (30.87) 20,367.2443 (22.66) 19,765.4018 (27.86) 167.1289 (10.53) 19,750.2729 (29.21) 188.9346 (36.37) 16;1 50.5935 (0.04) 49 1 test_log_melspect_gpu 1,197.9323 (5.68) 1,280.0004 (3.78) 1,214.0182 (5.58) 11.0918 (1.0) 1,211.6358 (5.66) 10.0820 (1.20) 84;31 823.7109 (0.18) 533 1
test_mfcc_cpu 19,881.3528 (31.45) 20,427.2158 (22.73) 20,104.6574 (28.34) 129.5621 (8.16) 20,075.8977 (29.69) 150.9022 (29.05) 12;2 49.7397 (0.04) 48 1 test_mfcc_gpu 1,337.0719 (6.35) 1,601.5675 (4.73) 1,355.4527 (6.23) 26.4458 (2.38) 1,348.6911 (6.30) 13.1410 (1.57) 16;17 737.7609 (0.16) 193 1
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- test_melspect_cpu_torchaudio 1,374.8817 (6.52) 3,937.5033 (11.63) 1,574.8930 (7.24) 355.4223 (32.04) 1,409.1432 (6.58) 193.7435 (23.15) 36;49 634.9638 (0.14) 291 1
test_log_melspect_cpu_torchaudio 1,390.2634 (6.60) 2,121.2976 (6.27) 1,559.3045 (7.17) 220.3090 (19.86) 1,409.4356 (6.59) 349.1524 (41.71) 106;0 641.3116 (0.14) 445 1
test_mfcc_cpu_torchaudio 1,445.6678 (6.86) 3,801.8432 (11.23) 1,680.8559 (7.73) 395.5443 (35.66) 1,469.8748 (6.87) 305.6149 (36.51) 38;35 594.9350 (0.13) 469 1
test_melspect_cpu 20,620.2641 (97.85) 20,984.0760 (61.98) 20,721.4942 (95.27) 70.2757 (6.34) 20,717.8025 (96.80) 57.8668 (6.91) 6;2 48.2591 (0.01) 30 1
test_log_melspect_cpu 21,025.3932 (99.78) 48,894.0198 (144.41) 23,057.7049 (106.01) 5,440.3207 (490.48) 21,190.5045 (99.01) 190.0699 (22.71) 4;9 43.3695 (0.01) 44 1
test_mfcc_cpu 21,127.2798 (100.26) 45,811.5358 (135.30) 23,176.4022 (106.56) 5,041.0751 (454.49) 21,319.1714 (99.61) 149.0396 (17.80) 5;9 43.1473 (0.01) 44 1
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Legend: Legend:
Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile. Outliers: 1 Standard Deviation from Mean; 1.5 IQR (InterQuartile Range) from 1st Quartile and 3rd Quartile.
OPS: Operations Per Second, computed as 1 / Mean OPS: Operations Per Second, computed as 1 / Mean
========================================================================== 6 passed in 20.51s =========================================================================== ========================================================================== 12 passed in 26.81s ==========================================================================
``` ```

145
paddleaudio/tests/benchmark/features.py
Unescape View File

@ -11,15 +11,28 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import urllib.request
import librosa import librosa
import numpy as np import numpy as np
import paddle import paddle
import torch
import torchaudio
import paddleaudio import paddleaudio
wav_url = 'https://paddlespeech.bj.bcebos.com/PaddleAudio/zh.wav'
if not os.path.isfile(os.path.basename(wav_url)):
urllib.request.urlretrieve(wav_url, os.path.basename(wav_url))
waveform, sr = paddleaudio.load(os.path.abspath(os.path.basename(wav_url)))
waveform_tensor = paddle.to_tensor(waveform).unsqueeze(0)
waveform_tensor_torch = torch.from_numpy(waveform).unsqueeze(0)
# Feature conf # Feature conf
mel_conf = { mel_conf = {
'sr': 16000, 'sr': sr,
'n_fft': 512, 'n_fft': 512,
'hop_length': 128, 'hop_length': 128,
'n_mels': 40, 'n_mels': 40,
@ -30,9 +43,18 @@ mfcc_conf = {
} }
mfcc_conf.update(mel_conf) mfcc_conf.update(mel_conf)
input_shape = (48000) mel_conf_torchaudio = {
waveform = np.random.random(size=input_shape) 'sample_rate': sr,
waveform_tensor = paddle.to_tensor(waveform).unsqueeze(0) 'n_fft': 512,
'hop_length': 128,
'n_mels': 40,
'norm': 'slaney',
'mel_scale': 'slaney',
}
mfcc_conf_torchaudio = {
'sample_rate': sr,
'n_mfcc': 20,
}
def enable_cpu_device(): def enable_cpu_device():
@ -56,7 +78,7 @@ def test_melspect_cpu(benchmark):
feature_paddleaudio = benchmark(melspectrogram) feature_paddleaudio = benchmark(melspectrogram)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
np.testing.assert_array_almost_equal( np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=4) feature_librosa, feature_paddleaudio, decimal=3)
def test_melspect_gpu(benchmark): def test_melspect_gpu(benchmark):
@ -64,11 +86,39 @@ def test_melspect_gpu(benchmark):
feature_paddleaudio = benchmark(melspectrogram) feature_paddleaudio = benchmark(melspectrogram)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
np.testing.assert_array_almost_equal( np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=4) feature_librosa, feature_paddleaudio, decimal=3)
mel_extractor_torchaudio = torchaudio.transforms.MelSpectrogram(
**mel_conf_torchaudio, f_min=0.0)
def melspectrogram_torchaudio():
return mel_extractor_torchaudio(waveform_tensor_torch).squeeze(0)
def test_melspect_cpu_torchaudio(benchmark):
global waveform_tensor_torch, mel_extractor_torchaudio
mel_extractor_torchaudio = mel_extractor_torchaudio.to('cpu')
waveform_tensor_torch = waveform_tensor_torch.to('cpu')
feature_paddleaudio = benchmark(melspectrogram_torchaudio)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=3)
def test_melspect_gpu_torchaudio(benchmark):
global waveform_tensor_torch, mel_extractor_torchaudio
mel_extractor_torchaudio = mel_extractor_torchaudio.to('cuda')
waveform_tensor_torch = waveform_tensor_torch.to('cuda')
feature_torchaudio = benchmark(melspectrogram_torchaudio)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
np.testing.assert_array_almost_equal(
feature_librosa, feature_torchaudio.cpu(), decimal=3)
log_mel_extractor = paddleaudio.features.LogMelSpectrogram( log_mel_extractor = paddleaudio.features.LogMelSpectrogram(
**mel_conf, f_min=0.0, dtype=waveform_tensor.dtype) **mel_conf, f_min=0.0, top_db=80.0, dtype=waveform_tensor.dtype)
def log_melspectrogram(): def log_melspectrogram():
@ -79,18 +129,54 @@ def test_log_melspect_cpu(benchmark):
enable_cpu_device() enable_cpu_device()
feature_paddleaudio = benchmark(log_melspectrogram) feature_paddleaudio = benchmark(log_melspectrogram)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
feature_librosa = librosa.power_to_db(feature_librosa, top_db=None) feature_librosa = librosa.power_to_db(feature_librosa, top_db=80.0)
np.testing.assert_array_almost_equal( np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=4) feature_librosa, feature_paddleaudio, decimal=3)
def test_log_melspect_gpu(benchmark): def test_log_melspect_gpu(benchmark):
enable_gpu_device() enable_gpu_device()
feature_paddleaudio = benchmark(log_melspectrogram) feature_paddleaudio = benchmark(log_melspectrogram)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf) feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
feature_librosa = librosa.power_to_db(feature_librosa, top_db=None) feature_librosa = librosa.power_to_db(feature_librosa, top_db=80.0)
np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=2)
amplitude_to_DB = torchaudio.transforms.AmplitudeToDB('power', top_db=80.0)
def log_melspectrogram_torchaudio():
mel_specgram = mel_extractor_torchaudio(waveform_tensor_torch)
return amplitude_to_DB(mel_specgram).squeeze(0)
def test_log_melspect_cpu_torchaudio(benchmark):
global waveform_tensor_torch, mel_extractor_torchaudio, amplitude_to_DB
mel_extractor_torchaudio = mel_extractor_torchaudio.to('cpu')
waveform_tensor_torch = waveform_tensor_torch.to('cpu')
amplitude_to_DB = amplitude_to_DB.to('cpu')
feature_paddleaudio = benchmark(log_melspectrogram_torchaudio)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
feature_librosa = librosa.power_to_db(feature_librosa, top_db=80.0)
np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=3)
def test_log_melspect_gpu_torchaudio(benchmark):
global waveform_tensor_torch, mel_extractor_torchaudio, amplitude_to_DB
mel_extractor_torchaudio = mel_extractor_torchaudio.to('cuda')
waveform_tensor_torch = waveform_tensor_torch.to('cuda')
amplitude_to_DB = amplitude_to_DB.to('cuda')
feature_torchaudio = benchmark(log_melspectrogram_torchaudio)
feature_librosa = librosa.feature.melspectrogram(waveform, **mel_conf)
feature_librosa = librosa.power_to_db(feature_librosa, top_db=80.0)
np.testing.assert_array_almost_equal( np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=4) feature_librosa, feature_torchaudio.cpu(), decimal=2)
mfcc_extractor = paddleaudio.features.MFCC( mfcc_extractor = paddleaudio.features.MFCC(
@ -106,7 +192,7 @@ def test_mfcc_cpu(benchmark):
feature_paddleaudio = benchmark(mfcc) feature_paddleaudio = benchmark(mfcc)
feature_librosa = librosa.feature.mfcc(waveform, **mel_conf) feature_librosa = librosa.feature.mfcc(waveform, **mel_conf)
np.testing.assert_array_almost_equal( np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=4) feature_librosa, feature_paddleaudio, decimal=3)
def test_mfcc_gpu(benchmark): def test_mfcc_gpu(benchmark):
@ -114,4 +200,37 @@ def test_mfcc_gpu(benchmark):
feature_paddleaudio = benchmark(mfcc) feature_paddleaudio = benchmark(mfcc)
feature_librosa = librosa.feature.mfcc(waveform, **mel_conf) feature_librosa = librosa.feature.mfcc(waveform, **mel_conf)
np.testing.assert_array_almost_equal( np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=4) feature_librosa, feature_paddleaudio, decimal=3)
del mel_conf_torchaudio['sample_rate']
mfcc_extractor_torchaudio = torchaudio.transforms.MFCC(
**mfcc_conf_torchaudio, melkwargs=mel_conf_torchaudio)
def mfcc_torchaudio():
return mfcc_extractor_torchaudio(waveform_tensor_torch).squeeze(0)
def test_mfcc_cpu_torchaudio(benchmark):
global waveform_tensor_torch, mfcc_extractor_torchaudio
mel_extractor_torchaudio = mfcc_extractor_torchaudio.to('cpu')
waveform_tensor_torch = waveform_tensor_torch.to('cpu')
feature_paddleaudio = benchmark(mfcc_torchaudio)
feature_librosa = librosa.feature.mfcc(waveform, **mel_conf)
np.testing.assert_array_almost_equal(
feature_librosa, feature_paddleaudio, decimal=3)
def test_mfcc_gpu_torchaudio(benchmark):
global waveform_tensor_torch, mfcc_extractor_torchaudio
mel_extractor_torchaudio = mfcc_extractor_torchaudio.to('cuda')
waveform_tensor_torch = waveform_tensor_torch.to('cuda')
feature_torchaudio = benchmark(mfcc_torchaudio)
feature_librosa = librosa.feature.mfcc(waveform, **mel_conf)
np.testing.assert_array_almost_equal(
feature_librosa, feature_torchaudio.cpu(), decimal=3)

Write Preview
Loading…
Cancel
Save