From 66d3bbf9affc3b62c4ba4b756404ca049451b8e5 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Wed, 6 Apr 2022 20:20:11 +0800 Subject: [PATCH 01/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1144d3ab5..a90498293 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ Via the easy-to-use, efficient, flexible and scalable implementation, our vision 2021.12.14: We would like to have an online courses to introduce basics and research of speech, as well as code practice with `paddlespeech`. Please pay attention to our [Calendar](https://www.paddlepaddle.org.cn/live). ---> - 👏🏻 2022.03.28: PaddleSpeech Server is available for Audio Classification, Automatic Speech Recognition and Text-to-Speech. -- 👏🏻 2022.03.28: PaddleSpeech CLI is available for Speaker Verfication. +- 👏🏻 2022.03.28: PaddleSpeech CLI is available for Speaker Verification. - 🤗 2021.12.14: Our PaddleSpeech [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) and [TTS](https://huggingface.co/spaces/KPatrick/PaddleSpeechTTS) Demos on Hugging Face Spaces are available! - 👏🏻 2021.12.10: PaddleSpeech CLI is available for Audio Classification, Automatic Speech Recognition, Speech Translation (English to Chinese) and Text-to-Speech. From d0b0c8f9b52aa998747b82a9318674314486f3af Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 12:08:38 +0800 Subject: [PATCH 02/23] Update mcd.py --- paddleaudio/paddleaudio/metric/mcd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddleaudio/paddleaudio/metric/mcd.py b/paddleaudio/paddleaudio/metric/mcd.py index 63a25fc23..c973c8522 100644 --- a/paddleaudio/paddleaudio/metric/mcd.py +++ b/paddleaudio/paddleaudio/metric/mcd.py @@ -13,7 +13,7 @@ # limitations under the License. from typing import Callable -import mcd.metrics_fast as mt +import mcd.metrics as mt import numpy as np from mcd import dtw From 079ac5caa01fd18e999adb5ac691be94dee12273 Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 12:11:29 +0800 Subject: [PATCH 03/23] Update README.md --- examples/esc50/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/esc50/README.md b/examples/esc50/README.md index 911a72ad7..9eab95d26 100644 --- a/examples/esc50/README.md +++ b/examples/esc50/README.md @@ -4,7 +4,7 @@ 对于声音分类任务，传统机器学习的一个常用做法是首先人工提取音频的时域和频域的多种特征并做特征选择、组合、变换等，然后基于SVM或决策树进行分类。而端到端的深度学习则通常利用深度网络如RNN，CNN等直接对声间波形(waveform)或时频特征(time-frequency)进行特征学习(representation learning)和分类预测。 -在IEEE ICASSP 2017 大会上，谷歌开放了一个大规模的音频数据集[Audioset](https://research.google.com/audioset/)。该数据集包含了 632 类的音频类别以及 2,084,320 条人工标记的每段 10 秒长度的声音剪辑片段（来源于YouTube视频）。目前该数据集已经有210万个已标注的视频数据，5800小时的音频数据，经过标记的声音样本的标签类别为527。 +在IEEE ICASSP 2017 大会上，谷歌开放了一个大规模的音频数据集[Audioset](https://research.google.com/audioset/)。该数据集包含了 632 类的音频类别以及 2,084,320 条人工标记的每段 **10 秒**长度的声音剪辑片段（来源于YouTube视频）。目前该数据集已经有 210万 个已标注的视频数据，5800 小时的音频数据，经过标记的声音样本的标签类别为 527。 `PANNs`([PANNs: Large-Scale Pretrained Audio Neural Networks for Audio Pattern Recognition](https://arxiv.org/pdf/1912.10211.pdf))是基于Audioset数据集训练的声音分类/识别的模型。经过预训练后，模型可以用于提取音频的embbedding。本示例将使用`PANNs`的预训练模型Finetune完成声音分类的任务。 @@ -12,14 +12,14 @@ ## 模型简介 PaddleAudio提供了PANNs的CNN14、CNN10和CNN6的预训练模型，可供用户选择使用： -- CNN14: 该模型主要包含12个卷积层和2个全连接层，模型参数的数量为79.6M，embbedding维度是2048。 -- CNN10: 该模型主要包含8个卷积层和2个全连接层，模型参数的数量为4.9M，embbedding维度是512。 -- CNN6: 该模型主要包含4个卷积层和2个全连接层，模型参数的数量为4.5M，embbedding维度是512。 +- CNN14: 该模型主要包含12个卷积层和2个全连接层，模型参数的数量为 79.6M，embbedding维度是 2048。 +- CNN10: 该模型主要包含8个卷积层和2个全连接层，模型参数的数量为 4.9M，embbedding维度是 512。 +- CNN6: 该模型主要包含4个卷积层和2个全连接层，模型参数的数量为 4.5M，embbedding维度是 512。 ## 数据集 -[ESC-50: Dataset for Environmental Sound Classification](https://github.com/karolpiczak/ESC-50) 是一个包含有 2000 个带标签的环境声音样本，音频样本采样率为 44,100Hz 的单通道音频文件，所有样本根据标签被划分为 50 个类别，每个类别有 40 个样本。 +[ESC-50: Dataset for Environmental Sound Classification](https://github.com/karolpiczak/ESC-50) 是一个包含有 2000 个带标签的时长为 **5 秒**的环境声音样本，音频样本采样率为 44,100Hz 的单通道音频文件，所有样本根据标签被划分为 50 个类别，每个类别有 40 个样本。 ## 模型指标 @@ -43,13 +43,13 @@ $ CUDA_VISIBLE_DEVICES=0 ./run.sh 1 conf/panns.yaml ``` 训练的参数可在 `conf/panns.yaml` 的 `training` 中配置，其中： -- `epochs`: 训练轮次，默认为50。 +- `epochs`: 训练轮次，默认为 50。 - `learning_rate`: Fine-tune的学习率；默认为5e-5。 -- `batch_size`: 批处理大小，请结合显存情况进行调整，若出现显存不足，请适当调低这一参数；默认为16。 +- `batch_size`: 批处理大小，请结合显存情况进行调整，若出现显存不足，请适当调低这一参数；默认为 16。 - `num_workers`: Dataloader获取数据的子进程数。默认为0，加载数据的流程在主进程执行。 - `checkpoint_dir`: 模型参数文件和optimizer参数文件的保存目录，默认为`./checkpoint`。 -- `save_freq`: 训练过程中的模型保存频率，默认为10。 -- `log_freq`: 训练过程中的信息打印频率，默认为10。 +- `save_freq`: 训练过程中的模型保存频率，默认为 10。 +- `log_freq`: 训练过程中的信息打印频率，默认为 10。 示例代码中使用的预训练模型为`CNN14`，如果想更换为其他预训练模型，可通过修改 `conf/panns.yaml` 的 `model` 中配置： ```yaml @@ -76,7 +76,7 @@ $ CUDA_VISIBLE_DEVICES=0 ./run.sh 2 conf/panns.yaml 训练的参数可在 `conf/panns.yaml` 的 `predicting` 中配置，其中： - `audio_file`: 指定预测的音频文件。 -- `top_k`: 预测显示的top k标签的得分，默认为1。 +- `top_k`: 预测显示的top k标签的得分，默认为 1。 - `checkpoint`: 模型参数checkpoint文件。 输出的预测结果如下： From d0ce1b3e6c51c24a092ee2984eb831575966190f Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 14:31:59 +0800 Subject: [PATCH 04/23] Delete mcd.py --- paddleaudio/paddleaudio/metric/mcd.py | 63 --------------------------- 1 file changed, 63 deletions(-) delete mode 100644 paddleaudio/paddleaudio/metric/mcd.py diff --git a/paddleaudio/paddleaudio/metric/mcd.py b/paddleaudio/paddleaudio/metric/mcd.py deleted file mode 100644 index c973c8522..000000000 --- a/paddleaudio/paddleaudio/metric/mcd.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Callable - -import mcd.metrics as mt -import numpy as np -from mcd import dtw - -__all__ = [ - 'mcd_distance', -] - - -def mcd_distance(xs: np.ndarray, - ys: np.ndarray, - cost_fn: Callable=mt.logSpecDbDist) -> float: - """Mel cepstral distortion (MCD), dtw distance. - - Dynamic Time Warping. - Uses dynamic programming to compute: - - Examples: - .. code-block:: python - - wps[i, j] = cost_fn(xs[i], ys[j]) + min( - wps[i-1, j ], // vertical / insertion / expansion - wps[i , j-1], // horizontal / deletion / compression - wps[i-1, j-1]) // diagonal / match - - dtw = sqrt(wps[-1, -1]) - - Cost Function: - Examples: - .. code-block:: python - - logSpecDbConst = 10.0 / math.log(10.0) * math.sqrt(2.0) - - def logSpecDbDist(x, y): - diff = x - y - return logSpecDbConst * math.sqrt(np.inner(diff, diff)) - - Args: - xs (np.ndarray): ref sequence, [T,D] - ys (np.ndarray): hyp sequence, [T,D] - cost_fn (Callable, optional): Cost function. Defaults to mt.logSpecDbDist. - - Returns: - float: dtw distance - """ - - min_cost, path = dtw.dtw(xs, ys, cost_fn) - return min_cost From 3a4bc2f3e9ecda589e82bba81a1c474fcdbcb5f4 Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 14:32:29 +0800 Subject: [PATCH 05/23] Update __init__.py --- paddleaudio/paddleaudio/metric/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/paddleaudio/paddleaudio/metric/__init__.py b/paddleaudio/paddleaudio/metric/__init__.py index 8e5ca9f75..d2b3a1360 100644 --- a/paddleaudio/paddleaudio/metric/__init__.py +++ b/paddleaudio/paddleaudio/metric/__init__.py @@ -14,4 +14,3 @@ from .dtw import dtw_distance from .eer import compute_eer from .eer import compute_minDCF -from .mcd import mcd_distance From 6c6262f4037be9a224a772a2ab9cb52429aaae32 Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 14:38:17 +0800 Subject: [PATCH 06/23] Update setup.py --- paddleaudio/setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/paddleaudio/setup.py b/paddleaudio/setup.py index e08b88a3b..c3550bf67 100644 --- a/paddleaudio/setup.py +++ b/paddleaudio/setup.py @@ -83,9 +83,8 @@ setuptools.setup( python_requires='>=3.6', install_requires=[ 'numpy >= 1.15.0', 'scipy >= 1.0.0', 'resampy >= 0.2.2', - 'soundfile >= 0.9.0', 'colorlog', 'dtaidistance == 2.3.1', 'mcd >= 0.4', - 'pathos' - ], + 'soundfile >= 0.9.0', 'colorlog', 'dtaidistance == 2.3.1', 'pathos' + ], extras_require={ 'test': [ 'nose', 'librosa==0.8.1', 'soundfile==0.10.3.post1', From c47c181e4edbc4e6204f450ff3e8651132cfc819 Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 14:40:35 +0800 Subject: [PATCH 07/23] Update setup.py --- paddleaudio/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddleaudio/setup.py b/paddleaudio/setup.py index c3550bf67..c92e5c73f 100644 --- a/paddleaudio/setup.py +++ b/paddleaudio/setup.py @@ -19,7 +19,7 @@ from setuptools.command.install import install from setuptools.command.test import test # set the version here -VERSION = '0.2.0' +VERSION = '0.2.1' # Inspired by the example at https://pytest.org/latest/goodpractises.html From a8f5990869e1aeec49fcae2d2168d7234f1c201c Mon Sep 17 00:00:00 2001 From: TianYuan Date: Thu, 7 Apr 2022 06:44:15 +0000 Subject: [PATCH 08/23] fix preprocess bug, test=tts --- paddlespeech/t2s/exps/fastspeech2/preprocess.py | 3 +++ paddlespeech/t2s/exps/speedyspeech/preprocess.py | 3 +++ paddlespeech/t2s/exps/tacotron2/preprocess.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/paddlespeech/t2s/exps/fastspeech2/preprocess.py b/paddlespeech/t2s/exps/fastspeech2/preprocess.py index 5bda75451..db1842b2e 100644 --- a/paddlespeech/t2s/exps/fastspeech2/preprocess.py +++ b/paddlespeech/t2s/exps/fastspeech2/preprocess.py @@ -86,6 +86,9 @@ def process_sentence(config: Dict[str, Any], logmel = mel_extractor.get_log_mel_fbank(wav) # change duration according to mel_length compare_duration_and_mel_length(sentences, utt_id, logmel) + # utt_id may be popped in compare_duration_and_mel_length + if utt_id not in sentences: + return None phones = sentences[utt_id][0] durations = sentences[utt_id][1] num_frames = logmel.shape[0] diff --git a/paddlespeech/t2s/exps/speedyspeech/preprocess.py b/paddlespeech/t2s/exps/speedyspeech/preprocess.py index 3f81c4e14..e833d1394 100644 --- a/paddlespeech/t2s/exps/speedyspeech/preprocess.py +++ b/paddlespeech/t2s/exps/speedyspeech/preprocess.py @@ -79,6 +79,9 @@ def process_sentence(config: Dict[str, Any], logmel = mel_extractor.get_log_mel_fbank(wav) # change duration according to mel_length compare_duration_and_mel_length(sentences, utt_id, logmel) + # utt_id may be popped in compare_duration_and_mel_length + if utt_id not in sentences: + return None labels = sentences[utt_id][0] # extract phone and duration phones = [] diff --git a/paddlespeech/t2s/exps/tacotron2/preprocess.py b/paddlespeech/t2s/exps/tacotron2/preprocess.py index 7f41089eb..14a0d7eae 100644 --- a/paddlespeech/t2s/exps/tacotron2/preprocess.py +++ b/paddlespeech/t2s/exps/tacotron2/preprocess.py @@ -82,6 +82,9 @@ def process_sentence(config: Dict[str, Any], logmel = mel_extractor.get_log_mel_fbank(wav) # change duration according to mel_length compare_duration_and_mel_length(sentences, utt_id, logmel) + # utt_id may be popped in compare_duration_and_mel_length + if utt_id not in sentences: + return None phones = sentences[utt_id][0] durations = sentences[utt_id][1] num_frames = logmel.shape[0] From d064c8196e174ddd91eb379a734bfad2d16b5453 Mon Sep 17 00:00:00 2001 From: xiongxinlei Date: Thu, 7 Apr 2022 15:21:49 +0800 Subject: [PATCH 09/23] update the speaker verification model, test=doc --- demos/speaker_verification/README.md | 223 +++++++++++++++--------- demos/speaker_verification/README_cn.md | 218 ++++++++++++++--------- demos/speaker_verification/run.sh | 4 +- docs/source/released_model.md | 2 +- examples/voxceleb/sv0/RESULT.md | 2 +- paddlespeech/cli/vector/infer.py | 4 +- 6 files changed, 287 insertions(+), 166 deletions(-) diff --git a/demos/speaker_verification/README.md b/demos/speaker_verification/README.md index 8739d402d..27413bd8d 100644 --- a/demos/speaker_verification/README.md +++ b/demos/speaker_verification/README.md @@ -30,6 +30,11 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav paddlespeech vector --task spk --input vec.job echo -e "demo2 85236145389.wav \n demo3 85236145389.wav" | paddlespeech vector --task spk + + paddlespeech vector --task score --input "./85236145389.wav ./123456789.wav" + + echo -e "demo4 85236145389.wav 85236145389.wav \n demo5 85236145389.wav 123456789.wav" > vec.job + paddlespeech vector --task score --input vec.job ``` Usage: @@ -38,6 +43,7 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav ``` Arguments: - `input`(required): Audio file to recognize. + - `task` (required): Specify `vector` task. Default `spk`。 - `model`: Model type of vector task. Default: `ecapatdnn_voxceleb12`. - `sample_rate`: Sample rate of the model. Default: `16000`. - `config`: Config of vector task. Use pretrained model when it is None. Default: `None`. @@ -47,45 +53,45 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav Output: ```bash - demo [ -5.749211 9.505463 -8.200284 -5.2075014 5.3940268 - -3.04878 1.611095 10.127234 -10.534177 -15.821609 - 1.2032688 -0.35080156 1.2629458 -12.643498 -2.5758228 - -11.343508 2.3385992 -8.719341 14.213509 15.404744 - -0.39327756 6.338786 2.688887 8.7104025 17.469526 - -8.77959 7.0576906 4.648855 -1.3089896 -23.294737 - 8.013747 13.891729 -9.926753 5.655307 -5.9422326 - -22.842539 0.6293588 -18.46266 -10.811862 9.8192625 - 3.0070958 3.8072643 -2.3861165 3.0821571 -14.739942 - 1.7594414 -0.6485091 4.485623 2.0207152 7.264915 - -6.40137 23.63524 2.9711294 -22.708025 9.93719 - 20.354511 -10.324688 -0.700492 -8.783211 -5.27593 - 15.999649 3.3004563 12.747926 15.429879 4.7849145 - 5.6699696 -2.3826702 10.605882 3.9112158 3.1500628 - 15.859915 -2.1832209 -23.908653 -6.4799504 -4.5365124 - -9.224193 14.568347 -10.568833 4.982321 -4.342062 - 0.0914714 12.645902 -5.74285 -3.2141201 -2.7173362 - -6.680575 0.4757669 -5.035051 -6.7964664 16.865469 - -11.54324 7.681869 0.44475392 9.708182 -8.932846 - 0.4123232 -4.361452 1.3948607 9.511665 0.11667654 - 2.9079323 6.049952 9.275183 -18.078873 6.2983274 - -0.7500531 -2.725033 -7.6027865 3.3404543 2.990815 - 4.010979 11.000591 -2.8873312 7.1352735 -16.79663 - 18.495346 -14.293832 7.89578 2.2714825 22.976387 - -4.875734 -3.0836344 -2.9999814 13.751918 6.448228 - -11.924197 2.171869 2.0423572 -6.173772 10.778437 - 25.77281 -4.9495463 14.57806 0.3044315 2.6132357 - -7.591999 -2.076944 9.025118 1.7834753 -3.1799617 - -4.9401326 23.465864 5.1685796 -9.018578 9.037825 - -4.4150195 6.859591 -12.274467 -0.88911164 5.186309 - -3.9988663 -13.638606 -9.925445 -0.06329413 -3.6709652 - -12.397416 -12.719869 -1.395601 2.1150916 5.7381287 - -4.4691963 -3.82819 -0.84233856 -1.1604277 -13.490127 - 8.731719 -20.778936 -11.495662 5.8033476 -4.752041 - 10.833007 -6.717991 4.504732 13.4244375 1.1306485 - 7.3435574 1.400918 14.704036 -9.501399 7.2315617 - -6.417456 1.3333273 11.872697 -0.30664724 8.8845 - 6.5569253 4.7948146 0.03662816 -8.704245 6.224871 - -3.2701402 -11.508579 ] + demo [ 1.4217498 5.626253 -5.342073 1.1773866 3.308055 + 1.756596 5.167894 10.80636 -3.8226728 -5.6141334 + 2.623845 -0.8072968 1.9635103 -7.3128724 0.01103897 + -9.723131 0.6619743 -6.976803 10.213478 7.494748 + 2.9105635 3.8949256 3.7999806 7.1061673 16.905321 + -7.1493764 8.733103 3.4230042 -4.831653 -11.403367 + 11.232214 7.1274667 -4.2828417 2.452362 -5.130748 + -18.177666 -2.6116815 -11.000337 -6.7314315 1.6564683 + 0.7618269 1.1253023 -2.083836 4.725744 -8.782597 + -3.539873 3.814236 5.1420674 2.162061 4.096431 + -6.4162116 12.747448 1.9429878 -15.152943 6.417416 + 16.097002 -9.716668 -1.9920526 -3.3649497 -1.871939 + 11.567354 3.69788 11.258265 7.442363 9.183411 + 4.5281515 -1.2417862 4.3959084 6.6727695 5.8898783 + 7.627124 -0.66919386 -11.889693 -9.208865 -7.4274073 + -3.7776625 6.917234 -9.848748 -2.0944717 -5.135116 + 0.49563864 9.317534 -5.9141874 -1.8098574 -0.11738578 + -7.169265 -1.0578263 -5.7216787 -5.1173844 16.137651 + -4.473626 7.6624317 -0.55381083 9.631587 -6.4704556 + -8.548508 4.3716145 -0.79702514 4.478997 -2.9758704 + 3.272176 2.8382776 5.134597 -9.190781 -0.5657382 + -4.8745747 2.3165567 -5.984303 -2.1798875 0.35541576 + -0.31784213 9.493548 2.1144536 4.358092 -12.089823 + 8.451689 -7.925461 4.6242585 4.4289427 18.692003 + -2.6204622 -5.149185 -0.35821092 8.488551 4.981496 + -9.32683 -2.2544234 6.6417594 1.2119585 10.977129 + 16.555033 3.3238444 9.551863 -1.6676947 -0.79539716 + -8.605674 -0.47356385 2.6741948 -5.359179 -2.6673796 + 0.66607 15.443222 4.740594 -3.4725387 11.592567 + -2.054497 1.7361217 -8.265324 -9.30447 5.4068313 + -1.5180256 -7.746615 -6.089606 0.07112726 -0.34904733 + -8.649895 -9.998958 -2.564841 -0.53999114 2.601808 + -0.31927416 -1.8815292 -2.07215 -3.4105783 -8.2998085 + 1.483641 -15.365992 -8.288208 3.8847756 -3.4876456 + 7.3629923 0.4657332 3.132599 12.438889 -1.8337058 + 4.532936 2.7264361 10.145339 -6.521951 2.897153 + -3.3925855 5.079156 7.759716 4.677565 5.8457737 + 2.402413 7.7071047 3.9711342 -6.390043 6.1268735 + -3.7760346 -11.118123 ] ``` - Python API @@ -97,56 +103,111 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav audio_emb = vector_executor( model='ecapatdnn_voxceleb12', sample_rate=16000, - config=None, + config=None, # Set `config` and `ckpt_path` to None to use pretrained model. ckpt_path=None, audio_file='./85236145389.wav', - force_yes=False, device=paddle.get_device()) print('Audio embedding Result: \n{}'.format(audio_emb)) + + test_emb = vector_executor( + model='ecapatdnn_voxceleb12', + sample_rate=16000, + config=None, # Set `config` and `ckpt_path` to None to use pretrained model. + ckpt_path=None, + audio_file='./123456789.wav', + device=paddle.get_device()) + print('Test embedding Result: \n{}'.format(test_emb)) + score = vector_executor.get_embeddings_score(audio_emb, test_emb) + print(f"Eembeddings Score: {score}") ``` - Output: + Output： + ```bash # Vector Result: - [ -5.749211 9.505463 -8.200284 -5.2075014 5.3940268 - -3.04878 1.611095 10.127234 -10.534177 -15.821609 - 1.2032688 -0.35080156 1.2629458 -12.643498 -2.5758228 - -11.343508 2.3385992 -8.719341 14.213509 15.404744 - -0.39327756 6.338786 2.688887 8.7104025 17.469526 - -8.77959 7.0576906 4.648855 -1.3089896 -23.294737 - 8.013747 13.891729 -9.926753 5.655307 -5.9422326 - -22.842539 0.6293588 -18.46266 -10.811862 9.8192625 - 3.0070958 3.8072643 -2.3861165 3.0821571 -14.739942 - 1.7594414 -0.6485091 4.485623 2.0207152 7.264915 - -6.40137 23.63524 2.9711294 -22.708025 9.93719 - 20.354511 -10.324688 -0.700492 -8.783211 -5.27593 - 15.999649 3.3004563 12.747926 15.429879 4.7849145 - 5.6699696 -2.3826702 10.605882 3.9112158 3.1500628 - 15.859915 -2.1832209 -23.908653 -6.4799504 -4.5365124 - -9.224193 14.568347 -10.568833 4.982321 -4.342062 - 0.0914714 12.645902 -5.74285 -3.2141201 -2.7173362 - -6.680575 0.4757669 -5.035051 -6.7964664 16.865469 - -11.54324 7.681869 0.44475392 9.708182 -8.932846 - 0.4123232 -4.361452 1.3948607 9.511665 0.11667654 - 2.9079323 6.049952 9.275183 -18.078873 6.2983274 - -0.7500531 -2.725033 -7.6027865 3.3404543 2.990815 - 4.010979 11.000591 -2.8873312 7.1352735 -16.79663 - 18.495346 -14.293832 7.89578 2.2714825 22.976387 - -4.875734 -3.0836344 -2.9999814 13.751918 6.448228 - -11.924197 2.171869 2.0423572 -6.173772 10.778437 - 25.77281 -4.9495463 14.57806 0.3044315 2.6132357 - -7.591999 -2.076944 9.025118 1.7834753 -3.1799617 - -4.9401326 23.465864 5.1685796 -9.018578 9.037825 - -4.4150195 6.859591 -12.274467 -0.88911164 5.186309 - -3.9988663 -13.638606 -9.925445 -0.06329413 -3.6709652 - -12.397416 -12.719869 -1.395601 2.1150916 5.7381287 - -4.4691963 -3.82819 -0.84233856 -1.1604277 -13.490127 - 8.731719 -20.778936 -11.495662 5.8033476 -4.752041 - 10.833007 -6.717991 4.504732 13.4244375 1.1306485 - 7.3435574 1.400918 14.704036 -9.501399 7.2315617 - -6.417456 1.3333273 11.872697 -0.30664724 8.8845 - 6.5569253 4.7948146 0.03662816 -8.704245 6.224871 - -3.2701402 -11.508579 ] + Audio embedding Result: + [ 1.4217498 5.626253 -5.342073 1.1773866 3.308055 + 1.756596 5.167894 10.80636 -3.8226728 -5.6141334 + 2.623845 -0.8072968 1.9635103 -7.3128724 0.01103897 + -9.723131 0.6619743 -6.976803 10.213478 7.494748 + 2.9105635 3.8949256 3.7999806 7.1061673 16.905321 + -7.1493764 8.733103 3.4230042 -4.831653 -11.403367 + 11.232214 7.1274667 -4.2828417 2.452362 -5.130748 + -18.177666 -2.6116815 -11.000337 -6.7314315 1.6564683 + 0.7618269 1.1253023 -2.083836 4.725744 -8.782597 + -3.539873 3.814236 5.1420674 2.162061 4.096431 + -6.4162116 12.747448 1.9429878 -15.152943 6.417416 + 16.097002 -9.716668 -1.9920526 -3.3649497 -1.871939 + 11.567354 3.69788 11.258265 7.442363 9.183411 + 4.5281515 -1.2417862 4.3959084 6.6727695 5.8898783 + 7.627124 -0.66919386 -11.889693 -9.208865 -7.4274073 + -3.7776625 6.917234 -9.848748 -2.0944717 -5.135116 + 0.49563864 9.317534 -5.9141874 -1.8098574 -0.11738578 + -7.169265 -1.0578263 -5.7216787 -5.1173844 16.137651 + -4.473626 7.6624317 -0.55381083 9.631587 -6.4704556 + -8.548508 4.3716145 -0.79702514 4.478997 -2.9758704 + 3.272176 2.8382776 5.134597 -9.190781 -0.5657382 + -4.8745747 2.3165567 -5.984303 -2.1798875 0.35541576 + -0.31784213 9.493548 2.1144536 4.358092 -12.089823 + 8.451689 -7.925461 4.6242585 4.4289427 18.692003 + -2.6204622 -5.149185 -0.35821092 8.488551 4.981496 + -9.32683 -2.2544234 6.6417594 1.2119585 10.977129 + 16.555033 3.3238444 9.551863 -1.6676947 -0.79539716 + -8.605674 -0.47356385 2.6741948 -5.359179 -2.6673796 + 0.66607 15.443222 4.740594 -3.4725387 11.592567 + -2.054497 1.7361217 -8.265324 -9.30447 5.4068313 + -1.5180256 -7.746615 -6.089606 0.07112726 -0.34904733 + -8.649895 -9.998958 -2.564841 -0.53999114 2.601808 + -0.31927416 -1.8815292 -2.07215 -3.4105783 -8.2998085 + 1.483641 -15.365992 -8.288208 3.8847756 -3.4876456 + 7.3629923 0.4657332 3.132599 12.438889 -1.8337058 + 4.532936 2.7264361 10.145339 -6.521951 2.897153 + -3.3925855 5.079156 7.759716 4.677565 5.8457737 + 2.402413 7.7071047 3.9711342 -6.390043 6.1268735 + -3.7760346 -11.118123 ] + # get the test embedding + Test embedding Result: + [ -1.902964 2.0690894 -8.034194 3.5472693 0.18089125 + 6.9085927 1.4097427 -1.9487704 -10.021278 -0.20755845 + -8.04332 4.344489 2.3200977 -14.306299 5.184692 + -11.55602 -3.8497238 0.6444722 1.2833948 2.6766639 + 0.5878921 0.7946299 1.7207596 2.5791872 14.998469 + -1.3385371 15.031221 -0.8006958 1.99287 -9.52007 + 2.435466 4.003221 -4.33817 -4.898601 -5.304714 + -18.033886 10.790787 -12.784645 -5.641755 2.9761686 + -10.566622 1.4839455 6.152458 -5.7195854 2.8603241 + 6.112133 8.489869 5.5958056 1.2836679 -1.2293907 + 0.89927405 7.0288725 -2.854029 -0.9782962 5.8255906 + 14.905906 -5.025907 0.7866458 -4.2444224 -16.354029 + 10.521315 0.9604709 -3.3257897 7.144871 -13.592733 + -8.568869 -1.7953678 0.26313916 10.916714 -6.9374123 + 1.857403 -6.2746415 2.8154466 -7.2338667 -2.293357 + -0.05452765 5.4287076 5.0849075 -6.690375 -1.6183422 + 3.654291 0.94352573 -9.200294 -5.4749465 -3.5235846 + 1.3420814 4.240421 -2.772944 -2.8451524 16.311104 + 4.2969875 -1.762936 -12.5758915 8.595198 -0.8835239 + -1.5708797 1.568961 1.1413603 3.5032008 -0.45251232 + -6.786333 16.89443 5.3366146 -8.789056 0.6355629 + 3.2579517 -3.328322 7.5969577 0.66025066 -6.550468 + -9.148656 2.020372 -0.4615173 1.1965656 -3.8764873 + 11.6562195 -6.0750933 12.182899 3.2218833 0.81969476 + 5.570001 -3.8459578 -7.205299 7.9262037 -7.6611166 + -5.249467 -2.2671914 7.2658715 -13.298164 4.821147 + -2.7263982 11.691089 -3.8918593 -2.838112 -1.0336838 + -3.8034165 2.8536487 -5.60398 -1.1972581 1.3455094 + -3.4903061 2.2408795 5.5010734 -3.970756 11.99696 + -7.8858757 0.43160373 -5.5059714 4.3426995 16.322706 + 11.635366 0.72157705 -9.245714 -3.91465 -4.449838 + -1.5716927 7.713747 -2.2430465 -6.198303 -13.481864 + 2.8156567 -5.7812386 5.1456156 2.7289324 -14.505571 + 13.270688 3.448231 -7.0659585 4.5886116 -4.466099 + -0.296428 -11.463529 -2.6076477 14.110243 -6.9725137 + -1.9962958 2.7119343 19.391657 0.01961198 14.607133 + -1.6695905 -4.391516 1.3131028 -6.670972 -5.888604 + 12.0612335 5.9285784 3.3715196 1.492534 10.723728 + -0.95514804 -12.085431 ] + # get the score between enroll and test + Eembeddings Score: 0.4292638301849365 ``` ### 4.Pretrained Models diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md index fe8949b3c..068802fd3 100644 --- a/demos/speaker_verification/README_cn.md +++ b/demos/speaker_verification/README_cn.md @@ -29,6 +29,11 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav paddlespeech vector --task spk --input vec.job echo -e "demo2 85236145389.wav \n demo3 85236145389.wav" | paddlespeech vector --task spk + + paddlespeech vector --task score --input "./85236145389.wav ./123456789.wav" + + echo -e "demo4 85236145389.wav 85236145389.wav \n demo5 85236145389.wav 123456789.wav" > vec.job + paddlespeech vector --task score --input vec.job ``` 使用方法： @@ -37,6 +42,7 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav ``` 参数： - `input`(必须输入)：用于识别的音频文件。 + - `task` (必须输入): 用于指定 `vector` 处理的具体任务，默认是 `spk`。 - `model`：声纹任务的模型，默认值：`ecapatdnn_voxceleb12`。 - `sample_rate`：音频采样率，默认值：`16000`。 - `config`：声纹任务的参数文件，若不设置则使用预训练模型中的默认配置，默认值：`None`。 @@ -45,45 +51,45 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav 输出： ```bash - demo [ -5.749211 9.505463 -8.200284 -5.2075014 5.3940268 - -3.04878 1.611095 10.127234 -10.534177 -15.821609 - 1.2032688 -0.35080156 1.2629458 -12.643498 -2.5758228 - -11.343508 2.3385992 -8.719341 14.213509 15.404744 - -0.39327756 6.338786 2.688887 8.7104025 17.469526 - -8.77959 7.0576906 4.648855 -1.3089896 -23.294737 - 8.013747 13.891729 -9.926753 5.655307 -5.9422326 - -22.842539 0.6293588 -18.46266 -10.811862 9.8192625 - 3.0070958 3.8072643 -2.3861165 3.0821571 -14.739942 - 1.7594414 -0.6485091 4.485623 2.0207152 7.264915 - -6.40137 23.63524 2.9711294 -22.708025 9.93719 - 20.354511 -10.324688 -0.700492 -8.783211 -5.27593 - 15.999649 3.3004563 12.747926 15.429879 4.7849145 - 5.6699696 -2.3826702 10.605882 3.9112158 3.1500628 - 15.859915 -2.1832209 -23.908653 -6.4799504 -4.5365124 - -9.224193 14.568347 -10.568833 4.982321 -4.342062 - 0.0914714 12.645902 -5.74285 -3.2141201 -2.7173362 - -6.680575 0.4757669 -5.035051 -6.7964664 16.865469 - -11.54324 7.681869 0.44475392 9.708182 -8.932846 - 0.4123232 -4.361452 1.3948607 9.511665 0.11667654 - 2.9079323 6.049952 9.275183 -18.078873 6.2983274 - -0.7500531 -2.725033 -7.6027865 3.3404543 2.990815 - 4.010979 11.000591 -2.8873312 7.1352735 -16.79663 - 18.495346 -14.293832 7.89578 2.2714825 22.976387 - -4.875734 -3.0836344 -2.9999814 13.751918 6.448228 - -11.924197 2.171869 2.0423572 -6.173772 10.778437 - 25.77281 -4.9495463 14.57806 0.3044315 2.6132357 - -7.591999 -2.076944 9.025118 1.7834753 -3.1799617 - -4.9401326 23.465864 5.1685796 -9.018578 9.037825 - -4.4150195 6.859591 -12.274467 -0.88911164 5.186309 - -3.9988663 -13.638606 -9.925445 -0.06329413 -3.6709652 - -12.397416 -12.719869 -1.395601 2.1150916 5.7381287 - -4.4691963 -3.82819 -0.84233856 -1.1604277 -13.490127 - 8.731719 -20.778936 -11.495662 5.8033476 -4.752041 - 10.833007 -6.717991 4.504732 13.4244375 1.1306485 - 7.3435574 1.400918 14.704036 -9.501399 7.2315617 - -6.417456 1.3333273 11.872697 -0.30664724 8.8845 - 6.5569253 4.7948146 0.03662816 -8.704245 6.224871 - -3.2701402 -11.508579 ] + demo [ 1.4217498 5.626253 -5.342073 1.1773866 3.308055 + 1.756596 5.167894 10.80636 -3.8226728 -5.6141334 + 2.623845 -0.8072968 1.9635103 -7.3128724 0.01103897 + -9.723131 0.6619743 -6.976803 10.213478 7.494748 + 2.9105635 3.8949256 3.7999806 7.1061673 16.905321 + -7.1493764 8.733103 3.4230042 -4.831653 -11.403367 + 11.232214 7.1274667 -4.2828417 2.452362 -5.130748 + -18.177666 -2.6116815 -11.000337 -6.7314315 1.6564683 + 0.7618269 1.1253023 -2.083836 4.725744 -8.782597 + -3.539873 3.814236 5.1420674 2.162061 4.096431 + -6.4162116 12.747448 1.9429878 -15.152943 6.417416 + 16.097002 -9.716668 -1.9920526 -3.3649497 -1.871939 + 11.567354 3.69788 11.258265 7.442363 9.183411 + 4.5281515 -1.2417862 4.3959084 6.6727695 5.8898783 + 7.627124 -0.66919386 -11.889693 -9.208865 -7.4274073 + -3.7776625 6.917234 -9.848748 -2.0944717 -5.135116 + 0.49563864 9.317534 -5.9141874 -1.8098574 -0.11738578 + -7.169265 -1.0578263 -5.7216787 -5.1173844 16.137651 + -4.473626 7.6624317 -0.55381083 9.631587 -6.4704556 + -8.548508 4.3716145 -0.79702514 4.478997 -2.9758704 + 3.272176 2.8382776 5.134597 -9.190781 -0.5657382 + -4.8745747 2.3165567 -5.984303 -2.1798875 0.35541576 + -0.31784213 9.493548 2.1144536 4.358092 -12.089823 + 8.451689 -7.925461 4.6242585 4.4289427 18.692003 + -2.6204622 -5.149185 -0.35821092 8.488551 4.981496 + -9.32683 -2.2544234 6.6417594 1.2119585 10.977129 + 16.555033 3.3238444 9.551863 -1.6676947 -0.79539716 + -8.605674 -0.47356385 2.6741948 -5.359179 -2.6673796 + 0.66607 15.443222 4.740594 -3.4725387 11.592567 + -2.054497 1.7361217 -8.265324 -9.30447 5.4068313 + -1.5180256 -7.746615 -6.089606 0.07112726 -0.34904733 + -8.649895 -9.998958 -2.564841 -0.53999114 2.601808 + -0.31927416 -1.8815292 -2.07215 -3.4105783 -8.2998085 + 1.483641 -15.365992 -8.288208 3.8847756 -3.4876456 + 7.3629923 0.4657332 3.132599 12.438889 -1.8337058 + 4.532936 2.7264361 10.145339 -6.521951 2.897153 + -3.3925855 5.079156 7.759716 4.677565 5.8457737 + 2.402413 7.7071047 3.9711342 -6.390043 6.1268735 + -3.7760346 -11.118123 ] ``` - Python API @@ -98,53 +104,107 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav config=None, # Set `config` and `ckpt_path` to None to use pretrained model. ckpt_path=None, audio_file='./85236145389.wav', - force_yes=False, device=paddle.get_device()) print('Audio embedding Result: \n{}'.format(audio_emb)) + + test_emb = vector_executor( + model='ecapatdnn_voxceleb12', + sample_rate=16000, + config=None, # Set `config` and `ckpt_path` to None to use pretrained model. + ckpt_path=None, + audio_file='./123456789.wav', + device=paddle.get_device()) + print('Test embedding Result: \n{}'.format(test_emb)) + score = vector_executor.get_embeddings_score(audio_emb, test_emb) + print(f"Eembeddings Score: {score}") ``` 输出： ```bash # Vector Result: - [ -5.749211 9.505463 -8.200284 -5.2075014 5.3940268 - -3.04878 1.611095 10.127234 -10.534177 -15.821609 - 1.2032688 -0.35080156 1.2629458 -12.643498 -2.5758228 - -11.343508 2.3385992 -8.719341 14.213509 15.404744 - -0.39327756 6.338786 2.688887 8.7104025 17.469526 - -8.77959 7.0576906 4.648855 -1.3089896 -23.294737 - 8.013747 13.891729 -9.926753 5.655307 -5.9422326 - -22.842539 0.6293588 -18.46266 -10.811862 9.8192625 - 3.0070958 3.8072643 -2.3861165 3.0821571 -14.739942 - 1.7594414 -0.6485091 4.485623 2.0207152 7.264915 - -6.40137 23.63524 2.9711294 -22.708025 9.93719 - 20.354511 -10.324688 -0.700492 -8.783211 -5.27593 - 15.999649 3.3004563 12.747926 15.429879 4.7849145 - 5.6699696 -2.3826702 10.605882 3.9112158 3.1500628 - 15.859915 -2.1832209 -23.908653 -6.4799504 -4.5365124 - -9.224193 14.568347 -10.568833 4.982321 -4.342062 - 0.0914714 12.645902 -5.74285 -3.2141201 -2.7173362 - -6.680575 0.4757669 -5.035051 -6.7964664 16.865469 - -11.54324 7.681869 0.44475392 9.708182 -8.932846 - 0.4123232 -4.361452 1.3948607 9.511665 0.11667654 - 2.9079323 6.049952 9.275183 -18.078873 6.2983274 - -0.7500531 -2.725033 -7.6027865 3.3404543 2.990815 - 4.010979 11.000591 -2.8873312 7.1352735 -16.79663 - 18.495346 -14.293832 7.89578 2.2714825 22.976387 - -4.875734 -3.0836344 -2.9999814 13.751918 6.448228 - -11.924197 2.171869 2.0423572 -6.173772 10.778437 - 25.77281 -4.9495463 14.57806 0.3044315 2.6132357 - -7.591999 -2.076944 9.025118 1.7834753 -3.1799617 - -4.9401326 23.465864 5.1685796 -9.018578 9.037825 - -4.4150195 6.859591 -12.274467 -0.88911164 5.186309 - -3.9988663 -13.638606 -9.925445 -0.06329413 -3.6709652 - -12.397416 -12.719869 -1.395601 2.1150916 5.7381287 - -4.4691963 -3.82819 -0.84233856 -1.1604277 -13.490127 - 8.731719 -20.778936 -11.495662 5.8033476 -4.752041 - 10.833007 -6.717991 4.504732 13.4244375 1.1306485 - 7.3435574 1.400918 14.704036 -9.501399 7.2315617 - -6.417456 1.3333273 11.872697 -0.30664724 8.8845 - 6.5569253 4.7948146 0.03662816 -8.704245 6.224871 - -3.2701402 -11.508579 ] + Audio embedding Result: + [ 1.4217498 5.626253 -5.342073 1.1773866 3.308055 + 1.756596 5.167894 10.80636 -3.8226728 -5.6141334 + 2.623845 -0.8072968 1.9635103 -7.3128724 0.01103897 + -9.723131 0.6619743 -6.976803 10.213478 7.494748 + 2.9105635 3.8949256 3.7999806 7.1061673 16.905321 + -7.1493764 8.733103 3.4230042 -4.831653 -11.403367 + 11.232214 7.1274667 -4.2828417 2.452362 -5.130748 + -18.177666 -2.6116815 -11.000337 -6.7314315 1.6564683 + 0.7618269 1.1253023 -2.083836 4.725744 -8.782597 + -3.539873 3.814236 5.1420674 2.162061 4.096431 + -6.4162116 12.747448 1.9429878 -15.152943 6.417416 + 16.097002 -9.716668 -1.9920526 -3.3649497 -1.871939 + 11.567354 3.69788 11.258265 7.442363 9.183411 + 4.5281515 -1.2417862 4.3959084 6.6727695 5.8898783 + 7.627124 -0.66919386 -11.889693 -9.208865 -7.4274073 + -3.7776625 6.917234 -9.848748 -2.0944717 -5.135116 + 0.49563864 9.317534 -5.9141874 -1.8098574 -0.11738578 + -7.169265 -1.0578263 -5.7216787 -5.1173844 16.137651 + -4.473626 7.6624317 -0.55381083 9.631587 -6.4704556 + -8.548508 4.3716145 -0.79702514 4.478997 -2.9758704 + 3.272176 2.8382776 5.134597 -9.190781 -0.5657382 + -4.8745747 2.3165567 -5.984303 -2.1798875 0.35541576 + -0.31784213 9.493548 2.1144536 4.358092 -12.089823 + 8.451689 -7.925461 4.6242585 4.4289427 18.692003 + -2.6204622 -5.149185 -0.35821092 8.488551 4.981496 + -9.32683 -2.2544234 6.6417594 1.2119585 10.977129 + 16.555033 3.3238444 9.551863 -1.6676947 -0.79539716 + -8.605674 -0.47356385 2.6741948 -5.359179 -2.6673796 + 0.66607 15.443222 4.740594 -3.4725387 11.592567 + -2.054497 1.7361217 -8.265324 -9.30447 5.4068313 + -1.5180256 -7.746615 -6.089606 0.07112726 -0.34904733 + -8.649895 -9.998958 -2.564841 -0.53999114 2.601808 + -0.31927416 -1.8815292 -2.07215 -3.4105783 -8.2998085 + 1.483641 -15.365992 -8.288208 3.8847756 -3.4876456 + 7.3629923 0.4657332 3.132599 12.438889 -1.8337058 + 4.532936 2.7264361 10.145339 -6.521951 2.897153 + -3.3925855 5.079156 7.759716 4.677565 5.8457737 + 2.402413 7.7071047 3.9711342 -6.390043 6.1268735 + -3.7760346 -11.118123 ] + # get the test embedding + Test embedding Result: + [ -1.902964 2.0690894 -8.034194 3.5472693 0.18089125 + 6.9085927 1.4097427 -1.9487704 -10.021278 -0.20755845 + -8.04332 4.344489 2.3200977 -14.306299 5.184692 + -11.55602 -3.8497238 0.6444722 1.2833948 2.6766639 + 0.5878921 0.7946299 1.7207596 2.5791872 14.998469 + -1.3385371 15.031221 -0.8006958 1.99287 -9.52007 + 2.435466 4.003221 -4.33817 -4.898601 -5.304714 + -18.033886 10.790787 -12.784645 -5.641755 2.9761686 + -10.566622 1.4839455 6.152458 -5.7195854 2.8603241 + 6.112133 8.489869 5.5958056 1.2836679 -1.2293907 + 0.89927405 7.0288725 -2.854029 -0.9782962 5.8255906 + 14.905906 -5.025907 0.7866458 -4.2444224 -16.354029 + 10.521315 0.9604709 -3.3257897 7.144871 -13.592733 + -8.568869 -1.7953678 0.26313916 10.916714 -6.9374123 + 1.857403 -6.2746415 2.8154466 -7.2338667 -2.293357 + -0.05452765 5.4287076 5.0849075 -6.690375 -1.6183422 + 3.654291 0.94352573 -9.200294 -5.4749465 -3.5235846 + 1.3420814 4.240421 -2.772944 -2.8451524 16.311104 + 4.2969875 -1.762936 -12.5758915 8.595198 -0.8835239 + -1.5708797 1.568961 1.1413603 3.5032008 -0.45251232 + -6.786333 16.89443 5.3366146 -8.789056 0.6355629 + 3.2579517 -3.328322 7.5969577 0.66025066 -6.550468 + -9.148656 2.020372 -0.4615173 1.1965656 -3.8764873 + 11.6562195 -6.0750933 12.182899 3.2218833 0.81969476 + 5.570001 -3.8459578 -7.205299 7.9262037 -7.6611166 + -5.249467 -2.2671914 7.2658715 -13.298164 4.821147 + -2.7263982 11.691089 -3.8918593 -2.838112 -1.0336838 + -3.8034165 2.8536487 -5.60398 -1.1972581 1.3455094 + -3.4903061 2.2408795 5.5010734 -3.970756 11.99696 + -7.8858757 0.43160373 -5.5059714 4.3426995 16.322706 + 11.635366 0.72157705 -9.245714 -3.91465 -4.449838 + -1.5716927 7.713747 -2.2430465 -6.198303 -13.481864 + 2.8156567 -5.7812386 5.1456156 2.7289324 -14.505571 + 13.270688 3.448231 -7.0659585 4.5886116 -4.466099 + -0.296428 -11.463529 -2.6076477 14.110243 -6.9725137 + -1.9962958 2.7119343 19.391657 0.01961198 14.607133 + -1.6695905 -4.391516 1.3131028 -6.670972 -5.888604 + 12.0612335 5.9285784 3.3715196 1.492534 10.723728 + -0.95514804 -12.085431 ] + # get the score between enroll and test + Eembeddings Score: 0.4292638301849365 ``` ### 4.预训练模型 diff --git a/demos/speaker_verification/run.sh b/demos/speaker_verification/run.sh index 856886d33..23ca8eb42 100644 --- a/demos/speaker_verification/run.sh +++ b/demos/speaker_verification/run.sh @@ -2,5 +2,5 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav -# asr -paddlespeech vector --task spk --input ./85236145389.wav \ No newline at end of file +# vector +paddlespeech vector --task spk --input ./85236145389.wav diff --git a/docs/source/released_model.md b/docs/source/released_model.md index 9a423e03e..48ceaf843 100644 --- a/docs/source/released_model.md +++ b/docs/source/released_model.md @@ -80,7 +80,7 @@ PANN | ESC-50 |[pann-esc50](../../examples/esc50/cls0)|[esc50_cnn6.tar.gz](https Model Type | Dataset| Example Link | Pretrained Models | Static Models :-------------:| :------------:| :-----: | :-----: | :-----: -PANN | VoxCeleb| [voxceleb_ecapatdnn](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/voxceleb/sv0) | [ecapatdnn.tar.gz](https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_1.tar.gz) | - +PANN | VoxCeleb| [voxceleb_ecapatdnn](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/voxceleb/sv0) | [ecapatdnn.tar.gz](https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_2_0.tar.gz) | - ## Punctuation Restoration Models Model Type | Dataset| Example Link | Pretrained Models diff --git a/examples/voxceleb/sv0/RESULT.md b/examples/voxceleb/sv0/RESULT.md index c37bcecef..3a3f67d09 100644 --- a/examples/voxceleb/sv0/RESULT.md +++ b/examples/voxceleb/sv0/RESULT.md @@ -4,4 +4,4 @@ | Model | Number of Params | Release | Config | dim | Test set | Cosine | Cosine + S-Norm | | --- | --- | --- | --- | --- | --- | --- | ---- | -| ECAPA-TDNN | 85M | 0.1.1 | conf/ecapa_tdnn.yaml |192 | test | 1.15 | 1.06 | +| ECAPA-TDNN | 85M | 0.2.0 | conf/ecapa_tdnn.yaml |192 | test | 1.02 | 0.95 | diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py index 175a9723e..52f4f207a 100644 --- a/paddlespeech/cli/vector/infer.py +++ b/paddlespeech/cli/vector/infer.py @@ -42,9 +42,9 @@ pretrained_models = { # "paddlespeech vector --task spk --model ecapatdnn_voxceleb12-16k --sr 16000 --input ./input.wav" "ecapatdnn_voxceleb12-16k": { 'url': - 'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_1_1.tar.gz', + 'https://paddlespeech.bj.bcebos.com/vector/voxceleb/sv0_ecapa_tdnn_voxceleb12_ckpt_0_2_0.tar.gz', 'md5': - 'a1c0dba7d4de997187786ff517d5b4ec', + 'cc33023c54ab346cd318408f43fcaf95', 'cfg_path': 'conf/model.yaml', # the yaml config path 'ckpt_path': From 2a095db22ef842344b2e1ffe170eb29aa0757a64 Mon Sep 17 00:00:00 2001 From: xiongxinlei Date: Thu, 7 Apr 2022 15:27:32 +0800 Subject: [PATCH 10/23] remove unuse content in readme, test=doc --- demos/speaker_verification/README.md | 59 ------------------------- demos/speaker_verification/README_cn.md | 54 ---------------------- 2 files changed, 113 deletions(-) diff --git a/demos/speaker_verification/README.md b/demos/speaker_verification/README.md index 27413bd8d..71fbbfe04 100644 --- a/demos/speaker_verification/README.md +++ b/demos/speaker_verification/README.md @@ -30,11 +30,6 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav paddlespeech vector --task spk --input vec.job echo -e "demo2 85236145389.wav \n demo3 85236145389.wav" | paddlespeech vector --task spk - - paddlespeech vector --task score --input "./85236145389.wav ./123456789.wav" - - echo -e "demo4 85236145389.wav 85236145389.wav \n demo5 85236145389.wav 123456789.wav" > vec.job - paddlespeech vector --task score --input vec.job ``` Usage: @@ -108,17 +103,6 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav audio_file='./85236145389.wav', device=paddle.get_device()) print('Audio embedding Result: \n{}'.format(audio_emb)) - - test_emb = vector_executor( - model='ecapatdnn_voxceleb12', - sample_rate=16000, - config=None, # Set `config` and `ckpt_path` to None to use pretrained model. - ckpt_path=None, - audio_file='./123456789.wav', - device=paddle.get_device()) - print('Test embedding Result: \n{}'.format(test_emb)) - score = vector_executor.get_embeddings_score(audio_emb, test_emb) - print(f"Eembeddings Score: {score}") ``` Output： @@ -165,49 +149,6 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav -3.3925855 5.079156 7.759716 4.677565 5.8457737 2.402413 7.7071047 3.9711342 -6.390043 6.1268735 -3.7760346 -11.118123 ] - # get the test embedding - Test embedding Result: - [ -1.902964 2.0690894 -8.034194 3.5472693 0.18089125 - 6.9085927 1.4097427 -1.9487704 -10.021278 -0.20755845 - -8.04332 4.344489 2.3200977 -14.306299 5.184692 - -11.55602 -3.8497238 0.6444722 1.2833948 2.6766639 - 0.5878921 0.7946299 1.7207596 2.5791872 14.998469 - -1.3385371 15.031221 -0.8006958 1.99287 -9.52007 - 2.435466 4.003221 -4.33817 -4.898601 -5.304714 - -18.033886 10.790787 -12.784645 -5.641755 2.9761686 - -10.566622 1.4839455 6.152458 -5.7195854 2.8603241 - 6.112133 8.489869 5.5958056 1.2836679 -1.2293907 - 0.89927405 7.0288725 -2.854029 -0.9782962 5.8255906 - 14.905906 -5.025907 0.7866458 -4.2444224 -16.354029 - 10.521315 0.9604709 -3.3257897 7.144871 -13.592733 - -8.568869 -1.7953678 0.26313916 10.916714 -6.9374123 - 1.857403 -6.2746415 2.8154466 -7.2338667 -2.293357 - -0.05452765 5.4287076 5.0849075 -6.690375 -1.6183422 - 3.654291 0.94352573 -9.200294 -5.4749465 -3.5235846 - 1.3420814 4.240421 -2.772944 -2.8451524 16.311104 - 4.2969875 -1.762936 -12.5758915 8.595198 -0.8835239 - -1.5708797 1.568961 1.1413603 3.5032008 -0.45251232 - -6.786333 16.89443 5.3366146 -8.789056 0.6355629 - 3.2579517 -3.328322 7.5969577 0.66025066 -6.550468 - -9.148656 2.020372 -0.4615173 1.1965656 -3.8764873 - 11.6562195 -6.0750933 12.182899 3.2218833 0.81969476 - 5.570001 -3.8459578 -7.205299 7.9262037 -7.6611166 - -5.249467 -2.2671914 7.2658715 -13.298164 4.821147 - -2.7263982 11.691089 -3.8918593 -2.838112 -1.0336838 - -3.8034165 2.8536487 -5.60398 -1.1972581 1.3455094 - -3.4903061 2.2408795 5.5010734 -3.970756 11.99696 - -7.8858757 0.43160373 -5.5059714 4.3426995 16.322706 - 11.635366 0.72157705 -9.245714 -3.91465 -4.449838 - -1.5716927 7.713747 -2.2430465 -6.198303 -13.481864 - 2.8156567 -5.7812386 5.1456156 2.7289324 -14.505571 - 13.270688 3.448231 -7.0659585 4.5886116 -4.466099 - -0.296428 -11.463529 -2.6076477 14.110243 -6.9725137 - -1.9962958 2.7119343 19.391657 0.01961198 14.607133 - -1.6695905 -4.391516 1.3131028 -6.670972 -5.888604 - 12.0612335 5.9285784 3.3715196 1.492534 10.723728 - -0.95514804 -12.085431 ] - # get the score between enroll and test - Eembeddings Score: 0.4292638301849365 ``` ### 4.Pretrained Models diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md index 068802fd3..183be9425 100644 --- a/demos/speaker_verification/README_cn.md +++ b/demos/speaker_verification/README_cn.md @@ -106,17 +106,6 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav audio_file='./85236145389.wav', device=paddle.get_device()) print('Audio embedding Result: \n{}'.format(audio_emb)) - - test_emb = vector_executor( - model='ecapatdnn_voxceleb12', - sample_rate=16000, - config=None, # Set `config` and `ckpt_path` to None to use pretrained model. - ckpt_path=None, - audio_file='./123456789.wav', - device=paddle.get_device()) - print('Test embedding Result: \n{}'.format(test_emb)) - score = vector_executor.get_embeddings_score(audio_emb, test_emb) - print(f"Eembeddings Score: {score}") ``` 输出： @@ -162,49 +151,6 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav -3.3925855 5.079156 7.759716 4.677565 5.8457737 2.402413 7.7071047 3.9711342 -6.390043 6.1268735 -3.7760346 -11.118123 ] - # get the test embedding - Test embedding Result: - [ -1.902964 2.0690894 -8.034194 3.5472693 0.18089125 - 6.9085927 1.4097427 -1.9487704 -10.021278 -0.20755845 - -8.04332 4.344489 2.3200977 -14.306299 5.184692 - -11.55602 -3.8497238 0.6444722 1.2833948 2.6766639 - 0.5878921 0.7946299 1.7207596 2.5791872 14.998469 - -1.3385371 15.031221 -0.8006958 1.99287 -9.52007 - 2.435466 4.003221 -4.33817 -4.898601 -5.304714 - -18.033886 10.790787 -12.784645 -5.641755 2.9761686 - -10.566622 1.4839455 6.152458 -5.7195854 2.8603241 - 6.112133 8.489869 5.5958056 1.2836679 -1.2293907 - 0.89927405 7.0288725 -2.854029 -0.9782962 5.8255906 - 14.905906 -5.025907 0.7866458 -4.2444224 -16.354029 - 10.521315 0.9604709 -3.3257897 7.144871 -13.592733 - -8.568869 -1.7953678 0.26313916 10.916714 -6.9374123 - 1.857403 -6.2746415 2.8154466 -7.2338667 -2.293357 - -0.05452765 5.4287076 5.0849075 -6.690375 -1.6183422 - 3.654291 0.94352573 -9.200294 -5.4749465 -3.5235846 - 1.3420814 4.240421 -2.772944 -2.8451524 16.311104 - 4.2969875 -1.762936 -12.5758915 8.595198 -0.8835239 - -1.5708797 1.568961 1.1413603 3.5032008 -0.45251232 - -6.786333 16.89443 5.3366146 -8.789056 0.6355629 - 3.2579517 -3.328322 7.5969577 0.66025066 -6.550468 - -9.148656 2.020372 -0.4615173 1.1965656 -3.8764873 - 11.6562195 -6.0750933 12.182899 3.2218833 0.81969476 - 5.570001 -3.8459578 -7.205299 7.9262037 -7.6611166 - -5.249467 -2.2671914 7.2658715 -13.298164 4.821147 - -2.7263982 11.691089 -3.8918593 -2.838112 -1.0336838 - -3.8034165 2.8536487 -5.60398 -1.1972581 1.3455094 - -3.4903061 2.2408795 5.5010734 -3.970756 11.99696 - -7.8858757 0.43160373 -5.5059714 4.3426995 16.322706 - 11.635366 0.72157705 -9.245714 -3.91465 -4.449838 - -1.5716927 7.713747 -2.2430465 -6.198303 -13.481864 - 2.8156567 -5.7812386 5.1456156 2.7289324 -14.505571 - 13.270688 3.448231 -7.0659585 4.5886116 -4.466099 - -0.296428 -11.463529 -2.6076477 14.110243 -6.9725137 - -1.9962958 2.7119343 19.391657 0.01961198 14.607133 - -1.6695905 -4.391516 1.3131028 -6.670972 -5.888604 - 12.0612335 5.9285784 3.3715196 1.492534 10.723728 - -0.95514804 -12.085431 ] - # get the score between enroll and test - Eembeddings Score: 0.4292638301849365 ``` ### 4.预训练模型 From 85e4e70605b7e76461f08917ce67c25abd8ad232 Mon Sep 17 00:00:00 2001 From: xiongxinlei Date: Thu, 7 Apr 2022 15:29:28 +0800 Subject: [PATCH 11/23] remove score content, test=doc --- demos/speaker_verification/README_cn.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/demos/speaker_verification/README_cn.md b/demos/speaker_verification/README_cn.md index 183be9425..8b542b203 100644 --- a/demos/speaker_verification/README_cn.md +++ b/demos/speaker_verification/README_cn.md @@ -29,11 +29,6 @@ wget -c https://paddlespeech.bj.bcebos.com/vector/audio/85236145389.wav paddlespeech vector --task spk --input vec.job echo -e "demo2 85236145389.wav \n demo3 85236145389.wav" | paddlespeech vector --task spk - - paddlespeech vector --task score --input "./85236145389.wav ./123456789.wav" - - echo -e "demo4 85236145389.wav 85236145389.wav \n demo5 85236145389.wav 123456789.wav" > vec.job - paddlespeech vector --task score --input vec.job ``` 使用方法： From faf21f033f28f0c094f4723251d6fe9dc5a229b4 Mon Sep 17 00:00:00 2001 From: huangyuxin Date: Thu, 7 Apr 2022 08:48:18 +0000 Subject: [PATCH 12/23] add duration limitation for asr --- paddlespeech/cli/asr/infer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/paddlespeech/cli/asr/infer.py b/paddlespeech/cli/asr/infer.py index 1fb4be434..8eea7ff5a 100644 --- a/paddlespeech/cli/asr/infer.py +++ b/paddlespeech/cli/asr/infer.py @@ -426,6 +426,11 @@ class ASRExecutor(BaseExecutor): try: audio, audio_sample_rate = soundfile.read( audio_file, dtype="int16", always_2d=True) + audio_duration = audio.shape[0] / audio_sample_rate + max_duration = 30.0 + if audio_duration >= max_duration: + logger.error("Please input audio file less then 30 seconds.\n") + return except Exception as e: logger.exception(e) logger.error( From 06f4169df8ad632a87ad957cc8d1e4ccbdd9fcdc Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 7 Apr 2022 17:04:39 +0800 Subject: [PATCH 13/23] [ASR] update ds2 online model --- docs/source/released_model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/released_model.md b/docs/source/released_model.md index 9a423e03e..91ed36673 100644 --- a/docs/source/released_model.md +++ b/docs/source/released_model.md @@ -6,7 +6,7 @@ ### Speech Recognition Model Acoustic Model | Training Data | Token-based | Size | Descriptions | CER | WER | Hours of speech | Example Link :-------------:| :------------:| :-----: | -----: | :-----: |:-----:| :-----: | :-----: | :-----: -[Ds2 Online Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_online_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 345 MB | 2 Conv + 5 LSTM layers with only forward direction | 0.080 |-| 151 h | [D2 Online Aishell ASR0](../../examples/aishell/asr0) +[Ds2 Online Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_online_aishell_ckpt_0.2.0.model.tar.gz) | Aishell Dataset | Char-based | 345 MB | 2 Conv + 5 LSTM layers with only forward direction | 0.078 |-| 151 h | [D2 Online Aishell ASR0](../../examples/aishell/asr0) [Ds2 Offline Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_aishell_ckpt_0.1.1.model.tar.gz)| Aishell Dataset | Char-based | 306 MB | 2 Conv + 3 bidirectional GRU layers| 0.064 |-| 151 h | [Ds2 Offline Aishell ASR0](../../examples/aishell/asr0) [Conformer Online Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_chunk_conformer_aishell_ckpt_0.1.2.model.tar.gz) | Aishell Dataset | Char-based | 189 MB | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0565 |-| 151 h | [Conformer Online Aishell ASR1](../../examples/aishell/asr1) [Conformer Offline Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_conformer_aishell_ckpt_0.1.2.model.tar.gz) | Aishell Dataset | Char-based | 189 MB | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0483 |-| 151 h | [Conformer Offline Aishell ASR1](../../examples/aishell/asr1) From eeae00cc040b853cbb02c94690d51b77a3f312c4 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 7 Apr 2022 17:10:18 +0800 Subject: [PATCH 14/23] test=doc --- examples/aishell/asr0/RESULTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/aishell/asr0/RESULTS.md b/examples/aishell/asr0/RESULTS.md index 5841a8522..1f942ddfd 100644 --- a/examples/aishell/asr0/RESULTS.md +++ b/examples/aishell/asr0/RESULTS.md @@ -4,6 +4,7 @@ | Model | Number of Params | Release | Config | Test set | Valid Loss | CER | | --- | --- | --- | --- | --- | --- | --- | +| DeepSpeech2 | 45.18M | 2.2.2 | conf/deepspeech2_online.yaml + spec aug | test | 7.708217620849609| 0.078 | | DeepSpeech2 | 45.18M | 2.2.0 | conf/deepspeech2_online.yaml + spec aug | test | 7.994938373565674 | 0.080 | ## Deepspeech2 Non-Streaming From 5bb36472e866cd2e7c9d4b0d9905591ff6eaba1d Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 7 Apr 2022 17:11:41 +0800 Subject: [PATCH 15/23] test=doc --- examples/aishell/asr0/README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/examples/aishell/asr0/README.md b/examples/aishell/asr0/README.md index bb45d8df0..ecc66adf8 100644 --- a/examples/aishell/asr0/README.md +++ b/examples/aishell/asr0/README.md @@ -173,12 +173,7 @@ bash local/data.sh --stage 2 --stop_stage 2 CUDA_VISIBLE_DEVICES= ./local/test.sh conf/deepspeech2.yaml exp/deepspeech2/checkpoints/avg_1 ``` -The performance of the released models are shown below: - -| Acoustic Model | Training Data | Token-based | Size | Descriptions | CER | WER | Hours of speech | -| :----------------------------: | :-------------: | :---------: | -----: | :------------------------------------------------- | :---- | :--- | :-------------- | -| Ds2 Online Aishell ASR0 Model | Aishell Dataset | Char-based | 345 MB | 2 Conv + 5 LSTM layers with only forward direction | 0.080 | - | 151 h | -| Ds2 Offline Aishell ASR0 Model | Aishell Dataset | Char-based | 306 MB | 2 Conv + 3 bidirectional GRU layers | 0.064 | - | 151 h | +The performance of the released models are shown in [this](./RESULT.md) ## Stage 4: Static graph model Export This stage is to transform dygraph to static graph. ```bash From 82cd7015d749eccc14e170b63e03ff36e125c6f2 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 7 Apr 2022 17:12:39 +0800 Subject: [PATCH 16/23] test=doc --- examples/aishell/asr0/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/aishell/asr0/README.md b/examples/aishell/asr0/README.md index ecc66adf8..16489992d 100644 --- a/examples/aishell/asr0/README.md +++ b/examples/aishell/asr0/README.md @@ -173,7 +173,7 @@ bash local/data.sh --stage 2 --stop_stage 2 CUDA_VISIBLE_DEVICES= ./local/test.sh conf/deepspeech2.yaml exp/deepspeech2/checkpoints/avg_1 ``` -The performance of the released models are shown in [this](./RESULT.md) +The performance of the released models are shown in [this](./RESULTS.md) ## Stage 4: Static graph model Export This stage is to transform dygraph to static graph. ```bash From bc23f621559ee039fd8c0d965652a0d8d3ebb2f2 Mon Sep 17 00:00:00 2001 From: iftaken Date: Thu, 7 Apr 2022 18:01:07 +0800 Subject: [PATCH 17/23] update README, test=doc --- README.md | 22 +++++++++++++++++++++- README_cn.md | 14 +++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a90498293..eccf70373 100644 --- a/README.md +++ b/README.md @@ -280,10 +280,14 @@ paddlespeech_client cls --server_ip 127.0.0.1 --port 8090 --input input.wav For more information about server command lines, please see: [speech server demos](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/demos/speech_server) + + ## Model List PaddleSpeech supports a series of most popular models. They are summarized in [released models](./docs/source/released_model.md) and attached with available pretrained models. + + **Speech-to-Text** contains *Acoustic Model*, *Language Model*, and *Speech Translation*, with the following details: @@ -357,6 +361,8 @@ PaddleSpeech supports a series of most popular models. They are summarized in [r
+ + **Text-to-Speech** in PaddleSpeech mainly contains three modules: *Text Frontend*, *Acoustic Model* and *Vocoder*. Acoustic Model and Vocoder models are listed as follow: @@ -473,6 +479,8 @@ PaddleSpeech supports a series of most popular models. They are summarized in [r
+ + **Audio Classification** @@ -496,6 +504,8 @@ PaddleSpeech supports a series of most popular models. They are summarized in [r
+ + **Speaker Verification** @@ -519,6 +529,8 @@ PaddleSpeech supports a series of most popular models. They are summarized in [r
+ + **Punctuation Restoration** @@ -559,10 +571,18 @@ Normally, [Speech SoTA](https://paperswithcode.com/area/speech), [Audio SoTA](ht - [Advanced Usage](./docs/source/tts/advanced_usage.md) - [Chinese Rule Based Text Frontend](./docs/source/tts/zh_text_frontend.md) - [Test Audio Samples](https://paddlespeech.readthedocs.io/en/latest/tts/demo.html) + - Speaker Verification + - [Audio Searching](./demos/audio_searching/README.md) + - [Speaker Verification](./demos/speaker_verification/README.md) - [Audio Classification](./demos/audio_tagging/README.md) - - [Speaker Verification](./demos/speaker_verification/README.md) - [Speech Translation](./demos/speech_translation/README.md) + - [Speech Server](./demos/speech_server/README.md) - [Released Models](./docs/source/released_model.md) + - [Speech-to-Text](#SpeechToText) + - [Text-to-Speech](#TextToSpeech) + - [Audio Classification](#AudioClassification) + - [Speaker Verification](#SpeakerVerification) + - [Punctuation Restoration](#PunctuationRestoration) - [Community](#Community) - [Welcome to contribute](#contribution) - [License](#License) diff --git a/README_cn.md b/README_cn.md index ab4ce6e6b..f8f84ca87 100644 --- a/README_cn.md +++ b/README_cn.md @@ -273,6 +273,8 @@ paddlespeech_client cls --server_ip 127.0.0.1 --port 8090 --input input.wav ## 模型列表 PaddleSpeech 支持很多主流的模型，并提供了预训练模型，详情请见[模型列表](./docs/source/released_model.md)。 + + PaddleSpeech 的 **语音转文本** 包含语音识别声学模型、语音识别语言模型和语音翻译, 详情如下：
@@ -347,6 +349,7 @@ PaddleSpeech 的 **语音转文本** 包含语音识别声学模型、语音识
+ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声学模型和声码器。声学模型和声码器模型如下： @@ -488,6 +491,8 @@ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声
+ + **声纹识别** @@ -511,6 +516,8 @@ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声
+ + **标点恢复** @@ -556,13 +563,18 @@ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声 - [进阶用法](./docs/source/tts/advanced_usage.md) - [中文文本前端](./docs/source/tts/zh_text_frontend.md) - [测试语音样本](https://paddlespeech.readthedocs.io/en/latest/tts/demo.html) + - 声纹识别 + - [声纹识别](./demos/speaker_verification/README_cn.md) + - [音频检索](./demos/audio_searching/README_cn.md) - [声音分类](./demos/audio_tagging/README_cn.md) - - [声纹识别](./demos/speaker_verification/README_cn.md) - [语音翻译](./demos/speech_translation/README_cn.md) + - [服务化部署](./demos/speech_server/README_cn.md) - [模型列表](#模型列表) - [语音识别](#语音识别模型) - [语音合成](#语音合成模型) - [声音分类](#声音分类模型) + - [声纹识别](#声纹识别模型) + - [标点恢复](#标点恢复模型) - [技术交流群](#技术交流群) - [欢迎贡献](#欢迎贡献) - [License](#License) From 2b7ca6f261c3260398bba4ef7cc494c8ba01c059 Mon Sep 17 00:00:00 2001 From: Hui Zhang Date: Thu, 7 Apr 2022 18:26:32 +0800 Subject: [PATCH 18/23] Update RESULTS.md --- examples/aishell/asr0/RESULTS.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/aishell/asr0/RESULTS.md b/examples/aishell/asr0/RESULTS.md index 1f942ddfd..8af3d66d1 100644 --- a/examples/aishell/asr0/RESULTS.md +++ b/examples/aishell/asr0/RESULTS.md @@ -4,16 +4,16 @@ | Model | Number of Params | Release | Config | Test set | Valid Loss | CER | | --- | --- | --- | --- | --- | --- | --- | -| DeepSpeech2 | 45.18M | 2.2.2 | conf/deepspeech2_online.yaml + spec aug | test | 7.708217620849609| 0.078 | -| DeepSpeech2 | 45.18M | 2.2.0 | conf/deepspeech2_online.yaml + spec aug | test | 7.994938373565674 | 0.080 | +| DeepSpeech2 | 45.18M | r0.2.0 | conf/deepspeech2_online.yaml + spec aug | test | 7.708217620849609| 0.078 | +| DeepSpeech2 | 45.18M | v2.2.0 | conf/deepspeech2_online.yaml + spec aug | test | 7.994938373565674 | 0.080 | ## Deepspeech2 Non-Streaming | Model | Number of Params | Release | Config | Test set | Valid Loss | CER | | --- | --- | --- | --- | --- | --- | --- | -| DeepSpeech2 | 58.4M | 2.2.0 | conf/deepspeech2.yaml + spec aug | test | 5.738585948944092 | 0.064000 | -| DeepSpeech2 | 58.4M | 2.1.0 | conf/deepspeech2.yaml + spec aug | test | 7.483316898345947 | 0.077860 | -| DeepSpeech2 | 58.4M | 2.1.0 | conf/deepspeech2.yaml | test | 7.299022197723389 | 0.078671 | -| DeepSpeech2 | 58.4M | 2.0.0 | conf/deepspeech2.yaml | test | - | 0.078977 | +| DeepSpeech2 | 58.4M | v2.2.0 | conf/deepspeech2.yaml + spec aug | test | 5.738585948944092 | 0.064000 | +| DeepSpeech2 | 58.4M | v2.1.0 | conf/deepspeech2.yaml + spec aug | test | 7.483316898345947 | 0.077860 | +| DeepSpeech2 | 58.4M | v2.1.0 | conf/deepspeech2.yaml | test | 7.299022197723389 | 0.078671 | +| DeepSpeech2 | 58.4M | v2.0.0 | conf/deepspeech2.yaml | test | - | 0.078977 | | --- | --- | --- | --- | --- | --- | --- | -| DeepSpeech2 | 58.4M | 1.8.5 | - | test | - | 0.080447 | +| DeepSpeech2 | 58.4M | v1.8.5 | - | test | - | 0.080447 | From 1c05d038064b10ec14acbbf176eb68654d3b14e6 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 7 Apr 2022 19:46:04 +0800 Subject: [PATCH 19/23] test=asr --- paddlespeech/cli/asr/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlespeech/cli/asr/infer.py b/paddlespeech/cli/asr/infer.py index 8eea7ff5a..bb8c07c96 100644 --- a/paddlespeech/cli/asr/infer.py +++ b/paddlespeech/cli/asr/infer.py @@ -427,7 +427,7 @@ class ASRExecutor(BaseExecutor): audio, audio_sample_rate = soundfile.read( audio_file, dtype="int16", always_2d=True) audio_duration = audio.shape[0] / audio_sample_rate - max_duration = 30.0 + max_duration = 50.0 if audio_duration >= max_duration: logger.error("Please input audio file less then 30 seconds.\n") return From 4922e697e18581f0b6ae6ccb5c8ae8e78d03f178 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 7 Apr 2022 19:53:12 +0800 Subject: [PATCH 20/23] update cli, test = asr --- paddlespeech/cli/asr/infer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddlespeech/cli/asr/infer.py b/paddlespeech/cli/asr/infer.py index 1fb4be434..9386ac834 100644 --- a/paddlespeech/cli/asr/infer.py +++ b/paddlespeech/cli/asr/infer.py @@ -80,9 +80,9 @@ pretrained_models = { }, "deepspeech2online_aishell-zh-16k": { 'url': - 'https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_online_aishell_ckpt_0.1.1.model.tar.gz', + 'https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_online_aishell_ckpt_0.2.0.model.tar.gz', 'md5': - 'd5e076217cf60486519f72c217d21b9b', + '23e16c69730a1cb5d735c98c83c21e16', 'cfg_path': 'model.yaml', 'ckpt_path': From 34b77a9db179bd538c2f0fce8b3bf87228049773 Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 19:59:17 +0800 Subject: [PATCH 21/23] Update RESULTS.md. test=doc --- examples/iwslt2012/punc0/README.md | 2 +- paddleaudio/paddleaudio/utils/numeric.py | 30 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 paddleaudio/paddleaudio/utils/numeric.py diff --git a/examples/iwslt2012/punc0/README.md b/examples/iwslt2012/punc0/README.md index 74d599a21..6caa9710b 100644 --- a/examples/iwslt2012/punc0/README.md +++ b/examples/iwslt2012/punc0/README.md @@ -21,7 +21,7 @@ The pretrained model can be downloaded here [ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip](https://paddlespeech.bj.bcebos.com/text/ernie_linear_p3_iwslt2012_zh_ckpt_0.1.1.zip). ### Test Result -- Ernie Linear +- Ernie | |COMMA | PERIOD | QUESTION | OVERALL| |:-----:|:-----:|:-----:|:-----:|:-----:| |Precision |0.510955 |0.526462 |0.820755 |0.619391| diff --git a/paddleaudio/paddleaudio/utils/numeric.py b/paddleaudio/paddleaudio/utils/numeric.py new file mode 100644 index 000000000..126cada50 --- /dev/null +++ b/paddleaudio/paddleaudio/utils/numeric.py @@ -0,0 +1,30 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + + +def pcm16to32(audio: np.ndarray) -> np.ndarray: + """pcm int16 to float32 + + Args: + audio (np.ndarray): Waveform with dtype of int16. + + Returns: + np.ndarray: Waveform with dtype of float32. + """ + if audio.dtype == np.int16: + audio = audio.astype("float32") + bits = np.iinfo(np.int16).bits + audio = audio / (2**(bits - 1)) + return audio From 80b1fb98392f49d3173c29940b0bade23a58426f Mon Sep 17 00:00:00 2001 From: KP <109694228@qq.com> Date: Thu, 7 Apr 2022 20:00:42 +0800 Subject: [PATCH 22/23] Update RESULTS.md. test=doc --- examples/iwslt2012/punc0/RESULTS.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 examples/iwslt2012/punc0/RESULTS.md diff --git a/examples/iwslt2012/punc0/RESULTS.md b/examples/iwslt2012/punc0/RESULTS.md new file mode 100644 index 000000000..2e22713d8 --- /dev/null +++ b/examples/iwslt2012/punc0/RESULTS.md @@ -0,0 +1,9 @@ +# iwslt2012 + +## Ernie + +| |COMMA | PERIOD | QUESTION | OVERALL| +|:-----:|:-----:|:-----:|:-----:|:-----:| +|Precision |0.510955 |0.526462 |0.820755 |0.619391| +|Recall |0.517433 |0.564179 |0.861386 |0.647666| +|F1 |0.514173 |0.544669 |0.840580 |0.633141| From c852776bc63548f08eccc588c1f449412002a8f6 Mon Sep 17 00:00:00 2001 From: Jackwaterveg <87408988+Jackwaterveg@users.noreply.github.com> Date: Thu, 7 Apr 2022 20:44:24 +0800 Subject: [PATCH 23/23] test=doc --- paddlespeech/cli/asr/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlespeech/cli/asr/infer.py b/paddlespeech/cli/asr/infer.py index bb8c07c96..469a57e4b 100644 --- a/paddlespeech/cli/asr/infer.py +++ b/paddlespeech/cli/asr/infer.py @@ -429,7 +429,7 @@ class ASRExecutor(BaseExecutor): audio_duration = audio.shape[0] / audio_sample_rate max_duration = 50.0 if audio_duration >= max_duration: - logger.error("Please input audio file less then 30 seconds.\n") + logger.error("Please input audio file less then 50 seconds.\n") return except Exception as e: logger.exception(e)