From e622f42d92e52e9f63c3cce74ababf68d676b366 Mon Sep 17 00:00:00 2001 From: TianYuan Date: Wed, 7 Sep 2022 11:20:41 +0000 Subject: [PATCH 1/5] add aishell3 voice cloning with ECAPA-TDNN spk encoder --- examples/aishell3/README.md | 1 + examples/aishell3/vc1/README.md | 4 +- examples/aishell3/vc2/README.md | 126 ++++++++++++++++++ examples/aishell3/vc2/conf/default.yaml | 104 +++++++++++++++ examples/aishell3/vc2/local/preprocess.sh | 85 ++++++++++++ examples/aishell3/vc2/local/synthesize.sh | 22 +++ examples/aishell3/vc2/local/train.sh | 13 ++ examples/aishell3/vc2/local/voice_cloning.sh | 23 ++++ examples/aishell3/vc2/path.sh | 13 ++ examples/aishell3/vc2/run.sh | 39 ++++++ paddlespeech/cli/asr/infer.py | 20 ++- paddlespeech/cli/vector/infer.py | 69 ++++++++-- .../engine/vector/python/vector_engine.py | 3 +- .../t2s/exps/fastspeech2/vc2_infer.py | 70 ++++++++++ paddlespeech/t2s/exps/voice_cloning.py | 126 +++++++++++------- 15 files changed, 651 insertions(+), 67 deletions(-) create mode 100644 examples/aishell3/vc2/README.md create mode 100644 examples/aishell3/vc2/conf/default.yaml create mode 100755 examples/aishell3/vc2/local/preprocess.sh create mode 100755 examples/aishell3/vc2/local/synthesize.sh create mode 100755 examples/aishell3/vc2/local/train.sh create mode 100755 examples/aishell3/vc2/local/voice_cloning.sh create mode 100755 examples/aishell3/vc2/path.sh create mode 100755 examples/aishell3/vc2/run.sh create mode 100644 paddlespeech/t2s/exps/fastspeech2/vc2_infer.py diff --git a/examples/aishell3/README.md b/examples/aishell3/README.md index e022cef4..dd09bdfb 100644 --- a/examples/aishell3/README.md +++ b/examples/aishell3/README.md @@ -10,4 +10,5 @@ * voc3 - MultiBand MelGAN * vc0 - Tacotron2 Voice Cloning with GE2E * vc1 - FastSpeech2 Voice Cloning with GE2E +* vc2 - FastSpeech2 Voice Cloning with ECAPA-TDNN * ernie_sat - ERNIE-SAT diff --git a/examples/aishell3/vc1/README.md b/examples/aishell3/vc1/README.md index aab52510..93e0fd7e 100644 --- a/examples/aishell3/vc1/README.md +++ b/examples/aishell3/vc1/README.md @@ -99,7 +99,7 @@ CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_p The synthesizing step is very similar to that one of [tts3](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/tts3), but we should set `--voice-cloning=True` when calling `${BIN_DIR}/../synthesize.py`. ### Voice Cloning -Assume there are some reference audios in `./ref_audio` +Assume there are some reference audios in `./ref_audio` ```text ref_audio ├── 001238.wav @@ -116,7 +116,7 @@ CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${conf_path} ${train_outpu Model | Step | eval/loss | eval/l1_loss | eval/duration_loss | eval/pitch_loss| eval/energy_loss :-------------:| :------------:| :-----: | :-----: | :--------: |:--------:|:---------: -default|2(gpu) x 96400|0.99699|0.62013|0.53057|0.11954| 0.20426| +default|2(gpu) x 96400|0.99699|0.62013|0.053057|0.11954| 0.20426| FastSpeech2 checkpoint contains files listed below. (There is no need for `speaker_id_map.txt` here ) diff --git a/examples/aishell3/vc2/README.md b/examples/aishell3/vc2/README.md new file mode 100644 index 00000000..77482367 --- /dev/null +++ b/examples/aishell3/vc2/README.md @@ -0,0 +1,126 @@ +# FastSpeech2 + AISHELL-3 Voice Cloning (ECAPA-TDNN) +This example contains code used to train a [FastSpeech2](https://arxiv.org/abs/2006.04558) model with [AISHELL-3](http://www.aishelltech.com/aishell_3). The trained model can be used in Voice Cloning Task, We refer to the model structure of [Transfer Learning from Speaker Verification to Multispeaker Text-To-Speech Synthesis](https://arxiv.org/pdf/1806.04558.pdf). The general steps are as follows: +1. Speaker Encoder: We use Speaker Verification to train a speaker encoder. Datasets used in this task are different from those used in `FastSpeech2` because the transcriptions are not needed, we use more datasets, refer to [ECAPA-TDNN](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/voxceleb/sv0). +2. Synthesizer: We use the trained speaker encoder to generate speaker embedding for each sentence in AISHELL-3. This embedding is an extra input of `FastSpeech2` which will be concated with encoder outputs. +3. Vocoder: We use [Parallel Wave GAN](http://arxiv.org/abs/1910.11480) as the neural Vocoder, refer to [voc1](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/voc1). + +## Dataset +### Download and Extract +Download AISHELL-3 from it's [Official Website](http://www.aishelltech.com/aishell_3) and extract it to `~/datasets`. Then the dataset is in the directory `~/datasets/data_aishell3`. + +### Get MFA Result and Extract +We use [MFA2.x](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) to get durations for aishell3_fastspeech2. +You can download from here [aishell3_alignment_tone.tar.gz](https://paddlespeech.bj.bcebos.com/MFA/AISHELL-3/with_tone/aishell3_alignment_tone.tar.gz), or train your MFA model reference to [mfa example](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/mfa) (use MFA1.x now) of our repo. + +## Get Started +Assume the path to the dataset is `~/datasets/data_aishell3`. +Assume the path to the MFA result of AISHELL-3 is `./aishell3_alignment_tone`. + +Run the command below to +1. **source path**. +2. preprocess the dataset. +3. train the model. +4. synthesize waveform from `metadata.jsonl`. +5. start a voice cloning inference. +```bash +./run.sh +``` +You can choose a range of stages you want to run, or set `stage` equal to `stop-stage` to use only one stage, for example, running the following command will only preprocess the dataset. +```bash +./run.sh --stage 0 --stop-stage 0 +``` +### Data Preprocessing +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/preprocess.sh ${conf_path} +``` +When it is done. A `dump` folder is created in the current directory. The structure of the dump folder is listed below. +```text +dump +├── dev +│ ├── norm +│ └── raw +├── embed +│ ├── SSB0005 +│ ├── SSB0009 +│ ├── ... +│ └── ... +├── phone_id_map.txt +├── speaker_id_map.txt +├── test +│ ├── norm +│ └── raw +└── train + ├── energy_stats.npy + ├── norm + ├── pitch_stats.npy + ├── raw + └── speech_stats.npy +``` +The `embed` contains the generated speaker embedding for each sentence in AISHELL-3, which has the same file structure with wav files and the format is `.npy`. + +The computing time of utterance embedding can be x hours. + +The dataset is split into 3 parts, namely `train`, `dev`, and` test`, each of which contains a `norm` and `raw` subfolder. The raw folder contains speech、pitch and energy features of each utterance, while the norm folder contains normalized ones. The statistics used to normalize features are computed from the training set, which is located in `dump/train/*_stats.npy`. + +Also, there is a `metadata.jsonl` in each subfolder. It is a table-like file that contains phones, text_lengths, speech_lengths, durations, the path of speech features, the path of pitch features, the path of energy features, speaker, and id of each utterance. + +The preprocessing step is very similar to that one of [tts3](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/tts3), but there is one more `ECAPA-TDNN/inference` step here. + +### Model Training +`./local/train.sh` calls `${BIN_DIR}/train.py`. +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} +``` +The training step is very similar to that one of [tts3](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/tts3), but we should set `--voice-cloning=True` when calling `${BIN_DIR}/train.py`. + +### Synthesizing +We use [parallel wavegan](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/voc1) as the neural vocoder. +Download pretrained parallel wavegan model from [pwg_aishell3_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/pwgan/pwg_aishell3_ckpt_0.5.zip) and unzip it. +```bash +unzip pwg_aishell3_ckpt_0.5.zip +``` +Parallel WaveGAN checkpoint contains files listed below. +```text +pwg_aishell3_ckpt_0.5 +├── default.yaml # default config used to train parallel wavegan +├── feats_stats.npy # statistics used to normalize spectrogram when training parallel wavegan +└── snapshot_iter_1000000.pdz # generator parameters of parallel wavegan +``` +`./local/synthesize.sh` calls `${BIN_DIR}/../synthesize.py`, which can synthesize waveform from `metadata.jsonl`. +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} +``` +The synthesizing step is very similar to that one of [tts3](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/tts3), but we should set `--voice-cloning=True` when calling `${BIN_DIR}/../synthesize.py`. + +### Voice Cloning +Assume there are some reference audios in `./ref_audio` (the format must be wav here) +```text +ref_audio +├── 001238.wav +├── LJ015-0254.wav +└── audio_self_test.wav +``` +`./local/voice_cloning.sh` calls `${BIN_DIR}/../voice_cloning.py` + +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${conf_path} ${train_output_path} ${ckpt_name} ${ref_audio_dir} +``` +## Pretrained Model +- [fastspeech2_aishell3_ckpt_vc2_1.2.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_aishell3_ckpt_vc2_1.2.0.zip) + +Model | Step | eval/loss | eval/l1_loss | eval/duration_loss | eval/pitch_loss| eval/energy_loss +:-------------:| :------------:| :-----: | :-----: | :--------: |:--------:|:---------: +default|2(gpu) x 96400|0.991855|0.599517|0.052142|0.094877| 0.245318| + +FastSpeech2 checkpoint contains files listed below. +(There is no need for `speaker_id_map.txt` here ) + +```text +fastspeech2_aishell3_ckpt_vc2_1.2.0 +├── default.yaml # default config used to train fastspeech2 +├── energy_stats.npy # statistics used to normalize energy when training fastspeech2 +├── phone_id_map.txt # phone vocabulary file when training fastspeech2 +├── pitch_stats.npy # statistics used to normalize pitch when training fastspeech2 +├── snapshot_iter_96400.pdz # model parameters and optimizer states +└── speech_stats.npy # statistics used to normalize spectrogram when training fastspeech2 +``` diff --git a/examples/aishell3/vc2/conf/default.yaml b/examples/aishell3/vc2/conf/default.yaml new file mode 100644 index 00000000..5ef37f81 --- /dev/null +++ b/examples/aishell3/vc2/conf/default.yaml @@ -0,0 +1,104 @@ +########################################################### +# FEATURE EXTRACTION SETTING # +########################################################### + +fs: 24000 # sr +n_fft: 2048 # FFT size (samples). +n_shift: 300 # Hop size (samples). 12.5ms +win_length: 1200 # Window length (samples). 50ms + # If set to null, it will be the same as fft_size. +window: "hann" # Window function. + +# Only used for feats_type != raw + +fmin: 80 # Minimum frequency of Mel basis. +fmax: 7600 # Maximum frequency of Mel basis. +n_mels: 80 # The number of mel basis. + +# Only used for the model using pitch features (e.g. FastSpeech2) +f0min: 80 # Minimum f0 for pitch extraction. +f0max: 400 # Maximum f0 for pitch extraction. + + +########################################################### +# DATA SETTING # +########################################################### +batch_size: 64 +num_workers: 2 + + +########################################################### +# MODEL SETTING # +########################################################### +model: + adim: 384 # attention dimension + aheads: 2 # number of attention heads + elayers: 4 # number of encoder layers + eunits: 1536 # number of encoder ff units + dlayers: 4 # number of decoder layers + dunits: 1536 # number of decoder ff units + positionwise_layer_type: conv1d # type of position-wise layer + positionwise_conv_kernel_size: 3 # kernel size of position wise conv layer + duration_predictor_layers: 2 # number of layers of duration predictor + duration_predictor_chans: 256 # number of channels of duration predictor + duration_predictor_kernel_size: 3 # filter size of duration predictor + postnet_layers: 5 # number of layers of postnset + postnet_filts: 5 # filter size of conv layers in postnet + postnet_chans: 256 # number of channels of conv layers in postnet + use_scaled_pos_enc: True # whether to use scaled positional encoding + encoder_normalize_before: True # whether to perform layer normalization before the input + decoder_normalize_before: True # whether to perform layer normalization before the input + reduction_factor: 1 # reduction factor + init_type: xavier_uniform # initialization type + init_enc_alpha: 1.0 # initial value of alpha of encoder scaled position encoding + init_dec_alpha: 1.0 # initial value of alpha of decoder scaled position encoding + transformer_enc_dropout_rate: 0.2 # dropout rate for transformer encoder layer + transformer_enc_positional_dropout_rate: 0.2 # dropout rate for transformer encoder positional encoding + transformer_enc_attn_dropout_rate: 0.2 # dropout rate for transformer encoder attention layer + transformer_dec_dropout_rate: 0.2 # dropout rate for transformer decoder layer + transformer_dec_positional_dropout_rate: 0.2 # dropout rate for transformer decoder positional encoding + transformer_dec_attn_dropout_rate: 0.2 # dropout rate for transformer decoder attention layer + pitch_predictor_layers: 5 # number of conv layers in pitch predictor + pitch_predictor_chans: 256 # number of channels of conv layers in pitch predictor + pitch_predictor_kernel_size: 5 # kernel size of conv leyers in pitch predictor + pitch_predictor_dropout: 0.5 # dropout rate in pitch predictor + pitch_embed_kernel_size: 1 # kernel size of conv embedding layer for pitch + pitch_embed_dropout: 0.0 # dropout rate after conv embedding layer for pitch + stop_gradient_from_pitch_predictor: True # whether to stop the gradient from pitch predictor to encoder + energy_predictor_layers: 2 # number of conv layers in energy predictor + energy_predictor_chans: 256 # number of channels of conv layers in energy predictor + energy_predictor_kernel_size: 3 # kernel size of conv leyers in energy predictor + energy_predictor_dropout: 0.5 # dropout rate in energy predictor + energy_embed_kernel_size: 1 # kernel size of conv embedding layer for energy + energy_embed_dropout: 0.0 # dropout rate after conv embedding layer for energy + stop_gradient_from_energy_predictor: False # whether to stop the gradient from energy predictor to encoder + spk_embed_dim: 192 # speaker embedding dimension + spk_embed_integration_type: concat # speaker embedding integration type + + + +########################################################### +# UPDATER SETTING # +########################################################### +updater: + use_masking: True # whether to apply masking for padded part in loss calculation + + +########################################################### +# OPTIMIZER SETTING # +########################################################### +optimizer: + optim: adam # optimizer type + learning_rate: 0.001 # learning rate + +########################################################### +# TRAINING SETTING # +########################################################### +max_epoch: 200 +num_snapshots: 5 + + +########################################################### +# OTHER SETTING # +########################################################### +seed: 10086 diff --git a/examples/aishell3/vc2/local/preprocess.sh b/examples/aishell3/vc2/local/preprocess.sh new file mode 100755 index 00000000..f5262a26 --- /dev/null +++ b/examples/aishell3/vc2/local/preprocess.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +stage=0 +stop_stage=100 + +config_path=$1 + +# gen speaker embedding +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + python3 ${BIN_DIR}/vc2_infer.py \ + --input=~/datasets/data_aishell3/train/wav/ \ + --output=dump/embed \ + --num-cpu=20 +fi + +# copy from tts3/preprocess +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + # get durations from MFA's result + echo "Generate durations.txt from MFA results ..." + python3 ${MAIN_ROOT}/utils/gen_duration_from_textgrid.py \ + --inputdir=./aishell3_alignment_tone \ + --output durations.txt \ + --config=${config_path} +fi + +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + # extract features + echo "Extract features ..." + python3 ${BIN_DIR}/preprocess.py \ + --dataset=aishell3 \ + --rootdir=~/datasets/data_aishell3/ \ + --dumpdir=dump \ + --dur-file=durations.txt \ + --config=${config_path} \ + --num-cpu=20 \ + --cut-sil=True \ + --spk_emb_dir=dump/embed +fi + +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # get features' stats(mean and std) + echo "Get features' stats ..." + python3 ${MAIN_ROOT}/utils/compute_statistics.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --field-name="speech" + + python3 ${MAIN_ROOT}/utils/compute_statistics.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --field-name="pitch" + + python3 ${MAIN_ROOT}/utils/compute_statistics.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --field-name="energy" +fi + +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + # normalize and covert phone/speaker to id, dev and test should use train's stats + echo "Normalize ..." + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --dumpdir=dump/train/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --pitch-stats=dump/train/pitch_stats.npy \ + --energy-stats=dump/train/energy_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt + + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/dev/raw/metadata.jsonl \ + --dumpdir=dump/dev/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --pitch-stats=dump/train/pitch_stats.npy \ + --energy-stats=dump/train/energy_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt + + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/test/raw/metadata.jsonl \ + --dumpdir=dump/test/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --pitch-stats=dump/train/pitch_stats.npy \ + --energy-stats=dump/train/energy_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt +fi diff --git a/examples/aishell3/vc2/local/synthesize.sh b/examples/aishell3/vc2/local/synthesize.sh new file mode 100755 index 00000000..8c61e3f3 --- /dev/null +++ b/examples/aishell3/vc2/local/synthesize.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +config_path=$1 +train_output_path=$2 +ckpt_name=$3 + +FLAGS_allocator_strategy=naive_best_fit \ +FLAGS_fraction_of_gpu_memory_to_use=0.01 \ +python3 ${BIN_DIR}/../synthesize.py \ + --am=fastspeech2_aishell3 \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=pwgan_aishell3 \ + --voc_config=pwg_aishell3_ckpt_0.5/default.yaml \ + --voc_ckpt=pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \ + --voc_stat=pwg_aishell3_ckpt_0.5/feats_stats.npy \ + --test_metadata=dump/test/norm/metadata.jsonl \ + --output_dir=${train_output_path}/test \ + --phones_dict=dump/phone_id_map.txt \ + --speaker_dict=dump/speaker_id_map.txt \ + --voice-cloning=True diff --git a/examples/aishell3/vc2/local/train.sh b/examples/aishell3/vc2/local/train.sh new file mode 100755 index 00000000..c775fcad --- /dev/null +++ b/examples/aishell3/vc2/local/train.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +config_path=$1 +train_output_path=$2 + +python3 ${BIN_DIR}/train.py \ + --train-metadata=dump/train/norm/metadata.jsonl \ + --dev-metadata=dump/dev/norm/metadata.jsonl \ + --config=${config_path} \ + --output-dir=${train_output_path} \ + --ngpu=2 \ + --phones-dict=dump/phone_id_map.txt \ + --voice-cloning=True \ No newline at end of file diff --git a/examples/aishell3/vc2/local/voice_cloning.sh b/examples/aishell3/vc2/local/voice_cloning.sh new file mode 100755 index 00000000..09c5e436 --- /dev/null +++ b/examples/aishell3/vc2/local/voice_cloning.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +config_path=$1 +train_output_path=$2 +ckpt_name=$3 +ref_audio_dir=$4 + +FLAGS_allocator_strategy=naive_best_fit \ +FLAGS_fraction_of_gpu_memory_to_use=0.01 \ +python3 ${BIN_DIR}/../voice_cloning.py \ + --am=fastspeech2_aishell3 \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=pwgan_aishell3 \ + --voc_config=pwg_aishell3_ckpt_0.5/default.yaml \ + --voc_ckpt=pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \ + --voc_stat=pwg_aishell3_ckpt_0.5/feats_stats.npy \ + --text="凯莫瑞安联合体的经济崩溃迫在眉睫。" \ + --input-dir=${ref_audio_dir} \ + --output-dir=${train_output_path}/vc_syn \ + --phones-dict=dump/phone_id_map.txt \ + --use_ecapa=True diff --git a/examples/aishell3/vc2/path.sh b/examples/aishell3/vc2/path.sh new file mode 100755 index 00000000..fb7e8411 --- /dev/null +++ b/examples/aishell3/vc2/path.sh @@ -0,0 +1,13 @@ +#!/bin/bash +export MAIN_ROOT=`realpath ${PWD}/../../../` + +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} +export LC_ALL=C + +export PYTHONDONTWRITEBYTECODE=1 +# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} + +MODEL=fastspeech2 +export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL} diff --git a/examples/aishell3/vc2/run.sh b/examples/aishell3/vc2/run.sh new file mode 100755 index 00000000..06d56298 --- /dev/null +++ b/examples/aishell3/vc2/run.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e +source path.sh + +gpus=0,1 +stage=0 +stop_stage=100 + +conf_path=conf/default.yaml +train_output_path=exp/default +ckpt_name=snapshot_iter_96400.pdz +ref_audio_dir=ref_audio + + +# with the following command, you can choose the stage range you want to run +# such as `./run.sh --stage 0 --stop-stage 0` +# this can not be mixed use with `$1`, `$2` ... +source ${MAIN_ROOT}/utils/parse_options.sh || exit 1 + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + # prepare data + CUDA_VISIBLE_DEVICES=${gpus} ./local/preprocess.sh ${conf_path} || exit -1 +fi + +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + # train model, all `ckpt` under `train_output_path/checkpoints/` dir + CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1 +fi + +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + # synthesize, vocoder is pwgan + CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1 +fi + +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # synthesize, vocoder is pwgan + CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${conf_path} ${train_output_path} ${ckpt_name} ${ref_audio_dir} || exit -1 +fi diff --git a/paddlespeech/cli/asr/infer.py b/paddlespeech/cli/asr/infer.py index f9b4439e..7296776f 100644 --- a/paddlespeech/cli/asr/infer.py +++ b/paddlespeech/cli/asr/infer.py @@ -99,8 +99,9 @@ class ASRExecutor(BaseExecutor): '-y', action="store_true", default=False, - help='No additional parameters required. Once set this parameter, it means accepting the request of the program by default, which includes transforming the audio sample rate' - ) + help='No additional parameters required. \ + Once set this parameter, it means accepting the request of the program by default, \ + which includes transforming the audio sample rate') self.parser.add_argument( '--rtf', action="store_true", @@ -340,7 +341,7 @@ class ASRExecutor(BaseExecutor): audio = np.round(audio).astype("int16") return audio - def _check(self, audio_file: str, sample_rate: int, force_yes: bool): + def _check(self, audio_file: str, sample_rate: int, force_yes: bool=False): self.sample_rate = sample_rate if self.sample_rate != 16000 and self.sample_rate != 8000: logger.error( @@ -434,8 +435,17 @@ class ASRExecutor(BaseExecutor): for id_, input_ in task_source.items(): try: - res = self(input_, model, lang, sample_rate, config, ckpt_path, - decode_method, force_yes, rtf, device) + res = self( + audio_file=input_, + model=model, + lang=lang, + sample_rate=sample_rate, + config=config, + ckpt_path=ckpt_path, + decode_method=decode_method, + force_yes=force_yes, + rtf=rtf, + device=device) task_results[id_] = res except Exception as e: has_exceptions = True diff --git a/paddlespeech/cli/vector/infer.py b/paddlespeech/cli/vector/infer.py index 48ca1f98..11198724 100644 --- a/paddlespeech/cli/vector/infer.py +++ b/paddlespeech/cli/vector/infer.py @@ -70,6 +70,14 @@ class VectorExecutor(BaseExecutor): type=str, default=None, help="Checkpoint file of model.") + self.parser.add_argument( + '--yes', + '-y', + action="store_true", + default=False, + help='No additional parameters required. \ + Once set this parameter, it means accepting the request of the program by default, \ + which includes transforming the audio sample rate') self.parser.add_argument( '--config', type=str, @@ -109,6 +117,7 @@ class VectorExecutor(BaseExecutor): sample_rate = parser_args.sample_rate config = parser_args.config ckpt_path = parser_args.ckpt_path + force_yes = parser_args.yes device = parser_args.device # stage 1: configurate the verbose flag @@ -128,8 +137,14 @@ class VectorExecutor(BaseExecutor): # extract the speaker audio embedding if parser_args.task == "spk": logger.debug("do vector spk task") - res = self(input_, model, sample_rate, config, ckpt_path, - device) + res = self( + audio_file=input_, + model=model, + sample_rate=sample_rate, + config=config, + ckpt_path=ckpt_path, + force_yes=force_yes, + device=device) task_result[id_] = res elif parser_args.task == "score": logger.debug("do vector score task") @@ -145,10 +160,22 @@ class VectorExecutor(BaseExecutor): logger.debug( f"score task, enroll audio: {enroll_audio}, test audio: {test_audio}" ) - enroll_embedding = self(enroll_audio, model, sample_rate, - config, ckpt_path, device) - test_embedding = self(test_audio, model, sample_rate, - config, ckpt_path, device) + enroll_embedding = self( + audio_file=enroll_audio, + model=model, + sample_rate=sample_rate, + config=config, + ckpt_path=ckpt_path, + force_yes=force_yes, + device=device) + test_embedding = self( + audio_file=test_audio, + model=model, + sample_rate=sample_rate, + config=config, + ckpt_path=ckpt_path, + force_yes=force_yes, + device=device) # get the score res = self.get_embeddings_score(enroll_embedding, @@ -222,6 +249,7 @@ class VectorExecutor(BaseExecutor): sample_rate: int=16000, config: os.PathLike=None, ckpt_path: os.PathLike=None, + force_yes: bool=False, device=paddle.get_device()): """Extract the audio embedding @@ -240,7 +268,7 @@ class VectorExecutor(BaseExecutor): """ # stage 0: check the audio format audio_file = os.path.abspath(audio_file) - if not self._check(audio_file, sample_rate): + if not self._check(audio_file, sample_rate, force_yes): sys.exit(-1) # stage 1: set the paddle runtime host device @@ -418,7 +446,7 @@ class VectorExecutor(BaseExecutor): logger.debug("audio extract the feat success") - def _check(self, audio_file: str, sample_rate: int): + def _check(self, audio_file: str, sample_rate: int, force_yes: bool=False): """Check if the model sample match the audio sample rate Args: @@ -462,13 +490,34 @@ class VectorExecutor(BaseExecutor): logger.debug(f"The sample rate is {audio_sample_rate}") if audio_sample_rate != self.sample_rate: - logger.error("The sample rate of the input file is not {}.\n \ + logger.debug("The sample rate of the input file is not {}.\n \ The program will resample the wav file to {}.\n \ If the result does not meet your expectations，\n \ Please input the 16k 16 bit 1 channel wav file. \ ".format(self.sample_rate, self.sample_rate)) - sys.exit(-1) + if force_yes is False: + while (True): + logger.debug( + "Whether to change the sample rate and the channel. Y: change the sample. N: exit the prgream." + ) + content = input("Input(Y/N):") + if content.strip() == "Y" or content.strip( + ) == "y" or content.strip() == "yes" or content.strip( + ) == "Yes": + logger.debug( + "change the sampele rate, channel to 16k and 1 channel" + ) + break + elif content.strip() == "N" or content.strip( + ) == "n" or content.strip() == "no" or content.strip( + ) == "No": + logger.debug("Exit the program") + return False + else: + logger.warning("Not regular input, please input again") + self.change_format = True else: logger.debug("The audio file format is right") + self.change_format = False return True diff --git a/paddlespeech/server/engine/vector/python/vector_engine.py b/paddlespeech/server/engine/vector/python/vector_engine.py index f7d60648..7b8f667d 100644 --- a/paddlespeech/server/engine/vector/python/vector_engine.py +++ b/paddlespeech/server/engine/vector/python/vector_engine.py @@ -105,7 +105,8 @@ class PaddleVectorConnectionHandler: # we can not reuse the cache io.BytesIO(audio) data, # because the soundfile will change the io.BytesIO(audio) to the end # thus we should convert the base64 string to io.BytesIO when we need the audio data - if not self.executor._check(io.BytesIO(audio), sample_rate): + if not self.executor._check( + io.BytesIO(audio), sample_rate, force_yes=True): logger.debug("check the audio sample rate occurs error") return np.array([0.0]) diff --git a/paddlespeech/t2s/exps/fastspeech2/vc2_infer.py b/paddlespeech/t2s/exps/fastspeech2/vc2_infer.py new file mode 100644 index 00000000..3d0a8366 --- /dev/null +++ b/paddlespeech/t2s/exps/fastspeech2/vc2_infer.py @@ -0,0 +1,70 @@ +import argparse +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np +import tqdm + +from paddlespeech.cli.vector import VectorExecutor + + +def _process_utterance(ifpath: Path, + input_dir: Path, + output_dir: Path, + vec_executor): + rel_path = ifpath.relative_to(input_dir) + ofpath = (output_dir / rel_path).with_suffix(".npy") + ofpath.parent.mkdir(parents=True, exist_ok=True) + embed = vec_executor(audio_file=ifpath, force_yes=True) + np.save(ofpath, embed) + return ofpath + + +def main(args): + # input output preparation + input_dir = Path(args.input).expanduser() + ifpaths = list(input_dir.rglob(args.pattern)) + print(f"{len(ifpaths)} utterances in total") + output_dir = Path(args.output).expanduser() + output_dir.mkdir(parents=True, exist_ok=True) + vec_executor = VectorExecutor() + nprocs = args.num_cpu + + # warm up + vec_executor(audio_file=ifpaths[0], force_yes=True) + + if nprocs == 1: + results = [] + for ifpath in tqdm.tqdm(ifpaths, total=len(ifpaths)): + _process_utterance( + ifpath=ifpath, + input_dir=input_dir, + output_dir=output_dir, + vec_executor=vec_executor) + else: + with ThreadPoolExecutor(nprocs) as pool: + with tqdm.tqdm(total=len(ifpaths)) as progress: + for ifpath in ifpaths: + future = pool.submit(_process_utterance, ifpath, input_dir, + output_dir, vec_executor) + future.add_done_callback(lambda p: progress.update()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="compute utterance embed.") + parser.add_argument( + "--input", type=str, help="path of the audio_file folder.") + parser.add_argument( + "--pattern", + type=str, + default="*.wav", + help="pattern to filter audio files.") + parser.add_argument( + "--output", + metavar="OUTPUT_DIR", + help="path to save spk embedding results.") + parser.add_argument( + "--num-cpu", type=int, default=1, help="number of process.") + args = parser.parse_args() + + main(args) diff --git a/paddlespeech/t2s/exps/voice_cloning.py b/paddlespeech/t2s/exps/voice_cloning.py index b51a4d7b..80cfea4a 100644 --- a/paddlespeech/t2s/exps/voice_cloning.py +++ b/paddlespeech/t2s/exps/voice_cloning.py @@ -21,13 +21,28 @@ import soundfile as sf import yaml from yacs.config import CfgNode +from paddlespeech.cli.vector import VectorExecutor from paddlespeech.t2s.exps.syn_utils import get_am_inference from paddlespeech.t2s.exps.syn_utils import get_voc_inference from paddlespeech.t2s.frontend.zh_frontend import Frontend +from paddlespeech.t2s.utils import str2bool from paddlespeech.vector.exps.ge2e.audio_processor import SpeakerVerificationPreprocessor from paddlespeech.vector.models.lstm_speaker_encoder import LSTMSpeakerEncoder +def gen_random_embed(use_ecapa: bool=False): + if use_ecapa: + # Randomly generate numbers of -25 ~ 25, 192 is the dim of spk_emb + random_spk_emb = (-1 + 2 * np.random.rand(192)) * 25 + + # GE2E + else: + # Randomly generate numbers of 0 ~ 0.2, 256 is the dim of spk_emb + random_spk_emb = np.random.rand(256) * 0.2 + random_spk_emb = paddle.to_tensor(random_spk_emb, dtype='float32') + return random_spk_emb + + def voice_cloning(args): # Init body. with open(args.am_config) as f: @@ -41,30 +56,47 @@ def voice_cloning(args): print(am_config) print(voc_config) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + input_dir = Path(args.input_dir) + # speaker encoder - p = SpeakerVerificationPreprocessor( - sampling_rate=16000, - audio_norm_target_dBFS=-30, - vad_window_length=30, - vad_moving_average_width=8, - vad_max_silence_length=6, - mel_window_length=25, - mel_window_step=10, - n_mels=40, - partial_n_frames=160, - min_pad_coverage=0.75, - partial_overlap_ratio=0.5) - print("Audio Processor Done!") - - speaker_encoder = LSTMSpeakerEncoder( - n_mels=40, num_layers=3, hidden_size=256, output_size=256) - speaker_encoder.set_state_dict(paddle.load(args.ge2e_params_path)) - speaker_encoder.eval() - print("GE2E Done!") + if args.use_ecapa: + vec_executor = VectorExecutor() + # warm up + vec_executor( + audio_file=input_dir / os.listdir(input_dir)[0], force_yes=True) + print("ECAPA-TDNN Done!") + # use GE2E + else: + p = SpeakerVerificationPreprocessor( + sampling_rate=16000, + audio_norm_target_dBFS=-30, + vad_window_length=30, + vad_moving_average_width=8, + vad_max_silence_length=6, + mel_window_length=25, + mel_window_step=10, + n_mels=40, + partial_n_frames=160, + min_pad_coverage=0.75, + partial_overlap_ratio=0.5) + print("Audio Processor Done!") + + speaker_encoder = LSTMSpeakerEncoder( + n_mels=40, num_layers=3, hidden_size=256, output_size=256) + speaker_encoder.set_state_dict(paddle.load(args.ge2e_params_path)) + speaker_encoder.eval() + print("GE2E Done!") frontend = Frontend(phone_vocab_path=args.phones_dict) print("frontend done!") + sentence = args.text + input_ids = frontend.get_input_ids(sentence, merge_sentences=True) + phone_ids = input_ids["phone_ids"][0] + # acoustic model am_inference = get_am_inference( am=args.am, @@ -80,26 +112,19 @@ def voice_cloning(args): voc_ckpt=args.voc_ckpt, voc_stat=args.voc_stat) - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - input_dir = Path(args.input_dir) - - sentence = args.text - - input_ids = frontend.get_input_ids(sentence, merge_sentences=True) - phone_ids = input_ids["phone_ids"][0] - for name in os.listdir(input_dir): utt_id = name.split(".")[0] ref_audio_path = input_dir / name - mel_sequences = p.extract_mel_partials(p.preprocess_wav(ref_audio_path)) - # print("mel_sequences: ", mel_sequences.shape) - with paddle.no_grad(): - spk_emb = speaker_encoder.embed_utterance( - paddle.to_tensor(mel_sequences)) - # print("spk_emb shape: ", spk_emb.shape) - + if args.use_ecapa: + spk_emb = vec_executor(audio_file=ref_audio_path, force_yes=True) + spk_emb = paddle.to_tensor(spk_emb) + # GE2E + else: + mel_sequences = p.extract_mel_partials( + p.preprocess_wav(ref_audio_path)) + with paddle.no_grad(): + spk_emb = speaker_encoder.embed_utterance( + paddle.to_tensor(mel_sequences)) with paddle.no_grad(): wav = voc_inference(am_inference(phone_ids, spk_emb=spk_emb)) @@ -108,16 +133,17 @@ def voice_cloning(args): wav.numpy(), samplerate=am_config.fs) print(f"{utt_id} done!") - # Randomly generate numbers of 0 ~ 0.2, 256 is the dim of spk_emb - random_spk_emb = np.random.rand(256) * 0.2 - random_spk_emb = paddle.to_tensor(random_spk_emb, dtype='float32') - utt_id = "random_spk_emb" - with paddle.no_grad(): - wav = voc_inference(am_inference(phone_ids, spk_emb=random_spk_emb)) - sf.write( - str(output_dir / (utt_id + ".wav")), - wav.numpy(), - samplerate=am_config.fs) + + # generate 5 random_spk_emb + for i in range(5): + random_spk_emb = gen_random_embed(args.use_ecapa) + utt_id = "random_spk_emb" + with paddle.no_grad(): + wav = voc_inference(am_inference(phone_ids, spk_emb=random_spk_emb)) + sf.write( + str(output_dir / (utt_id + "_" + str(i) + ".wav")), + wav.numpy(), + samplerate=am_config.fs) print(f"{utt_id} done!") @@ -171,13 +197,15 @@ def parse_args(): type=str, default="每当你觉得，想要批评什么人的时候，你切要记着，这个世界上的人，并非都具备你禀有的条件。", help="text to synthesize, a line") - parser.add_argument( "--ge2e_params_path", type=str, help="ge2e params path.") - + parser.add_argument( + "--use_ecapa", + type=str2bool, + default=False, + help="whether to use ECAPA-TDNN as speaker encoder.") parser.add_argument( "--ngpu", type=int, default=1, help="if ngpu=0, use cpu.") - parser.add_argument( "--input-dir", type=str, From 5ae5e6819c575fc7169c0c5f47d4ab7f4216c73d Mon Sep 17 00:00:00 2001 From: TianYuan Date: Thu, 8 Sep 2022 06:45:48 +0000 Subject: [PATCH 2/5] update readme, test=tts --- README.md | 57 +++++++++++++-------- README_cn.md | 46 +++++++++++------ demos/text_to_speech/README.md | 82 +++++++++++++++++++---------- demos/text_to_speech/README_cn.md | 85 ++++++++++++++++++++----------- 4 files changed, 175 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 5c62925e..2fb281e7 100644 --- a/README.md +++ b/README.md @@ -159,15 +159,20 @@ Via the easy-to-use, efficient, flexible and scalable implementation, our vision - 🧩 *Cascaded models application*: as an extension of the typical traditional audio tasks, we combine the workflows of the aforementioned tasks with other fields like Natural language processing (NLP) and Computer Vision (CV). ### Recent Update -- 👑 2022.05.13: Release [PP-ASR](./docs/source/asr/PPASR.md)、[PP-TTS](./docs/source/tts/PPTTS.md)、[PP-VPR](docs/source/vpr/PPVPR.md) -- 👏🏻 2022.05.06: `Streaming ASR` with `Punctuation Restoration` and `Token Timestamp`. -- 👏🏻 2022.05.06: `Server` is available for `Speaker Verification`, and `Punctuation Restoration`. -- 👏🏻 2022.04.28: `Streaming Server` is available for `Automatic Speech Recognition` and `Text-to-Speech`. -- 👏🏻 2022.03.28: `Server` is available for `Audio Classification`, `Automatic Speech Recognition` and `Text-to-Speech`. -- 👏🏻 2022.03.28: `CLI` is available for `Speaker Verification`. +- ⚡ 2022.08.25: Release TTS [finetune](./examples/other/tts_finetune/tts3) example. +- 🔥 2022.08.22: Add ERNIE-SAT models: [ERNIE-SAT-vctk](./examples/vctk/ernie_sat)、[ERNIE-SAT-aishell3](./examples/aishell3/ernie_sat)、[ERNIE-SAT-zh_en](./examples/aishell3_vctk/ernie_sat). +- 🔥 2022.08.15: Add [g2pW](https://github.com/GitYCC/g2pW) into TTS Chinese Text Frontend. +- 🔥 2022.08.09: Release [Chinese English mixed TTS](./examples/zh_en_tts/tts3). +- ⚡ 2022.08.03: Add ONNXRuntime infer for TTS CLI. +- 🎉 2022.07.18: Release VITS: [VITS-csmsc](./examples/csmsc/vits)、[VITS-aishell3](./examples/aishell3/vits)、[VITS-VC](./examples/aishell3/vits-vc). +- 🎉 2022.06.22: All TTS models support ONNX format. +- 🍀 2022.06.17: Add [PaddleSpeech Web Demo](./demos/speech_web). +- 👑 2022.05.13: Release [PP-ASR](./docs/source/asr/PPASR.md)、[PP-TTS](./docs/source/tts/PPTTS.md)、[PP-VPR](docs/source/vpr/PPVPR.md). +- 👏🏻 2022.05.06: `PaddleSpeech Streaming Server` is available for `Streaming ASR` with `Punctuation Restoration` and `Token Timestamp` and `Text-to-Speech`. +- 👏🏻 2022.05.06: `PaddleSpeech Server` is available for `Audio Classification`, `Automatic Speech Recognition` and `Text-to-Speech`, `Speaker Verification` and `Punctuation Restoration`. +- 👏🏻 2022.03.28: `PaddleSpeech CLI` is available for `Speaker Verification`. - 🤗 2021.12.14: [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) and [TTS](https://huggingface.co/spaces/KPatrick/PaddleSpeechTTS) Demos on Hugging Face Spaces are available! -- 👏🏻 2021.12.10: `CLI` is available for `Audio Classification`, `Automatic Speech Recognition`, `Speech Translation (English to Chinese)` and `Text-to-Speech`. - +- 👏🏻 2021.12.10: `PaddleSpeech CLI` is available for `Audio Classification`, `Automatic Speech Recognition`, `Speech Translation (English to Chinese)` and `Text-to-Speech`. ### Community - Scan the QR code below with your Wechat, you can access to official technical exchange group and get the bonus ( more than 20GB learning materials, such as papers, codes and videos ) and the live link of the lessons. Look forward to your participation. @@ -599,49 +604,56 @@ PaddleSpeech supports a series of most popular models. They are summarized in [r - HiFiGAN - LJSpeech / VCTK / CSMSC / AISHELL-3 + HiFiGAN + LJSpeech / VCTK / CSMSC / AISHELL-3 HiFiGAN-ljspeech / HiFiGAN-vctk / HiFiGAN-csmsc / HiFiGAN-aishell3 - WaveRNN - CSMSC + WaveRNN + CSMSC WaveRNN-csmsc - Voice Cloning + Voice Cloning GE2E Librispeech, etc. - ge2e + GE2E + + + + SV2TTS (GE2E + Tacotron2) + AISHELL-3 + + VC0 - GE2E + Tacotron2 + SV2TTS (GE2E + FastSpeech2) AISHELL-3 - ge2e-tacotron2-aishell3 + VC1 - GE2E + FastSpeech2 + SV2TTS (ECAPA-TDNN + FastSpeech2) AISHELL-3 - ge2e-fastspeech2-aishell3 + VC2 GE2E + VITS AISHELL-3 - ge2e-vits-aishell3 + VITS-VC - + End-to-End VITS CSMSC / AISHELL-3 @@ -876,8 +888,9 @@ You are warmly welcome to submit questions in [discussions](https://github.com/P

## Acknowledgement -- Many thanks to [david-95](https://github.com/david-95) improved TTS, fixed multi-punctuation bug, and contributed to multiple program and data. -- Many thanks to [BarryKCL](https://github.com/BarryKCL) improved TTS Chinses frontend based on [G2PW](https://github.com/GitYCC/g2pW) +- Many thanks to [HighCWu](https://github.com/HighCWu)for adding [VITS-aishell3](./examples/aishell3/vits) and [VITS-VC](./examples/aishell3/vits-vc) examples. +- Many thanks to [david-95](https://github.com/david-95) improved TTS, fixed multi-punctuation bug, and contributed to multiple program and data. +- Many thanks to [BarryKCL](https://github.com/BarryKCL) improved TTS Chinses frontend based on [G2PW](https://github.com/GitYCC/g2pW). - Many thanks to [yeyupiaoling](https://github.com/yeyupiaoling)/[PPASR](https://github.com/yeyupiaoling/PPASR)/[PaddlePaddle-DeepSpeech](https://github.com/yeyupiaoling/PaddlePaddle-DeepSpeech)/[VoiceprintRecognition-PaddlePaddle](https://github.com/yeyupiaoling/VoiceprintRecognition-PaddlePaddle)/[AudioClassification-PaddlePaddle](https://github.com/yeyupiaoling/AudioClassification-PaddlePaddle) for years of attention, constructive advice and great help. - Many thanks to [mymagicpower](https://github.com/mymagicpower) for the Java implementation of ASR upon [short](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_sdk) and [long](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_long_audio_sdk) audio files. - Many thanks to [JiehangXie](https://github.com/JiehangXie)/[PaddleBoBo](https://github.com/JiehangXie/PaddleBoBo) for developing Virtual Uploader(VUP)/Virtual YouTuber(VTuber) with PaddleSpeech TTS function. diff --git a/README_cn.md b/README_cn.md index 21cd00a9..59012464 100644 --- a/README_cn.md +++ b/README_cn.md @@ -181,12 +181,20 @@ ### 近期更新 - +- ⚡ 2022.08.25: 发布 TTS [finetune](./examples/other/tts_finetune/tts3) 示例。 +- 🔥 2022.08.22: 新增 ERNIE-SAT 模型: [ERNIE-SAT-vctk](./examples/vctk/ernie_sat)、[ERNIE-SAT-aishell3](./examples/aishell3/ernie_sat)、[ERNIE-SAT-zh_en](./examples/aishell3_vctk/ernie_sat)。 +- 🔥 2022.08.15: 将 [g2pW](https://github.com/GitYCC/g2pW) 引入 TTS 中文文本前端。 +- 🔥 2022.08.09: 发布[中英文混合 TTS](./examples/zh_en_tts/tts3)。 +- ⚡ 2022.08.03: TTS CLI 新增 ONNXRuntime 推理方式。 +- 🎉 2022.07.18: 发布 VITS 模型: [VITS-csmsc](./examples/csmsc/vits)、[VITS-aishell3](./examples/aishell3/vits)、[VITS-VC](./examples/aishell3/vits-vc)。 +- 🎉 2022.06.22: 所有 TTS 模型支持了 ONNX 格式。 +- 🍀 2022.06.17: 新增 [PaddleSpeech 网页应用](./demos/speech_web)。 - 👑 2022.05.13: PaddleSpeech 发布 [PP-ASR](./docs/source/asr/PPASR_cn.md) 流式语音识别系统、[PP-TTS](./docs/source/tts/PPTTS_cn.md) 流式语音合成系统、[PP-VPR](docs/source/vpr/PPVPR_cn.md) 全链路声纹识别系统 -- 👏🏻 2022.05.06: PaddleSpeech Streaming Server 上线! 覆盖了语音识别（标点恢复、时间戳），和语音合成。 -- 👏🏻 2022.05.06: PaddleSpeech Server 上线! 覆盖了声音分类、语音识别、语音合成、声纹识别，标点恢复。 -- 👏🏻 2022.03.28: PaddleSpeech CLI 覆盖声音分类、语音识别、语音翻译（英译中）、语音合成，声纹验证。 -- 🤗 2021.12.14: PaddleSpeech [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) and [TTS](https://huggingface.co/spaces/KPatrick/PaddleSpeechTTS) Demos on Hugging Face Spaces are available! +- 👏🏻 2022.05.06: PaddleSpeech Streaming Server 上线！覆盖了语音识别（标点恢复、时间戳）和语音合成。 +- 👏🏻 2022.05.06: PaddleSpeech Server 上线！覆盖了声音分类、语音识别、语音合成、声纹识别，标点恢复。 +- 👏🏻 2022.03.28: PaddleSpeech CLI 覆盖声音分类、语音识别、语音翻译（英译中）、语音合成和声纹验证。 +- 🤗 2021.12.14: PaddleSpeech [ASR](https://huggingface.co/spaces/KPatrick/PaddleSpeechASR) 和 [TTS](https://huggingface.co/spaces/KPatrick/PaddleSpeechTTS) 可在 Hugging Face Spaces 上体验！ +- 👏🏻 2021.12.10: PaddleSpeech CLI 支持语音分类, 语音识别, 语音翻译（英译中）和语音合成。 ### 🔥 加入技术交流群获取入群福利 @@ -237,7 +245,6 @@ pip install . ## 快速开始 - 安装完成后，开发者可以通过命令行或者 Python 快速开始，命令行模式下改变 `--input` 可以尝试用自己的音频或文本测试，支持 16k wav 格式音频。 你也可以在 `aistudio` 中快速体验 👉🏻[一键预测，快速上手 Speech 开发任务](https://aistudio.baidu.com/aistudio/projectdetail/4353348?sUid=2470186&shared=1&ts=1660878142250)。 @@ -624,34 +631,40 @@ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声 - 声音克隆 + 声音克隆 GE2E Librispeech, etc. - ge2e + GE2E - GE2E + Tacotron2 + SV2TTS (GE2E + Tacotron2) AISHELL-3 - ge2e-tacotron2-aishell3 + VC0 - GE2E + FastSpeech2 + SV2TTS (GE2E + FastSpeech2) AISHELL-3 - ge2e-fastspeech2-aishell3 + VC1 - GE2E + VITS + SV2TTS (ECAPA-TDNN + FastSpeech2) AISHELL-3 - ge2e-vits-aishell3 + VC2 + + GE2E + VITS + AISHELL-3 + + VITS-VC + 端到端 @@ -896,8 +909,9 @@ PaddleSpeech 的 **语音合成** 主要包含三个模块：文本前端、声

## 致谢 -- 非常感谢 [david-95](https://github.com/david-95)修复句尾多标点符号出错的问题，补充frontend语音polyphonic 数据，贡献补充多条程序和数据 -- 非常感谢 [BarryKCL](https://github.com/BarryKCL)基于[G2PW](https://github.com/GitYCC/g2pW)对TTS中文文本前端的优化。 +- 非常感谢 [HighCWu](https://github.com/HighCWu) 新增 [VITS-aishell3](./examples/aishell3/vits) 和 [VITS-VC](./examples/aishell3/vits-vc) 代码示例。 +- 非常感谢 [david-95](https://github.com/david-95) 修复句尾多标点符号出错的问题，贡献补充多条程序和数据。 +- 非常感谢 [BarryKCL](https://github.com/BarryKCL) 基于 [G2PW](https://github.com/GitYCC/g2pW) 对 TTS 中文文本前端的优化。 - 非常感谢 [yeyupiaoling](https://github.com/yeyupiaoling)/[PPASR](https://github.com/yeyupiaoling/PPASR)/[PaddlePaddle-DeepSpeech](https://github.com/yeyupiaoling/PaddlePaddle-DeepSpeech)/[VoiceprintRecognition-PaddlePaddle](https://github.com/yeyupiaoling/VoiceprintRecognition-PaddlePaddle)/[AudioClassification-PaddlePaddle](https://github.com/yeyupiaoling/AudioClassification-PaddlePaddle) 多年来的关注和建议，以及在诸多问题上的帮助。 - 非常感谢 [mymagicpower](https://github.com/mymagicpower) 采用PaddleSpeech 对 ASR 的[短语音](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_sdk)及[长语音](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_long_audio_sdk)进行 Java 实现。 - 非常感谢 [JiehangXie](https://github.com/JiehangXie)/[PaddleBoBo](https://github.com/JiehangXie/PaddleBoBo) 采用 PaddleSpeech 语音合成功能实现 Virtual Uploader(VUP)/Virtual YouTuber(VTuber) 虚拟主播。 diff --git a/demos/text_to_speech/README.md b/demos/text_to_speech/README.md index 3288ecf2..41dcf820 100644 --- a/demos/text_to_speech/README.md +++ b/demos/text_to_speech/README.md @@ -16,8 +16,8 @@ You can choose one way from easy, meduim and hard to install paddlespeech. The input of this demo should be a text of the specific language that can be passed via argument. ### 3. Usage - Command Line (Recommended) + The default acoustic model is `Fastspeech2`, and the default vocoder is `HiFiGAN`, the default inference method is dygraph inference. - Chinese - The default acoustic model is `Fastspeech2`, and the default vocoder is `Parallel WaveGAN`. ```bash paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！" ``` @@ -58,6 +58,20 @@ The input of this demo should be a text of the specific language that can be pas paddlespeech tts --am fastspeech2_mix --voc pwgan_csmsc --lang mix --input "我们的声学模型使用了 Fast Speech Two, 声码器使用了 Parallel Wave GAN and Hifi GAN." --spk_id 175 --output mix_spk175_pwgan.wav paddlespeech tts --am fastspeech2_mix --voc hifigan_csmsc --lang mix --input "我们的声学模型使用了 Fast Speech Two, 声码器使用了 Parallel Wave GAN and Hifi GAN." --spk_id 175 --output mix_spk175.wav ``` + - Use ONNXRuntime infer： + ```bash + paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！" --output default.wav --use_onnx True + paddlespeech tts --am speedyspeech_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！" --output ss.wav --use_onnx True + paddlespeech tts --voc mb_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！" --output mb.wav --use_onnx True + paddlespeech tts --voc pwgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！" --output pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_aishell3 --voc pwgan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0 --output aishell3_fs2_pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_aishell3 --voc hifigan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0 --output aishell3_fs2_hifigan.wav --use_onnx True + paddlespeech tts --am fastspeech2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get." --output lj_fs2_pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_ljspeech --voc hifigan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get." --output lj_fs2_hifigan.wav --use_onnx True + paddlespeech tts --am fastspeech2_vctk --voc pwgan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0 --output vctk_fs2_pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_vctk --voc hifigan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0 --output vctk_fs2_hifigan.wav --use_onnx True + ``` + Usage: ```bash @@ -80,6 +94,8 @@ The input of this demo should be a text of the specific language that can be pas - `lang`: Language of tts task. Default: `zh`. - `device`: Choose device to execute model inference. Default: default device of paddlepaddle in current environment. - `output`: Output wave filepath. Default: `output.wav`. + - `use_onnx`: whether to usen ONNXRuntime inference. + - `fs`: sample rate for ONNX models when use specified model files. Output: ```bash @@ -87,38 +103,50 @@ The input of this demo should be a text of the specific language that can be pas ``` - Python API - ```python - import paddle - from paddlespeech.cli.tts import TTSExecutor - - tts_executor = TTSExecutor() - wav_file = tts_executor( - text='今天的天气不错啊', - output='output.wav', - am='fastspeech2_csmsc', - am_config=None, - am_ckpt=None, - am_stat=None, - spk_id=0, - phones_dict=None, - tones_dict=None, - speaker_dict=None, - voc='pwgan_csmsc', - voc_config=None, - voc_ckpt=None, - voc_stat=None, - lang='zh', - device=paddle.get_device()) - print('Wave file has been generated: {}'.format(wav_file)) - ``` - + - Dygraph infer: + ```python + import paddle + from paddlespeech.cli.tts import TTSExecutor + tts_executor = TTSExecutor() + wav_file = tts_executor( + text='今天的天气不错啊', + output='output.wav', + am='fastspeech2_csmsc', + am_config=None, + am_ckpt=None, + am_stat=None, + spk_id=0, + phones_dict=None, + tones_dict=None, + speaker_dict=None, + voc='pwgan_csmsc', + voc_config=None, + voc_ckpt=None, + voc_stat=None, + lang='zh', + device=paddle.get_device()) + print('Wave file has been generated: {}'.format(wav_file)) + ``` + - ONNXRuntime infer: + ```python + from paddlespeech.cli.tts import TTSExecutor + tts_executor = TTSExecutor() + wav_file = tts_executor( + text='对数据集进行预处理', + output='output.wav', + am='fastspeech2_csmsc', + voc='hifigan_csmsc', + lang='zh', + use_onnx=True, + cpu_threads=2) + ``` + Output: ```bash Wave file has been generated: output.wav ``` ### 4. Pretrained Models - Here is a list of pretrained models released by PaddleSpeech that can be used by command and python API: - Acoustic model diff --git a/demos/text_to_speech/README_cn.md b/demos/text_to_speech/README_cn.md index ec5eb5ae..4a413223 100644 --- a/demos/text_to_speech/README_cn.md +++ b/demos/text_to_speech/README_cn.md @@ -1,26 +1,23 @@ (简体中文|[English](./README.md)) # 语音合成 - ## 介绍 语音合成是一种自然语言建模过程，其将文本转换为语音以进行音频演示。 这个 demo 是一个从给定文本生成音频的实现，它可以通过使用 `PaddleSpeech` 的单个命令或 python 中的几行代码来实现。 - ## 使用方法 ### 1. 安装 请看[安装文档](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/docs/source/install_cn.md)。 -你可以从 easy，medium，hard 三中方式中选择一种方式安装。 +你可以从 easy，medium，hard 三种方式中选择一种方式安装。 ### 2. 准备输入 这个 demo 的输入是通过参数传递的特定语言的文本。 ### 3. 使用方法 - 命令行 (推荐使用) + 默认的声学模型是 `Fastspeech2`，默认的声码器是 `HiFiGAN`，默认推理方式是动态图推理。 - 中文 - - 默认的声学模型是 `Fastspeech2`，默认的声码器是 `Parallel WaveGAN`. ```bash paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！" ``` @@ -61,6 +58,19 @@ paddlespeech tts --am fastspeech2_mix --voc pwgan_csmsc --lang mix --input "我们的声学模型使用了 Fast Speech Two, 声码器使用了 Parallel Wave GAN and Hifi GAN." --spk_id 175 --output mix_spk175_pwgan.wav paddlespeech tts --am fastspeech2_mix --voc hifigan_csmsc --lang mix --input "我们的声学模型使用了 Fast Speech Two, 声码器使用了 Parallel Wave GAN and Hifi GAN." --spk_id 175 --output mix_spk175.wav ``` + - 使用 ONNXRuntime 推理： + ```bash + paddlespeech tts --input "你好，欢迎使用百度飞桨深度学习框架！" --output default.wav --use_onnx True + paddlespeech tts --am speedyspeech_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！" --output ss.wav --use_onnx True + paddlespeech tts --voc mb_melgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！" --output mb.wav --use_onnx True + paddlespeech tts --voc pwgan_csmsc --input "你好，欢迎使用百度飞桨深度学习框架！" --output pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_aishell3 --voc pwgan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0 --output aishell3_fs2_pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_aishell3 --voc hifigan_aishell3 --input "你好，欢迎使用百度飞桨深度学习框架！" --spk_id 0 --output aishell3_fs2_hifigan.wav --use_onnx True + paddlespeech tts --am fastspeech2_ljspeech --voc pwgan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get." --output lj_fs2_pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_ljspeech --voc hifigan_ljspeech --lang en --input "Life was like a box of chocolates, you never know what you're gonna get." --output lj_fs2_hifigan.wav --use_onnx True + paddlespeech tts --am fastspeech2_vctk --voc pwgan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0 --output vctk_fs2_pwgan.wav --use_onnx True + paddlespeech tts --am fastspeech2_vctk --voc hifigan_vctk --input "Life was like a box of chocolates, you never know what you're gonna get." --lang en --spk_id 0 --output vctk_fs2_hifigan.wav --use_onnx True + ``` 使用方法： @@ -84,6 +94,8 @@ - `lang`：TTS 任务的语言， 默认值：`zh`。 - `device`：执行预测的设备， 默认值：当前系统下 paddlepaddle 的默认 device。 - `output`：输出音频的路径， 默认值：`output.wav`。 + - `use_onnx`: 是否使用 ONNXRuntime 进行推理。 + - `fs`: 使用特定 ONNX 模型时的采样率。 输出： ```bash @@ -91,31 +103,44 @@ ``` - Python API - ```python - import paddle - from paddlespeech.cli.tts import TTSExecutor - - tts_executor = TTSExecutor() - wav_file = tts_executor( - text='今天的天气不错啊', - output='output.wav', - am='fastspeech2_csmsc', - am_config=None, - am_ckpt=None, - am_stat=None, - spk_id=0, - phones_dict=None, - tones_dict=None, - speaker_dict=None, - voc='pwgan_csmsc', - voc_config=None, - voc_ckpt=None, - voc_stat=None, - lang='zh', - device=paddle.get_device()) - print('Wave file has been generated: {}'.format(wav_file)) - ``` - + - 动态图推理: + ```python + import paddle + from paddlespeech.cli.tts import TTSExecutor + tts_executor = TTSExecutor() + wav_file = tts_executor( + text='今天的天气不错啊', + output='output.wav', + am='fastspeech2_csmsc', + am_config=None, + am_ckpt=None, + am_stat=None, + spk_id=0, + phones_dict=None, + tones_dict=None, + speaker_dict=None, + voc='pwgan_csmsc', + voc_config=None, + voc_ckpt=None, + voc_stat=None, + lang='zh', + device=paddle.get_device()) + print('Wave file has been generated: {}'.format(wav_file)) + ``` + - ONNXRuntime 推理: + ```python + from paddlespeech.cli.tts import TTSExecutor + tts_executor = TTSExecutor() + wav_file = tts_executor( + text='对数据集进行预处理', + output='output.wav', + am='fastspeech2_csmsc', + voc='hifigan_csmsc', + lang='zh', + use_onnx=True, + cpu_threads=2) + ``` + 输出： ```bash Wave file has been generated: output.wav From 056622d5f637565d90527407ca6c93e595b4c202 Mon Sep 17 00:00:00 2001 From: TianYuan Date: Thu, 8 Sep 2022 07:53:48 +0000 Subject: [PATCH 3/5] update speech_server config, test=tts --- README.md | 2 +- README_cn.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2fb281e7..f17cec13 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,7 @@ Developers can have a try of our speech server with [PaddleSpeech Server Command **Start server** ```shell -paddlespeech_server start --config_file ./paddlespeech/server/conf/application.yaml +paddlespeech_server start --config_file ./demos/speech_server/conf/application.yaml ``` **Access Speech Recognition Services** diff --git a/README_cn.md b/README_cn.md index 59012464..070a656a 100644 --- a/README_cn.md +++ b/README_cn.md @@ -400,7 +400,7 @@ python API 一键预测 **启动服务** ```shell -paddlespeech_server start --config_file ./paddlespeech/server/conf/application.yaml +paddlespeech_server start --config_file ./demos/speech_server/conf/application.yaml ``` **访问语音识别服务** From 881618dc2a8f5aff10e38abed6cde72a3f4454d5 Mon Sep 17 00:00:00 2001 From: "david.95" Date: Fri, 9 Sep 2022 10:54:29 +0800 Subject: [PATCH 4/5] add tool to compare two tests results, show difference -t tts --- examples/other/g2p/compare_badcase.py | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 examples/other/g2p/compare_badcase.py diff --git a/examples/other/g2p/compare_badcase.py b/examples/other/g2p/compare_badcase.py new file mode 100644 index 00000000..a4a78a23 --- /dev/null +++ b/examples/other/g2p/compare_badcase.py @@ -0,0 +1,63 @@ +# -*- encoding:utf-8 -*- +import re +import sys +''' +@arthur: david_95 + +Assum you executed g2p test twice, the WER rate have some gap, you would like to see what sentences error cause your rate up. +so you may get test result ( exp/g2p )into two directories, as exp/prefolder and exp/curfolder +run this program as "python compare_badcase.py prefolder curfolder" +then you will get diffrences between two run, uuid, phonetics, chinese samples + +''' + + +def compare(prefolder, curfolder): + ''' + compare file of text.g2p.pra in two folders + result P1 will be prefolder ; P2 will be curfolder, just about the sequence you input in argvs + ''' + + linecnt = 0 + pre_block = [] + cur_block = [] + zh_lines = [] + with open(prefolder + "/text.g2p.pra", "r") as pre_file, open( + curfolder + "/text.g2p.pra", "r") as cur_file: + for pre_line, cur_line in zip(pre_file, cur_file): + linecnt += 1 + + if linecnt < 11: #skip non-data head in files + continue + else: + pre_block.append(pre_line.strip()) + cur_block.append(cur_line.strip()) + if pre_line.strip().startswith( + "Eval:") and pre_line.strip() != cur_line.strip(): + uuid = pre_block[-5].replace("id: (baker_", "").replace(")", + "") + with open("data/g2p/text", 'r') as txt: + conlines = txt.readlines() + + for line in conlines: + if line.strip().startswith(uuid.strip()): + print(line) + zh_lines.append(re.sub(r"#[1234]", "", line)) + break + + print("*" + cur_block[-3]) # ref + print("P1 " + pre_block[-2]) + print("P2 " + cur_block[-2]) + print("P1 " + pre_block[-1]) + print("P2 " + cur_block[-1] + "\n\n") + pre_block = [] + cur_block = [] + + print("\n") + print(str.join("\n", zh_lines)) + + +if __name__ == '__main__': + assert len( + sys.argv) == 3, "Usage: python compare_badcase.py %prefolder %curfolder" + compare(sys.argv[1], sys.argv[2]) From 61422e71e32782b57f4deccec737365e5b4418c7 Mon Sep 17 00:00:00 2001 From: "david.95" Date: Fri, 9 Sep 2022 11:34:03 +0800 Subject: [PATCH 5/5] add tool to compare test badcase and add run examples,test=tts --- examples/other/g2p/compare_badcase.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/other/g2p/compare_badcase.py b/examples/other/g2p/compare_badcase.py index a4a78a23..d762459c 100644 --- a/examples/other/g2p/compare_badcase.py +++ b/examples/other/g2p/compare_badcase.py @@ -9,6 +9,9 @@ so you may get test result ( exp/g2p )into two directories, as exp/prefolder and run this program as "python compare_badcase.py prefolder curfolder" then you will get diffrences between two run, uuid, phonetics, chinese samples +examples: python compare_badcase.py exp/g2p_laotouzi exp/g2p +in this example: exp/g2p_laotouzi and exp/g2p are two folders with two g2p tests result + '''