Hui Zhang
3 years ago
committed by
GitHub
commit
fe29f74a1c
|
After Width: | Height: | Size: 441 KiB |
@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
export MAIN_ROOT=`realpath ${PWD}/../../`
|
||||||
|
|
||||||
|
export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
|
||||||
|
export LC_ALL=C
|
||||||
|
|
||||||
|
export PYTHONDONTWRITEBYTECODE=1
|
||||||
|
# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
|
||||||
|
export PYTHONIOENCODING=UTF-8
|
||||||
|
export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
|
||||||
|
|
||||||
|
MODEL=fastspeech2
|
||||||
|
export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL}
|
||||||
@ -0,0 +1,61 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
source path.sh
|
||||||
|
|
||||||
|
gpus=0
|
||||||
|
stage=0
|
||||||
|
stop_stage=100
|
||||||
|
|
||||||
|
# with the following command, you can choice the stage range you want to run
|
||||||
|
# such as `./run.sh --stage 0 --stop-stage 0`
|
||||||
|
# this can not be mixed use with `$1`, `$2` ...
|
||||||
|
source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
|
||||||
|
|
||||||
|
mkdir -p download
|
||||||
|
|
||||||
|
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||||
|
# install PaddleGAN
|
||||||
|
git clone https://github.com/PaddlePaddle/PaddleGAN.git
|
||||||
|
pip install -e PaddleGAN/
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||||
|
# download pretrained PaddleGAN model
|
||||||
|
wget -P download https://paddlegan.bj.bcebos.com/models/wav2lip_hq.pdparams
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||||
|
# download pretrained tts models and unzip
|
||||||
|
wget -P download https://paddlespeech.bj.bcebos.com/Parakeet/pwg_baker_ckpt_0.4.zip
|
||||||
|
unzip -d download download/pwg_baker_ckpt_0.4.zip
|
||||||
|
wget -P download https://paddlespeech.bj.bcebos.com/Parakeet/fastspeech2_nosil_baker_ckpt_0.4.zip
|
||||||
|
unzip -d download download/fastspeech2_nosil_baker_ckpt_0.4.zip
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
||||||
|
# run tts
|
||||||
|
CUDA_VISIBLE_DEVICES=${gpus} \
|
||||||
|
python3 ${BIN_DIR}/synthesize_e2e.py \
|
||||||
|
--fastspeech2-config=download/fastspeech2_nosil_baker_ckpt_0.4/default.yaml \
|
||||||
|
--fastspeech2-checkpoint=download/fastspeech2_nosil_baker_ckpt_0.4/snapshot_iter_76000.pdz \
|
||||||
|
--fastspeech2-stat=download/fastspeech2_nosil_baker_ckpt_0.4/speech_stats.npy \
|
||||||
|
--pwg-config=download/pwg_baker_ckpt_0.4/pwg_default.yaml \
|
||||||
|
--pwg-checkpoint=download/pwg_baker_ckpt_0.4/pwg_snapshot_iter_400000.pdz \
|
||||||
|
--pwg-stat=download/pwg_baker_ckpt_0.4/pwg_stats.npy \
|
||||||
|
--text=sentences.txt \
|
||||||
|
--output-dir=output/wavs \
|
||||||
|
--inference-dir=output/inference \
|
||||||
|
--phones-dict=download/fastspeech2_nosil_baker_ckpt_0.4/phone_id_map.txt
|
||||||
|
# output/inference is not needed here, which save the static models
|
||||||
|
rm -rf output/inference
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
|
||||||
|
# We only test one audio here, cause it's slow
|
||||||
|
CUDA_VISIBLE_DEVICES=${gpus} \
|
||||||
|
python3 PaddleGAN/applications/tools/wav2lip.py \
|
||||||
|
--checkpoint_path download/wav2lip_hq.pdparams \
|
||||||
|
--face Lamarr.png \
|
||||||
|
--audio output/wavs/000.wav \
|
||||||
|
--outfile output/tts_lips.mp4 \
|
||||||
|
--face_enhancement
|
||||||
|
fi
|
||||||
@ -0,0 +1 @@
|
|||||||
|
000 谁知青蛙一落地，竟变成了一位英俊的王子。于是遵照国王的意思，他做了公主的亲密伴侣。
|
||||||
|
After Width: | Height: | Size: 1.5 MiB |
@ -0,0 +1,74 @@
|
|||||||
|
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import paddle
|
||||||
|
from paddleocr import draw_ocr
|
||||||
|
from paddleocr import PaddleOCR
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate(args, ocr):
|
||||||
|
img_dir = Path(args.img_dir)
|
||||||
|
output_dir = Path(args.output_dir)
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
img_out_dir = output_dir / "imgs"
|
||||||
|
img_out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(output_dir / "sentences.txt", "w") as wf:
|
||||||
|
for name in os.listdir(img_dir):
|
||||||
|
id = name.split(".")[0]
|
||||||
|
img_path = img_dir / name
|
||||||
|
result = ocr.ocr(str(img_path), cls=True)
|
||||||
|
# draw result
|
||||||
|
image = Image.open(img_path).convert('RGB')
|
||||||
|
boxes = [line[0] for line in result]
|
||||||
|
txts = [line[1][0] for line in result]
|
||||||
|
scores = [line[1][1] for line in result]
|
||||||
|
im_show = draw_ocr(
|
||||||
|
image, boxes, txts, scores, font_path=args.font_path)
|
||||||
|
im_show = Image.fromarray(im_show)
|
||||||
|
paragraph = "".join(txts)
|
||||||
|
# 过滤出中文结果
|
||||||
|
pattern = re.compile(r'[^(\u4e00-\u9fa5)+，。？、]')
|
||||||
|
sentence = re.sub(pattern, '', paragraph)
|
||||||
|
im_show.save(img_out_dir / name)
|
||||||
|
wf.write(id + " " + sentence + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# parse args and config and redirect to train_sp
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Synthesize with fastspeech2 & parallel wavegan.")
|
||||||
|
parser.add_argument("--img-dir", default="imgs", type=str, help="img_dir.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-dir",
|
||||||
|
type=str,
|
||||||
|
default="output",
|
||||||
|
help="output sentences path.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--font-path", type=str, default="simfang.ttf", help="font path")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
paddle.set_device("gpu")
|
||||||
|
# need to run only once to download and load model into memory
|
||||||
|
ocr = PaddleOCR(use_angle_cls=True, lang='ch')
|
||||||
|
|
||||||
|
evaluate(args, ocr)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
export MAIN_ROOT=`realpath ${PWD}/../../`
|
||||||
|
|
||||||
|
export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
|
||||||
|
export LC_ALL=C
|
||||||
|
|
||||||
|
export PYTHONDONTWRITEBYTECODE=1
|
||||||
|
# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
|
||||||
|
export PYTHONIOENCODING=UTF-8
|
||||||
|
export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
|
||||||
|
|
||||||
|
MODEL=fastspeech2
|
||||||
|
export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL}
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
source path.sh
|
||||||
|
|
||||||
|
gpus=0
|
||||||
|
stage=0
|
||||||
|
stop_stage=100
|
||||||
|
|
||||||
|
# with the following command, you can choice the stage range you want to run
|
||||||
|
# such as `./run.sh --stage 0 --stop-stage 0`
|
||||||
|
# this can not be mixed use with `$1`, `$2` ...
|
||||||
|
source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
|
||||||
|
|
||||||
|
mkdir -p download
|
||||||
|
|
||||||
|
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||||
|
# install PaddleOCR
|
||||||
|
pip install "paddleocr>=2.0.1"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||||
|
# download pretrained tts models and unzip
|
||||||
|
wget -P download https://paddlespeech.bj.bcebos.com/Parakeet/pwg_baker_ckpt_0.4.zip
|
||||||
|
unzip -d download download/pwg_baker_ckpt_0.4.zip
|
||||||
|
wget -P download https://paddlespeech.bj.bcebos.com/Parakeet/fastspeech2_nosil_baker_ckpt_0.4.zip
|
||||||
|
unzip -d download download/fastspeech2_nosil_baker_ckpt_0.4.zip
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||||
|
# run ocr
|
||||||
|
CUDA_VISIBLE_DEVICES=${gpus} \
|
||||||
|
python3 ocr.py --img-dir=imgs --output-dir=output --font-path=simfang.ttf
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
||||||
|
# run tts
|
||||||
|
CUDA_VISIBLE_DEVICES=${gpus} \
|
||||||
|
python3 ${BIN_DIR}/synthesize_e2e.py \
|
||||||
|
--fastspeech2-config=download/fastspeech2_nosil_baker_ckpt_0.4/default.yaml \
|
||||||
|
--fastspeech2-checkpoint=download/fastspeech2_nosil_baker_ckpt_0.4/snapshot_iter_76000.pdz \
|
||||||
|
--fastspeech2-stat=download/fastspeech2_nosil_baker_ckpt_0.4/speech_stats.npy \
|
||||||
|
--pwg-config=download/pwg_baker_ckpt_0.4/pwg_default.yaml \
|
||||||
|
--pwg-checkpoint=download/pwg_baker_ckpt_0.4/pwg_snapshot_iter_400000.pdz \
|
||||||
|
--pwg-stat=download/pwg_baker_ckpt_0.4/pwg_stats.npy \
|
||||||
|
--text=output/sentences.txt \
|
||||||
|
--output-dir=output/wavs \
|
||||||
|
--inference-dir=output/inference \
|
||||||
|
--phones-dict=download/fastspeech2_nosil_baker_ckpt_0.4/phone_id_map.txt
|
||||||
|
# output/inference is not needed here, which save the static models
|
||||||
|
rm -rf output/inference
|
||||||
|
fi
|
||||||
Binary file not shown.
@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
export MAIN_ROOT=`realpath ${PWD}/../../`
|
||||||
|
|
||||||
|
export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
|
||||||
|
export LC_ALL=C
|
||||||
|
|
||||||
|
export PYTHONDONTWRITEBYTECODE=1
|
||||||
|
# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
|
||||||
|
export PYTHONIOENCODING=UTF-8
|
||||||
|
export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
|
||||||
|
MODEL=fastspeech2
|
||||||
|
export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL}
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
source path.sh
|
||||||
|
|
||||||
|
gpus=0
|
||||||
|
stage=0
|
||||||
|
stop_stage=100
|
||||||
|
|
||||||
|
# with the following command, you can choice the stage range you want to run
|
||||||
|
# such as `./run.sh --stage 0 --stop-stage 0`
|
||||||
|
# this can not be mixed use with `$1`, `$2` ...
|
||||||
|
source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
|
||||||
|
|
||||||
|
mkdir -p download
|
||||||
|
|
||||||
|
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||||
|
# download pretrained tts models and unzip
|
||||||
|
wget -P download https://paddlespeech.bj.bcebos.com/Parakeet/pwg_baker_ckpt_0.4.zip
|
||||||
|
unzip -d download download/pwg_baker_ckpt_0.4.zip
|
||||||
|
wget -P download https://paddlespeech.bj.bcebos.com/Parakeet/fastspeech2_nosil_baker_ckpt_0.4.zip
|
||||||
|
unzip -d download download/fastspeech2_nosil_baker_ckpt_0.4.zip
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||||
|
# run tts
|
||||||
|
CUDA_VISIBLE_DEVICES=${gpus} \
|
||||||
|
python3 style_syn.py \
|
||||||
|
--fastspeech2-config=download/fastspeech2_nosil_baker_ckpt_0.4/default.yaml \
|
||||||
|
--fastspeech2-checkpoint=download/fastspeech2_nosil_baker_ckpt_0.4/snapshot_iter_76000.pdz \
|
||||||
|
--fastspeech2-stat=download/fastspeech2_nosil_baker_ckpt_0.4/speech_stats.npy \
|
||||||
|
--fastspeech2-pitch-stat=download/fastspeech2_nosil_baker_ckpt_0.4/pitch_stats.npy \
|
||||||
|
--fastspeech2-energy-stat=download/fastspeech2_nosil_baker_ckpt_0.4/energy_stats.npy \
|
||||||
|
--pwg-config=download/pwg_baker_ckpt_0.4/pwg_default.yaml \
|
||||||
|
--pwg-checkpoint=download/pwg_baker_ckpt_0.4/pwg_snapshot_iter_400000.pdz \
|
||||||
|
--pwg-stat=download/pwg_baker_ckpt_0.4/pwg_stats.npy \
|
||||||
|
--text=${BIN_DIR}/../sentences.txt \
|
||||||
|
--output-dir=output \
|
||||||
|
--phones-dict=download/fastspeech2_nosil_baker_ckpt_0.4/phone_id_map.txt
|
||||||
|
fi
|
||||||
@ -0,0 +1 @@
|
|||||||
|
000 谁知青蛙一落地，竟变成了一位英俊的王子。于是遵照国王的意思，他做了公主的亲密伴侣。
|
||||||
@ -0,0 +1,328 @@
|
|||||||
|
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import paddle
|
||||||
|
import soundfile as sf
|
||||||
|
import yaml
|
||||||
|
from yacs.config import CfgNode
|
||||||
|
|
||||||
|
from paddlespeech.t2s.frontend.zh_frontend import Frontend
|
||||||
|
from paddlespeech.t2s.models.fastspeech2 import FastSpeech2
|
||||||
|
from paddlespeech.t2s.models.fastspeech2 import FastSpeech2Inference
|
||||||
|
from paddlespeech.t2s.models.parallel_wavegan import PWGGenerator
|
||||||
|
from paddlespeech.t2s.models.parallel_wavegan import PWGInference
|
||||||
|
from paddlespeech.t2s.modules.normalizer import ZScore
|
||||||
|
|
||||||
|
|
||||||
|
class StyleFastSpeech2Inference(FastSpeech2Inference):
|
||||||
|
def __init__(self, normalizer, model, pitch_stats_path, energy_stats_path):
|
||||||
|
super().__init__(normalizer, model)
|
||||||
|
pitch_mean, pitch_std = np.load(pitch_stats_path)
|
||||||
|
self.pitch_mean = paddle.to_tensor(pitch_mean)
|
||||||
|
self.pitch_std = paddle.to_tensor(pitch_std)
|
||||||
|
energy_mean, energy_std = np.load(energy_stats_path)
|
||||||
|
self.energy_mean = paddle.to_tensor(energy_mean)
|
||||||
|
self.energy_std = paddle.to_tensor(energy_std)
|
||||||
|
|
||||||
|
def denorm(self, data, mean, std):
|
||||||
|
return data * std + mean
|
||||||
|
|
||||||
|
def norm(self, data, mean, std):
|
||||||
|
return (data - mean) / std
|
||||||
|
|
||||||
|
def forward(self,
|
||||||
|
text: paddle.Tensor,
|
||||||
|
durations: Union[paddle.Tensor, np.ndarray]=None,
|
||||||
|
durations_scale: Union[int, float]=None,
|
||||||
|
durations_bias: Union[int, float]=None,
|
||||||
|
pitch: Union[paddle.Tensor, np.ndarray]=None,
|
||||||
|
pitch_scale: Union[int, float]=None,
|
||||||
|
pitch_bias: Union[int, float]=None,
|
||||||
|
energy: Union[paddle.Tensor, np.ndarray]=None,
|
||||||
|
energy_scale: Union[int, float]=None,
|
||||||
|
energy_bias: Union[int, float]=None,
|
||||||
|
robot: bool=False):
|
||||||
|
"""
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
text : Tensor(int64)
|
||||||
|
Input sequence of characters (T,).
|
||||||
|
speech : Tensor, optional
|
||||||
|
Feature sequence to extract style (N, idim).
|
||||||
|
durations : paddle.Tensor/np.ndarray, optional (int64)
|
||||||
|
Groundtruth of duration (T,), this will overwrite the set of durations_scale and durations_bias
|
||||||
|
durations_scale: int/float, optional
|
||||||
|
durations_bias: int/float, optional
|
||||||
|
pitch : paddle.Tensor/np.ndarray, optional
|
||||||
|
Groundtruth of token-averaged pitch (T, 1), this will overwrite the set of pitch_scale and pitch_bias
|
||||||
|
pitch_scale: int/float, optional
|
||||||
|
In denormed HZ domain.
|
||||||
|
pitch_bias: int/float, optional
|
||||||
|
In denormed HZ domain.
|
||||||
|
energy : paddle.Tensor/np.ndarray, optional
|
||||||
|
Groundtruth of token-averaged energy (T, 1), this will overwrite the set of energy_scale and energy_bias
|
||||||
|
energy_scale: int/float, optional
|
||||||
|
In denormed domain.
|
||||||
|
energy_bias: int/float, optional
|
||||||
|
In denormed domain.
|
||||||
|
robot : bool, optional
|
||||||
|
Weather output robot style
|
||||||
|
Returns
|
||||||
|
----------
|
||||||
|
Tensor
|
||||||
|
Output sequence of features (L, odim).
|
||||||
|
"""
|
||||||
|
normalized_mel, d_outs, p_outs, e_outs = self.acoustic_model.inference(
|
||||||
|
text, durations=None, pitch=None, energy=None)
|
||||||
|
# priority: groundtruth > scale/bias > previous output
|
||||||
|
# set durations
|
||||||
|
if isinstance(durations, np.ndarray):
|
||||||
|
durations = paddle.to_tensor(durations)
|
||||||
|
elif isinstance(durations, paddle.Tensor):
|
||||||
|
durations = durations
|
||||||
|
elif durations_scale or durations_bias:
|
||||||
|
durations_scale = durations_scale if durations_scale is not None else 1
|
||||||
|
durations_bias = durations_bias if durations_bias is not None else 0
|
||||||
|
durations = durations_scale * d_outs + durations_bias
|
||||||
|
else:
|
||||||
|
durations = d_outs
|
||||||
|
|
||||||
|
if robot:
|
||||||
|
# set normed pitch to zeros have the same effect with set denormd ones to mean
|
||||||
|
pitch = paddle.zeros(p_outs.shape)
|
||||||
|
|
||||||
|
# set pitch, can overwrite robot set
|
||||||
|
if isinstance(pitch, np.ndarray):
|
||||||
|
pitch = paddle.to_tensor(pitch)
|
||||||
|
elif isinstance(pitch, paddle.Tensor):
|
||||||
|
pitch = pitch
|
||||||
|
elif pitch_scale or pitch_bias:
|
||||||
|
pitch_scale = pitch_scale if pitch_scale is not None else 1
|
||||||
|
pitch_bias = pitch_bias if pitch_bias is not None else 0
|
||||||
|
p_Hz = paddle.exp(
|
||||||
|
self.denorm(p_outs, self.pitch_mean, self.pitch_std))
|
||||||
|
p_HZ = pitch_scale * p_Hz + pitch_bias
|
||||||
|
pitch = self.norm(paddle.log(p_HZ), self.pitch_mean, self.pitch_std)
|
||||||
|
else:
|
||||||
|
pitch = p_outs
|
||||||
|
|
||||||
|
# set energy
|
||||||
|
if isinstance(energy, np.ndarray):
|
||||||
|
energy = paddle.to_tensor(energy)
|
||||||
|
elif isinstance(energy, paddle.Tensor):
|
||||||
|
energy = energy
|
||||||
|
elif energy_scale or energy_bias:
|
||||||
|
energy_scale = energy_scale if energy_scale is not None else 1
|
||||||
|
energy_bias = energy_bias if energy_bias is not None else 0
|
||||||
|
e_dnorm = self.denorm(e_outs, self.energy_mean, self.energy_std)
|
||||||
|
e_dnorm = energy_scale * e_dnorm + energy_bias
|
||||||
|
energy = self.norm(e_dnorm, self.energy_mean, self.energy_std)
|
||||||
|
else:
|
||||||
|
energy = e_outs
|
||||||
|
|
||||||
|
normalized_mel, d_outs, p_outs, e_outs = self.acoustic_model.inference(
|
||||||
|
text,
|
||||||
|
durations=durations,
|
||||||
|
pitch=pitch,
|
||||||
|
energy=energy,
|
||||||
|
use_teacher_forcing=True)
|
||||||
|
|
||||||
|
logmel = self.normalizer.inverse(normalized_mel)
|
||||||
|
return logmel
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate(args, fastspeech2_config, pwg_config):
|
||||||
|
|
||||||
|
# construct dataset for evaluation
|
||||||
|
sentences = []
|
||||||
|
with open(args.text, 'rt') as f:
|
||||||
|
for line in f:
|
||||||
|
utt_id, sentence = line.strip().split()
|
||||||
|
sentences.append((utt_id, sentence))
|
||||||
|
|
||||||
|
with open(args.phones_dict, "r") as f:
|
||||||
|
phn_id = [line.strip().split() for line in f.readlines()]
|
||||||
|
vocab_size = len(phn_id)
|
||||||
|
print("vocab_size:", vocab_size)
|
||||||
|
|
||||||
|
odim = fastspeech2_config.n_mels
|
||||||
|
model = FastSpeech2(
|
||||||
|
idim=vocab_size, odim=odim, **fastspeech2_config["model"])
|
||||||
|
|
||||||
|
model.set_state_dict(
|
||||||
|
paddle.load(args.fastspeech2_checkpoint)["main_params"])
|
||||||
|
model.eval()
|
||||||
|
|
||||||
|
vocoder = PWGGenerator(**pwg_config["generator_params"])
|
||||||
|
vocoder.set_state_dict(paddle.load(args.pwg_checkpoint)["generator_params"])
|
||||||
|
vocoder.remove_weight_norm()
|
||||||
|
vocoder.eval()
|
||||||
|
print("model done!")
|
||||||
|
|
||||||
|
frontend = Frontend(phone_vocab_path=args.phones_dict)
|
||||||
|
print("frontend done!")
|
||||||
|
|
||||||
|
stat = np.load(args.fastspeech2_stat)
|
||||||
|
mu, std = stat
|
||||||
|
mu = paddle.to_tensor(mu)
|
||||||
|
std = paddle.to_tensor(std)
|
||||||
|
fastspeech2_normalizer = ZScore(mu, std)
|
||||||
|
|
||||||
|
stat = np.load(args.pwg_stat)
|
||||||
|
mu, std = stat
|
||||||
|
mu = paddle.to_tensor(mu)
|
||||||
|
std = paddle.to_tensor(std)
|
||||||
|
pwg_normalizer = ZScore(mu, std)
|
||||||
|
|
||||||
|
fastspeech2_inference = StyleFastSpeech2Inference(
|
||||||
|
fastspeech2_normalizer, model, args.fastspeech2_pitch_stat,
|
||||||
|
args.fastspeech2_energy_stat)
|
||||||
|
fastspeech2_inference.eval()
|
||||||
|
|
||||||
|
pwg_inference = PWGInference(pwg_normalizer, vocoder)
|
||||||
|
pwg_inference.eval()
|
||||||
|
|
||||||
|
output_dir = Path(args.output_dir)
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
styles = ["normal", "robot", "1.2xspeed", "0.8xspeed", "child_voice"]
|
||||||
|
for style in styles:
|
||||||
|
robot = False
|
||||||
|
durations = None
|
||||||
|
durations_scale = None
|
||||||
|
durations_bias = None
|
||||||
|
pitch = None
|
||||||
|
pitch_scale = None
|
||||||
|
pitch_bias = None
|
||||||
|
energy = None
|
||||||
|
energy_scale = None
|
||||||
|
energy_bias = None
|
||||||
|
if style == "robot":
|
||||||
|
# all tones in phones be `1`
|
||||||
|
# all pitch should be the same, we use mean here
|
||||||
|
robot = True
|
||||||
|
if style == "1.2xspeed":
|
||||||
|
durations_scale = 1 / 1.2
|
||||||
|
if style == "0.8xspeed":
|
||||||
|
durations_scale = 1 / 0.8
|
||||||
|
if style == "child_voice":
|
||||||
|
pitch_scale = 1.3
|
||||||
|
sub_output_dir = output_dir / style
|
||||||
|
sub_output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for utt_id, sentence in sentences:
|
||||||
|
input_ids = frontend.get_input_ids(
|
||||||
|
sentence, merge_sentences=True, robot=robot)
|
||||||
|
phone_ids = input_ids["phone_ids"][0]
|
||||||
|
|
||||||
|
with paddle.no_grad():
|
||||||
|
mel = fastspeech2_inference(
|
||||||
|
phone_ids,
|
||||||
|
durations=durations,
|
||||||
|
durations_scale=durations_scale,
|
||||||
|
durations_bias=durations_bias,
|
||||||
|
pitch=pitch,
|
||||||
|
pitch_scale=pitch_scale,
|
||||||
|
pitch_bias=pitch_bias,
|
||||||
|
energy=energy,
|
||||||
|
energy_scale=energy_scale,
|
||||||
|
energy_bias=energy_bias,
|
||||||
|
robot=robot)
|
||||||
|
wav = pwg_inference(mel)
|
||||||
|
|
||||||
|
sf.write(
|
||||||
|
str(sub_output_dir / (utt_id + ".wav")),
|
||||||
|
wav.numpy(),
|
||||||
|
samplerate=fastspeech2_config.fs)
|
||||||
|
print(f"{style}_{utt_id} done!")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# parse args and config and redirect to train_sp
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Synthesize with fastspeech2 & parallel wavegan.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fastspeech2-config", type=str, help="fastspeech2 config file.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fastspeech2-checkpoint",
|
||||||
|
type=str,
|
||||||
|
help="fastspeech2 checkpoint to load.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fastspeech2-stat",
|
||||||
|
type=str,
|
||||||
|
help="mean and standard deviation used to normalize spectrogram when training fastspeech2."
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fastspeech2-pitch-stat",
|
||||||
|
type=str,
|
||||||
|
help="mean and standard deviation used to normalize pitch when training fastspeech2"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fastspeech2-energy-stat",
|
||||||
|
type=str,
|
||||||
|
help="mean and standard deviation used to normalize energy when training fastspeech2."
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--pwg-config", type=str, help="parallel wavegan config file.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--pwg-checkpoint",
|
||||||
|
type=str,
|
||||||
|
help="parallel wavegan generator parameters to load.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--pwg-stat",
|
||||||
|
type=str,
|
||||||
|
help="mean and standard deviation used to normalize spectrogram when training parallel wavegan."
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--phones-dict",
|
||||||
|
type=str,
|
||||||
|
default="phone_id_map.txt",
|
||||||
|
help="phone vocabulary file.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--text",
|
||||||
|
type=str,
|
||||||
|
help="text to synthesize, a 'utt_id sentence' pair per line.")
|
||||||
|
parser.add_argument("--output-dir", type=str, help="output dir.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.")
|
||||||
|
parser.add_argument("--verbose", type=int, default=1, help="verbose.")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.ngpu == 0:
|
||||||
|
paddle.set_device("cpu")
|
||||||
|
elif args.ngpu > 0:
|
||||||
|
paddle.set_device("gpu")
|
||||||
|
else:
|
||||||
|
print("ngpu should >= 0 !")
|
||||||
|
|
||||||
|
with open(args.fastspeech2_config) as f:
|
||||||
|
fastspeech2_config = CfgNode(yaml.safe_load(f))
|
||||||
|
with open(args.pwg_config) as f:
|
||||||
|
pwg_config = CfgNode(yaml.safe_load(f))
|
||||||
|
|
||||||
|
print("========Args========")
|
||||||
|
print(yaml.safe_dump(vars(args)))
|
||||||
|
print("========Config========")
|
||||||
|
print(fastspeech2_config)
|
||||||
|
print(pwg_config)
|
||||||
|
|
||||||
|
evaluate(args, fastspeech2_config, pwg_config)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
After Width: | Height: | Size: 47 KiB |
|
After Width: | Height: | Size: 117 KiB |
|
After Width: | Height: | Size: 1.5 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 108 KiB |
|
After Width: | Height: | Size: 224 KiB |
|
After Width: | Height: | Size: 1.5 MiB |
|
After Width: | Height: | Size: 581 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 368 KiB |
@ -0,0 +1,146 @@
|
|||||||
|
# Parallel WaveGAN with AISHELL-3
|
||||||
|
This example contains code used to train a [parallel wavegan](http://arxiv.org/abs/1910.11480) model with [AISHELL-3](http://www.aishelltech.com/aishell_3).
|
||||||
|
|
||||||
|
AISHELL-3 is a large-scale and high-fidelity multi-speaker Mandarin speech corpus which could be used to train multi-speaker Text-to-Speech (TTS) systems.
|
||||||
|
## Dataset
|
||||||
|
### Download and Extract the datasaet
|
||||||
|
Download AISHELL-3.
|
||||||
|
```bash
|
||||||
|
wget https://www.openslr.org/resources/93/data_aishell3.tgz
|
||||||
|
```
|
||||||
|
Extract AISHELL-3.
|
||||||
|
```bash
|
||||||
|
mkdir data_aishell3
|
||||||
|
tar zxvf data_aishell3.tgz -C data_aishell3
|
||||||
|
```
|
||||||
|
### Get MFA result of AISHELL-3 and Extract it
|
||||||
|
We use [MFA2.x](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) to get durations for aishell3_fastspeech2.
|
||||||
|
You can download from here [aishell3_alignment_tone.tar.gz](https://paddlespeech.bj.bcebos.com/MFA/AISHELL-3/with_tone/aishell3_alignment_tone.tar.gz), or train your own MFA model reference to [use_mfa example](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/use_mfa) (use MFA1.x now) of our repo.
|
||||||
|
|
||||||
|
## Get Started
|
||||||
|
Assume the path to the dataset is `~/datasets/data_aishell3`.
|
||||||
|
Assume the path to the MFA result of AISHELL-3 is `./aishell3_alignment_tone`.
|
||||||
|
Run the command below to
|
||||||
|
1. **source path**.
|
||||||
|
2. preprocess the dataset,
|
||||||
|
3. train the model.
|
||||||
|
4. synthesize wavs.
|
||||||
|
- synthesize waveform from `metadata.jsonl`.
|
||||||
|
```bash
|
||||||
|
./run.sh
|
||||||
|
```
|
||||||
|
### Preprocess the dataset
|
||||||
|
```bash
|
||||||
|
./local/preprocess.sh ${conf_path}
|
||||||
|
```
|
||||||
|
When it is done. A `dump` folder is created in the current directory. The structure of the dump folder is listed below.
|
||||||
|
|
||||||
|
```text
|
||||||
|
dump
|
||||||
|
├── dev
|
||||||
|
│ ├── norm
|
||||||
|
│ └── raw
|
||||||
|
├── test
|
||||||
|
│ ├── norm
|
||||||
|
│ └── raw
|
||||||
|
└── train
|
||||||
|
├── norm
|
||||||
|
├── raw
|
||||||
|
└── feats_stats.npy
|
||||||
|
```
|
||||||
|
|
||||||
|
The dataset is split into 3 parts, namely `train`, `dev` and `test`, each of which contains a `norm` and `raw` subfolder. The `raw` folder contains log magnitude of mel spectrogram of each utterances, while the norm folder contains normalized spectrogram. The statistics used to normalize the spectrogram is computed from the training set, which is located in `dump/train/feats_stats.npy`.
|
||||||
|
|
||||||
|
Also there is a `metadata.jsonl` in each subfolder. It is a table-like file which contains id and paths to spectrogam of each utterance.
|
||||||
|
|
||||||
|
### Train the model
|
||||||
|
```bash
|
||||||
|
CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path}
|
||||||
|
```
|
||||||
|
`./local/train.sh` calls `${BIN_DIR}/train.py`.
|
||||||
|
Here's the complete help message.
|
||||||
|
|
||||||
|
```text
|
||||||
|
usage: train.py [-h] [--config CONFIG] [--train-metadata TRAIN_METADATA]
|
||||||
|
[--dev-metadata DEV_METADATA] [--output-dir OUTPUT_DIR]
|
||||||
|
[--ngpu NGPU] [--verbose VERBOSE] [--batch-size BATCH_SIZE]
|
||||||
|
[--max-iter MAX_ITER] [--run-benchmark RUN_BENCHMARK]
|
||||||
|
[--profiler_options PROFILER_OPTIONS]
|
||||||
|
|
||||||
|
Train a ParallelWaveGAN model.
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
--config CONFIG config file to overwrite default config.
|
||||||
|
--train-metadata TRAIN_METADATA
|
||||||
|
training data.
|
||||||
|
--dev-metadata DEV_METADATA
|
||||||
|
dev data.
|
||||||
|
--output-dir OUTPUT_DIR
|
||||||
|
output dir.
|
||||||
|
--ngpu NGPU if ngpu == 0, use cpu.
|
||||||
|
--verbose VERBOSE verbose.
|
||||||
|
|
||||||
|
benchmark:
|
||||||
|
arguments related to benchmark.
|
||||||
|
|
||||||
|
--batch-size BATCH_SIZE
|
||||||
|
batch size.
|
||||||
|
--max-iter MAX_ITER train max steps.
|
||||||
|
--run-benchmark RUN_BENCHMARK
|
||||||
|
runing benchmark or not, if True, use the --batch-size
|
||||||
|
and --max-iter.
|
||||||
|
--profiler_options PROFILER_OPTIONS
|
||||||
|
The option of profiler, which should be in format
|
||||||
|
"key1=value1;key2=value2;key3=value3".
|
||||||
|
```
|
||||||
|
|
||||||
|
1. `--config` is a config file in yaml format to overwrite the default config, which can be found at `conf/default.yaml`.
|
||||||
|
2. `--train-metadata` and `--dev-metadata` should be the metadata file in the normalized subfolder of `train` and `dev` in the `dump` folder.
|
||||||
|
3. `--output-dir` is the directory to save the results of the experiment. Checkpoints are save in `checkpoints/` inside this directory.
|
||||||
|
4. `--ngpu` is the number of gpus to use, if ngpu == 0, use cpu.
|
||||||
|
|
||||||
|
### Synthesize
|
||||||
|
`./local/synthesize.sh` calls `${BIN_DIR}/synthesize.py`, which can synthesize waveform from `metadata.jsonl`.
|
||||||
|
```bash
|
||||||
|
CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name}
|
||||||
|
```
|
||||||
|
```text
|
||||||
|
usage: synthesize.py [-h] [--config CONFIG] [--checkpoint CHECKPOINT]
|
||||||
|
[--test-metadata TEST_METADATA] [--output-dir OUTPUT_DIR]
|
||||||
|
[--ngpu NGPU] [--verbose VERBOSE]
|
||||||
|
|
||||||
|
Synthesize with parallel wavegan.
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
--config CONFIG parallel wavegan config file.
|
||||||
|
--checkpoint CHECKPOINT
|
||||||
|
snapshot to load.
|
||||||
|
--test-metadata TEST_METADATA
|
||||||
|
dev data.
|
||||||
|
--output-dir OUTPUT_DIR
|
||||||
|
output dir.
|
||||||
|
--ngpu NGPU if ngpu == 0, use cpu.
|
||||||
|
--verbose VERBOSE verbose.
|
||||||
|
```
|
||||||
|
|
||||||
|
1. `--config` parallel wavegan config file. You should use the same config with which the model is trained.
|
||||||
|
2. `--checkpoint` is the checkpoint to load. Pick one of the checkpoints from `checkpoints` inside the training output directory. If you use the pretrained model, use the `snapshot_iter_1000000.pdz `.
|
||||||
|
3. `--test-metadata` is the metadata of the test dataset. Use the `metadata.jsonl` in the `dev/norm` subfolder from the processed directory.
|
||||||
|
4. `--output-dir` is the directory to save the synthesized audio files.
|
||||||
|
5. `--ngpu` is the number of gpus to use, if ngpu == 0, use cpu.
|
||||||
|
|
||||||
|
## Pretrained Models
|
||||||
|
Pretrained models can be downloaded here [pwg_aishell3_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/pwg_aishell3_ckpt_0.5.zip).
|
||||||
|
|
||||||
|
Parallel WaveGAN checkpoint contains files listed below.
|
||||||
|
|
||||||
|
```text
|
||||||
|
pwg_aishell3_ckpt_0.5
|
||||||
|
├── default.yaml # default config used to train parallel wavegan
|
||||||
|
├── feats_stats.npy # statistics used to normalize spectrogram when training parallel wavegan
|
||||||
|
└── snapshot_iter_1000000.pdz # generator parameters of parallel wavegan
|
||||||
|
```
|
||||||
|
## Acknowledgement
|
||||||
|
We adapted some code from https://github.com/kan-bayashi/ParallelWaveGAN.
|
||||||
Loading…
Reference in new issue