TianYuan
3 years ago
parent
0ea9def0b8
commit
94688264c7
@ -0,0 +1,282 @@
|
||||
###########################################################
|
||||
# FEATURE EXTRACTION SETTING #
|
||||
###########################################################
|
||||
|
||||
fs: 24000 # sr
|
||||
n_fft: 2048 # FFT size (samples).
|
||||
n_shift: 300 # Hop size (samples). 12.5ms
|
||||
win_length: 1200 # Window length (samples). 50ms
|
||||
# If set to null, it will be the same as fft_size.
|
||||
window: "hann" # Window function.
|
||||
|
||||
# Only used for feats_type != raw
|
||||
|
||||
fmin: 80 # Minimum frequency of Mel basis.
|
||||
fmax: 7600 # Maximum frequency of Mel basis.
|
||||
n_mels: 80 # The number of mel basis.
|
||||
|
||||
mean_phn_span: 8
|
||||
mlm_prob: 0.8
|
||||
|
||||
###########################################################
|
||||
# DATA SETTING #
|
||||
###########################################################
|
||||
batch_size: 64
|
||||
num_workers: 2
|
||||
|
||||
###########################################################
|
||||
# MODEL SETTING #
|
||||
###########################################################
|
||||
model:
|
||||
text_masking: false
|
||||
postnet_layers: 5
|
||||
postnet_filts: 5
|
||||
postnet_chans: 256
|
||||
encoder_type: conformer
|
||||
decoder_type: conformer
|
||||
enc_input_layer: sega_mlm
|
||||
enc_pre_speech_layer: 0
|
||||
enc_cnn_module_kernel: 7
|
||||
enc_attention_dim: 384
|
||||
enc_attention_heads: 2
|
||||
enc_linear_units: 1536
|
||||
enc_num_blocks: 4
|
||||
enc_dropout_rate: 0.2
|
||||
enc_positional_dropout_rate: 0.2
|
||||
enc_attention_dropout_rate: 0.2
|
||||
enc_normalize_before: true
|
||||
enc_macaron_style: true
|
||||
enc_use_cnn_module: true
|
||||
enc_selfattention_layer_type: legacy_rel_selfattn
|
||||
enc_activation_type: swish
|
||||
enc_pos_enc_layer_type: legacy_rel_pos
|
||||
enc_positionwise_layer_type: conv1d
|
||||
enc_positionwise_conv_kernel_size: 3
|
||||
dec_cnn_module_kernel: 31
|
||||
dec_attention_dim: 384
|
||||
dec_attention_heads: 2
|
||||
dec_linear_units: 1536
|
||||
dec_num_blocks: 4
|
||||
dec_dropout_rate: 0.2
|
||||
dec_positional_dropout_rate: 0.2
|
||||
dec_attention_dropout_rate: 0.2
|
||||
dec_macaron_style: true
|
||||
dec_use_cnn_module: true
|
||||
dec_selfattention_layer_type: legacy_rel_selfattn
|
||||
dec_activation_type: swish
|
||||
dec_pos_enc_layer_type: legacy_rel_pos
|
||||
dec_positionwise_layer_type: conv1d
|
||||
dec_positionwise_conv_kernel_size: 3
|
||||
|
||||
###########################################################
|
||||
# OPTIMIZER SETTING #
|
||||
###########################################################
|
||||
optimizer:
|
||||
optim: adam # optimizer type
|
||||
learning_rate: 0.001 # learning rate
|
||||
|
||||
###########################################################
|
||||
# TRAINING SETTING #
|
||||
###########################################################
|
||||
max_epoch: 200
|
||||
num_snapshots: 5
|
||||
|
||||
###########################################################
|
||||
# OTHER SETTING #
|
||||
###########################################################
|
||||
seed: 10086
|
||||
|
||||
token_list:
|
||||
- <blank>
|
||||
- <unk>
|
||||
- d
|
||||
- sp
|
||||
- sh
|
||||
- ii
|
||||
- j
|
||||
- zh
|
||||
- l
|
||||
- x
|
||||
- b
|
||||
- g
|
||||
- uu
|
||||
- e5
|
||||
- h
|
||||
- q
|
||||
- m
|
||||
- i1
|
||||
- t
|
||||
- z
|
||||
- ch
|
||||
- f
|
||||
- s
|
||||
- u4
|
||||
- ix4
|
||||
- i4
|
||||
- n
|
||||
- i3
|
||||
- iu3
|
||||
- vv
|
||||
- ian4
|
||||
- ix2
|
||||
- r
|
||||
- e4
|
||||
- ai4
|
||||
- k
|
||||
- ing2
|
||||
- a1
|
||||
- en2
|
||||
- ui4
|
||||
- ong1
|
||||
- uo3
|
||||
- u2
|
||||
- u3
|
||||
- ao4
|
||||
- ee
|
||||
- p
|
||||
- an1
|
||||
- eng2
|
||||
- i2
|
||||
- in1
|
||||
- c
|
||||
- ai2
|
||||
- ian2
|
||||
- e2
|
||||
- an4
|
||||
- ing4
|
||||
- v4
|
||||
- ai3
|
||||
- a5
|
||||
- ian3
|
||||
- eng1
|
||||
- ong4
|
||||
- ang4
|
||||
- ian1
|
||||
- ing1
|
||||
- iy4
|
||||
- ao3
|
||||
- ang1
|
||||
- uo4
|
||||
- u1
|
||||
- iao4
|
||||
- iu4
|
||||
- a4
|
||||
- van2
|
||||
- ie4
|
||||
- ang2
|
||||
- ou4
|
||||
- iang4
|
||||
- ix1
|
||||
- er4
|
||||
- iy1
|
||||
- e1
|
||||
- en1
|
||||
- ui2
|
||||
- an3
|
||||
- ei4
|
||||
- ong2
|
||||
- uo1
|
||||
- ou3
|
||||
- uo2
|
||||
- iao1
|
||||
- ou1
|
||||
- an2
|
||||
- uan4
|
||||
- ia4
|
||||
- ia1
|
||||
- ang3
|
||||
- v3
|
||||
- iu2
|
||||
- iao3
|
||||
- in4
|
||||
- a3
|
||||
- ei3
|
||||
- iang3
|
||||
- v2
|
||||
- eng4
|
||||
- en3
|
||||
- aa
|
||||
- uan1
|
||||
- v1
|
||||
- ao1
|
||||
- ve4
|
||||
- ie3
|
||||
- ai1
|
||||
- ing3
|
||||
- iang1
|
||||
- a2
|
||||
- ui1
|
||||
- en4
|
||||
- en5
|
||||
- in3
|
||||
- uan3
|
||||
- e3
|
||||
- ie1
|
||||
- ve2
|
||||
- ei2
|
||||
- in2
|
||||
- ix3
|
||||
- uan2
|
||||
- iang2
|
||||
- ie2
|
||||
- ua4
|
||||
- ou2
|
||||
- uai4
|
||||
- er2
|
||||
- eng3
|
||||
- uang3
|
||||
- un1
|
||||
- ong3
|
||||
- uang4
|
||||
- vn4
|
||||
- un2
|
||||
- iy3
|
||||
- iz4
|
||||
- ui3
|
||||
- iao2
|
||||
- iong4
|
||||
- un4
|
||||
- van4
|
||||
- ao2
|
||||
- uang1
|
||||
- iy5
|
||||
- o2
|
||||
- ei1
|
||||
- ua1
|
||||
- iu1
|
||||
- uang2
|
||||
- er5
|
||||
- o1
|
||||
- un3
|
||||
- vn1
|
||||
- vn2
|
||||
- o4
|
||||
- ve1
|
||||
- van3
|
||||
- ua2
|
||||
- er3
|
||||
- iong3
|
||||
- van1
|
||||
- ia2
|
||||
- iy2
|
||||
- ia3
|
||||
- iong1
|
||||
- uo5
|
||||
- oo
|
||||
- ve3
|
||||
- ou5
|
||||
- uai3
|
||||
- ian5
|
||||
- iong2
|
||||
- uai2
|
||||
- uai1
|
||||
- ua3
|
||||
- vn3
|
||||
- ia5
|
||||
- ie5
|
||||
- ueng1
|
||||
- o5
|
||||
- o3
|
||||
- iang5
|
||||
- ei5
|
||||
- <sos/eos>
|
||||
@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
stage=0
|
||||
stop_stage=100
|
||||
|
||||
config_path=$1
|
||||
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# get durations from MFA's result
|
||||
echo "Generate durations.txt from MFA results ..."
|
||||
python3 ${MAIN_ROOT}/utils/gen_duration_from_textgrid.py \
|
||||
--inputdir=./aishell3_alignment_tone \
|
||||
--output durations.txt \
|
||||
--config=${config_path}
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# extract features
|
||||
echo "Extract features ..."
|
||||
python3 ${BIN_DIR}/preprocess.py \
|
||||
--dataset=aishell3 \
|
||||
--rootdir=~/datasets/data_aishell3/ \
|
||||
--dumpdir=dump \
|
||||
--dur-file=durations.txt \
|
||||
--config=${config_path} \
|
||||
--num-cpu=20 \
|
||||
--cut-sil=True
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||
# get features' stats(mean and std)
|
||||
echo "Get features' stats ..."
|
||||
python3 ${MAIN_ROOT}/utils/compute_statistics.py \
|
||||
--metadata=dump/train/raw/metadata.jsonl \
|
||||
--field-name="speech"
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
||||
# normalize and covert phone/speaker to id, dev and test should use train's stats
|
||||
echo "Normalize ..."
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/train/raw/metadata.jsonl \
|
||||
--dumpdir=dump/train/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/dev/raw/metadata.jsonl \
|
||||
--dumpdir=dump/dev/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/test/raw/metadata.jsonl \
|
||||
--dumpdir=dump/test/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
fi
|
||||
@ -0,0 +1 @@
|
||||
#!/bin/bash
|
||||
@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
config_path=$1
|
||||
train_output_path=$2
|
||||
|
||||
python3 ${BIN_DIR}/train.py \
|
||||
--train-metadata=dump/train/norm/metadata.jsonl \
|
||||
--dev-metadata=dump/dev/norm/metadata.jsonl \
|
||||
--config=${config_path} \
|
||||
--output-dir=${train_output_path} \
|
||||
--ngpu=1 \
|
||||
--phones-dict=dump/phone_id_map.txt
|
||||
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
export MAIN_ROOT=`realpath ${PWD}/../../../`
|
||||
|
||||
export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
|
||||
export LC_ALL=C
|
||||
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
|
||||
export PYTHONIOENCODING=UTF-8
|
||||
export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
|
||||
|
||||
MODEL=ernie_sat
|
||||
export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL}
|
||||
@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
source path.sh
|
||||
|
||||
gpus=0,1
|
||||
stage=0
|
||||
stop_stage=100
|
||||
|
||||
conf_path=conf/default.yaml
|
||||
train_output_path=exp/default
|
||||
ckpt_name=snapshot_iter_153.pdz
|
||||
|
||||
# with the following command, you can choose the stage range you want to run
|
||||
# such as `./run.sh --stage 0 --stop-stage 0`
|
||||
# this can not be mixed use with `$1`, `$2` ...
|
||||
source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
|
||||
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# prepare data
|
||||
./local/preprocess.sh ${conf_path} || exit -1
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# train model, all `ckpt` under `train_output_path/checkpoints/` dir
|
||||
CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||
# synthesize, vocoder is pwgan
|
||||
CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1
|
||||
fi
|
||||
@ -0,0 +1,351 @@
|
||||
###########################################################
|
||||
# FEATURE EXTRACTION SETTING #
|
||||
###########################################################
|
||||
|
||||
fs: 24000 # sr
|
||||
n_fft: 2048 # FFT size (samples).
|
||||
n_shift: 300 # Hop size (samples). 12.5ms
|
||||
win_length: 1200 # Window length (samples). 50ms
|
||||
# If set to null, it will be the same as fft_size.
|
||||
window: "hann" # Window function.
|
||||
|
||||
# Only used for feats_type != raw
|
||||
|
||||
fmin: 80 # Minimum frequency of Mel basis.
|
||||
fmax: 7600 # Maximum frequency of Mel basis.
|
||||
n_mels: 80 # The number of mel basis.
|
||||
|
||||
mean_phn_span: 8
|
||||
mlm_prob: 0.8
|
||||
|
||||
###########################################################
|
||||
# DATA SETTING #
|
||||
###########################################################
|
||||
batch_size: 64
|
||||
num_workers: 2
|
||||
|
||||
###########################################################
|
||||
# MODEL SETTING #
|
||||
###########################################################
|
||||
model:
|
||||
text_masking: true
|
||||
postnet_layers: 5
|
||||
postnet_filts: 5
|
||||
postnet_chans: 256
|
||||
encoder_type: conformer
|
||||
decoder_type: conformer
|
||||
enc_input_layer: sega_mlm
|
||||
enc_pre_speech_layer: 0
|
||||
enc_cnn_module_kernel: 7
|
||||
enc_attention_dim: 384
|
||||
enc_attention_heads: 2
|
||||
enc_linear_units: 1536
|
||||
enc_num_blocks: 4
|
||||
enc_dropout_rate: 0.2
|
||||
enc_positional_dropout_rate: 0.2
|
||||
enc_attention_dropout_rate: 0.2
|
||||
enc_normalize_before: true
|
||||
enc_macaron_style: true
|
||||
enc_use_cnn_module: true
|
||||
enc_selfattention_layer_type: legacy_rel_selfattn
|
||||
enc_activation_type: swish
|
||||
enc_pos_enc_layer_type: legacy_rel_pos
|
||||
enc_positionwise_layer_type: conv1d
|
||||
enc_positionwise_conv_kernel_size: 3
|
||||
dec_cnn_module_kernel: 31
|
||||
dec_attention_dim: 384
|
||||
dec_attention_heads: 2
|
||||
dec_linear_units: 1536
|
||||
dec_num_blocks: 4
|
||||
dec_dropout_rate: 0.2
|
||||
dec_positional_dropout_rate: 0.2
|
||||
dec_attention_dropout_rate: 0.2
|
||||
dec_macaron_style: true
|
||||
dec_use_cnn_module: true
|
||||
dec_selfattention_layer_type: legacy_rel_selfattn
|
||||
dec_activation_type: swish
|
||||
dec_pos_enc_layer_type: legacy_rel_pos
|
||||
dec_positionwise_layer_type: conv1d
|
||||
dec_positionwise_conv_kernel_size: 3
|
||||
|
||||
###########################################################
|
||||
# OPTIMIZER SETTING #
|
||||
###########################################################
|
||||
optimizer:
|
||||
optim: adam # optimizer type
|
||||
learning_rate: 0.001 # learning rate
|
||||
|
||||
###########################################################
|
||||
# TRAINING SETTING #
|
||||
###########################################################
|
||||
max_epoch: 100
|
||||
num_snapshots: 5
|
||||
|
||||
###########################################################
|
||||
# OTHER SETTING #
|
||||
###########################################################
|
||||
seed: 10086
|
||||
|
||||
token_list:
|
||||
- <blank>
|
||||
- <unk>
|
||||
- AH0
|
||||
- T
|
||||
- N
|
||||
- sp
|
||||
- S
|
||||
- R
|
||||
- D
|
||||
- L
|
||||
- Z
|
||||
- DH
|
||||
- IH1
|
||||
- K
|
||||
- W
|
||||
- M
|
||||
- EH1
|
||||
- AE1
|
||||
- ER0
|
||||
- B
|
||||
- IY1
|
||||
- P
|
||||
- V
|
||||
- IY0
|
||||
- F
|
||||
- HH
|
||||
- AA1
|
||||
- AY1
|
||||
- AH1
|
||||
- EY1
|
||||
- IH0
|
||||
- AO1
|
||||
- OW1
|
||||
- UW1
|
||||
- G
|
||||
- NG
|
||||
- SH
|
||||
- Y
|
||||
- TH
|
||||
- ER1
|
||||
- JH
|
||||
- UH1
|
||||
- AW1
|
||||
- CH
|
||||
- IH2
|
||||
- OW0
|
||||
- OW2
|
||||
- EY2
|
||||
- EH2
|
||||
- UW0
|
||||
- OY1
|
||||
- ZH
|
||||
- EH0
|
||||
- AY2
|
||||
- AW2
|
||||
- AA2
|
||||
- AE2
|
||||
- IY2
|
||||
- AH2
|
||||
- AE0
|
||||
- AO2
|
||||
- AY0
|
||||
- AO0
|
||||
- UW2
|
||||
- UH2
|
||||
- AA0
|
||||
- EY0
|
||||
- AW0
|
||||
- UH0
|
||||
- ER2
|
||||
- OY2
|
||||
- OY0
|
||||
- d
|
||||
- sh
|
||||
- ii
|
||||
- j
|
||||
- zh
|
||||
- l
|
||||
- x
|
||||
- b
|
||||
- g
|
||||
- uu
|
||||
- e5
|
||||
- h
|
||||
- q
|
||||
- m
|
||||
- i1
|
||||
- t
|
||||
- z
|
||||
- ch
|
||||
- f
|
||||
- s
|
||||
- u4
|
||||
- ix4
|
||||
- i4
|
||||
- n
|
||||
- i3
|
||||
- iu3
|
||||
- vv
|
||||
- ian4
|
||||
- ix2
|
||||
- r
|
||||
- e4
|
||||
- ai4
|
||||
- k
|
||||
- ing2
|
||||
- a1
|
||||
- en2
|
||||
- ui4
|
||||
- ong1
|
||||
- uo3
|
||||
- u2
|
||||
- u3
|
||||
- ao4
|
||||
- ee
|
||||
- p
|
||||
- an1
|
||||
- eng2
|
||||
- i2
|
||||
- in1
|
||||
- c
|
||||
- ai2
|
||||
- ian2
|
||||
- e2
|
||||
- an4
|
||||
- ing4
|
||||
- v4
|
||||
- ai3
|
||||
- a5
|
||||
- ian3
|
||||
- eng1
|
||||
- ong4
|
||||
- ang4
|
||||
- ian1
|
||||
- ing1
|
||||
- iy4
|
||||
- ao3
|
||||
- ang1
|
||||
- uo4
|
||||
- u1
|
||||
- iao4
|
||||
- iu4
|
||||
- a4
|
||||
- van2
|
||||
- ie4
|
||||
- ang2
|
||||
- ou4
|
||||
- iang4
|
||||
- ix1
|
||||
- er4
|
||||
- iy1
|
||||
- e1
|
||||
- en1
|
||||
- ui2
|
||||
- an3
|
||||
- ei4
|
||||
- ong2
|
||||
- uo1
|
||||
- ou3
|
||||
- uo2
|
||||
- iao1
|
||||
- ou1
|
||||
- an2
|
||||
- uan4
|
||||
- ia4
|
||||
- ia1
|
||||
- ang3
|
||||
- v3
|
||||
- iu2
|
||||
- iao3
|
||||
- in4
|
||||
- a3
|
||||
- ei3
|
||||
- iang3
|
||||
- v2
|
||||
- eng4
|
||||
- en3
|
||||
- aa
|
||||
- uan1
|
||||
- v1
|
||||
- ao1
|
||||
- ve4
|
||||
- ie3
|
||||
- ai1
|
||||
- ing3
|
||||
- iang1
|
||||
- a2
|
||||
- ui1
|
||||
- en4
|
||||
- en5
|
||||
- in3
|
||||
- uan3
|
||||
- e3
|
||||
- ie1
|
||||
- ve2
|
||||
- ei2
|
||||
- in2
|
||||
- ix3
|
||||
- uan2
|
||||
- iang2
|
||||
- ie2
|
||||
- ua4
|
||||
- ou2
|
||||
- uai4
|
||||
- er2
|
||||
- eng3
|
||||
- uang3
|
||||
- un1
|
||||
- ong3
|
||||
- uang4
|
||||
- vn4
|
||||
- un2
|
||||
- iy3
|
||||
- iz4
|
||||
- ui3
|
||||
- iao2
|
||||
- iong4
|
||||
- un4
|
||||
- van4
|
||||
- ao2
|
||||
- uang1
|
||||
- iy5
|
||||
- o2
|
||||
- ei1
|
||||
- ua1
|
||||
- iu1
|
||||
- uang2
|
||||
- er5
|
||||
- o1
|
||||
- un3
|
||||
- vn1
|
||||
- vn2
|
||||
- o4
|
||||
- ve1
|
||||
- van3
|
||||
- ua2
|
||||
- er3
|
||||
- iong3
|
||||
- van1
|
||||
- ia2
|
||||
- iy2
|
||||
- ia3
|
||||
- iong1
|
||||
- uo5
|
||||
- oo
|
||||
- ve3
|
||||
- ou5
|
||||
- uai3
|
||||
- ian5
|
||||
- iong2
|
||||
- uai2
|
||||
- uai1
|
||||
- ua3
|
||||
- vn3
|
||||
- ia5
|
||||
- ie5
|
||||
- ueng1
|
||||
- o5
|
||||
- o3
|
||||
- iang5
|
||||
- ei5
|
||||
- <sos/eos>
|
||||
@ -0,0 +1,67 @@
|
||||
#!/bin/bash
|
||||
|
||||
stage=0
|
||||
stop_stage=100
|
||||
|
||||
config_path=$1
|
||||
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# get durations from MFA's result
|
||||
echo "Generate durations.txt from MFA results ..."
|
||||
python3 ${MAIN_ROOT}/utils/gen_duration_from_textgrid.py \
|
||||
--inputdir=./aishell3_alignment_tone \
|
||||
--output durations.txt \
|
||||
--config=${config_path}
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# extract features
|
||||
echo "Extract features ..."
|
||||
python3 ${BIN_DIR}/preprocess.py \
|
||||
--dataset=aishell3 \
|
||||
--rootdir=~/datasets/data_aishell3/ \
|
||||
--dumpdir=dump \
|
||||
--dur-file=durations.txt \
|
||||
--config=${config_path} \
|
||||
--num-cpu=20 \
|
||||
--cut-sil=True
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||
# get features' stats(mean and std)
|
||||
echo "Get features' stats ..."
|
||||
python3 ${MAIN_ROOT}/utils/compute_statistics.py \
|
||||
--metadata=dump/train/raw/metadata.jsonl \
|
||||
--field-name="speech"
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
||||
# normalize and covert phone/speaker to id, dev and test should use train's stats
|
||||
echo "Normalize ..."
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/train/raw/metadata.jsonl \
|
||||
--dumpdir=dump/train/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--pitch-stats=dump/train/pitch_stats.npy \
|
||||
--energy-stats=dump/train/energy_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/dev/raw/metadata.jsonl \
|
||||
--dumpdir=dump/dev/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--pitch-stats=dump/train/pitch_stats.npy \
|
||||
--energy-stats=dump/train/energy_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/test/raw/metadata.jsonl \
|
||||
--dumpdir=dump/test/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--pitch-stats=dump/train/pitch_stats.npy \
|
||||
--energy-stats=dump/train/energy_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
fi
|
||||
@ -0,0 +1 @@
|
||||
#!/bin/bash
|
||||
@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
config_path=$1
|
||||
train_output_path=$2
|
||||
|
||||
python3 ${BIN_DIR}/train.py \
|
||||
--train-metadata=dump/train/norm/metadata.jsonl \
|
||||
--dev-metadata=dump/dev/norm/metadata.jsonl \
|
||||
--config=${config_path} \
|
||||
--output-dir=${train_output_path} \
|
||||
--ngpu=1 \
|
||||
--phones-dict=dump/phone_id_map.txt
|
||||
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
export MAIN_ROOT=`realpath ${PWD}/../../`
|
||||
|
||||
export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
|
||||
export LC_ALL=C
|
||||
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
|
||||
export PYTHONIOENCODING=UTF-8
|
||||
export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
|
||||
|
||||
MODEL=ernie_sat
|
||||
export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL}
|
||||
@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
source path.sh
|
||||
|
||||
gpus=0,1
|
||||
stage=0
|
||||
stop_stage=100
|
||||
|
||||
conf_path=conf/default.yaml
|
||||
train_output_path=exp/default
|
||||
ckpt_name=snapshot_iter_153.pdz
|
||||
|
||||
# with the following command, you can choose the stage range you want to run
|
||||
# such as `./run.sh --stage 0 --stop-stage 0`
|
||||
# this can not be mixed use with `$1`, `$2` ...
|
||||
source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
|
||||
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# prepare data
|
||||
./local/preprocess.sh ${conf_path} || exit -1
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# train model, all `ckpt` under `train_output_path/checkpoints/` dir
|
||||
CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||
# synthesize, vocoder is pwgan
|
||||
CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1
|
||||
fi
|
||||
@ -0,0 +1,622 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import random
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import paddle
|
||||
import soundfile as sf
|
||||
import yaml
|
||||
from align import alignment
|
||||
from align import alignment_zh
|
||||
from align import words2phns
|
||||
from align import words2phns_zh
|
||||
from paddle import nn
|
||||
from sedit_arg_parser import parse_args
|
||||
from utils import eval_durs
|
||||
from utils import get_voc_out
|
||||
from utils import is_chinese
|
||||
from utils import load_num_sequence_text
|
||||
from utils import read_2col_text
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from paddlespeech.t2s.datasets.am_batch_fn import build_mlm_collate_fn
|
||||
from paddlespeech.t2s.models.ernie_sat.ernie_sat import ErnieSAT
|
||||
|
||||
random.seed(0)
|
||||
np.random.seed(0)
|
||||
|
||||
|
||||
def get_wav(wav_path: str,
|
||||
source_lang: str='english',
|
||||
target_lang: str='english',
|
||||
model_name: str="paddle_checkpoint_en",
|
||||
old_str: str="",
|
||||
new_str: str="",
|
||||
non_autoreg: bool=True):
|
||||
wav_org, output_feat, old_span_bdy, new_span_bdy, fs, hop_length = get_mlm_output(
|
||||
source_lang=source_lang,
|
||||
target_lang=target_lang,
|
||||
model_name=model_name,
|
||||
wav_path=wav_path,
|
||||
old_str=old_str,
|
||||
new_str=new_str,
|
||||
use_teacher_forcing=non_autoreg)
|
||||
|
||||
masked_feat = output_feat[new_span_bdy[0]:new_span_bdy[1]]
|
||||
|
||||
alt_wav = get_voc_out(masked_feat)
|
||||
|
||||
old_time_bdy = [hop_length * x for x in old_span_bdy]
|
||||
|
||||
wav_replaced = np.concatenate(
|
||||
[wav_org[:old_time_bdy[0]], alt_wav, wav_org[old_time_bdy[1]:]])
|
||||
|
||||
data_dict = {"origin": wav_org, "output": wav_replaced}
|
||||
|
||||
return data_dict
|
||||
|
||||
|
||||
def load_model(model_name: str="paddle_checkpoint_en"):
|
||||
config_path = './pretrained_model/{}/default.yaml'.format(model_name)
|
||||
model_path = './pretrained_model/{}/model.pdparams'.format(model_name)
|
||||
with open(config_path) as f:
|
||||
conf = CfgNode(yaml.safe_load(f))
|
||||
token_list = list(conf.token_list)
|
||||
vocab_size = len(token_list)
|
||||
odim = conf.n_mels
|
||||
mlm_model = ErnieSAT(idim=vocab_size, odim=odim, **conf["model"])
|
||||
state_dict = paddle.load(model_path)
|
||||
new_state_dict = {}
|
||||
for key, value in state_dict.items():
|
||||
new_key = "model." + key
|
||||
new_state_dict[new_key] = value
|
||||
mlm_model.set_state_dict(new_state_dict)
|
||||
mlm_model.eval()
|
||||
|
||||
return mlm_model, conf
|
||||
|
||||
|
||||
def read_data(uid: str, prefix: os.PathLike):
|
||||
# 获取 uid 对应的文本
|
||||
mfa_text = read_2col_text(prefix + '/text')[uid]
|
||||
# 获取 uid 对应的音频路径
|
||||
mfa_wav_path = read_2col_text(prefix + '/wav.scp')[uid]
|
||||
if not os.path.isabs(mfa_wav_path):
|
||||
mfa_wav_path = prefix + mfa_wav_path
|
||||
return mfa_text, mfa_wav_path
|
||||
|
||||
|
||||
def get_align_data(uid: str, prefix: os.PathLike):
|
||||
mfa_path = prefix + "mfa_"
|
||||
mfa_text = read_2col_text(mfa_path + 'text')[uid]
|
||||
mfa_start = load_num_sequence_text(
|
||||
mfa_path + 'start', loader_type='text_float')[uid]
|
||||
mfa_end = load_num_sequence_text(
|
||||
mfa_path + 'end', loader_type='text_float')[uid]
|
||||
mfa_wav_path = read_2col_text(mfa_path + 'wav.scp')[uid]
|
||||
return mfa_text, mfa_start, mfa_end, mfa_wav_path
|
||||
|
||||
|
||||
# 获取需要被 mask 的 mel 帧的范围
|
||||
def get_masked_mel_bdy(mfa_start: List[float],
|
||||
mfa_end: List[float],
|
||||
fs: int,
|
||||
hop_length: int,
|
||||
span_to_repl: List[List[int]]):
|
||||
align_start = np.array(mfa_start)
|
||||
align_end = np.array(mfa_end)
|
||||
align_start = np.floor(fs * align_start / hop_length).astype('int')
|
||||
align_end = np.floor(fs * align_end / hop_length).astype('int')
|
||||
if span_to_repl[0] >= len(mfa_start):
|
||||
span_bdy = [align_end[-1], align_end[-1]]
|
||||
else:
|
||||
span_bdy = [
|
||||
align_start[span_to_repl[0]], align_end[span_to_repl[1] - 1]
|
||||
]
|
||||
return span_bdy, align_start, align_end
|
||||
|
||||
|
||||
def recover_dict(word2phns: Dict[str, str], tp_word2phns: Dict[str, str]):
|
||||
dic = {}
|
||||
keys_to_del = []
|
||||
exist_idx = []
|
||||
sp_count = 0
|
||||
add_sp_count = 0
|
||||
for key in word2phns.keys():
|
||||
idx, wrd = key.split('_')
|
||||
if wrd == 'sp':
|
||||
sp_count += 1
|
||||
exist_idx.append(int(idx))
|
||||
else:
|
||||
keys_to_del.append(key)
|
||||
|
||||
for key in keys_to_del:
|
||||
del word2phns[key]
|
||||
|
||||
cur_id = 0
|
||||
for key in tp_word2phns.keys():
|
||||
if cur_id in exist_idx:
|
||||
dic[str(cur_id) + "_sp"] = 'sp'
|
||||
cur_id += 1
|
||||
add_sp_count += 1
|
||||
idx, wrd = key.split('_')
|
||||
dic[str(cur_id) + "_" + wrd] = tp_word2phns[key]
|
||||
cur_id += 1
|
||||
|
||||
if add_sp_count + 1 == sp_count:
|
||||
dic[str(cur_id) + "_sp"] = 'sp'
|
||||
add_sp_count += 1
|
||||
|
||||
assert add_sp_count == sp_count, "sp are not added in dic"
|
||||
return dic
|
||||
|
||||
|
||||
def get_max_idx(dic):
|
||||
return sorted([int(key.split('_')[0]) for key in dic.keys()])[-1]
|
||||
|
||||
|
||||
def get_phns_and_spans(wav_path: str,
|
||||
old_str: str="",
|
||||
new_str: str="",
|
||||
source_lang: str="english",
|
||||
target_lang: str="english"):
|
||||
is_append = (old_str == new_str[:len(old_str)])
|
||||
old_phns, mfa_start, mfa_end = [], [], []
|
||||
# source
|
||||
if source_lang == "english":
|
||||
intervals, word2phns = alignment(wav_path, old_str)
|
||||
elif source_lang == "chinese":
|
||||
intervals, word2phns = alignment_zh(wav_path, old_str)
|
||||
_, tp_word2phns = words2phns_zh(old_str)
|
||||
|
||||
for key, value in tp_word2phns.items():
|
||||
idx, wrd = key.split('_')
|
||||
cur_val = " ".join(value)
|
||||
tp_word2phns[key] = cur_val
|
||||
|
||||
word2phns = recover_dict(word2phns, tp_word2phns)
|
||||
else:
|
||||
assert source_lang == "chinese" or source_lang == "english", \
|
||||
"source_lang is wrong..."
|
||||
|
||||
for item in intervals:
|
||||
old_phns.append(item[0])
|
||||
mfa_start.append(float(item[1]))
|
||||
mfa_end.append(float(item[2]))
|
||||
# target
|
||||
if is_append and (source_lang != target_lang):
|
||||
cross_lingual_clone = True
|
||||
else:
|
||||
cross_lingual_clone = False
|
||||
|
||||
if cross_lingual_clone:
|
||||
str_origin = new_str[:len(old_str)]
|
||||
str_append = new_str[len(old_str):]
|
||||
|
||||
if target_lang == "chinese":
|
||||
phns_origin, origin_word2phns = words2phns(str_origin)
|
||||
phns_append, append_word2phns_tmp = words2phns_zh(str_append)
|
||||
|
||||
elif target_lang == "english":
|
||||
# 原始句子
|
||||
phns_origin, origin_word2phns = words2phns_zh(str_origin)
|
||||
# clone 句子
|
||||
phns_append, append_word2phns_tmp = words2phns(str_append)
|
||||
else:
|
||||
assert target_lang == "chinese" or target_lang == "english", \
|
||||
"cloning is not support for this language, please check it."
|
||||
|
||||
new_phns = phns_origin + phns_append
|
||||
|
||||
append_word2phns = {}
|
||||
length = len(origin_word2phns)
|
||||
for key, value in append_word2phns_tmp.items():
|
||||
idx, wrd = key.split('_')
|
||||
append_word2phns[str(int(idx) + length) + '_' + wrd] = value
|
||||
new_word2phns = origin_word2phns.copy()
|
||||
new_word2phns.update(append_word2phns)
|
||||
|
||||
else:
|
||||
if source_lang == target_lang and target_lang == "english":
|
||||
new_phns, new_word2phns = words2phns(new_str)
|
||||
elif source_lang == target_lang and target_lang == "chinese":
|
||||
new_phns, new_word2phns = words2phns_zh(new_str)
|
||||
else:
|
||||
assert source_lang == target_lang, \
|
||||
"source language is not same with target language..."
|
||||
|
||||
span_to_repl = [0, len(old_phns) - 1]
|
||||
span_to_add = [0, len(new_phns) - 1]
|
||||
left_idx = 0
|
||||
new_phns_left = []
|
||||
sp_count = 0
|
||||
# find the left different index
|
||||
for key in word2phns.keys():
|
||||
idx, wrd = key.split('_')
|
||||
if wrd == 'sp':
|
||||
sp_count += 1
|
||||
new_phns_left.append('sp')
|
||||
else:
|
||||
idx = str(int(idx) - sp_count)
|
||||
if idx + '_' + wrd in new_word2phns:
|
||||
left_idx += len(new_word2phns[idx + '_' + wrd])
|
||||
new_phns_left.extend(word2phns[key].split())
|
||||
else:
|
||||
span_to_repl[0] = len(new_phns_left)
|
||||
span_to_add[0] = len(new_phns_left)
|
||||
break
|
||||
|
||||
# reverse word2phns and new_word2phns
|
||||
right_idx = 0
|
||||
new_phns_right = []
|
||||
sp_count = 0
|
||||
word2phns_max_idx = get_max_idx(word2phns)
|
||||
new_word2phns_max_idx = get_max_idx(new_word2phns)
|
||||
new_phns_mid = []
|
||||
if is_append:
|
||||
new_phns_right = []
|
||||
new_phns_mid = new_phns[left_idx:]
|
||||
span_to_repl[0] = len(new_phns_left)
|
||||
span_to_add[0] = len(new_phns_left)
|
||||
span_to_add[1] = len(new_phns_left) + len(new_phns_mid)
|
||||
span_to_repl[1] = len(old_phns) - len(new_phns_right)
|
||||
# speech edit
|
||||
else:
|
||||
for key in list(word2phns.keys())[::-1]:
|
||||
idx, wrd = key.split('_')
|
||||
if wrd == 'sp':
|
||||
sp_count += 1
|
||||
new_phns_right = ['sp'] + new_phns_right
|
||||
else:
|
||||
idx = str(new_word2phns_max_idx - (word2phns_max_idx - int(idx)
|
||||
- sp_count))
|
||||
if idx + '_' + wrd in new_word2phns:
|
||||
right_idx -= len(new_word2phns[idx + '_' + wrd])
|
||||
new_phns_right = word2phns[key].split() + new_phns_right
|
||||
else:
|
||||
span_to_repl[1] = len(old_phns) - len(new_phns_right)
|
||||
new_phns_mid = new_phns[left_idx:right_idx]
|
||||
span_to_add[1] = len(new_phns_left) + len(new_phns_mid)
|
||||
if len(new_phns_mid) == 0:
|
||||
span_to_add[1] = min(span_to_add[1] + 1, len(new_phns))
|
||||
span_to_add[0] = max(0, span_to_add[0] - 1)
|
||||
span_to_repl[0] = max(0, span_to_repl[0] - 1)
|
||||
span_to_repl[1] = min(span_to_repl[1] + 1,
|
||||
len(old_phns))
|
||||
break
|
||||
new_phns = new_phns_left + new_phns_mid + new_phns_right
|
||||
'''
|
||||
For that reason cover should not be given.
|
||||
For that reason cover is impossible to be given.
|
||||
span_to_repl: [17, 23] "should not"
|
||||
span_to_add: [17, 30] "is impossible to"
|
||||
'''
|
||||
return mfa_start, mfa_end, old_phns, new_phns, span_to_repl, span_to_add
|
||||
|
||||
|
||||
# mfa 获得的 duration 和 fs2 的 duration_predictor 获取的 duration 可能不同
|
||||
# 此处获得一个缩放比例, 用于预测值和真实值之间的缩放
|
||||
def get_dur_adj_factor(orig_dur: List[int],
|
||||
pred_dur: List[int],
|
||||
phns: List[str]):
|
||||
length = 0
|
||||
factor_list = []
|
||||
for orig, pred, phn in zip(orig_dur, pred_dur, phns):
|
||||
if pred == 0 or phn == 'sp':
|
||||
continue
|
||||
else:
|
||||
factor_list.append(orig / pred)
|
||||
factor_list = np.array(factor_list)
|
||||
factor_list.sort()
|
||||
if len(factor_list) < 5:
|
||||
return 1
|
||||
length = 2
|
||||
avg = np.average(factor_list[length:-length])
|
||||
return avg
|
||||
|
||||
|
||||
def prep_feats_with_dur(wav_path: str,
|
||||
source_lang: str="English",
|
||||
target_lang: str="English",
|
||||
old_str: str="",
|
||||
new_str: str="",
|
||||
mask_reconstruct: bool=False,
|
||||
duration_adjust: bool=True,
|
||||
start_end_sp: bool=False,
|
||||
fs: int=24000,
|
||||
hop_length: int=300):
|
||||
'''
|
||||
Returns:
|
||||
np.ndarray: new wav, replace the part to be edited in original wav with 0
|
||||
List[str]: new phones
|
||||
List[float]: mfa start of new wav
|
||||
List[float]: mfa end of new wav
|
||||
List[int]: masked mel boundary of original wav
|
||||
List[int]: masked mel boundary of new wav
|
||||
'''
|
||||
wav_org, _ = librosa.load(wav_path, sr=fs)
|
||||
|
||||
mfa_start, mfa_end, old_phns, new_phns, span_to_repl, span_to_add = get_phns_and_spans(
|
||||
wav_path=wav_path,
|
||||
old_str=old_str,
|
||||
new_str=new_str,
|
||||
source_lang=source_lang,
|
||||
target_lang=target_lang)
|
||||
|
||||
if start_end_sp:
|
||||
if new_phns[-1] != 'sp':
|
||||
new_phns = new_phns + ['sp']
|
||||
# 中文的 phns 不一定都在 fastspeech2 的字典里, 用 sp 代替
|
||||
if target_lang == "english" or target_lang == "chinese":
|
||||
old_durs = eval_durs(old_phns, target_lang=source_lang)
|
||||
else:
|
||||
assert target_lang == "chinese" or target_lang == "english", \
|
||||
"calculate duration_predict is not support for this language..."
|
||||
|
||||
orig_old_durs = [e - s for e, s in zip(mfa_end, mfa_start)]
|
||||
if '[MASK]' in new_str:
|
||||
new_phns = old_phns
|
||||
span_to_add = span_to_repl
|
||||
d_factor_left = get_dur_adj_factor(
|
||||
orig_dur=orig_old_durs[:span_to_repl[0]],
|
||||
pred_dur=old_durs[:span_to_repl[0]],
|
||||
phns=old_phns[:span_to_repl[0]])
|
||||
d_factor_right = get_dur_adj_factor(
|
||||
orig_dur=orig_old_durs[span_to_repl[1]:],
|
||||
pred_dur=old_durs[span_to_repl[1]:],
|
||||
phns=old_phns[span_to_repl[1]:])
|
||||
d_factor = (d_factor_left + d_factor_right) / 2
|
||||
new_durs_adjusted = [d_factor * i for i in old_durs]
|
||||
else:
|
||||
if duration_adjust:
|
||||
d_factor = get_dur_adj_factor(
|
||||
orig_dur=orig_old_durs, pred_dur=old_durs, phns=old_phns)
|
||||
d_factor = d_factor * 1.25
|
||||
else:
|
||||
d_factor = 1
|
||||
|
||||
if target_lang == "english" or target_lang == "chinese":
|
||||
new_durs = eval_durs(new_phns, target_lang=target_lang)
|
||||
else:
|
||||
assert target_lang == "chinese" or target_lang == "english", \
|
||||
"calculate duration_predict is not support for this language..."
|
||||
|
||||
new_durs_adjusted = [d_factor * i for i in new_durs]
|
||||
|
||||
new_span_dur_sum = sum(new_durs_adjusted[span_to_add[0]:span_to_add[1]])
|
||||
old_span_dur_sum = sum(orig_old_durs[span_to_repl[0]:span_to_repl[1]])
|
||||
dur_offset = new_span_dur_sum - old_span_dur_sum
|
||||
new_mfa_start = mfa_start[:span_to_repl[0]]
|
||||
new_mfa_end = mfa_end[:span_to_repl[0]]
|
||||
for i in new_durs_adjusted[span_to_add[0]:span_to_add[1]]:
|
||||
if len(new_mfa_end) == 0:
|
||||
new_mfa_start.append(0)
|
||||
new_mfa_end.append(i)
|
||||
else:
|
||||
new_mfa_start.append(new_mfa_end[-1])
|
||||
new_mfa_end.append(new_mfa_end[-1] + i)
|
||||
new_mfa_start += [i + dur_offset for i in mfa_start[span_to_repl[1]:]]
|
||||
new_mfa_end += [i + dur_offset for i in mfa_end[span_to_repl[1]:]]
|
||||
|
||||
# 3. get new wav
|
||||
# 在原始句子后拼接
|
||||
if span_to_repl[0] >= len(mfa_start):
|
||||
left_idx = len(wav_org)
|
||||
right_idx = left_idx
|
||||
# 在原始句子中间替换
|
||||
else:
|
||||
left_idx = int(np.floor(mfa_start[span_to_repl[0]] * fs))
|
||||
right_idx = int(np.ceil(mfa_end[span_to_repl[1] - 1] * fs))
|
||||
blank_wav = np.zeros(
|
||||
(int(np.ceil(new_span_dur_sum * fs)), ), dtype=wav_org.dtype)
|
||||
# 原始音频，需要编辑的部分替换成空音频，空音频的时间由 fs2 的 duration_predictor 决定
|
||||
new_wav = np.concatenate(
|
||||
[wav_org[:left_idx], blank_wav, wav_org[right_idx:]])
|
||||
|
||||
# 4. get old and new mel span to be mask
|
||||
# [92, 92]
|
||||
|
||||
old_span_bdy, mfa_start, mfa_end = get_masked_mel_bdy(
|
||||
mfa_start=mfa_start,
|
||||
mfa_end=mfa_end,
|
||||
fs=fs,
|
||||
hop_length=hop_length,
|
||||
span_to_repl=span_to_repl)
|
||||
# [92, 174]
|
||||
# new_mfa_start, new_mfa_end 时间级别的开始和结束时间 -> 帧级别
|
||||
new_span_bdy, new_mfa_start, new_mfa_end = get_masked_mel_bdy(
|
||||
mfa_start=new_mfa_start,
|
||||
mfa_end=new_mfa_end,
|
||||
fs=fs,
|
||||
hop_length=hop_length,
|
||||
span_to_repl=span_to_add)
|
||||
|
||||
# old_span_bdy, new_span_bdy 是帧级别的范围
|
||||
return new_wav, new_phns, new_mfa_start, new_mfa_end, old_span_bdy, new_span_bdy
|
||||
|
||||
|
||||
def prep_feats(wav_path: str,
|
||||
source_lang: str="english",
|
||||
target_lang: str="english",
|
||||
old_str: str="",
|
||||
new_str: str="",
|
||||
duration_adjust: bool=True,
|
||||
start_end_sp: bool=False,
|
||||
mask_reconstruct: bool=False,
|
||||
fs: int=24000,
|
||||
hop_length: int=300,
|
||||
token_list: List[str]=[]):
|
||||
wav, phns, mfa_start, mfa_end, old_span_bdy, new_span_bdy = prep_feats_with_dur(
|
||||
source_lang=source_lang,
|
||||
target_lang=target_lang,
|
||||
old_str=old_str,
|
||||
new_str=new_str,
|
||||
wav_path=wav_path,
|
||||
duration_adjust=duration_adjust,
|
||||
start_end_sp=start_end_sp,
|
||||
mask_reconstruct=mask_reconstruct,
|
||||
fs=fs,
|
||||
hop_length=hop_length)
|
||||
|
||||
token_to_id = {item: i for i, item in enumerate(token_list)}
|
||||
text = np.array(
|
||||
list(map(lambda x: token_to_id.get(x, token_to_id['<unk>']), phns)))
|
||||
span_bdy = np.array(new_span_bdy)
|
||||
|
||||
batch = [('1', {
|
||||
"speech": wav,
|
||||
"align_start": mfa_start,
|
||||
"align_end": mfa_end,
|
||||
"text": text,
|
||||
"span_bdy": span_bdy
|
||||
})]
|
||||
|
||||
return batch, old_span_bdy, new_span_bdy
|
||||
|
||||
|
||||
def decode_with_model(mlm_model: nn.Layer,
|
||||
collate_fn,
|
||||
wav_path: str,
|
||||
source_lang: str="english",
|
||||
target_lang: str="english",
|
||||
old_str: str="",
|
||||
new_str: str="",
|
||||
use_teacher_forcing: bool=False,
|
||||
duration_adjust: bool=True,
|
||||
start_end_sp: bool=False,
|
||||
fs: int=24000,
|
||||
hop_length: int=300,
|
||||
token_list: List[str]=[]):
|
||||
batch, old_span_bdy, new_span_bdy = prep_feats(
|
||||
source_lang=source_lang,
|
||||
target_lang=target_lang,
|
||||
wav_path=wav_path,
|
||||
old_str=old_str,
|
||||
new_str=new_str,
|
||||
duration_adjust=duration_adjust,
|
||||
start_end_sp=start_end_sp,
|
||||
fs=fs,
|
||||
hop_length=hop_length,
|
||||
token_list=token_list)
|
||||
|
||||
feats = collate_fn(batch)[1]
|
||||
|
||||
if 'text_masked_pos' in feats.keys():
|
||||
feats.pop('text_masked_pos')
|
||||
|
||||
output = mlm_model.inference(
|
||||
text=feats['text'],
|
||||
speech=feats['speech'],
|
||||
masked_pos=feats['masked_pos'],
|
||||
speech_mask=feats['speech_mask'],
|
||||
text_mask=feats['text_mask'],
|
||||
speech_seg_pos=feats['speech_seg_pos'],
|
||||
text_seg_pos=feats['text_seg_pos'],
|
||||
span_bdy=new_span_bdy,
|
||||
use_teacher_forcing=use_teacher_forcing)
|
||||
|
||||
# 拼接音频
|
||||
output_feat = paddle.concat(x=output, axis=0)
|
||||
wav_org, _ = librosa.load(wav_path, sr=fs)
|
||||
return wav_org, output_feat, old_span_bdy, new_span_bdy, fs, hop_length
|
||||
|
||||
|
||||
def get_mlm_output(wav_path: str,
|
||||
model_name: str="paddle_checkpoint_en",
|
||||
source_lang: str="english",
|
||||
target_lang: str="english",
|
||||
old_str: str="",
|
||||
new_str: str="",
|
||||
use_teacher_forcing: bool=False,
|
||||
duration_adjust: bool=True,
|
||||
start_end_sp: bool=False):
|
||||
mlm_model, train_conf = load_model(model_name)
|
||||
|
||||
collate_fn = build_mlm_collate_fn(
|
||||
sr=train_conf.fs,
|
||||
n_fft=train_conf.n_fft,
|
||||
hop_length=train_conf.n_shift,
|
||||
win_length=train_conf.win_length,
|
||||
n_mels=train_conf.n_mels,
|
||||
fmin=train_conf.fmin,
|
||||
fmax=train_conf.fmax,
|
||||
mlm_prob=train_conf.mlm_prob,
|
||||
mean_phn_span=train_conf.mean_phn_span,
|
||||
seg_emb=train_conf.model['enc_input_layer'] == 'sega_mlm')
|
||||
|
||||
return decode_with_model(
|
||||
source_lang=source_lang,
|
||||
target_lang=target_lang,
|
||||
mlm_model=mlm_model,
|
||||
collate_fn=collate_fn,
|
||||
wav_path=wav_path,
|
||||
old_str=old_str,
|
||||
new_str=new_str,
|
||||
use_teacher_forcing=use_teacher_forcing,
|
||||
duration_adjust=duration_adjust,
|
||||
start_end_sp=start_end_sp,
|
||||
fs=train_conf.fs,
|
||||
hop_length=train_conf.n_shift,
|
||||
token_list=train_conf.token_list)
|
||||
|
||||
|
||||
def evaluate(uid: str,
|
||||
source_lang: str="english",
|
||||
target_lang: str="english",
|
||||
prefix: os.PathLike="./prompt/dev/",
|
||||
model_name: str="paddle_checkpoint_en",
|
||||
new_str: str="",
|
||||
prompt_decoding: bool=False,
|
||||
task_name: str=None):
|
||||
|
||||
# get origin text and path of origin wav
|
||||
old_str, wav_path = read_data(uid=uid, prefix=prefix)
|
||||
|
||||
if task_name == 'edit':
|
||||
new_str = new_str
|
||||
elif task_name == 'synthesize':
|
||||
new_str = old_str + new_str
|
||||
else:
|
||||
new_str = old_str + ' '.join([ch for ch in new_str if is_chinese(ch)])
|
||||
|
||||
print('new_str is ', new_str)
|
||||
|
||||
results_dict = get_wav(
|
||||
source_lang=source_lang,
|
||||
target_lang=target_lang,
|
||||
model_name=model_name,
|
||||
wav_path=wav_path,
|
||||
old_str=old_str,
|
||||
new_str=new_str)
|
||||
return results_dict
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# parse config and args
|
||||
args = parse_args()
|
||||
|
||||
data_dict = evaluate(
|
||||
uid=args.uid,
|
||||
source_lang=args.source_lang,
|
||||
target_lang=args.target_lang,
|
||||
prefix=args.prefix,
|
||||
model_name=args.model_name,
|
||||
new_str=args.new_str,
|
||||
task_name=args.task_name)
|
||||
sf.write(args.output_name, data_dict['output'], samplerate=24000)
|
||||
print("finished...")
|
||||
@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
source path.sh
|
||||
|
||||
# en --> zh 的 语音合成
|
||||
# 根据 Prompt_003_new 作为提示语音: This was not the show for me. 来合成: '今天天气很好'
|
||||
# 注: 输入的 new_str 需为中文汉字, 否则会通过预处理只保留中文汉字, 即合成预处理后的中文语音。
|
||||
|
||||
python local/inference_new.py \
|
||||
--task_name=cross-lingual_clone \
|
||||
--model_name=paddle_checkpoint_dual_mask_enzh \
|
||||
--uid=Prompt_003_new \
|
||||
--new_str='今天天气很好.' \
|
||||
--prefix='./prompt/dev/' \
|
||||
--source_lang=english \
|
||||
--target_lang=chinese \
|
||||
--output_name=pred_clone.wav \
|
||||
--voc=pwgan_aishell3 \
|
||||
--voc_config=download/pwg_aishell3_ckpt_0.5/default.yaml \
|
||||
--voc_ckpt=download/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \
|
||||
--voc_stat=download/pwg_aishell3_ckpt_0.5/feats_stats.npy \
|
||||
--am=fastspeech2_csmsc \
|
||||
--am_config=download/fastspeech2_conformer_baker_ckpt_0.5/conformer.yaml \
|
||||
--am_ckpt=download/fastspeech2_conformer_baker_ckpt_0.5/snapshot_iter_76000.pdz \
|
||||
--am_stat=download/fastspeech2_conformer_baker_ckpt_0.5/speech_stats.npy \
|
||||
--phones_dict=download/fastspeech2_conformer_baker_ckpt_0.5/phone_id_map.txt
|
||||
@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
source path.sh
|
||||
|
||||
# 纯英文的语音合成
|
||||
# 样例为根据 p299_096 对应的语音作为提示语音: This was not the show for me. 来合成: 'I enjoy my life.'
|
||||
|
||||
python local/inference_new.py \
|
||||
--task_name=synthesize \
|
||||
--model_name=paddle_checkpoint_en \
|
||||
--uid=p299_096 \
|
||||
--new_str='I enjoy my life, do you?' \
|
||||
--prefix='./prompt/dev/' \
|
||||
--source_lang=english \
|
||||
--target_lang=english \
|
||||
--output_name=pred_gen.wav \
|
||||
--voc=pwgan_aishell3 \
|
||||
--voc_config=download/pwg_aishell3_ckpt_0.5/default.yaml \
|
||||
--voc_ckpt=download/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \
|
||||
--voc_stat=download/pwg_aishell3_ckpt_0.5/feats_stats.npy \
|
||||
--am=fastspeech2_ljspeech \
|
||||
--am_config=download/fastspeech2_nosil_ljspeech_ckpt_0.5/default.yaml \
|
||||
--am_ckpt=download/fastspeech2_nosil_ljspeech_ckpt_0.5/snapshot_iter_100000.pdz \
|
||||
--am_stat=download/fastspeech2_nosil_ljspeech_ckpt_0.5/speech_stats.npy \
|
||||
--phones_dict=download/fastspeech2_nosil_ljspeech_ckpt_0.5/phone_id_map.txt
|
||||
@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
source path.sh
|
||||
|
||||
# 纯英文的语音编辑
|
||||
# 样例为把 p243_new 对应的原始语音: For that reason cover should not be given.编辑成 'for that reason cover is impossible to be given.' 对应的语音
|
||||
# NOTE: 语音编辑任务暂支持句子中 1 个位置的替换或者插入文本操作
|
||||
|
||||
python local/inference_new.py \
|
||||
--task_name=edit \
|
||||
--model_name=paddle_checkpoint_en \
|
||||
--uid=p243_new \
|
||||
--new_str='for that reason cover is impossible to be given.' \
|
||||
--prefix='./prompt/dev/' \
|
||||
--source_lang=english \
|
||||
--target_lang=english \
|
||||
--output_name=pred_edit.wav \
|
||||
--voc=pwgan_aishell3 \
|
||||
--voc_config=download/pwg_aishell3_ckpt_0.5/default.yaml \
|
||||
--voc_ckpt=download/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \
|
||||
--voc_stat=download/pwg_aishell3_ckpt_0.5/feats_stats.npy \
|
||||
--am=fastspeech2_ljspeech \
|
||||
--am_config=download/fastspeech2_nosil_ljspeech_ckpt_0.5/default.yaml \
|
||||
--am_ckpt=download/fastspeech2_nosil_ljspeech_ckpt_0.5/snapshot_iter_100000.pdz \
|
||||
--am_stat=download/fastspeech2_nosil_ljspeech_ckpt_0.5/speech_stats.npy \
|
||||
--phones_dict=download/fastspeech2_nosil_ljspeech_ckpt_0.5/phone_id_map.txt
|
||||
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf *.wav
|
||||
./run_sedit_en_new.sh # 语音编辑任务(英文)
|
||||
./run_gen_en_new.sh # 个性化语音合成任务(英文)
|
||||
./run_clone_en_to_zh_new.sh # 跨语言语音合成任务(英文到中文的语音克隆)
|
||||
@ -0,0 +1,162 @@
|
||||
###########################################################
|
||||
# FEATURE EXTRACTION SETTING #
|
||||
###########################################################
|
||||
|
||||
fs: 24000 # sr
|
||||
n_fft: 2048 # FFT size (samples).
|
||||
n_shift: 300 # Hop size (samples). 12.5ms
|
||||
win_length: 1200 # Window length (samples). 50ms
|
||||
# If set to null, it will be the same as fft_size.
|
||||
window: "hann" # Window function.
|
||||
|
||||
# Only used for feats_type != raw
|
||||
|
||||
fmin: 80 # Minimum frequency of Mel basis.
|
||||
fmax: 7600 # Maximum frequency of Mel basis.
|
||||
n_mels: 80 # The number of mel basis.
|
||||
|
||||
mean_phn_span: 8
|
||||
mlm_prob: 0.8
|
||||
|
||||
###########################################################
|
||||
# DATA SETTING #
|
||||
###########################################################
|
||||
batch_size: 20
|
||||
num_workers: 2
|
||||
|
||||
###########################################################
|
||||
# MODEL SETTING #
|
||||
###########################################################
|
||||
model:
|
||||
text_masking: false
|
||||
postnet_layers: 5
|
||||
postnet_filts: 5
|
||||
postnet_chans: 256
|
||||
encoder_type: conformer
|
||||
decoder_type: conformer
|
||||
enc_input_layer: sega_mlm
|
||||
enc_pre_speech_layer: 0
|
||||
enc_cnn_module_kernel: 7
|
||||
enc_attention_dim: 384
|
||||
enc_attention_heads: 2
|
||||
enc_linear_units: 1536
|
||||
enc_num_blocks: 4
|
||||
enc_dropout_rate: 0.2
|
||||
enc_positional_dropout_rate: 0.2
|
||||
enc_attention_dropout_rate: 0.2
|
||||
enc_normalize_before: true
|
||||
enc_macaron_style: true
|
||||
enc_use_cnn_module: true
|
||||
enc_selfattention_layer_type: legacy_rel_selfattn
|
||||
enc_activation_type: swish
|
||||
enc_pos_enc_layer_type: legacy_rel_pos
|
||||
enc_positionwise_layer_type: conv1d
|
||||
enc_positionwise_conv_kernel_size: 3
|
||||
dec_cnn_module_kernel: 31
|
||||
dec_attention_dim: 384
|
||||
dec_attention_heads: 2
|
||||
dec_linear_units: 1536
|
||||
dec_num_blocks: 4
|
||||
dec_dropout_rate: 0.2
|
||||
dec_positional_dropout_rate: 0.2
|
||||
dec_attention_dropout_rate: 0.2
|
||||
dec_macaron_style: true
|
||||
dec_use_cnn_module: true
|
||||
dec_selfattention_layer_type: legacy_rel_selfattn
|
||||
dec_activation_type: swish
|
||||
dec_pos_enc_layer_type: legacy_rel_pos
|
||||
dec_positionwise_layer_type: conv1d
|
||||
dec_positionwise_conv_kernel_size: 3
|
||||
|
||||
###########################################################
|
||||
# OPTIMIZER SETTING #
|
||||
###########################################################
|
||||
optimizer:
|
||||
optim: adam # optimizer type
|
||||
learning_rate: 0.001 # learning rate
|
||||
|
||||
###########################################################
|
||||
# TRAINING SETTING #
|
||||
###########################################################
|
||||
max_epoch: 200
|
||||
num_snapshots: 5
|
||||
|
||||
###########################################################
|
||||
# OTHER SETTING #
|
||||
###########################################################
|
||||
seed: 10086
|
||||
|
||||
token_list:
|
||||
- <blank>
|
||||
- <unk>
|
||||
- AH0
|
||||
- T
|
||||
- N
|
||||
- sp
|
||||
- D
|
||||
- S
|
||||
- R
|
||||
- L
|
||||
- IH1
|
||||
- DH
|
||||
- AE1
|
||||
- M
|
||||
- EH1
|
||||
- K
|
||||
- Z
|
||||
- W
|
||||
- HH
|
||||
- ER0
|
||||
- AH1
|
||||
- IY1
|
||||
- P
|
||||
- V
|
||||
- F
|
||||
- B
|
||||
- AY1
|
||||
- IY0
|
||||
- EY1
|
||||
- AA1
|
||||
- AO1
|
||||
- UW1
|
||||
- IH0
|
||||
- OW1
|
||||
- NG
|
||||
- G
|
||||
- SH
|
||||
- ER1
|
||||
- Y
|
||||
- TH
|
||||
- AW1
|
||||
- CH
|
||||
- UH1
|
||||
- IH2
|
||||
- JH
|
||||
- OW0
|
||||
- EH2
|
||||
- OY1
|
||||
- AY2
|
||||
- EH0
|
||||
- EY2
|
||||
- UW0
|
||||
- AE2
|
||||
- AA2
|
||||
- OW2
|
||||
- AH2
|
||||
- ZH
|
||||
- AO2
|
||||
- IY2
|
||||
- AE0
|
||||
- UW2
|
||||
- AY0
|
||||
- AA0
|
||||
- AO0
|
||||
- AW2
|
||||
- EY0
|
||||
- UH2
|
||||
- ER2
|
||||
- OY2
|
||||
- UH0
|
||||
- AW0
|
||||
- OY0
|
||||
- <sos/eos>
|
||||
@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
stage=0
|
||||
stop_stage=100
|
||||
|
||||
config_path=$1
|
||||
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# get durations from MFA's result
|
||||
echo "Generate durations.txt from MFA results ..."
|
||||
python3 ${MAIN_ROOT}/utils/gen_duration_from_textgrid.py \
|
||||
--inputdir=./vctk_alignment \
|
||||
--output durations.txt \
|
||||
--config=${config_path}
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# extract features
|
||||
echo "Extract features ..."
|
||||
python3 ${BIN_DIR}/preprocess.py \
|
||||
--dataset=vctk \
|
||||
--rootdir=~/datasets/VCTK-Corpus-0.92/ \
|
||||
--dumpdir=dump \
|
||||
--dur-file=durations.txt \
|
||||
--config=${config_path} \
|
||||
--num-cpu=20 \
|
||||
--cut-sil=True
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||
# get features' stats(mean and std)
|
||||
echo "Get features' stats ..."
|
||||
python3 ${MAIN_ROOT}/utils/compute_statistics.py \
|
||||
--metadata=dump/train/raw/metadata.jsonl \
|
||||
--field-name="speech"
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
||||
# normalize and covert phone/speaker to id, dev and test should use train's stats
|
||||
echo "Normalize ..."
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/train/raw/metadata.jsonl \
|
||||
--dumpdir=dump/train/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/dev/raw/metadata.jsonl \
|
||||
--dumpdir=dump/dev/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
|
||||
python3 ${BIN_DIR}/normalize.py \
|
||||
--metadata=dump/test/raw/metadata.jsonl \
|
||||
--dumpdir=dump/test/norm \
|
||||
--speech-stats=dump/train/speech_stats.npy \
|
||||
--phones-dict=dump/phone_id_map.txt \
|
||||
--speaker-dict=dump/speaker_id_map.txt
|
||||
fi
|
||||
@ -0,0 +1 @@
|
||||
#!/bin/bash
|
||||
@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
config_path=$1
|
||||
train_output_path=$2
|
||||
|
||||
python3 ${BIN_DIR}/train.py \
|
||||
--train-metadata=dump/train/norm/metadata.jsonl \
|
||||
--dev-metadata=dump/dev/norm/metadata.jsonl \
|
||||
--config=${config_path} \
|
||||
--output-dir=${train_output_path} \
|
||||
--ngpu=1 \
|
||||
--phones-dict=dump/phone_id_map.txt
|
||||
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
export MAIN_ROOT=`realpath ${PWD}/../../../`
|
||||
|
||||
export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH}
|
||||
export LC_ALL=C
|
||||
|
||||
export PYTHONDONTWRITEBYTECODE=1
|
||||
# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
|
||||
export PYTHONIOENCODING=UTF-8
|
||||
export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH}
|
||||
|
||||
MODEL=ernie_sat
|
||||
export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL}
|
||||
@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
source path.sh
|
||||
|
||||
gpus=0,1
|
||||
stage=0
|
||||
stop_stage=100
|
||||
|
||||
conf_path=conf/default.yaml
|
||||
train_output_path=exp/default
|
||||
ckpt_name=snapshot_iter_153.pdz
|
||||
|
||||
# with the following command, you can choose the stage range you want to run
|
||||
# such as `./run.sh --stage 0 --stop-stage 0`
|
||||
# this can not be mixed use with `$1`, `$2` ...
|
||||
source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
|
||||
|
||||
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||
# prepare data
|
||||
./local/preprocess.sh ${conf_path} || exit -1
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||
# train model, all `ckpt` under `train_output_path/checkpoints/` dir
|
||||
CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1
|
||||
fi
|
||||
|
||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||
# synthesize, vocoder is pwgan
|
||||
CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1
|
||||
fi
|
||||
@ -0,0 +1,130 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Normalize feature files and dump them."""
|
||||
import argparse
|
||||
import logging
|
||||
from operator import itemgetter
|
||||
from pathlib import Path
|
||||
|
||||
import jsonlines
|
||||
import numpy as np
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from tqdm import tqdm
|
||||
|
||||
from paddlespeech.t2s.datasets.data_table import DataTable
|
||||
|
||||
|
||||
def main():
|
||||
"""Run preprocessing process."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Normalize dumped raw features (See detail in parallel_wavegan/bin/normalize.py)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--metadata",
|
||||
type=str,
|
||||
required=True,
|
||||
help="directory including feature files to be normalized. "
|
||||
"you need to specify either *-scp or rootdir.")
|
||||
|
||||
parser.add_argument(
|
||||
"--dumpdir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="directory to dump normalized feature files.")
|
||||
parser.add_argument(
|
||||
"--speech-stats",
|
||||
type=str,
|
||||
required=True,
|
||||
help="speech statistics file.")
|
||||
parser.add_argument(
|
||||
"--phones-dict", type=str, default=None, help="phone vocabulary file.")
|
||||
parser.add_argument(
|
||||
"--speaker-dict", type=str, default=None, help="speaker id map file.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
dumpdir = Path(args.dumpdir).expanduser()
|
||||
# use absolute path
|
||||
dumpdir = dumpdir.resolve()
|
||||
dumpdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# get dataset
|
||||
with jsonlines.open(args.metadata, 'r') as reader:
|
||||
metadata = list(reader)
|
||||
dataset = DataTable(
|
||||
metadata, converters={
|
||||
"speech": np.load,
|
||||
})
|
||||
logging.info(f"The number of files = {len(dataset)}.")
|
||||
|
||||
# restore scaler
|
||||
speech_scaler = StandardScaler()
|
||||
speech_scaler.mean_ = np.load(args.speech_stats)[0]
|
||||
speech_scaler.scale_ = np.load(args.speech_stats)[1]
|
||||
speech_scaler.n_features_in_ = speech_scaler.mean_.shape[0]
|
||||
|
||||
vocab_phones = {}
|
||||
with open(args.phones_dict, 'rt') as f:
|
||||
phn_id = [line.strip().split() for line in f.readlines()]
|
||||
for phn, id in phn_id:
|
||||
vocab_phones[phn] = int(id)
|
||||
|
||||
vocab_speaker = {}
|
||||
with open(args.speaker_dict, 'rt') as f:
|
||||
spk_id = [line.strip().split() for line in f.readlines()]
|
||||
for spk, id in spk_id:
|
||||
vocab_speaker[spk] = int(id)
|
||||
|
||||
# process each file
|
||||
output_metadata = []
|
||||
|
||||
for item in tqdm(dataset):
|
||||
utt_id = item['utt_id']
|
||||
speech = item['speech']
|
||||
|
||||
# normalize
|
||||
speech = speech_scaler.transform(speech)
|
||||
speech_dir = dumpdir / "data_speech"
|
||||
speech_dir.mkdir(parents=True, exist_ok=True)
|
||||
speech_path = speech_dir / f"{utt_id}_speech.npy"
|
||||
np.save(speech_path, speech.astype(np.float32), allow_pickle=False)
|
||||
|
||||
phone_ids = [vocab_phones[p] for p in item['phones']]
|
||||
spk_id = vocab_speaker[item["speaker"]]
|
||||
record = {
|
||||
"utt_id": item['utt_id'],
|
||||
"spk_id": spk_id,
|
||||
"text": phone_ids,
|
||||
"text_lengths": item['text_lengths'],
|
||||
"speech_lengths": item['speech_lengths'],
|
||||
"durations": item['durations'],
|
||||
"speech": str(speech_path),
|
||||
"align_start": item['align_start'],
|
||||
"align_end": item['align_end'],
|
||||
}
|
||||
# add spk_emb for voice cloning
|
||||
if "spk_emb" in item:
|
||||
record["spk_emb"] = str(item["spk_emb"])
|
||||
|
||||
output_metadata.append(record)
|
||||
output_metadata.sort(key=itemgetter('utt_id'))
|
||||
output_metadata_path = Path(args.dumpdir) / "metadata.jsonl"
|
||||
with jsonlines.open(output_metadata_path, 'w') as writer:
|
||||
for item in output_metadata:
|
||||
writer.write(item)
|
||||
logging.info(f"metadata dumped into {output_metadata_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,341 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import argparse
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from operator import itemgetter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
|
||||
import jsonlines
|
||||
import librosa
|
||||
import numpy as np
|
||||
import tqdm
|
||||
import yaml
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from paddlespeech.t2s.datasets.get_feats import LogMelFBank
|
||||
from paddlespeech.t2s.datasets.preprocess_utils import compare_duration_and_mel_length
|
||||
from paddlespeech.t2s.datasets.preprocess_utils import get_input_token
|
||||
from paddlespeech.t2s.datasets.preprocess_utils import get_phn_dur
|
||||
from paddlespeech.t2s.datasets.preprocess_utils import get_spk_id_map
|
||||
from paddlespeech.t2s.datasets.preprocess_utils import merge_silence
|
||||
from paddlespeech.t2s.utils import str2bool
|
||||
|
||||
|
||||
def process_sentence(config: Dict[str, Any],
|
||||
fp: Path,
|
||||
sentences: Dict,
|
||||
output_dir: Path,
|
||||
mel_extractor=None,
|
||||
cut_sil: bool=True,
|
||||
spk_emb_dir: Path=None):
|
||||
utt_id = fp.stem
|
||||
# for vctk
|
||||
if utt_id.endswith("_mic2"):
|
||||
utt_id = utt_id[:-5]
|
||||
record = None
|
||||
if utt_id in sentences:
|
||||
# reading, resampling may occur
|
||||
wav, _ = librosa.load(str(fp), sr=config.fs)
|
||||
if len(wav.shape) != 1:
|
||||
return record
|
||||
max_value = np.abs(wav).max()
|
||||
if max_value > 1.0:
|
||||
wav = wav / max_value
|
||||
assert len(wav.shape) == 1, f"{utt_id} is not a mono-channel audio."
|
||||
assert np.abs(wav).max(
|
||||
) <= 1.0, f"{utt_id} is seems to be different that 16 bit PCM."
|
||||
phones = sentences[utt_id][0]
|
||||
durations = sentences[utt_id][1]
|
||||
speaker = sentences[utt_id][2]
|
||||
d_cumsum = np.pad(np.array(durations).cumsum(0), (1, 0), 'constant')
|
||||
|
||||
# little imprecise than use *.TextGrid directly
|
||||
times = librosa.frames_to_time(
|
||||
d_cumsum, sr=config.fs, hop_length=config.n_shift)
|
||||
if cut_sil:
|
||||
start = 0
|
||||
end = d_cumsum[-1]
|
||||
if phones[0] == "sil" and len(durations) > 1:
|
||||
start = times[1]
|
||||
durations = durations[1:]
|
||||
phones = phones[1:]
|
||||
if phones[-1] == 'sil' and len(durations) > 1:
|
||||
end = times[-2]
|
||||
durations = durations[:-1]
|
||||
phones = phones[:-1]
|
||||
sentences[utt_id][0] = phones
|
||||
sentences[utt_id][1] = durations
|
||||
start, end = librosa.time_to_samples([start, end], sr=config.fs)
|
||||
wav = wav[start:end]
|
||||
|
||||
# extract mel feats
|
||||
logmel = mel_extractor.get_log_mel_fbank(wav)
|
||||
# change duration according to mel_length
|
||||
compare_duration_and_mel_length(sentences, utt_id, logmel)
|
||||
# utt_id may be popped in compare_duration_and_mel_length
|
||||
if utt_id not in sentences:
|
||||
return None
|
||||
phones = sentences[utt_id][0]
|
||||
durations = sentences[utt_id][1]
|
||||
num_frames = logmel.shape[0]
|
||||
assert sum(durations) == num_frames
|
||||
|
||||
new_d_cumsum = np.pad(np.array(durations).cumsum(0), (1, 0), 'constant')
|
||||
align_start = new_d_cumsum[:-1]
|
||||
align_end = new_d_cumsum[1:]
|
||||
assert len(align_start) == len(align_end) == len(durations)
|
||||
|
||||
mel_dir = output_dir / "data_speech"
|
||||
mel_dir.mkdir(parents=True, exist_ok=True)
|
||||
mel_path = mel_dir / (utt_id + "_speech.npy")
|
||||
np.save(mel_path, logmel)
|
||||
# align_start_lengths == text_lengths
|
||||
record = {
|
||||
"utt_id": utt_id,
|
||||
"phones": phones,
|
||||
"text_lengths": len(phones),
|
||||
"speech_lengths": num_frames,
|
||||
"durations": durations,
|
||||
"speech": str(mel_path),
|
||||
"speaker": speaker,
|
||||
"align_start": align_start.tolist(),
|
||||
"align_end": align_end.tolist(),
|
||||
}
|
||||
if spk_emb_dir:
|
||||
if speaker in os.listdir(spk_emb_dir):
|
||||
embed_name = utt_id + ".npy"
|
||||
embed_path = spk_emb_dir / speaker / embed_name
|
||||
if embed_path.is_file():
|
||||
record["spk_emb"] = str(embed_path)
|
||||
else:
|
||||
return None
|
||||
return record
|
||||
|
||||
|
||||
def process_sentences(config,
|
||||
fps: List[Path],
|
||||
sentences: Dict,
|
||||
output_dir: Path,
|
||||
mel_extractor=None,
|
||||
nprocs: int=1,
|
||||
cut_sil: bool=True,
|
||||
spk_emb_dir: Path=None):
|
||||
if nprocs == 1:
|
||||
results = []
|
||||
for fp in tqdm.tqdm(fps, total=len(fps)):
|
||||
record = process_sentence(
|
||||
config=config,
|
||||
fp=fp,
|
||||
sentences=sentences,
|
||||
output_dir=output_dir,
|
||||
mel_extractor=mel_extractor,
|
||||
cut_sil=cut_sil,
|
||||
spk_emb_dir=spk_emb_dir)
|
||||
if record:
|
||||
results.append(record)
|
||||
else:
|
||||
with ThreadPoolExecutor(nprocs) as pool:
|
||||
futures = []
|
||||
with tqdm.tqdm(total=len(fps)) as progress:
|
||||
for fp in fps:
|
||||
future = pool.submit(process_sentence, config, fp,
|
||||
sentences, output_dir, mel_extractor,
|
||||
cut_sil, spk_emb_dir)
|
||||
future.add_done_callback(lambda p: progress.update())
|
||||
futures.append(future)
|
||||
|
||||
results = []
|
||||
for ft in futures:
|
||||
record = ft.result()
|
||||
if record:
|
||||
results.append(record)
|
||||
|
||||
results.sort(key=itemgetter("utt_id"))
|
||||
with jsonlines.open(output_dir / "metadata.jsonl", 'w') as writer:
|
||||
for item in results:
|
||||
writer.write(item)
|
||||
print("Done")
|
||||
|
||||
|
||||
def main():
|
||||
# parse config and args
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Preprocess audio and then extract features.")
|
||||
|
||||
parser.add_argument(
|
||||
"--dataset",
|
||||
default="baker",
|
||||
type=str,
|
||||
help="name of dataset, should in {baker, aishell3, ljspeech, vctk} now")
|
||||
|
||||
parser.add_argument(
|
||||
"--rootdir", default=None, type=str, help="directory to dataset.")
|
||||
|
||||
parser.add_argument(
|
||||
"--dumpdir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="directory to dump feature files.")
|
||||
parser.add_argument(
|
||||
"--dur-file", default=None, type=str, help="path to durations.txt.")
|
||||
|
||||
parser.add_argument("--config", type=str, help="fastspeech2 config file.")
|
||||
|
||||
parser.add_argument(
|
||||
"--num-cpu", type=int, default=1, help="number of process.")
|
||||
|
||||
parser.add_argument(
|
||||
"--cut-sil",
|
||||
type=str2bool,
|
||||
default=True,
|
||||
help="whether cut sil in the edge of audio")
|
||||
|
||||
parser.add_argument(
|
||||
"--spk_emb_dir",
|
||||
default=None,
|
||||
type=str,
|
||||
help="directory to speaker embedding files.")
|
||||
args = parser.parse_args()
|
||||
|
||||
rootdir = Path(args.rootdir).expanduser()
|
||||
dumpdir = Path(args.dumpdir).expanduser()
|
||||
# use absolute path
|
||||
dumpdir = dumpdir.resolve()
|
||||
dumpdir.mkdir(parents=True, exist_ok=True)
|
||||
dur_file = Path(args.dur_file).expanduser()
|
||||
|
||||
if args.spk_emb_dir:
|
||||
spk_emb_dir = Path(args.spk_emb_dir).expanduser().resolve()
|
||||
else:
|
||||
spk_emb_dir = None
|
||||
|
||||
assert rootdir.is_dir()
|
||||
assert dur_file.is_file()
|
||||
|
||||
with open(args.config, 'rt') as f:
|
||||
config = CfgNode(yaml.safe_load(f))
|
||||
|
||||
sentences, speaker_set = get_phn_dur(dur_file)
|
||||
|
||||
merge_silence(sentences)
|
||||
phone_id_map_path = dumpdir / "phone_id_map.txt"
|
||||
speaker_id_map_path = dumpdir / "speaker_id_map.txt"
|
||||
get_input_token(sentences, phone_id_map_path, args.dataset)
|
||||
get_spk_id_map(speaker_set, speaker_id_map_path)
|
||||
|
||||
if args.dataset == "baker":
|
||||
wav_files = sorted(list((rootdir / "Wave").rglob("*.wav")))
|
||||
# split data into 3 sections
|
||||
num_train = 9800
|
||||
num_dev = 100
|
||||
train_wav_files = wav_files[:num_train]
|
||||
dev_wav_files = wav_files[num_train:num_train + num_dev]
|
||||
test_wav_files = wav_files[num_train + num_dev:]
|
||||
elif args.dataset == "aishell3":
|
||||
sub_num_dev = 5
|
||||
wav_dir = rootdir / "train" / "wav"
|
||||
train_wav_files = []
|
||||
dev_wav_files = []
|
||||
test_wav_files = []
|
||||
for speaker in os.listdir(wav_dir):
|
||||
wav_files = sorted(list((wav_dir / speaker).rglob("*.wav")))
|
||||
if len(wav_files) > 100:
|
||||
train_wav_files += wav_files[:-sub_num_dev * 2]
|
||||
dev_wav_files += wav_files[-sub_num_dev * 2:-sub_num_dev]
|
||||
test_wav_files += wav_files[-sub_num_dev:]
|
||||
else:
|
||||
train_wav_files += wav_files
|
||||
|
||||
elif args.dataset == "ljspeech":
|
||||
wav_files = sorted(list((rootdir / "wavs").rglob("*.wav")))
|
||||
# split data into 3 sections
|
||||
num_train = 12900
|
||||
num_dev = 100
|
||||
train_wav_files = wav_files[:num_train]
|
||||
dev_wav_files = wav_files[num_train:num_train + num_dev]
|
||||
test_wav_files = wav_files[num_train + num_dev:]
|
||||
elif args.dataset == "vctk":
|
||||
sub_num_dev = 5
|
||||
wav_dir = rootdir / "wav48_silence_trimmed"
|
||||
train_wav_files = []
|
||||
dev_wav_files = []
|
||||
test_wav_files = []
|
||||
for speaker in os.listdir(wav_dir):
|
||||
wav_files = sorted(list((wav_dir / speaker).rglob("*_mic2.flac")))
|
||||
if len(wav_files) > 100:
|
||||
train_wav_files += wav_files[:-sub_num_dev * 2]
|
||||
dev_wav_files += wav_files[-sub_num_dev * 2:-sub_num_dev]
|
||||
test_wav_files += wav_files[-sub_num_dev:]
|
||||
else:
|
||||
train_wav_files += wav_files
|
||||
|
||||
else:
|
||||
print("dataset should in {baker, aishell3, ljspeech, vctk} now!")
|
||||
|
||||
train_dump_dir = dumpdir / "train" / "raw"
|
||||
train_dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
dev_dump_dir = dumpdir / "dev" / "raw"
|
||||
dev_dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
test_dump_dir = dumpdir / "test" / "raw"
|
||||
test_dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Extractor
|
||||
mel_extractor = LogMelFBank(
|
||||
sr=config.fs,
|
||||
n_fft=config.n_fft,
|
||||
hop_length=config.n_shift,
|
||||
win_length=config.win_length,
|
||||
window=config.window,
|
||||
n_mels=config.n_mels,
|
||||
fmin=config.fmin,
|
||||
fmax=config.fmax)
|
||||
|
||||
# process for the 3 sections
|
||||
if train_wav_files:
|
||||
process_sentences(
|
||||
config=config,
|
||||
fps=train_wav_files,
|
||||
sentences=sentences,
|
||||
output_dir=train_dump_dir,
|
||||
mel_extractor=mel_extractor,
|
||||
nprocs=args.num_cpu,
|
||||
cut_sil=args.cut_sil,
|
||||
spk_emb_dir=spk_emb_dir)
|
||||
if dev_wav_files:
|
||||
process_sentences(
|
||||
config=config,
|
||||
fps=dev_wav_files,
|
||||
sentences=sentences,
|
||||
output_dir=dev_dump_dir,
|
||||
mel_extractor=mel_extractor,
|
||||
cut_sil=args.cut_sil,
|
||||
spk_emb_dir=spk_emb_dir)
|
||||
if test_wav_files:
|
||||
process_sentences(
|
||||
config=config,
|
||||
fps=test_wav_files,
|
||||
sentences=sentences,
|
||||
output_dir=test_dump_dir,
|
||||
mel_extractor=mel_extractor,
|
||||
nprocs=args.num_cpu,
|
||||
cut_sil=args.cut_sil,
|
||||
spk_emb_dir=spk_emb_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,194 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import jsonlines
|
||||
import numpy as np
|
||||
import paddle
|
||||
import yaml
|
||||
from paddle import DataParallel
|
||||
from paddle import distributed as dist
|
||||
from paddle.io import DataLoader
|
||||
from paddle.io import DistributedBatchSampler
|
||||
from yacs.config import CfgNode
|
||||
|
||||
from paddlespeech.t2s.datasets.am_batch_fn import build_erniesat_collate_fn
|
||||
from paddlespeech.t2s.datasets.data_table import DataTable
|
||||
from paddlespeech.t2s.models.ernie_sat import ErnieSAT
|
||||
from paddlespeech.t2s.models.ernie_sat import ErnieSATEvaluator
|
||||
from paddlespeech.t2s.models.ernie_sat import ErnieSATUpdater
|
||||
from paddlespeech.t2s.training.extensions.snapshot import Snapshot
|
||||
from paddlespeech.t2s.training.extensions.visualizer import VisualDL
|
||||
from paddlespeech.t2s.training.optimizer import build_optimizers
|
||||
from paddlespeech.t2s.training.seeding import seed_everything
|
||||
from paddlespeech.t2s.training.trainer import Trainer
|
||||
|
||||
|
||||
def train_sp(args, config):
|
||||
# decides device type and whether to run in parallel
|
||||
# setup running environment correctly
|
||||
if (not paddle.is_compiled_with_cuda()) or args.ngpu == 0:
|
||||
paddle.set_device("cpu")
|
||||
else:
|
||||
paddle.set_device("gpu")
|
||||
world_size = paddle.distributed.get_world_size()
|
||||
if world_size > 1:
|
||||
paddle.distributed.init_parallel_env()
|
||||
|
||||
# set the random seed, it is a must for multiprocess training
|
||||
seed_everything(config.seed)
|
||||
|
||||
print(
|
||||
f"rank: {dist.get_rank()}, pid: {os.getpid()}, parent_pid: {os.getppid()}",
|
||||
)
|
||||
fields = [
|
||||
"text", "text_lengths", "speech", "speech_lengths", "align_start",
|
||||
"align_end"
|
||||
]
|
||||
converters = {"speech": np.load}
|
||||
spk_num = None
|
||||
|
||||
# dataloader has been too verbose
|
||||
logging.getLogger("DataLoader").disabled = True
|
||||
|
||||
# construct dataset for training and validation
|
||||
with jsonlines.open(args.train_metadata, 'r') as reader:
|
||||
train_metadata = list(reader)
|
||||
train_dataset = DataTable(
|
||||
data=train_metadata,
|
||||
fields=fields,
|
||||
converters=converters, )
|
||||
with jsonlines.open(args.dev_metadata, 'r') as reader:
|
||||
dev_metadata = list(reader)
|
||||
dev_dataset = DataTable(
|
||||
data=dev_metadata,
|
||||
fields=fields,
|
||||
converters=converters, )
|
||||
|
||||
# collate function and dataloader
|
||||
collate_fn = build_erniesat_collate_fn(
|
||||
mlm_prob=config.mlm_prob,
|
||||
mean_phn_span=config.mean_phn_span,
|
||||
seg_emb=config.model['enc_input_layer'] == 'sega_mlm',
|
||||
text_masking=config["model"]["text_masking"],
|
||||
epoch=config["max_epoch"])
|
||||
|
||||
train_sampler = DistributedBatchSampler(
|
||||
train_dataset,
|
||||
batch_size=config.batch_size,
|
||||
shuffle=True,
|
||||
drop_last=True)
|
||||
|
||||
print("samplers done!")
|
||||
|
||||
train_dataloader = DataLoader(
|
||||
train_dataset,
|
||||
batch_sampler=train_sampler,
|
||||
collate_fn=collate_fn,
|
||||
num_workers=config.num_workers)
|
||||
|
||||
dev_dataloader = DataLoader(
|
||||
dev_dataset,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
batch_size=config.batch_size,
|
||||
collate_fn=collate_fn,
|
||||
num_workers=config.num_workers)
|
||||
print("dataloaders done!")
|
||||
|
||||
with open(args.phones_dict, "r") as f:
|
||||
phn_id = [line.strip().split() for line in f.readlines()]
|
||||
vocab_size = len(phn_id)
|
||||
print("vocab_size:", vocab_size)
|
||||
|
||||
odim = config.n_mels
|
||||
model = ErnieSAT(idim=vocab_size, odim=odim, **config["model"])
|
||||
|
||||
if world_size > 1:
|
||||
model = DataParallel(model)
|
||||
print("model done!")
|
||||
|
||||
optimizer = build_optimizers(model, **config["optimizer"])
|
||||
print("optimizer done!")
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
if dist.get_rank() == 0:
|
||||
config_name = args.config.split("/")[-1]
|
||||
# copy conf to output_dir
|
||||
shutil.copyfile(args.config, output_dir / config_name)
|
||||
|
||||
updater = ErnieSATUpdater(
|
||||
model=model,
|
||||
optimizer=optimizer,
|
||||
dataloader=train_dataloader,
|
||||
text_masking=config["model"]["text_masking"],
|
||||
odim=odim,
|
||||
output_dir=output_dir)
|
||||
|
||||
trainer = Trainer(updater, (config.max_epoch, 'epoch'), output_dir)
|
||||
|
||||
evaluator = ErnieSATEvaluator(
|
||||
model=model,
|
||||
dataloader=dev_dataloader,
|
||||
text_masking=config["model"]["text_masking"],
|
||||
odim=odim,
|
||||
output_dir=output_dir, )
|
||||
|
||||
if dist.get_rank() == 0:
|
||||
trainer.extend(evaluator, trigger=(1, "epoch"))
|
||||
trainer.extend(VisualDL(output_dir), trigger=(1, "iteration"))
|
||||
trainer.extend(
|
||||
Snapshot(max_size=config.num_snapshots), trigger=(1, 'epoch'))
|
||||
trainer.run()
|
||||
|
||||
|
||||
def main():
|
||||
# parse args and config and redirect to train_sp
|
||||
parser = argparse.ArgumentParser(description="Train an ErnieSAT model.")
|
||||
parser.add_argument("--config", type=str, help="ErnieSAT config file.")
|
||||
parser.add_argument("--train-metadata", type=str, help="training data.")
|
||||
parser.add_argument("--dev-metadata", type=str, help="dev data.")
|
||||
parser.add_argument("--output-dir", type=str, help="output dir.")
|
||||
parser.add_argument(
|
||||
"--ngpu", type=int, default=1, help="if ngpu=0, use cpu.")
|
||||
parser.add_argument(
|
||||
"--phones-dict", type=str, default=None, help="phone vocabulary file.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.config) as f:
|
||||
config = CfgNode(yaml.safe_load(f))
|
||||
|
||||
print("========Args========")
|
||||
print(yaml.safe_dump(vars(args)))
|
||||
print("========Config========")
|
||||
print(config)
|
||||
print(
|
||||
f"master see the word size: {dist.get_world_size()}, from pid: {os.getpid()}"
|
||||
)
|
||||
|
||||
# dispatch
|
||||
if args.ngpu > 1:
|
||||
dist.spawn(train_sp, (args, config), nprocs=args.ngpu)
|
||||
else:
|
||||
train_sp(args, config)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,670 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
|
||||
import paddle
|
||||
from paddle import nn
|
||||
|
||||
from paddlespeech.t2s.modules.activation import get_activation
|
||||
from paddlespeech.t2s.modules.conformer.convolution import ConvolutionModule
|
||||
from paddlespeech.t2s.modules.conformer.encoder_layer import EncoderLayer
|
||||
from paddlespeech.t2s.modules.layer_norm import LayerNorm
|
||||
from paddlespeech.t2s.modules.masked_fill import masked_fill
|
||||
from paddlespeech.t2s.modules.nets_utils import initialize
|
||||
from paddlespeech.t2s.modules.tacotron2.decoder import Postnet
|
||||
from paddlespeech.t2s.modules.transformer.attention import LegacyRelPositionMultiHeadedAttention
|
||||
from paddlespeech.t2s.modules.transformer.attention import MultiHeadedAttention
|
||||
from paddlespeech.t2s.modules.transformer.attention import RelPositionMultiHeadedAttention
|
||||
from paddlespeech.t2s.modules.transformer.embedding import LegacyRelPositionalEncoding
|
||||
from paddlespeech.t2s.modules.transformer.embedding import PositionalEncoding
|
||||
from paddlespeech.t2s.modules.transformer.embedding import RelPositionalEncoding
|
||||
from paddlespeech.t2s.modules.transformer.embedding import ScaledPositionalEncoding
|
||||
from paddlespeech.t2s.modules.transformer.multi_layer_conv import Conv1dLinear
|
||||
from paddlespeech.t2s.modules.transformer.multi_layer_conv import MultiLayeredConv1d
|
||||
from paddlespeech.t2s.modules.transformer.positionwise_feed_forward import PositionwiseFeedForward
|
||||
from paddlespeech.t2s.modules.transformer.repeat import repeat
|
||||
from paddlespeech.t2s.modules.transformer.subsampling import Conv2dSubsampling
|
||||
|
||||
|
||||
# MLM -> Mask Language Model
|
||||
class mySequential(nn.Sequential):
|
||||
def forward(self, *inputs):
|
||||
for module in self._sub_layers.values():
|
||||
if type(inputs) == tuple:
|
||||
inputs = module(*inputs)
|
||||
else:
|
||||
inputs = module(inputs)
|
||||
return inputs
|
||||
|
||||
|
||||
class MaskInputLayer(nn.Layer):
|
||||
def __init__(self, out_features: int) -> None:
|
||||
super().__init__()
|
||||
self.mask_feature = paddle.create_parameter(
|
||||
shape=(1, 1, out_features),
|
||||
dtype=paddle.float32,
|
||||
default_initializer=paddle.nn.initializer.Assign(
|
||||
paddle.normal(shape=(1, 1, out_features))))
|
||||
|
||||
def forward(self, input: paddle.Tensor,
|
||||
masked_pos: paddle.Tensor=None) -> paddle.Tensor:
|
||||
masked_pos = paddle.expand_as(paddle.unsqueeze(masked_pos, -1), input)
|
||||
masked_input = masked_fill(input, masked_pos, 0) + masked_fill(
|
||||
paddle.expand_as(self.mask_feature, input), ~masked_pos, 0)
|
||||
return masked_input
|
||||
|
||||
|
||||
class MLMEncoder(nn.Layer):
|
||||
"""Conformer encoder module.
|
||||
|
||||
Args:
|
||||
idim (int): Input dimension.
|
||||
attention_dim (int): Dimension of attention.
|
||||
attention_heads (int): The number of heads of multi head attention.
|
||||
linear_units (int): The number of units of position-wise feed forward.
|
||||
num_blocks (int): The number of decoder blocks.
|
||||
dropout_rate (float): Dropout rate.
|
||||
positional_dropout_rate (float): Dropout rate after adding positional encoding.
|
||||
attention_dropout_rate (float): Dropout rate in attention.
|
||||
input_layer (Union[str, paddle.nn.Layer]): Input layer type.
|
||||
normalize_before (bool): Whether to use layer_norm before the first block.
|
||||
concat_after (bool): Whether to concat attention layer's input and output.
|
||||
if True, additional linear will be applied.
|
||||
i.e. x -> x + linear(concat(x, att(x)))
|
||||
if False, no additional linear will be applied. i.e. x -> x + att(x)
|
||||
positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear".
|
||||
positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer.
|
||||
macaron_style (bool): Whether to use macaron style for positionwise layer.
|
||||
pos_enc_layer_type (str): Encoder positional encoding layer type.
|
||||
selfattention_layer_type (str): Encoder attention layer type.
|
||||
activation_type (str): Encoder activation function type.
|
||||
use_cnn_module (bool): Whether to use convolution module.
|
||||
zero_triu (bool): Whether to zero the upper triangular part of attention matrix.
|
||||
cnn_module_kernel (int): Kernerl size of convolution module.
|
||||
padding_idx (int): Padding idx for input_layer=embed.
|
||||
stochastic_depth_rate (float): Maximum probability to skip the encoder layer.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
idim: int,
|
||||
vocab_size: int=0,
|
||||
pre_speech_layer: int=0,
|
||||
attention_dim: int=256,
|
||||
attention_heads: int=4,
|
||||
linear_units: int=2048,
|
||||
num_blocks: int=6,
|
||||
dropout_rate: float=0.1,
|
||||
positional_dropout_rate: float=0.1,
|
||||
attention_dropout_rate: float=0.0,
|
||||
input_layer: str="conv2d",
|
||||
normalize_before: bool=True,
|
||||
concat_after: bool=False,
|
||||
positionwise_layer_type: str="linear",
|
||||
positionwise_conv_kernel_size: int=1,
|
||||
macaron_style: bool=False,
|
||||
pos_enc_layer_type: str="abs_pos",
|
||||
pos_enc_class=None,
|
||||
selfattention_layer_type: str="selfattn",
|
||||
activation_type: str="swish",
|
||||
use_cnn_module: bool=False,
|
||||
zero_triu: bool=False,
|
||||
cnn_module_kernel: int=31,
|
||||
padding_idx: int=-1,
|
||||
stochastic_depth_rate: float=0.0,
|
||||
text_masking: bool=False):
|
||||
"""Construct an Encoder object."""
|
||||
super().__init__()
|
||||
self._output_size = attention_dim
|
||||
self.text_masking = text_masking
|
||||
if self.text_masking:
|
||||
self.text_masking_layer = MaskInputLayer(attention_dim)
|
||||
activation = get_activation(activation_type)
|
||||
if pos_enc_layer_type == "abs_pos":
|
||||
pos_enc_class = PositionalEncoding
|
||||
elif pos_enc_layer_type == "scaled_abs_pos":
|
||||
pos_enc_class = ScaledPositionalEncoding
|
||||
elif pos_enc_layer_type == "rel_pos":
|
||||
assert selfattention_layer_type == "rel_selfattn"
|
||||
pos_enc_class = RelPositionalEncoding
|
||||
elif pos_enc_layer_type == "legacy_rel_pos":
|
||||
pos_enc_class = LegacyRelPositionalEncoding
|
||||
assert selfattention_layer_type == "legacy_rel_selfattn"
|
||||
else:
|
||||
raise ValueError("unknown pos_enc_layer: " + pos_enc_layer_type)
|
||||
|
||||
self.conv_subsampling_factor = 1
|
||||
if input_layer == "linear":
|
||||
self.embed = nn.Sequential(
|
||||
nn.Linear(idim, attention_dim),
|
||||
nn.LayerNorm(attention_dim),
|
||||
nn.Dropout(dropout_rate),
|
||||
nn.ReLU(),
|
||||
pos_enc_class(attention_dim, positional_dropout_rate), )
|
||||
elif input_layer == "conv2d":
|
||||
self.embed = Conv2dSubsampling(
|
||||
idim,
|
||||
attention_dim,
|
||||
dropout_rate,
|
||||
pos_enc_class(attention_dim, positional_dropout_rate), )
|
||||
self.conv_subsampling_factor = 4
|
||||
elif input_layer == "embed":
|
||||
self.embed = nn.Sequential(
|
||||
nn.Embedding(idim, attention_dim, padding_idx=padding_idx),
|
||||
pos_enc_class(attention_dim, positional_dropout_rate), )
|
||||
elif input_layer == "mlm":
|
||||
self.segment_emb = None
|
||||
self.speech_embed = mySequential(
|
||||
MaskInputLayer(idim),
|
||||
nn.Linear(idim, attention_dim),
|
||||
nn.LayerNorm(attention_dim),
|
||||
nn.ReLU(),
|
||||
pos_enc_class(attention_dim, positional_dropout_rate))
|
||||
self.text_embed = nn.Sequential(
|
||||
nn.Embedding(
|
||||
vocab_size, attention_dim, padding_idx=padding_idx),
|
||||
pos_enc_class(attention_dim, positional_dropout_rate), )
|
||||
elif input_layer == "sega_mlm":
|
||||
self.segment_emb = nn.Embedding(
|
||||
500, attention_dim, padding_idx=padding_idx)
|
||||
self.speech_embed = mySequential(
|
||||
MaskInputLayer(idim),
|
||||
nn.Linear(idim, attention_dim),
|
||||
nn.LayerNorm(attention_dim),
|
||||
nn.ReLU(),
|
||||
pos_enc_class(attention_dim, positional_dropout_rate))
|
||||
self.text_embed = nn.Sequential(
|
||||
nn.Embedding(
|
||||
vocab_size, attention_dim, padding_idx=padding_idx),
|
||||
pos_enc_class(attention_dim, positional_dropout_rate), )
|
||||
elif isinstance(input_layer, nn.Layer):
|
||||
self.embed = nn.Sequential(
|
||||
input_layer,
|
||||
pos_enc_class(attention_dim, positional_dropout_rate), )
|
||||
elif input_layer is None:
|
||||
self.embed = nn.Sequential(
|
||||
pos_enc_class(attention_dim, positional_dropout_rate))
|
||||
else:
|
||||
raise ValueError("unknown input_layer: " + input_layer)
|
||||
self.normalize_before = normalize_before
|
||||
|
||||
# self-attention module definition
|
||||
if selfattention_layer_type == "selfattn":
|
||||
encoder_selfattn_layer = MultiHeadedAttention
|
||||
encoder_selfattn_layer_args = (attention_heads, attention_dim,
|
||||
attention_dropout_rate, )
|
||||
elif selfattention_layer_type == "legacy_rel_selfattn":
|
||||
assert pos_enc_layer_type == "legacy_rel_pos"
|
||||
encoder_selfattn_layer = LegacyRelPositionMultiHeadedAttention
|
||||
encoder_selfattn_layer_args = (attention_heads, attention_dim,
|
||||
attention_dropout_rate, )
|
||||
elif selfattention_layer_type == "rel_selfattn":
|
||||
assert pos_enc_layer_type == "rel_pos"
|
||||
encoder_selfattn_layer = RelPositionMultiHeadedAttention
|
||||
encoder_selfattn_layer_args = (attention_heads, attention_dim,
|
||||
attention_dropout_rate, zero_triu, )
|
||||
else:
|
||||
raise ValueError("unknown encoder_attn_layer: " +
|
||||
selfattention_layer_type)
|
||||
|
||||
# feed-forward module definition
|
||||
if positionwise_layer_type == "linear":
|
||||
positionwise_layer = PositionwiseFeedForward
|
||||
positionwise_layer_args = (attention_dim, linear_units,
|
||||
dropout_rate, activation, )
|
||||
elif positionwise_layer_type == "conv1d":
|
||||
positionwise_layer = MultiLayeredConv1d
|
||||
positionwise_layer_args = (attention_dim, linear_units,
|
||||
positionwise_conv_kernel_size,
|
||||
dropout_rate, )
|
||||
elif positionwise_layer_type == "conv1d-linear":
|
||||
positionwise_layer = Conv1dLinear
|
||||
positionwise_layer_args = (attention_dim, linear_units,
|
||||
positionwise_conv_kernel_size,
|
||||
dropout_rate, )
|
||||
else:
|
||||
raise NotImplementedError("Support only linear or conv1d.")
|
||||
|
||||
# convolution module definition
|
||||
convolution_layer = ConvolutionModule
|
||||
convolution_layer_args = (attention_dim, cnn_module_kernel, activation)
|
||||
|
||||
self.encoders = repeat(
|
||||
num_blocks,
|
||||
lambda lnum: EncoderLayer(
|
||||
attention_dim,
|
||||
encoder_selfattn_layer(*encoder_selfattn_layer_args),
|
||||
positionwise_layer(*positionwise_layer_args),
|
||||
positionwise_layer(*positionwise_layer_args) if macaron_style else None,
|
||||
convolution_layer(*convolution_layer_args) if use_cnn_module else None,
|
||||
dropout_rate,
|
||||
normalize_before,
|
||||
concat_after,
|
||||
stochastic_depth_rate * float(1 + lnum) / num_blocks, ), )
|
||||
self.pre_speech_layer = pre_speech_layer
|
||||
self.pre_speech_encoders = repeat(
|
||||
self.pre_speech_layer,
|
||||
lambda lnum: EncoderLayer(
|
||||
attention_dim,
|
||||
encoder_selfattn_layer(*encoder_selfattn_layer_args),
|
||||
positionwise_layer(*positionwise_layer_args),
|
||||
positionwise_layer(*positionwise_layer_args) if macaron_style else None,
|
||||
convolution_layer(*convolution_layer_args) if use_cnn_module else None,
|
||||
dropout_rate,
|
||||
normalize_before,
|
||||
concat_after,
|
||||
stochastic_depth_rate * float(1 + lnum) / self.pre_speech_layer, ),
|
||||
)
|
||||
if self.normalize_before:
|
||||
self.after_norm = LayerNorm(attention_dim)
|
||||
|
||||
def forward(self,
|
||||
speech: paddle.Tensor,
|
||||
text: paddle.Tensor,
|
||||
masked_pos: paddle.Tensor,
|
||||
speech_mask: paddle.Tensor=None,
|
||||
text_mask: paddle.Tensor=None,
|
||||
speech_seg_pos: paddle.Tensor=None,
|
||||
text_seg_pos: paddle.Tensor=None):
|
||||
"""Encode input sequence.
|
||||
|
||||
"""
|
||||
if masked_pos is not None:
|
||||
speech = self.speech_embed(speech, masked_pos)
|
||||
else:
|
||||
speech = self.speech_embed(speech)
|
||||
if text is not None:
|
||||
text = self.text_embed(text)
|
||||
if speech_seg_pos is not None and text_seg_pos is not None and self.segment_emb:
|
||||
speech_seg_emb = self.segment_emb(speech_seg_pos)
|
||||
text_seg_emb = self.segment_emb(text_seg_pos)
|
||||
text = (text[0] + text_seg_emb, text[1])
|
||||
speech = (speech[0] + speech_seg_emb, speech[1])
|
||||
if self.pre_speech_encoders:
|
||||
speech, _ = self.pre_speech_encoders(speech, speech_mask)
|
||||
|
||||
if text is not None:
|
||||
xs = paddle.concat([speech[0], text[0]], axis=1)
|
||||
xs_pos_emb = paddle.concat([speech[1], text[1]], axis=1)
|
||||
masks = paddle.concat([speech_mask, text_mask], axis=-1)
|
||||
else:
|
||||
xs = speech[0]
|
||||
xs_pos_emb = speech[1]
|
||||
masks = speech_mask
|
||||
|
||||
xs, masks = self.encoders((xs, xs_pos_emb), masks)
|
||||
|
||||
if isinstance(xs, tuple):
|
||||
xs = xs[0]
|
||||
if self.normalize_before:
|
||||
xs = self.after_norm(xs)
|
||||
|
||||
return xs, masks
|
||||
|
||||
|
||||
class MLMDecoder(MLMEncoder):
|
||||
def forward(self, xs: paddle.Tensor, masks: paddle.Tensor):
|
||||
"""Encode input sequence.
|
||||
|
||||
Args:
|
||||
xs (paddle.Tensor): Input tensor (#batch, time, idim).
|
||||
masks (paddle.Tensor): Mask tensor (#batch, time).
|
||||
|
||||
Returns:
|
||||
paddle.Tensor: Output tensor (#batch, time, attention_dim).
|
||||
paddle.Tensor: Mask tensor (#batch, time).
|
||||
|
||||
"""
|
||||
xs = self.embed(xs)
|
||||
xs, masks = self.encoders(xs, masks)
|
||||
|
||||
if isinstance(xs, tuple):
|
||||
xs = xs[0]
|
||||
if self.normalize_before:
|
||||
xs = self.after_norm(xs)
|
||||
|
||||
return xs, masks
|
||||
|
||||
|
||||
# encoder and decoder is nn.Layer, not str
|
||||
class MLM(nn.Layer):
|
||||
def __init__(self,
|
||||
odim: int,
|
||||
encoder: nn.Layer,
|
||||
decoder: Optional[nn.Layer],
|
||||
postnet_layers: int=0,
|
||||
postnet_chans: int=0,
|
||||
postnet_filts: int=0,
|
||||
text_masking: bool=False):
|
||||
|
||||
super().__init__()
|
||||
self.odim = odim
|
||||
self.encoder = encoder
|
||||
self.decoder = decoder
|
||||
self.vocab_size = encoder.text_embed[0]._num_embeddings
|
||||
|
||||
if self.decoder is None or not (hasattr(self.decoder,
|
||||
'output_layer') and
|
||||
self.decoder.output_layer is not None):
|
||||
self.sfc = nn.Linear(self.encoder._output_size, odim)
|
||||
else:
|
||||
self.sfc = None
|
||||
if text_masking:
|
||||
self.text_sfc = nn.Linear(
|
||||
self.encoder.text_embed[0]._embedding_dim,
|
||||
self.vocab_size,
|
||||
weight_attr=self.encoder.text_embed[0]._weight_attr)
|
||||
else:
|
||||
self.text_sfc = None
|
||||
|
||||
self.postnet = (None if postnet_layers == 0 else Postnet(
|
||||
idim=self.encoder._output_size,
|
||||
odim=odim,
|
||||
n_layers=postnet_layers,
|
||||
n_chans=postnet_chans,
|
||||
n_filts=postnet_filts,
|
||||
use_batch_norm=True,
|
||||
dropout_rate=0.5, ))
|
||||
|
||||
def inference(
|
||||
self,
|
||||
speech: paddle.Tensor,
|
||||
text: paddle.Tensor,
|
||||
masked_pos: paddle.Tensor,
|
||||
speech_mask: paddle.Tensor,
|
||||
text_mask: paddle.Tensor,
|
||||
speech_seg_pos: paddle.Tensor,
|
||||
text_seg_pos: paddle.Tensor,
|
||||
span_bdy: List[int],
|
||||
use_teacher_forcing: bool=False, ) -> Dict[str, paddle.Tensor]:
|
||||
'''
|
||||
Args:
|
||||
speech (paddle.Tensor): input speech (1, Tmax, D).
|
||||
text (paddle.Tensor): input text (1, Tmax2).
|
||||
masked_pos (paddle.Tensor): masked position of input speech (1, Tmax)
|
||||
speech_mask (paddle.Tensor): mask of speech (1, 1, Tmax).
|
||||
text_mask (paddle.Tensor): mask of text (1, 1, Tmax2).
|
||||
speech_seg_pos (paddle.Tensor): n-th phone of each mel, 0<=n<=Tmax2 (1, Tmax).
|
||||
text_seg_pos (paddle.Tensor): n-th phone of each phone, 0<=n<=Tmax2 (1, Tmax2).
|
||||
span_bdy (List[int]): masked mel boundary of input speech (2,)
|
||||
use_teacher_forcing (bool): whether to use teacher forcing
|
||||
Returns:
|
||||
List[Tensor]:
|
||||
eg:
|
||||
[Tensor(shape=[1, 181, 80]), Tensor(shape=[80, 80]), Tensor(shape=[1, 67, 80])]
|
||||
'''
|
||||
|
||||
z_cache = None
|
||||
if use_teacher_forcing:
|
||||
before_outs, zs, *_ = self.forward(
|
||||
speech=speech,
|
||||
text=text,
|
||||
masked_pos=masked_pos,
|
||||
speech_mask=speech_mask,
|
||||
text_mask=text_mask,
|
||||
speech_seg_pos=speech_seg_pos,
|
||||
text_seg_pos=text_seg_pos)
|
||||
if zs is None:
|
||||
zs = before_outs
|
||||
|
||||
speech = speech.squeeze(0)
|
||||
outs = [speech[:span_bdy[0]]]
|
||||
outs += [zs[0][span_bdy[0]:span_bdy[1]]]
|
||||
outs += [speech[span_bdy[1]:]]
|
||||
return outs
|
||||
return None
|
||||
|
||||
|
||||
class MLMEncAsDecoder(MLM):
|
||||
def forward(self,
|
||||
speech: paddle.Tensor,
|
||||
text: paddle.Tensor,
|
||||
masked_pos: paddle.Tensor,
|
||||
speech_mask: paddle.Tensor,
|
||||
text_mask: paddle.Tensor,
|
||||
speech_seg_pos: paddle.Tensor,
|
||||
text_seg_pos: paddle.Tensor):
|
||||
# feats: (Batch, Length, Dim)
|
||||
# -> encoder_out: (Batch, Length2, Dim2)
|
||||
encoder_out, h_masks = self.encoder(
|
||||
speech=speech,
|
||||
text=text,
|
||||
masked_pos=masked_pos,
|
||||
speech_mask=speech_mask,
|
||||
text_mask=text_mask,
|
||||
speech_seg_pos=speech_seg_pos,
|
||||
text_seg_pos=text_seg_pos)
|
||||
if self.decoder is not None:
|
||||
zs, _ = self.decoder(encoder_out, h_masks)
|
||||
else:
|
||||
zs = encoder_out
|
||||
speech_hidden_states = zs[:, :paddle.shape(speech)[1], :]
|
||||
if self.sfc is not None:
|
||||
before_outs = paddle.reshape(
|
||||
self.sfc(speech_hidden_states),
|
||||
(paddle.shape(speech_hidden_states)[0], -1, self.odim))
|
||||
else:
|
||||
before_outs = speech_hidden_states
|
||||
if self.postnet is not None:
|
||||
after_outs = before_outs + paddle.transpose(
|
||||
self.postnet(paddle.transpose(before_outs, [0, 2, 1])),
|
||||
[0, 2, 1])
|
||||
else:
|
||||
after_outs = None
|
||||
return before_outs, after_outs, None
|
||||
|
||||
|
||||
class MLMDualMaksing(MLM):
|
||||
def forward(self,
|
||||
speech: paddle.Tensor,
|
||||
text: paddle.Tensor,
|
||||
masked_pos: paddle.Tensor,
|
||||
speech_mask: paddle.Tensor,
|
||||
text_mask: paddle.Tensor,
|
||||
speech_seg_pos: paddle.Tensor,
|
||||
text_seg_pos: paddle.Tensor):
|
||||
# feats: (Batch, Length, Dim)
|
||||
# -> encoder_out: (Batch, Length2, Dim2)
|
||||
encoder_out, h_masks = self.encoder(
|
||||
speech=speech,
|
||||
text=text,
|
||||
masked_pos=masked_pos,
|
||||
speech_mask=speech_mask,
|
||||
text_mask=text_mask,
|
||||
speech_seg_pos=speech_seg_pos,
|
||||
text_seg_pos=text_seg_pos)
|
||||
if self.decoder is not None:
|
||||
zs, _ = self.decoder(encoder_out, h_masks)
|
||||
else:
|
||||
zs = encoder_out
|
||||
speech_hidden_states = zs[:, :paddle.shape(speech)[1], :]
|
||||
if self.text_sfc:
|
||||
text_hiddent_states = zs[:, paddle.shape(speech)[1]:, :]
|
||||
text_outs = paddle.reshape(
|
||||
self.text_sfc(text_hiddent_states),
|
||||
(paddle.shape(text_hiddent_states)[0], -1, self.vocab_size))
|
||||
if self.sfc is not None:
|
||||
before_outs = paddle.reshape(
|
||||
self.sfc(speech_hidden_states),
|
||||
(paddle.shape(speech_hidden_states)[0], -1, self.odim))
|
||||
else:
|
||||
before_outs = speech_hidden_states
|
||||
if self.postnet is not None:
|
||||
after_outs = before_outs + paddle.transpose(
|
||||
self.postnet(paddle.transpose(before_outs, [0, 2, 1])),
|
||||
[0, 2, 1])
|
||||
else:
|
||||
after_outs = None
|
||||
return before_outs, after_outs, text_outs
|
||||
|
||||
|
||||
class ErnieSAT(nn.Layer):
|
||||
def __init__(
|
||||
self,
|
||||
# network structure related
|
||||
idim: int,
|
||||
odim: int,
|
||||
postnet_layers: int=5,
|
||||
postnet_filts: int=5,
|
||||
postnet_chans: int=256,
|
||||
use_scaled_pos_enc: bool=False,
|
||||
encoder_type: str='conformer',
|
||||
decoder_type: str='conformer',
|
||||
enc_input_layer: str='sega_mlm',
|
||||
enc_pre_speech_layer: int=0,
|
||||
enc_cnn_module_kernel: int=7,
|
||||
enc_attention_dim: int=384,
|
||||
enc_attention_heads: int=2,
|
||||
enc_linear_units: int=1536,
|
||||
enc_num_blocks: int=4,
|
||||
enc_dropout_rate: float=0.2,
|
||||
enc_positional_dropout_rate: float=0.2,
|
||||
enc_attention_dropout_rate: float=0.2,
|
||||
enc_normalize_before: bool=True,
|
||||
enc_macaron_style: bool=True,
|
||||
enc_use_cnn_module: bool=True,
|
||||
enc_selfattention_layer_type: str='legacy_rel_selfattn',
|
||||
enc_activation_type: str='swish',
|
||||
enc_pos_enc_layer_type: str='legacy_rel_pos',
|
||||
enc_positionwise_layer_type: str='conv1d',
|
||||
enc_positionwise_conv_kernel_size: int=3,
|
||||
text_masking: bool=False,
|
||||
dec_cnn_module_kernel: int=31,
|
||||
dec_attention_dim: int=384,
|
||||
dec_attention_heads: int=2,
|
||||
dec_linear_units: int=1536,
|
||||
dec_num_blocks: int=4,
|
||||
dec_dropout_rate: float=0.2,
|
||||
dec_positional_dropout_rate: float=0.2,
|
||||
dec_attention_dropout_rate: float=0.2,
|
||||
dec_macaron_style: bool=True,
|
||||
dec_use_cnn_module: bool=True,
|
||||
dec_selfattention_layer_type: str='legacy_rel_selfattn',
|
||||
dec_activation_type: str='swish',
|
||||
dec_pos_enc_layer_type: str='legacy_rel_pos',
|
||||
dec_positionwise_layer_type: str='conv1d',
|
||||
dec_positionwise_conv_kernel_size: int=3,
|
||||
init_type: str="xavier_uniform", ):
|
||||
super().__init__()
|
||||
# store hyperparameters
|
||||
self.odim = odim
|
||||
|
||||
self.use_scaled_pos_enc = use_scaled_pos_enc
|
||||
|
||||
# initialize parameters
|
||||
initialize(self, init_type)
|
||||
|
||||
# Encoder
|
||||
if encoder_type == "conformer":
|
||||
encoder = MLMEncoder(
|
||||
idim=odim,
|
||||
vocab_size=idim,
|
||||
pre_speech_layer=enc_pre_speech_layer,
|
||||
attention_dim=enc_attention_dim,
|
||||
attention_heads=enc_attention_heads,
|
||||
linear_units=enc_linear_units,
|
||||
num_blocks=enc_num_blocks,
|
||||
dropout_rate=enc_dropout_rate,
|
||||
positional_dropout_rate=enc_positional_dropout_rate,
|
||||
attention_dropout_rate=enc_attention_dropout_rate,
|
||||
input_layer=enc_input_layer,
|
||||
normalize_before=enc_normalize_before,
|
||||
positionwise_layer_type=enc_positionwise_layer_type,
|
||||
positionwise_conv_kernel_size=enc_positionwise_conv_kernel_size,
|
||||
macaron_style=enc_macaron_style,
|
||||
pos_enc_layer_type=enc_pos_enc_layer_type,
|
||||
selfattention_layer_type=enc_selfattention_layer_type,
|
||||
activation_type=enc_activation_type,
|
||||
use_cnn_module=enc_use_cnn_module,
|
||||
cnn_module_kernel=enc_cnn_module_kernel,
|
||||
text_masking=text_masking)
|
||||
else:
|
||||
raise ValueError(f"{encoder_type} is not supported.")
|
||||
|
||||
# Decoder
|
||||
if decoder_type != 'no_decoder':
|
||||
decoder = MLMDecoder(
|
||||
idim=0,
|
||||
input_layer=None,
|
||||
cnn_module_kernel=dec_cnn_module_kernel,
|
||||
attention_dim=dec_attention_dim,
|
||||
attention_heads=dec_attention_heads,
|
||||
linear_units=dec_linear_units,
|
||||
num_blocks=dec_num_blocks,
|
||||
dropout_rate=dec_dropout_rate,
|
||||
positional_dropout_rate=dec_positional_dropout_rate,
|
||||
macaron_style=dec_macaron_style,
|
||||
use_cnn_module=dec_use_cnn_module,
|
||||
selfattention_layer_type=dec_selfattention_layer_type,
|
||||
activation_type=dec_activation_type,
|
||||
pos_enc_layer_type=dec_pos_enc_layer_type,
|
||||
positionwise_layer_type=dec_positionwise_layer_type,
|
||||
positionwise_conv_kernel_size=dec_positionwise_conv_kernel_size)
|
||||
|
||||
else:
|
||||
decoder = None
|
||||
|
||||
model_class = MLMDualMaksing if text_masking else MLMEncAsDecoder
|
||||
|
||||
self.model = model_class(
|
||||
odim=odim,
|
||||
encoder=encoder,
|
||||
decoder=decoder,
|
||||
postnet_layers=postnet_layers,
|
||||
postnet_filts=postnet_filts,
|
||||
postnet_chans=postnet_chans,
|
||||
text_masking=text_masking)
|
||||
|
||||
nn.initializer.set_global_initializer(None)
|
||||
|
||||
def forward(self,
|
||||
speech: paddle.Tensor,
|
||||
text: paddle.Tensor,
|
||||
masked_pos: paddle.Tensor,
|
||||
speech_mask: paddle.Tensor,
|
||||
text_mask: paddle.Tensor,
|
||||
speech_seg_pos: paddle.Tensor,
|
||||
text_seg_pos: paddle.Tensor):
|
||||
return self.model(
|
||||
speech=speech,
|
||||
text=text,
|
||||
masked_pos=masked_pos,
|
||||
speech_mask=speech_mask,
|
||||
text_mask=text_mask,
|
||||
speech_seg_pos=speech_seg_pos,
|
||||
text_seg_pos=text_seg_pos)
|
||||
|
||||
def inference(
|
||||
self,
|
||||
speech: paddle.Tensor,
|
||||
text: paddle.Tensor,
|
||||
masked_pos: paddle.Tensor,
|
||||
speech_mask: paddle.Tensor,
|
||||
text_mask: paddle.Tensor,
|
||||
speech_seg_pos: paddle.Tensor,
|
||||
text_seg_pos: paddle.Tensor,
|
||||
span_bdy: List[int],
|
||||
use_teacher_forcing: bool=False, ) -> Dict[str, paddle.Tensor]:
|
||||
return self.model.inference(
|
||||
speech=speech,
|
||||
text=text,
|
||||
masked_pos=masked_pos,
|
||||
speech_mask=speech_mask,
|
||||
text_mask=text_mask,
|
||||
speech_seg_pos=speech_seg_pos,
|
||||
text_seg_pos=text_seg_pos,
|
||||
span_bdy=span_bdy,
|
||||
use_teacher_forcing=use_teacher_forcing)
|
||||
@ -0,0 +1,148 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from paddle import distributed as dist
|
||||
from paddle.io import DataLoader
|
||||
from paddle.nn import Layer
|
||||
from paddle.optimizer import Optimizer
|
||||
|
||||
from paddlespeech.t2s.modules.losses import MLMLoss
|
||||
from paddlespeech.t2s.training.extensions.evaluator import StandardEvaluator
|
||||
from paddlespeech.t2s.training.reporter import report
|
||||
from paddlespeech.t2s.training.updaters.standard_updater import StandardUpdater
|
||||
logging.basicConfig(
|
||||
format='%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s',
|
||||
datefmt='[%Y-%m-%d %H:%M:%S]')
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
|
||||
class ErnieSATUpdater(StandardUpdater):
|
||||
def __init__(self,
|
||||
model: Layer,
|
||||
optimizer: Optimizer,
|
||||
dataloader: DataLoader,
|
||||
init_state=None,
|
||||
text_masking: bool=False,
|
||||
odim: int=80,
|
||||
output_dir: Path=None):
|
||||
super().__init__(model, optimizer, dataloader, init_state=None)
|
||||
|
||||
self.criterion = MLMLoss(text_masking=text_masking, odim=odim)
|
||||
|
||||
log_file = output_dir / 'worker_{}.log'.format(dist.get_rank())
|
||||
self.filehandler = logging.FileHandler(str(log_file))
|
||||
logger.addHandler(self.filehandler)
|
||||
self.logger = logger
|
||||
self.msg = ""
|
||||
|
||||
def update_core(self, batch):
|
||||
self.msg = "Rank: {}, ".format(dist.get_rank())
|
||||
losses_dict = {}
|
||||
|
||||
before_outs, after_outs, text_outs = self.model(
|
||||
speech=batch["speech"],
|
||||
text=batch["text"],
|
||||
masked_pos=batch["masked_pos"],
|
||||
speech_mask=batch["speech_mask"],
|
||||
text_mask=batch["text_mask"],
|
||||
speech_seg_pos=batch["speech_seg_pos"],
|
||||
text_seg_pos=batch["text_seg_pos"])
|
||||
|
||||
mlm_loss, text_mlm_loss = self.criterion(
|
||||
speech=batch["speech"],
|
||||
before_outs=before_outs,
|
||||
after_outs=after_outs,
|
||||
masked_pos=batch["masked_pos"],
|
||||
text=batch["text"],
|
||||
# maybe None
|
||||
text_outs=text_outs,
|
||||
# maybe None
|
||||
text_masked_pos=batch["text_masked_pos"])
|
||||
|
||||
loss = mlm_loss + text_mlm_loss if text_mlm_loss is not None else mlm_loss
|
||||
|
||||
optimizer = self.optimizer
|
||||
optimizer.clear_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
report("train/loss", float(loss))
|
||||
report("train/mlm_loss", float(mlm_loss))
|
||||
if text_mlm_loss is not None:
|
||||
report("train/text_mlm_loss", float(text_mlm_loss))
|
||||
losses_dict["text_mlm_loss"] = float(text_mlm_loss)
|
||||
|
||||
losses_dict["mlm_loss"] = float(mlm_loss)
|
||||
losses_dict["loss"] = float(loss)
|
||||
self.msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in losses_dict.items())
|
||||
|
||||
|
||||
class ErnieSATEvaluator(StandardEvaluator):
|
||||
def __init__(self,
|
||||
model: Layer,
|
||||
dataloader: DataLoader,
|
||||
text_masking: bool=False,
|
||||
odim: int=80,
|
||||
output_dir: Path=None):
|
||||
super().__init__(model, dataloader)
|
||||
|
||||
log_file = output_dir / 'worker_{}.log'.format(dist.get_rank())
|
||||
self.filehandler = logging.FileHandler(str(log_file))
|
||||
logger.addHandler(self.filehandler)
|
||||
self.logger = logger
|
||||
self.msg = ""
|
||||
|
||||
self.criterion = MLMLoss(text_masking=text_masking, odim=odim)
|
||||
|
||||
def evaluate_core(self, batch):
|
||||
self.msg = "Evaluate: "
|
||||
losses_dict = {}
|
||||
|
||||
before_outs, after_outs, text_outs = self.model(
|
||||
speech=batch["speech"],
|
||||
text=batch["text"],
|
||||
masked_pos=batch["masked_pos"],
|
||||
speech_mask=batch["speech_mask"],
|
||||
text_mask=batch["text_mask"],
|
||||
speech_seg_pos=batch["speech_seg_pos"],
|
||||
text_seg_pos=batch["text_seg_pos"])
|
||||
|
||||
mlm_loss, text_mlm_loss = self.criterion(
|
||||
speech=batch["speech"],
|
||||
before_outs=before_outs,
|
||||
after_outs=after_outs,
|
||||
masked_pos=batch["masked_pos"],
|
||||
text=batch["text"],
|
||||
# maybe None
|
||||
text_outs=text_outs,
|
||||
# maybe None
|
||||
text_masked_pos=batch["text_masked_pos"])
|
||||
loss = mlm_loss + text_mlm_loss if text_mlm_loss is not None else mlm_loss
|
||||
|
||||
report("eval/loss", float(loss))
|
||||
report("eval/mlm_loss", float(mlm_loss))
|
||||
if text_mlm_loss is not None:
|
||||
report("eval/text_mlm_loss", float(text_mlm_loss))
|
||||
losses_dict["text_mlm_loss"] = float(text_mlm_loss)
|
||||
|
||||
losses_dict["mlm_loss"] = float(mlm_loss)
|
||||
losses_dict["loss"] = float(loss)
|
||||
|
||||
self.msg += ', '.join('{}: {:>.6f}'.format(k, v)
|
||||
for k, v in losses_dict.items())
|
||||
self.logger.info(self.msg)
|
||||
Loading…
Reference in new issue