PaddleSpeech/paddlespeech/t2s/exps/starganv2_vc/vc.py

# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import os
import time
from pathlib import Path

import librosa
import paddle
import soundfile as sf
import yaml
from yacs.config import CfgNode

from paddlespeech.cli.utils import download_and_decompress
from paddlespeech.resource.pretrained_models import StarGANv2VC_source
from paddlespeech.t2s.datasets.get_feats import LogMelFBank
from paddlespeech.t2s.models.parallel_wavegan import PWGGenerator
from paddlespeech.t2s.models.starganv2_vc import Generator
from paddlespeech.t2s.models.starganv2_vc import JDCNet
from paddlespeech.t2s.models.starganv2_vc import MappingNetwork
from paddlespeech.t2s.models.starganv2_vc import StyleEncoder
from paddlespeech.utils.env import MODEL_HOME


def get_mel_extractor():
    sr = 16000
    n_fft = 2048
    win_length = 1200
    hop_length = 300
    n_mels = 80
    fmin = 0
    fmax = sr // 2

    mel_extractor = LogMelFBank(
        sr=sr,
        n_fft=n_fft,
        hop_length=hop_length,
        win_length=win_length,
        n_mels=n_mels,
        fmin=fmin,
        fmax=fmax,
        norm=None,
        htk=True,
        power=2.0)
    return mel_extractor


def preprocess(wave, mel_extractor):
    logmel = mel_extractor.get_log_mel_fbank(wave, base='e')
    # [1, 80, 1011]
    mean, std = -4, 4
    mel_tensor = (paddle.to_tensor(logmel.T).unsqueeze(0) - mean) / std
    return mel_tensor


def compute_style(speaker_dicts, mel_extractor, style_encoder, mapping_network):
    reference_embeddings = {}
    for key, (path, speaker) in speaker_dicts.items():
        if path == '':
            label = paddle.to_tensor([speaker], dtype=paddle.int64)
            latent_dim = mapping_network.shared[0].weight.shape[0]
            ref = mapping_network(paddle.randn([1, latent_dim]), label)
        else:
            wave, sr = librosa.load(path, sr=24000)
            audio, index = librosa.effects.trim(wave, top_db=30)
            if sr != 24000:
                wave = librosa.resample(wave, sr, 24000)
            mel_tensor = preprocess(wave=wave, mel_extractor=mel_extractor)
            with paddle.no_grad():
                label = paddle.to_tensor([speaker], dtype=paddle.int64)
                ref = style_encoder(mel_tensor.unsqueeze(1), label)
        reference_embeddings[key] = (ref, label)

    return reference_embeddings


def get_models(args, uncompress_path):
    model_dict = {}
    jdc_model_dir = os.path.join(uncompress_path, 'jdcnet.pdz')
    voc_model_dir = os.path.join(uncompress_path, 'Vocoder/')
    starganv2vc_model_dir = os.path.join(uncompress_path, 'starganv2vc.pdz')

    F0_model = JDCNet(num_class=1, seq_len=192)
    F0_model.set_state_dict(paddle.load(jdc_model_dir)['main_params'])
    F0_model.eval()

    voc_config_path = os.path.join(voc_model_dir, 'config.yml')
    with open(voc_config_path) as f:
        voc_config = CfgNode(yaml.safe_load(f))
    voc_config["generator_params"].pop("upsample_net")
    voc_config["generator_params"]["upsample_scales"] = voc_config[
        "generator_params"].pop("upsample_params")["upsample_scales"]
    vocoder = PWGGenerator(**voc_config["generator_params"])
    vocoder.remove_weight_norm()
    vocoder.eval()
    voc_model_path = os.path.join(voc_model_dir, 'checkpoint-400000steps.pd')
    vocoder.set_state_dict(paddle.load(voc_model_path))

    with open(args.config_path) as f:
        config = CfgNode(yaml.safe_load(f))

    generator = Generator(**config['generator_params'])
    mapping_network = MappingNetwork(**config['mapping_network_params'])
    style_encoder = StyleEncoder(**config['style_encoder_params'])

    starganv2vc_model_param = paddle.load(starganv2vc_model_dir)

    generator.set_state_dict(starganv2vc_model_param['generator_params'])
    mapping_network.set_state_dict(
        starganv2vc_model_param['mapping_network_params'])
    style_encoder.set_state_dict(
        starganv2vc_model_param['style_encoder_params'])

    generator.eval()
    mapping_network.eval()
    style_encoder.eval()

    model_dict['F0_model'] = F0_model
    model_dict['vocoder'] = vocoder
    model_dict['generator'] = generator
    model_dict['mapping_network'] = mapping_network
    model_dict['style_encoder'] = style_encoder
    return model_dict


def voice_conversion(args, uncompress_path):
    speakers = [
        225, 228, 229, 230, 231, 233, 236, 239, 240, 244, 226, 227, 232, 243,
        254, 256, 258, 259, 270, 273
    ]
    demo_dir = os.path.join(uncompress_path, 'Demo/VCTK-corpus/')
    model_dict = get_models(args, uncompress_path=uncompress_path)
    style_encoder = model_dict['style_encoder']
    mapping_network = model_dict['mapping_network']
    generator = model_dict['generator']
    vocoder = model_dict['vocoder']
    F0_model = model_dict['F0_model']

    # 计算 Demo 文件夹下的说话人的风格
    speaker_dicts = {}
    selected_speakers = [273, 259, 258, 243, 254, 244, 236, 233, 230, 228]
    for s in selected_speakers:
        k = s
        speaker_dicts['p' + str(s)] = (
            demo_dir + 'p' + str(k) + '/p' + str(k) + '_023.wav',
            speakers.index(s))
    mel_extractor = get_mel_extractor()
    reference_embeddings = compute_style(
        speaker_dicts=speaker_dicts,
        mel_extractor=mel_extractor,
        style_encoder=style_encoder,
        mapping_network=mapping_network)

    wave, sr = librosa.load(args.source_path, sr=24000)
    source = preprocess(wave=wave, mel_extractor=mel_extractor)
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)
    orig_wav_name = str(output_dir / 'orig_voc.wav')
    print('原始语音 (使用声码器解码): %s' % orig_wav_name)
    c = source.transpose([0, 2, 1]).squeeze()
    with paddle.no_grad():
        recon = vocoder.inference(c)
        recon = recon.reshape([-1]).numpy()
    sf.write(orig_wav_name, recon, samplerate=24000)

    keys = []
    converted_samples = {}
    reconstructed_samples = {}
    converted_mels = {}
    start = time.time()

    for key, (ref, _) in reference_embeddings.items():
        with paddle.no_grad():
            # F0_model 输入的特征是否可以不带 norm，或者 norm 是否一定要和 stargan 原作保持一致？
            # !! 需要，ASR 和 F0_model 用的是一样的数据预处理方式
            # 如果不想要重新训练 ASR 和 F0_model, 则我们的数据预处理需要和 stargan 原作保持一致
            # 但是 vocoder 就无法复用
            # 是否因为 asr 的输入是 16k 的，所以 torchaudio 的参数也是 16k 的？
            f0_feat = F0_model.get_feature_GAN(source.unsqueeze(1))
            # 输出是带 norm 的 mel, 所以可以直接用 vocoder.inference
            out = generator(source.unsqueeze(1), ref, F0=f0_feat)
            c = out.transpose([0, 1, 3, 2]).squeeze()
            y_out = vocoder.inference(c)
            y_out = y_out.reshape([-1])
            if key not in speaker_dicts or speaker_dicts[key][0] == "":
                recon = None
            else:
                wave, sr = librosa.load(speaker_dicts[key][0], sr=24000)
                mel = preprocess(wave=wave, mel_extractor=mel_extractor)
                c = mel.transpose([0, 2, 1]).squeeze()
                recon = vocoder.inference(c)
                recon = recon.reshape([-1]).numpy()

        converted_samples[key] = y_out.numpy()
        reconstructed_samples[key] = recon
        converted_mels[key] = out
        keys.append(key)
    end = time.time()
    print('总共花费时间: %.3f sec' % (end - start))
    for key, wave in converted_samples.items():
        wav_name = str(output_dir / ('vc_result_' + key + '.wav'))
        print('语音转换结果: %s' % wav_name)
        sf.write(wav_name, wave, samplerate=24000)
        ref_wav_name = str(output_dir / ('ref_voc_' + key + '.wav'))
        print('参考的说话人 (使用声码器解码): %s' % ref_wav_name)
        if reconstructed_samples[key] is not None:
            sf.write(ref_wav_name, reconstructed_samples[key], samplerate=24000)


def parse_args():
    # parse args and config  
    parser = argparse.ArgumentParser(
        description="StarGANv2-VC Voice Conversion.")
    parser.add_argument("--source_path", type=str, help="source audio's path.")
    parser.add_argument("--output_dir", type=str, help="output dir.")
    parser.add_argument(
        '--config_path',
        type=str,
        default=None,
        help='Config of StarGANv2-VC model.')
    parser.add_argument(
        "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.")
    args = parser.parse_args()
    return args


def main():
    args = parse_args()
    if args.ngpu == 0:
        paddle.set_device("cpu")
    elif args.ngpu > 0:
        paddle.set_device("gpu")
    else:
        print("ngpu should >= 0 !")
    model_version = '1.0'
    uncompress_path = download_and_decompress(StarGANv2VC_source[model_version],
                                              MODEL_HOME)
    voice_conversion(args, uncompress_path=uncompress_path)


if __name__ == "__main__":
    main()