From 03f6d9ed6ead1c5a8d08544d0ef00e7be714cdf6 Mon Sep 17 00:00:00 2001 From: supotato6 Date: Fri, 23 Jan 2026 14:45:43 +0800 Subject: [PATCH] fix flow matching diff --- paddlespeech/audio/functional/mel_extract.py | 39 +- paddlespeech/cli/tts/cosyvoice.py | 130 +++ .../frontend/CosyVoiceFrontEnd/__init__.py | 0 .../frontend/CosyVoiceFrontEnd/file_utils.py | 110 +++ .../frontend/CosyVoiceFrontEnd/frontend.py | 464 +++++++++++ .../CosyVoiceFrontEnd/frontend_utils.py | 121 +++ .../t2s/frontend/CosyVoiceFrontEnd/func.py | 747 ++++++++++++++++++ .../frontend/CosyVoiceFrontEnd/tokenizer.py | 578 ++++++++++++++ paddlespeech/t2s/models/CosyVoice/llm.py | 1 + .../t2s/models/hifigan/cosy_hifigan.py | 286 +++++-- .../t2s/models/hifigan/f0_predictor.py | 26 +- paddlespeech/t2s/modules/flow/decoder.py | 6 +- paddlespeech/t2s/modules/flow/flow.py | 6 +- .../t2s/modules/flow/flow_matching.py | 3 +- .../t2s/modules/flow/matcha_transformer.py | 15 +- .../t2s/modules/transformer/subsampling.py | 1 - .../modules/transformer/upsample_encoder.py | 11 +- 17 files changed, 2433 insertions(+), 111 deletions(-) create mode 100644 paddlespeech/cli/tts/cosyvoice.py create mode 100644 paddlespeech/t2s/frontend/CosyVoiceFrontEnd/__init__.py create mode 100644 paddlespeech/t2s/frontend/CosyVoiceFrontEnd/file_utils.py create mode 100644 paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend.py create mode 100644 paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend_utils.py create mode 100644 paddlespeech/t2s/frontend/CosyVoiceFrontEnd/func.py create mode 100644 paddlespeech/t2s/frontend/CosyVoiceFrontEnd/tokenizer.py diff --git a/paddlespeech/audio/functional/mel_extract.py b/paddlespeech/audio/functional/mel_extract.py index 11d881e05..8b8830aff 100644 --- a/paddlespeech/audio/functional/mel_extract.py +++ b/paddlespeech/audio/functional/mel_extract.py @@ -20,7 +20,7 @@ def dynamic_range_decompression(x, C=1): def dynamic_range_compression_torch(x, C=1, clip_val=1e-05): - return paddle.log(paddle.clamp(x, min=clip_val) * C) + return paddle.log(paddle.clip(x, min=clip_val) * C) def dynamic_range_decompression_torch(x, C=1): @@ -44,39 +44,40 @@ hann_window = {} def mel_spectrogram( y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False ): - if paddle.compat.min(y) < -1.0: - print("min value is ", paddle.compat.min(y)) - if paddle.compat.max(y) > 1.0: - print("max value is ", paddle.compat.max(y)) + y = paddle.to_tensor(y.detach().cpu().numpy()) + if paddle.min(paddle.to_tensor(y)) < -1.0: + print("min value is ", paddle.min(paddle.to_tensor(y))) + if paddle.max(paddle.to_tensor(y)) > 1.0: + print("max value is ", paddle.max(paddle.to_tensor(y))) global mel_basis, hann_window if f"{str(fmax)}_{str(y.place)}" not in mel_basis: mel = librosa_mel_fn( sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax ) mel_basis[str(fmax) + "_" + str(y.place)] = ( - paddle.from_numpy(mel).float().to(y.place) + paddle.to_tensor(mel).float().to(y.place) ) hann_window[str(y.place)] = paddle.audio.functional.get_window( win_length=win_size, dtype="float32", window="hann" ).to(y.place) - y = paddle.compat.pad( + + y = paddle.nn.functional.pad( y.unsqueeze(1), (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), mode="reflect", ) y = y.squeeze(1) - spec = paddle.view_as_real( - paddle.signal.stft( - x=y, - n_fft=n_fft, - hop_length=hop_size, - win_length=win_size, - window=hann_window[str(y.place)], - center=center, - pad_mode="reflect", - normalized=False, - onesided=True, - ) + window = paddle.load("/root/paddlejob/workspace/zhangjinghong/test/PaddleSpeech/matcha_window.pdparams").cuda() + + stft = paddle.signal.stft( + y.cuda(), + n_fft=1920, + hop_length=480, + window=window + ) + + spec = paddle.as_real( + stft ) spec = paddle.sqrt(spec.pow(2).sum(-1) + 1e-09) spec = paddle.matmul(mel_basis[str(fmax) + "_" + str(y.place)], spec) diff --git a/paddlespeech/cli/tts/cosyvoice.py b/paddlespeech/cli/tts/cosyvoice.py new file mode 100644 index 000000000..13e7768bf --- /dev/null +++ b/paddlespeech/cli/tts/cosyvoice.py @@ -0,0 +1,130 @@ +from paddlespeech.t2s.models.CosyVoice.cosyvoice import CosyVoice2 +import sys +from paddlenlp.transformers import AutoTokenizer, AutoModelForCausalLM +from pathlib import Path +import paddle +import torch +paddle.seed(42) +from paddlespeech.t2s.frontend.CosyVoiceFrontEnd.frontend import CosyVoiceFrontEnd +from paddlespeech.t2s.models.CosyVoice.llm import Qwen2LM,Qwen2Encoder +from paddlespeech.t2s.models.CosyVoice.common import ras_sampling +from paddlespeech.t2s.frontend.CosyVoiceFrontEnd.tokenizer import get_qwen_tokenizer +from hyperpyyaml import load_hyperpyyaml +hyper_yaml_path = "/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B_paddle/cosyvoice2.yaml" +with open(hyper_yaml_path, 'r') as f: + configs = load_hyperpyyaml(f) + +# frontend = CosyVoiceFrontEnd( +# lambda:get_qwen_tokenizer('/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B/CosyVoice-BlankEN',skip_special_tokens=True), +# configs['feat_extractor'], +# "/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B/campplus.onnx", +# "/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B/speech_tokenizer_v2.onnx", +# "/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B/spk2info.pt", +# configs['allowed_special'] +# ) +# prompt_wav = "../CosyVoice/gaoziyuan_10.wav" +# tts_text = frontend.text_normalize("定位解决了云存储服务和 data loader 等多个环节的性能波动问题", split=True, text_frontend=True) +# prompt_text = frontend.text_normalize('清晨的阳光透过树叶洒在地面上，微风轻轻吹过，带来花草的香气。街边的咖啡店刚开门，传来阵阵烘焙的香味，让人感到放松与愉快。', split=True, text_frontend=True) +# model_input = frontend.frontend_zero_shot(tts_text, prompt_text, prompt_wav, 24000,'') +# paddle.save(model_input,'model_input.pdparams') +# # cosyvoice_model = CosyVoice2("../CosyVoice/pretrained_models/CosyVoice2-0.5B_paddle") +# model = AutoModelForCausalLM.from_pretrained('/root/paddlejob/workspace/zhangjinghong/test/pretrained/Qwen/Qwen2-0.5B') +# print(type(model)) +# llm = Qwen2Encoder(model) +# qwen_lm = Qwen2LM(896,896,6561,llm,ras_sampling) +# state_dict = paddle.load("/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B_paddle/llm.pdparams") +# qwen_lm.set_state_dict(state_dict) + + +# new_dict = torch.load("/root/paddlejob/workspace/zhangjinghong/CosyVoice/data.pt") +# text = new_dict['text'] +# text_len = new_dict['text_len'] +# prompt_text = new_dict['prompt_text'] +# prompt_text_len = new_dict['prompt_text_len'] +# prompt_speech_token = new_dict['prompt_speech_token'] +# prompt_speech_token_len = new_dict['prompt_speech_token_len'] +# embedding = new_dict['embedding'] +# uuid = new_dict['uuid'] + + +# text = model_input['text'] +# text_len = model_input['text_len'] +# prompt_text = model_input['prompt_text'] +# prompt_text_len = model_input['prompt_text_len'] +# prompt_speech_token = model_input['llm_prompt_speech_token'] +# prompt_speech_token_len = model_input['llm_prompt_speech_token_len'] +# embedding = model_input['llm_embedding'] +# uuid = new_dict['uuid'] + +# # 统一设备并转换为Paddle张量 +# device = paddle.CUDAPlace(0) # 使用GPU设备 +# text_tensor = paddle.to_tensor(text).cuda() +# prompt_text_tensor = paddle.to_tensor(prompt_text).cuda() +# prompt_speech_token_tensor = paddle.to_tensor(prompt_speech_token).cuda() +# embedding_tensor = paddle.to_tensor(embedding, dtype='float32').cuda() +# # 确保长度张量也统一设备并正确转换 +# text_len_tensor = text_len.cuda() if hasattr(text_len, 'cuda') else paddle.to_tensor(text_len).cuda() +# prompt_text_len_tensor = prompt_text_len.cuda() if hasattr(prompt_text_len, 'cuda') else paddle.to_tensor(prompt_text_len).cuda() +# prompt_speech_token_len_tensor = prompt_speech_token_len.cuda() if hasattr(prompt_speech_token_len, 'cuda') else paddle.to_tensor(prompt_speech_token_len).cuda() +# token=[] +# for i in qwen_lm.inference(text=text_tensor, +# text_len=text_len_tensor, +# prompt_text=prompt_text_tensor, +# prompt_text_len=prompt_text_len_tensor, +# prompt_speech_token=prompt_speech_token_tensor, +# prompt_speech_token_len=prompt_speech_token_len_tensor, +# embedding=embedding_tensor, +# uuid=uuid): +# token.append(i) +# # print(text) +# print("token: ",i) + +############################################################################################################################ + + +flow = configs['flow'] +flow_state_dict = paddle.load("/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B_paddle/flow.pdparams") +flow.set_state_dict(flow_state_dict) +input_dict = torch.load("/root/paddlejob/workspace/zhangjinghong/test/CosyVoice/data.pt") +flow.eval() +tts_mel, _ = flow.inference( + token = paddle.to_tensor(input_dict['token']), + token_len = paddle.to_tensor(input_dict['token_len']), + prompt_token = paddle.to_tensor(input_dict['prompt_token'].cpu().numpy(), dtype = 'int32'), + prompt_token_len = paddle.to_tensor(input_dict['prompt_token_len'].cpu().numpy()), + prompt_feat = paddle.to_tensor(input_dict['prompt_feat'].cpu().numpy()), + prompt_feat_len = paddle.to_tensor(input_dict['prompt_feat_len'].cpu().numpy()), + embedding = paddle.to_tensor(input_dict['embedding'].cpu().numpy()), + streaming = input_dict['streaming'], + finalize = input_dict['finalize'] +) +paddle.save(tts_mel,"tts_mel.pdparams") + +############################################################################################################################ + +from paddlespeech.t2s.models.hifigan.cosy_hifigan import HiFTGenerator +from paddlespeech.t2s.models.hifigan.f0_predictor import ConvRNNF0Predictor +hift_state_dict = paddle.load("/root/paddlejob/workspace/zhangjinghong/CosyVoice/pretrained_models/CosyVoice2-0.5B_paddle/hift.pdparams") +input_mel = paddle.to_tensor(torch.load("../CosyVoice/tts_mel.pt").detach().cpu().numpy()).cuda() +hift_cache_source= paddle.to_tensor(torch.load("../CosyVoice/hift_cache_source.pt").detach().cpu().numpy()).cuda() +# hift_cache_source = paddle.zeros([1, 1, 0]) +hift_configs = configs['hift'] +f0_config = configs['f0_predictor'] +f0_predictor = ConvRNNF0Predictor(**f0_config) + +hift_configs['f0_predictor'] = f0_predictor +hift = HiFTGenerator(**hift_configs) +hift.set_state_dict(hift_state_dict) +# for k,v in hift.state_dict().items(): +# print(k,v.shape) +# print("---"*40) +for k,v in hift_state_dict.items(): + print(k,v.shape) +tts_speech, tts_source = hift.inference(speech_feat=input_mel, cache_source=hift_cache_source) +paddle.save(tts_speech,"speech.pdparams") +# tts_speech,_ = hift.inference(input_dict['tts_mel'],input_dict['cache_source']) + +import torchaudio +import torch +torchaudio.save("paddle.wav",torch.tensor(tts_speech.numpy()),24000) +# sf.write("paddle.wav",tts_speech[0],24000) \ No newline at end of file diff --git a/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/__init__.py b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/file_utils.py b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/file_utils.py new file mode 100644 index 000000000..5679f31a5 --- /dev/null +++ b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/file_utils.py @@ -0,0 +1,110 @@ +import json +import logging +import os + +import paddle +import torchaudio +import paddlespeech + +logging.getLogger("matplotlib").setLevel(logging.WARNING) +logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") + + +def read_lists(list_file): + lists = [] + with open(list_file, "r", encoding="utf8") as fin: + for line in fin: + lists.append(line.strip()) + return lists + + +def read_json_lists(list_file): + lists = read_lists(list_file) + results = {} + for fn in lists: + with open(fn, "r", encoding="utf8") as fin: + results.update(json.load(fin)) + return results + + +def load_wav(wav, target_sr, min_sr=16000): + speech, sample_rate = torchaudio.load(wav, backend="soundfile") + speech = speech.mean(dim=0, keepdim=True) + if sample_rate != target_sr: + assert ( + sample_rate >= min_sr + ), "wav sample rate {} must be greater than {}".format(sample_rate, target_sr) + speech = torchaudio.transforms.Resample( + orig_freq=sample_rate, new_freq=target_sr + )(speech) + return speech + + +def convert_onnx_to_trt(trt_model, trt_kwargs, onnx_model, fp16): + import tensorrt as trt + + logging.info("Converting onnx to trt...") + network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH) + logger = trt.Logger(trt.Logger.INFO) + builder = trt.Builder(logger) + network = builder.create_network(network_flags) + parser = trt.OnnxParser(network, logger) + config = builder.create_builder_config() + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 32) + if fp16: + config.set_flag(trt.BuilderFlag.FP16) + profile = builder.create_optimization_profile() + with open(onnx_model, "rb") as f: + if not parser.parse(f.read()): + for error in range(parser.num_errors): + print(parser.get_error(error)) + raise ValueError("failed to parse {}".format(onnx_model)) + for i in range(len(trt_kwargs["input_names"])): + profile.set_shape( + trt_kwargs["input_names"][i], + trt_kwargs["min_shape"][i], + trt_kwargs["opt_shape"][i], + trt_kwargs["max_shape"][i], + ) + tensor_dtype = trt.DataType.HALF if fp16 else trt.DataType.FLOAT + for i in range(network.num_inputs): + input_tensor = network.get_input(i) + input_tensor.dtype = tensor_dtype + for i in range(network.num_outputs): + output_tensor = network.get_output(i) + output_tensor.dtype = tensor_dtype + config.add_optimization_profile(profile) + engine_bytes = builder.build_serialized_network(network, config) + with open(trt_model, "wb") as f: + f.write(engine_bytes) + logging.info("Succesfully convert onnx to trt...") + + +def export_cosyvoice2_vllm(model, model_path, device): + if os.path.exists(model_path): + return + dtype = paddle.bfloat16 + use_bias = True if model.llm_decoder.bias is not None else False + model.llm.model.lm_head = model.llm_decoder + embed_tokens = model.llm.model.model.embed_tokens + model.llm.model.set_input_embeddings(model.speech_embedding) + model.llm.model.to(device) + model.llm.model.to(dtype) + tmp_vocab_size = model.llm.model.config.vocab_size + tmp_tie_embedding = model.llm.model.config.tie_word_embeddings + del model.llm.model.generation_config.eos_token_id + del model.llm.model.config.bos_token_id + del model.llm.model.config.eos_token_id + model.llm.model.config.vocab_size = model.speech_embedding.num_embeddings + model.llm.model.config.tie_word_embeddings = False + model.llm.model.config.use_bias = use_bias + model.llm.model.save_pretrained(model_path) + if use_bias is True: + os.system( + "sed -i s@Qwen2ForCausalLM@CosyVoice2ForCausalLM@g {}/config.json".format( + os.path.abspath(model_path) + ) + ) + model.llm.model.config.vocab_size = tmp_vocab_size + model.llm.model.config.tie_word_embeddings = tmp_tie_embedding + model.llm.model.set_input_embeddings(embed_tokens) diff --git a/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend.py b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend.py new file mode 100644 index 000000000..6780f81be --- /dev/null +++ b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend.py @@ -0,0 +1,464 @@ +import json +import os +import re +from functools import partial +from typing import Callable, Generator +import librosa +import inflect +import numpy as np +import onnxruntime +import paddle +import paddle +import paddle.nn.functional as F +import numpy as np +from typing import Union, Optional +import paddlespeech +from .func import fbank +def _stft(x, + n_fft, + n_shift, + win_length=None, + window="hann", + center=True, + pad_mode="reflect"): + # x: [Time, Channel] + window = window.cpu().numpy() + x = x.cpu().numpy() + if x.ndim == 1: + single_channel = True + # x: [Time] -> [Time, Channel] + x = x[:, None] + else: + single_channel = False + x = x.astype(np.float32) + + # FIXME(kamo): librosa.stft can't use multi-channel? + # x: [Time, Channel, Freq] + x = np.stack( + [ + librosa.stft( + y=x[:, ch], + n_fft=n_fft, + hop_length=n_shift, + win_length=win_length, + window=window, + center=center, + pad_mode=pad_mode, ).T for ch in range(x.shape[1]) + ], + axis=1, ) + + if single_channel: + # x: [Time, Channel, Freq] -> [Time, Freq] + x = x[:, 0] + return x +def log_mel_spectrogram( + audio: Union[str, np.ndarray, paddle.Tensor], + n_mels: int = 80, + padding: int = 0, + device: Optional[str] = None, +): + """ + Compute the log-Mel spectrogram of audio using PaddlePaddle + + Parameters + ---------- + audio: Union[str, np.ndarray, paddle.Tensor], shape = (*) + The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz + + n_mels: int + The number of Mel-frequency filters, only 80 is supported + + padding: int + Number of zero samples to pad to the right + + device: Optional[str] + If given, the audio tensor is moved to this device (e.g., 'gpu:0') before STFT + + Returns + ------- + paddle.Tensor, shape = (80, n_frames) + A Tensor that contains the Mel spectrogram + """ + N_FFT = 400 + HOP_LENGTH = 160 + SAMPLE_RATE = 16000 + + if not paddle.is_tensor(audio): + if isinstance(audio, str): + audio = load_audio(audio) + audio = paddle.to_tensor(audio.detach().cpu().numpy(), dtype='float32') + + if device is not None: + if 'gpu' in device: + place = paddle.CUDAPlace(int(device.split(':')[-1])) + else: + place = paddle.CPUPlace() + audio = audio.place(place) + if padding > 0: + audio = F.pad(audio.unsqueeze(0), [0, padding]).squeeze(0) + import torch + window = paddle.to_tensor(torch.hann_window(N_FFT).cpu().numpy()) + + # window = paddle.audio.functional.get_window( + # 'hann', + # N_FFT, + # dtype='float32' + # ).cuda() + # stft = _stft( + # audio, + # N_FFT, + # HOP_LENGTH, + # window=window + # ) + stft = paddle.signal.stft( + audio, + n_fft=N_FFT, + hop_length=HOP_LENGTH, + window=window + ) + # stft = paddle.to_tensor(stft) + magnitudes = stft[..., :-1].abs().square() + if magnitudes.shape[1] > N_FFT // 2: + magnitudes = magnitudes[:, :N_FFT // 2 + 1, :] + filters = mel_filters(audio.place, n_mels, N_FFT, SAMPLE_RATE) + mel_spec = paddle.matmul(filters, magnitudes) + log_spec = paddle.clip(mel_spec, min=1e-10).log10() + log_spec = paddle.maximum(log_spec, log_spec.max() - 8.0) + log_spec = (log_spec + 4.0) / 4.0 + return log_spec.squeeze(0) + +def mel_filters(device: str, n_mels: int = 80, n_fft: int = 400, sr: int = 16000): + + assert n_mels in {80, 128}, f"Unsupported n_mels: {n_mels}" + + filters_path = "/root/paddlejob/workspace/zhangjinghong/venv_cosy/lib/python3.10/site-packages/whisper/assets/mel_filters.npz" + with np.load(filters_path, allow_pickle=False) as f: + return paddle.to_tensor(f[f"mel_{n_mels}"]).to(device) + + + +try: + import ttsfrd + + use_ttsfrd = True +except ImportError: + print("failed to import ttsfrd, use wetext instead") + from wetext import Normalizer as EnNormalizer + from wetext import Normalizer as ZhNormalizer + + use_ttsfrd = False +from .file_utils import load_wav, logging +from .frontend_utils import (contains_chinese, + is_only_punctuation, + remove_bracket, replace_blank, + replace_corner_mark, + spell_out_number, split_paragraph) + + +class CosyVoiceFrontEnd: + def __init__( + self, + get_tokenizer: Callable, + feat_extractor: Callable, + campplus_model: str, + speech_tokenizer_model: str, + spk2info: str = "", + allowed_special: str = "all", + ): + self.tokenizer = get_tokenizer() + self.feat_extractor = feat_extractor + self.device = 'gpu:0' + option = onnxruntime.SessionOptions() + option.graph_optimization_level = ( + onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + ) + option.intra_op_num_threads = 1 + self.campplus_session = onnxruntime.InferenceSession( + campplus_model, sess_options=option, providers=["CPUExecutionProvider"] + ) + self.speech_tokenizer_session = onnxruntime.InferenceSession( + speech_tokenizer_model, + sess_options=option, + providers=[ + "CUDAExecutionProvider" + if paddle.device.is_compiled_with_cuda() + else "CPUExecutionProvider" + ], + ) + if os.path.exists(spk2info): + self.spk2info = paddle.load(path=str(spk2info)) + else: + self.spk2info = {} + self.allowed_special = allowed_special + self.use_ttsfrd = use_ttsfrd + if self.use_ttsfrd: + self.frd = ttsfrd.TtsFrontendEngine() + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + assert ( + self.frd.initialize( + "{}/../../pretrained_models/CosyVoice-ttsfrd/resource".format( + ROOT_DIR + ) + ) + is True + ), "failed to initialize ttsfrd resource" + self.frd.set_lang_type("pinyinvg") + else: + self.zh_tn_model = ZhNormalizer(remove_erhua=False) + self.en_tn_model = EnNormalizer() + self.inflect_parser = inflect.engine() + + def _extract_text_token(self, text): + if isinstance(text, Generator): + logging.info( + "get tts_text generator, will return _extract_text_token_generator!" + ) + return self._extract_text_token_generator(text), paddle.to_tensor( + [0], dtype=paddle.int32 + ).to(self.device) + else: + + text_token = self.tokenizer.encode( + text, allowed_special=self.allowed_special + ) + text_token = paddle.to_tensor([text_token], dtype=paddle.int32).to(self.device) + text_token_len = paddle.to_tensor( + [text_token.shape[1]], dtype=paddle.int32 + ).to(self.device) + return text_token, text_token_len + + def _extract_text_token_generator(self, text_generator): + for text in text_generator: + text_token, _ = self._extract_text_token(text) + for i in range(text_token.shape[1]): + yield text_token[:, i : i + 1] + + def _extract_speech_token(self, prompt_wav): + speech = load_wav(prompt_wav, 16000) + + assert ( + speech.shape[1] / 16000 <= 30 + ), "do not support extract speech token for audio longer than 30s" + feat =log_mel_spectrogram(speech, n_mels=128) + feat = feat.unsqueeze(0) + speech_token = ( + self.speech_tokenizer_session.run( + None, + { + self.speech_tokenizer_session.get_inputs()[0] + .name: feat.detach() + .cpu() + .numpy(), + self.speech_tokenizer_session.get_inputs()[1].name: np.array( + [feat.shape[2]], dtype=np.int32 + ), + }, + )[0] + .flatten() + .tolist() + ) + speech_token = paddle.to_tensor([speech_token], dtype=paddle.int32).to(self.device) + speech_token_len = paddle.to_tensor( + [speech_token.shape[1]], dtype=paddle.int32 + ).to(self.device) + return speech_token, speech_token_len + + def _extract_spk_embedding(self, prompt_wav): + speech = load_wav(prompt_wav, 16000) + speech = paddle.to_tensor(speech.detach().cpu().numpy()).cuda() + feat = fbank( + speech, num_mel_bins=80, dither=0, sample_frequency=16000 + ) + feat = feat - feat.mean(axis=0, keepdim=True) + embedding = ( + self.campplus_session.run( + None, + { + self.campplus_session.get_inputs()[0] + .name: feat.unsqueeze(axis=0) + .cpu() + .numpy() + }, + )[0] + .flatten() + .tolist() + ) + embedding = paddle.to_tensor([embedding]).to(self.device) + return embedding + + def _extract_speech_feat(self, prompt_wav): + speech = load_wav(prompt_wav, 24000) + speech_feat = ( + paddle.transpose(self.feat_extractor(speech).squeeze(axis=0),perm=[0, 1]).to(self.device) + ) + + speech_feat = speech_feat.unsqueeze(axis=0) + speech_feat_len = paddle.to_tensor([speech_feat.shape[1]], dtype=paddle.int32).to( + self.device + ) + return speech_feat, speech_feat_len + + def text_normalize(self, text, split=True, text_frontend=True): + if isinstance(text, Generator): + logging.info("get tts_text generator, will skip text_normalize!") + return [text] + if "<|" in text and "|>" in text: + text_frontend = False + if text_frontend is False or text == "": + return [text] if split is True else text + text = text.strip() + if self.use_ttsfrd: + texts = [ + i["text"] + for i in json.loads(self.frd.do_voicegen_frd(text))["sentences"] + ] + text = "".join(texts) + elif contains_chinese(text): + text = self.zh_tn_model.normalize(text) + text = text.replace("\n", "") + text = replace_blank(text) + text = replace_corner_mark(text) + text = text.replace(".", "。") + text = text.replace(" - ", "，") + text = remove_bracket(text) + text = re.sub("[，,、]+$", "。", text) + texts = list( + split_paragraph( + text, + partial( + self.tokenizer.encode, allowed_special=self.allowed_special + ), + "zh", + token_max_n=80, + token_min_n=60, + merge_len=20, + comma_split=False, + ) + ) + else: + text = self.en_tn_model.normalize(text) + text = spell_out_number(text, self.inflect_parser) + texts = list( + split_paragraph( + text, + partial( + self.tokenizer.encode, allowed_special=self.allowed_special + ), + "en", + token_max_n=80, + token_min_n=60, + merge_len=20, + comma_split=False, + ) + ) + texts = [i for i in texts if not is_only_punctuation(i)] + return texts if split is True else text + + def frontend_sft(self, tts_text, spk_id): + tts_text_token, tts_text_token_len = self._extract_text_token(tts_text) + embedding = self.spk2info[spk_id]["embedding"] + model_input = { + "text": tts_text_token, + "text_len": tts_text_token_len, + "llm_embedding": embedding, + "flow_embedding": embedding, + } + return model_input + + def frontend_zero_shot( + self, tts_text, prompt_text, prompt_wav, resample_rate, zero_shot_spk_id + ): + tts_text_token, tts_text_token_len = self._extract_text_token(tts_text) + + if zero_shot_spk_id == "": + + prompt_text_token, prompt_text_token_len = self._extract_text_token( + prompt_text + ) + + speech_feat, speech_feat_len = self._extract_speech_feat(prompt_wav) + speech_feat=paddle.transpose(speech_feat,perm =[0,2,1]) + speech_token, speech_token_len = self._extract_speech_token(prompt_wav) + if resample_rate == 24000: + token_len = min(int(speech_feat.shape[1] / 2), speech_token.shape[1]) + speech_feat, speech_feat_len[:] = ( + speech_feat[:, : 2 * token_len], + 2 * token_len, + ) + speech_token, speech_token_len[:] = ( + speech_token[:, :token_len], + token_len, + ) + + embedding = self._extract_spk_embedding(prompt_wav) + model_input = { + "prompt_text": prompt_text_token, + "prompt_text_len": prompt_text_token_len, + "llm_prompt_speech_token": speech_token, + "llm_prompt_speech_token_len": speech_token_len, + "flow_prompt_speech_token": speech_token, + "flow_prompt_speech_token_len": speech_token_len, + "prompt_speech_feat": speech_feat, + "prompt_speech_feat_len": speech_feat_len, + "llm_embedding": embedding, + "flow_embedding": embedding, + } + else: + model_input = self.spk2info[zero_shot_spk_id] + model_input["text"] = tts_text_token + model_input["text_len"] = tts_text_token_len + return model_input + + def frontend_cross_lingual( + self, tts_text, prompt_wav, resample_rate, zero_shot_spk_id + ): + model_input = self.frontend_zero_shot( + tts_text, "", prompt_wav, resample_rate, zero_shot_spk_id + ) + del model_input["prompt_text"] + del model_input["prompt_text_len"] + del model_input["llm_prompt_speech_token"] + del model_input["llm_prompt_speech_token_len"] + return model_input + + def frontend_instruct(self, tts_text, spk_id, instruct_text): + model_input = self.frontend_sft(tts_text, spk_id) + del model_input["llm_embedding"] + instruct_text_token, instruct_text_token_len = self._extract_text_token( + instruct_text + ) + model_input["prompt_text"] = instruct_text_token + model_input["prompt_text_len"] = instruct_text_token_len + return model_input + + def frontend_instruct2( + self, tts_text, instruct_text, prompt_wav, resample_rate, zero_shot_spk_id + ): + model_input = self.frontend_zero_shot( + tts_text, instruct_text, prompt_wav, resample_rate, zero_shot_spk_id + ) + del model_input["llm_prompt_speech_token"] + del model_input["llm_prompt_speech_token_len"] + return model_input + + def frontend_vc(self, source_speech_16k, prompt_wav, resample_rate): + prompt_speech_token, prompt_speech_token_len = self._extract_speech_token( + prompt_wav + ) + prompt_speech_feat, prompt_speech_feat_len = self._extract_speech_feat( + prompt_wav + ) + embedding = self._extract_spk_embedding(prompt_wav) + source_speech_token, source_speech_token_len = self._extract_speech_token( + source_speech_16k + ) + model_input = { + "source_speech_token": source_speech_token, + "source_speech_token_len": source_speech_token_len, + "flow_prompt_speech_token": prompt_speech_token, + "flow_prompt_speech_token_len": prompt_speech_token_len, + "prompt_speech_feat": prompt_speech_feat, + "prompt_speech_feat_len": prompt_speech_feat_len, + "flow_embedding": embedding, + } + return model_input diff --git a/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend_utils.py b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend_utils.py new file mode 100644 index 000000000..8ec6fd552 --- /dev/null +++ b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/frontend_utils.py @@ -0,0 +1,121 @@ +import re + +import regex + +chinese_char_pattern = re.compile("[\\u4e00-\\u9fff]+") + + +def contains_chinese(text): + return bool(chinese_char_pattern.search(text)) + + +def replace_corner_mark(text): + text = text.replace("²", "平方") + text = text.replace("³", "立方") + return text + + +def remove_bracket(text): + text = text.replace("（", "").replace("）", "") + text = text.replace("【", "").replace("】", "") + text = text.replace("`", "").replace("`", "") + text = text.replace("——", " ") + return text + + +def spell_out_number(text: str, inflect_parser): + new_text = [] + st = None + for i, c in enumerate(text): + if not c.isdigit(): + if st is not None: + num_str = inflect_parser.number_to_words(text[st:i]) + new_text.append(num_str) + st = None + new_text.append(c) + elif st is None: + st = i + if st is not None and st < len(text): + num_str = inflect_parser.number_to_words(text[st:]) + new_text.append(num_str) + return "".join(new_text) + + +def split_paragraph( + text: str, + tokenize, + lang="zh", + token_max_n=80, + token_min_n=60, + merge_len=20, + comma_split=False, +): + def calc_utt_length(_text: str): + if lang == "zh": + return len(_text) + else: + return len(tokenize(_text)) + + def should_merge(_text: str): + if lang == "zh": + return len(_text) < merge_len + else: + return len(tokenize(_text)) < merge_len + + if lang == "zh": + pounc = ["。", "？", "！", "；", "：", "、", ".", "?", "!", ";"] + else: + pounc = [".", "?", "!", ";", ":"] + if comma_split: + pounc.extend(["，", ","]) + if text[-1] not in pounc: + if lang == "zh": + text += "。" + else: + text += "." + st = 0 + utts = [] + for i, c in enumerate(text): + if c in pounc: + if len(text[st:i]) > 0: + utts.append(text[st:i] + c) + if i + 1 < len(text) and text[i + 1] in ['"', "”"]: + tmp = utts.pop(-1) + utts.append(tmp + text[i + 1]) + st = i + 2 + else: + st = i + 1 + final_utts = [] + cur_utt = "" + for utt in utts: + if ( + calc_utt_length(cur_utt + utt) > token_max_n + and calc_utt_length(cur_utt) > token_min_n + ): + final_utts.append(cur_utt) + cur_utt = "" + cur_utt = cur_utt + utt + if len(cur_utt) > 0: + if should_merge(cur_utt) and len(final_utts) != 0: + final_utts[-1] = final_utts[-1] + cur_utt + else: + final_utts.append(cur_utt) + return final_utts + + +def replace_blank(text: str): + out_str = [] + for i, c in enumerate(text): + if c == " ": + if (text[i + 1].isascii() and text[i + 1] != " ") and ( + text[i - 1].isascii() and text[i - 1] != " " + ): + out_str.append(c) + else: + out_str.append(c) + return "".join(out_str) + + +def is_only_punctuation(text): + punctuation_pattern = "^[\\p{P}\\p{S}]*$" + return bool(regex.fullmatch(punctuation_pattern, text)) diff --git a/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/func.py b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/func.py new file mode 100644 index 000000000..82a4ca6da --- /dev/null +++ b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/func.py @@ -0,0 +1,747 @@ +import math +from typing import Tuple + +import paddle + +import paddlespeech + +__all__ = [ + "get_mel_banks", + "inverse_mel_scale", + "inverse_mel_scale_scalar", + "mel_scale", + "mel_scale_scalar", + "spectrogram", + "fbank", + "mfcc", + "vtln_warp_freq", + "vtln_warp_mel_freq", +] +EPSILON = paddle.to_tensor(paddle.finfo(paddle.float32).eps) +MILLISECONDS_TO_SECONDS = 0.001 +HAMMING = "hamming" +HANNING = "hanning" +POVEY = "povey" +RECTANGULAR = "rectangular" +BLACKMAN = "blackman" +WINDOWS = [HAMMING, HANNING, POVEY, RECTANGULAR, BLACKMAN] + + +def _get_epsilon(device, dtype): + return EPSILON.to(device=device, dtype=dtype) + + +def _next_power_of_2(x: int) -> int: + """Returns the smallest power of 2 that is greater than x""" + return 1 if x == 0 else 2 ** (x - 1).bit_length() + + +def _get_strided( + waveform: paddle.Tensor, window_size: int, window_shift: int, snip_edges: bool +) -> paddle.Tensor: + """Given a waveform (1D tensor of size ``num_samples``), it returns a 2D tensor (m, ``window_size``) + representing how the window is shifted along the waveform. Each row is a frame. + + Args: + waveform (Tensor): Tensor of size ``num_samples`` + window_size (int): Frame length + window_shift (int): Frame shift + snip_edges (bool): If True, end effects will be handled by outputting only frames that completely fit + in the file, and the number of frames depends on the frame_length. If False, the number of frames + depends only on the frame_shift, and we reflect the data at the ends. + + Returns: + Tensor: 2D tensor of size (m, ``window_size``) where each row is a frame + """ + assert waveform.dim() == 1 + num_samples = waveform.shape[0] + strides = window_shift * waveform.strides[0], waveform.strides[0] + if snip_edges: + if num_samples < window_size: + return paddle.empty((0, 0), dtype=waveform.dtype, device=waveform.device) + else: + m = 1 + (num_samples - window_size) // window_shift + else: + reversed_waveform = paddle.flip(x=waveform, axis=[0]) + m = (num_samples + window_shift // 2) // window_shift + pad = window_size // 2 - window_shift // 2 + pad_right = reversed_waveform + if pad > 0: + pad_left = reversed_waveform[-pad:] + waveform = paddle.cat((pad_left, waveform, pad_right), axis=0) + else: + waveform = paddle.cat((waveform[-pad:], pad_right), axis=0) + sizes = m, window_size + return waveform.as_strided(shape=sizes, stride=strides) + + +def _feature_window_function( + window_type: str, + window_size: int, + blackman_coeff: float, + device: paddle.device, + dtype: int, +) -> paddle.Tensor: + """Returns a window function with the given type and size""" + if window_type == HANNING: + return paddle.audio.functional.get_window( + win_length=window_size, fftbins=False, dtype=dtype, window="hann" + ) + elif window_type == HAMMING: + return paddle.hamming_window( + window_size, + periodic=False, + alpha=0.54, + beta=0.46, + device=device, + dtype=dtype, + ) + elif window_type == POVEY: + return paddle.audio.functional.get_window( + win_length=window_size, fftbins=False, dtype=dtype, window="hann" + ).pow(0.85) + elif window_type == RECTANGULAR: + return paddle.ones(window_size, device=device, dtype=dtype) + elif window_type == BLACKMAN: + a = 2 * math.pi / (window_size - 1) + window_function = paddle.arange(window_size, device=device, dtype=dtype) + return ( + blackman_coeff + - 0.5 * paddle.cos(a * window_function) + + (0.5 - blackman_coeff) * paddle.cos(2 * a * window_function) + ).to(device=device, dtype=dtype) + else: + raise Exception("Invalid window type " + window_type) + + +def _get_log_energy( + strided_input: paddle.Tensor, epsilon: paddle.Tensor, energy_floor: float +) -> paddle.Tensor: + """Returns the log energy of size (m) for a strided_input (m,*)""" + place, dtype = strided_input.place, strided_input.dtype + log_energy = paddle.maximum(strided_input.pow(2).sum(axis=1), epsilon).log() + if energy_floor == 0.0: + return log_energy + return paddle.maximum( + log_energy, paddle.to_tensor(math.log(energy_floor), place=place, dtype=dtype) + ) + + +def _get_waveform_and_window_properties( + waveform: paddle.Tensor, + channel: int, + sample_frequency: float, + frame_shift: float, + frame_length: float, + round_to_power_of_two: bool, + preemphasis_coefficient: float, +) -> Tuple[paddle.Tensor, int, int, int]: + """Gets the waveform and window properties""" + channel = max(channel, 0) + assert channel < waveform.shape[0], "Invalid channel {} for size {}".format( + channel, waveform.shape[0] + ) + waveform = waveform[channel, :] + window_shift = int(sample_frequency * frame_shift * MILLISECONDS_TO_SECONDS) + window_size = int(sample_frequency * frame_length * MILLISECONDS_TO_SECONDS) + padded_window_size = ( + _next_power_of_2(window_size) if round_to_power_of_two else window_size + ) + assert ( + 2 <= window_size <= len(waveform) + ), "choose a window size {} that is [2, {}]".format(window_size, len(waveform)) + assert 0 < window_shift, "`window_shift` must be greater than 0" + assert ( + padded_window_size % 2 == 0 + ), "the padded `window_size` must be divisible by two. use `round_to_power_of_two` or change `frame_length`" + assert ( + 0.0 <= preemphasis_coefficient <= 1.0 + ), "`preemphasis_coefficient` must be between [0,1]" + assert sample_frequency > 0, "`sample_frequency` must be greater than zero" + return waveform, window_shift, window_size, padded_window_size + + +def _get_window( + waveform: paddle.Tensor, + padded_window_size: int, + window_size: int, + window_shift: int, + window_type: str, + blackman_coeff: float, + snip_edges: bool, + raw_energy: bool, + energy_floor: float, + dither: float, + remove_dc_offset: bool, + preemphasis_coefficient: float, +) -> Tuple[paddle.Tensor, paddle.Tensor]: + """Gets a window and its log energy + + Returns: + (Tensor, Tensor): strided_input of size (m, ``padded_window_size``) and signal_log_energy of size (m) + """ + + place, dtype = waveform.place, waveform.dtype + epsilon = _get_epsilon(place, dtype) + strided_input = _get_strided(waveform, window_size, window_shift, snip_edges) + if dither != 0.0: + rand_gauss = paddle.randn(strided_input.shape, place=place, dtype=dtype) + strided_input = strided_input + rand_gauss * dither + if remove_dc_offset: + row_means = paddle.mean(strided_input, axis=1).unsqueeze(1) + strided_input = strided_input - row_means + if raw_energy: + signal_log_energy = _get_log_energy(strided_input, epsilon, energy_floor) + if preemphasis_coefficient != 0.0: + offset_strided_input = paddle.nn.functional.pad( + strided_input.unsqueeze(0), (1, 0), mode="replicate" + ).squeeze(0) + strided_input = ( + strided_input - preemphasis_coefficient * offset_strided_input[:, :-1] + ) + window_function = _feature_window_function( + window_type, window_size, blackman_coeff, place, dtype + ).unsqueeze(0) + strided_input = strided_input * window_function + if padded_window_size != window_size: + padding_right = padded_window_size - window_size + strided_input = paddle.nn.functional.pad( + strided_input.unsqueeze(0), (0, padding_right), mode="constant", value=0 + ).squeeze(0) + if not raw_energy: + signal_log_energy = _get_log_energy(strided_input, epsilon, energy_floor) + return strided_input, signal_log_energy + + +def _subtract_column_mean(tensor: paddle.Tensor, subtract_mean: bool) -> paddle.Tensor: + if subtract_mean: + col_means = paddle.mean(tensor, axis=0).unsqueeze(0) + tensor = tensor - col_means + return tensor + + +def spectrogram( + waveform: paddle.Tensor, + blackman_coeff: float = 0.42, + channel: int = -1, + dither: float = 0.0, + energy_floor: float = 1.0, + frame_length: float = 25.0, + frame_shift: float = 10.0, + min_duration: float = 0.0, + preemphasis_coefficient: float = 0.97, + raw_energy: bool = True, + remove_dc_offset: bool = True, + round_to_power_of_two: bool = True, + sample_frequency: float = 16000.0, + snip_edges: bool = True, + subtract_mean: bool = False, + window_type: str = POVEY, +) -> paddle.Tensor: + """Create a spectrogram from a raw audio signal. This matches the input/output of Kaldi's + compute-spectrogram-feats. + + Args: + waveform (Tensor): Tensor of audio of size (c, n) where c is in the range [0,2) + blackman_coeff (float, optional): Constant coefficient for generalized Blackman window. (Default: ``0.42``) + channel (int, optional): Channel to extract (-1 -> expect mono, 0 -> left, 1 -> right) (Default: ``-1``) + dither (float, optional): Dithering constant (0.0 means no dither). If you turn this off, you should set + the energy_floor option, e.g. to 1.0 or 0.1 (Default: ``0.0``) + energy_floor (float, optional): Floor on energy (absolute, not relative) in Spectrogram computation. Caution: + this floor is applied to the zeroth component, representing the total signal energy. The floor on the + individual spectrogram elements is fixed at std::numeric_limits::epsilon(). (Default: ``1.0``) + frame_length (float, optional): Frame length in milliseconds (Default: ``25.0``) + frame_shift (float, optional): Frame shift in milliseconds (Default: ``10.0``) + min_duration (float, optional): Minimum duration of segments to process (in seconds). (Default: ``0.0``) + preemphasis_coefficient (float, optional): Coefficient for use in signal preemphasis (Default: ``0.97``) + raw_energy (bool, optional): If True, compute energy before preemphasis and windowing (Default: ``True``) + remove_dc_offset (bool, optional): Subtract mean from waveform on each frame (Default: ``True``) + round_to_power_of_two (bool, optional): If True, round window size to power of two by zero-padding input + to FFT. (Default: ``True``) + sample_frequency (float, optional): Waveform data sample frequency (must match the waveform file, if + specified there) (Default: ``16000.0``) + snip_edges (bool, optional): If True, end effects will be handled by outputting only frames that completely fit + in the file, and the number of frames depends on the frame_length. If False, the number of frames + depends only on the frame_shift, and we reflect the data at the ends. (Default: ``True``) + subtract_mean (bool, optional): Subtract mean of each feature file [CMS]; not recommended to do + it this way. (Default: ``False``) + window_type (str, optional): Type of window ('hamming'|'hanning'|'povey'|'rectangular'|'blackman') + (Default: ``'povey'``) + + Returns: + Tensor: A spectrogram identical to what Kaldi would output. The shape is + (m, ``padded_window_size // 2 + 1``) where m is calculated in _get_strided + """ + device, dtype = waveform.device, waveform.dtype + epsilon = _get_epsilon(device, dtype) + ( + waveform, + window_shift, + window_size, + padded_window_size, + ) = _get_waveform_and_window_properties( + waveform, + channel, + sample_frequency, + frame_shift, + frame_length, + round_to_power_of_two, + preemphasis_coefficient, + ) + if len(waveform) < min_duration * sample_frequency: + return paddle.empty(0) + strided_input, signal_log_energy = _get_window( + waveform, + padded_window_size, + window_size, + window_shift, + window_type, + blackman_coeff, + snip_edges, + raw_energy, + energy_floor, + dither, + remove_dc_offset, + preemphasis_coefficient, + ) + fft = paddle.fft.rfft(strided_input) + power_spectrum = paddle.maximum(fft.abs().pow(2.0), epsilon).log() + power_spectrum[:, 0] = signal_log_energy + power_spectrum = _subtract_column_mean(power_spectrum, subtract_mean) + return power_spectrum + + +def inverse_mel_scale_scalar(mel_freq: float) -> float: + return 700.0 * (math.exp(mel_freq / 1127.0) - 1.0) + + +def inverse_mel_scale(mel_freq: paddle.Tensor) -> paddle.Tensor: + return 700.0 * ((mel_freq / 1127.0).exp() - 1.0) + + +def mel_scale_scalar(freq: float) -> float: + return 1127.0 * math.log(1.0 + freq / 700.0) + + +def mel_scale(freq: paddle.Tensor) -> paddle.Tensor: + return 1127.0 * (1.0 + freq / 700.0).log() + + +def vtln_warp_freq( + vtln_low_cutoff: float, + vtln_high_cutoff: float, + low_freq: float, + high_freq: float, + vtln_warp_factor: float, + freq: paddle.Tensor, +) -> paddle.Tensor: + """This computes a VTLN warping function that is not the same as HTK's one, + but has similar inputs (this function has the advantage of never producing + empty bins). + + This function computes a warp function F(freq), defined between low_freq + and high_freq inclusive, with the following properties: + F(low_freq) == low_freq + F(high_freq) == high_freq + The function is continuous and piecewise linear with two inflection + points. + The lower inflection point (measured in terms of the unwarped + frequency) is at frequency l, determined as described below. + The higher inflection point is at a frequency h, determined as + described below. + If l <= f <= h, then F(f) = f/vtln_warp_factor. + If the higher inflection point (measured in terms of the unwarped + frequency) is at h, then max(h, F(h)) == vtln_high_cutoff. + Since (by the last point) F(h) == h/vtln_warp_factor, then + max(h, h/vtln_warp_factor) == vtln_high_cutoff, so + h = vtln_high_cutoff / max(1, 1/vtln_warp_factor). + = vtln_high_cutoff * min(1, vtln_warp_factor). + If the lower inflection point (measured in terms of the unwarped + frequency) is at l, then min(l, F(l)) == vtln_low_cutoff + This implies that l = vtln_low_cutoff / min(1, 1/vtln_warp_factor) + = vtln_low_cutoff * max(1, vtln_warp_factor) + Args: + vtln_low_cutoff (float): Lower frequency cutoffs for VTLN + vtln_high_cutoff (float): Upper frequency cutoffs for VTLN + low_freq (float): Lower frequency cutoffs in mel computation + high_freq (float): Upper frequency cutoffs in mel computation + vtln_warp_factor (float): Vtln warp factor + freq (Tensor): given frequency in Hz + + Returns: + Tensor: Freq after vtln warp + """ + assert ( + vtln_low_cutoff > low_freq + ), "be sure to set the vtln_low option higher than low_freq" + assert ( + vtln_high_cutoff < high_freq + ), "be sure to set the vtln_high option lower than high_freq [or negative]" + l = vtln_low_cutoff * max(1.0, vtln_warp_factor) + h = vtln_high_cutoff * min(1.0, vtln_warp_factor) + scale = 1.0 / vtln_warp_factor + Fl = scale * l + Fh = scale * h + assert l > low_freq and h < high_freq + scale_left = (Fl - low_freq) / (l - low_freq) + scale_right = (high_freq - Fh) / (high_freq - h) + res = paddle.empty_like(freq) + outside_low_high_freq = paddle.lt(freq, low_freq) | paddle.gt(freq, high_freq) + before_l = paddle.lt(freq, l) + before_h = paddle.lt(freq, h) + after_h = paddle.ge(freq, h) + res[after_h] = high_freq + scale_right * (freq[after_h] - high_freq) + res[before_h] = scale * freq[before_h] + res[before_l] = low_freq + scale_left * (freq[before_l] - low_freq) + res[outside_low_high_freq] = freq[outside_low_high_freq] + return res + + +def vtln_warp_mel_freq( + vtln_low_cutoff: float, + vtln_high_cutoff: float, + low_freq, + high_freq: float, + vtln_warp_factor: float, + mel_freq: paddle.Tensor, +) -> paddle.Tensor: + """ + Args: + vtln_low_cutoff (float): Lower frequency cutoffs for VTLN + vtln_high_cutoff (float): Upper frequency cutoffs for VTLN + low_freq (float): Lower frequency cutoffs in mel computation + high_freq (float): Upper frequency cutoffs in mel computation + vtln_warp_factor (float): Vtln warp factor + mel_freq (Tensor): Given frequency in Mel + + Returns: + Tensor: ``mel_freq`` after vtln warp + """ + return mel_scale( + vtln_warp_freq( + vtln_low_cutoff, + vtln_high_cutoff, + low_freq, + high_freq, + vtln_warp_factor, + inverse_mel_scale(mel_freq), + ) + ) + + +def get_mel_banks( + num_bins: int, + window_length_padded: int, + sample_freq: float, + low_freq: float, + high_freq: float, + vtln_low: float, + vtln_high: float, + vtln_warp_factor: float, +) -> Tuple[paddle.Tensor, paddle.Tensor]: + """ + Returns: + (Tensor, Tensor): The tuple consists of ``bins`` (which is + melbank of size (``num_bins``, ``num_fft_bins``)) and ``center_freqs`` (which is + center frequencies of bins of size (``num_bins``)). + """ + assert num_bins > 3, "Must have at least 3 mel bins" + assert window_length_padded % 2 == 0 + num_fft_bins = window_length_padded / 2 + nyquist = 0.5 * sample_freq + if high_freq <= 0.0: + high_freq += nyquist + assert ( + 0.0 <= low_freq < nyquist + and 0.0 < high_freq <= nyquist + and low_freq < high_freq + ), "Bad values in options: low-freq {} and high-freq {} vs. nyquist {}".format( + low_freq, high_freq, nyquist + ) + fft_bin_width = sample_freq / window_length_padded + mel_low_freq = mel_scale_scalar(low_freq) + mel_high_freq = mel_scale_scalar(high_freq) + mel_freq_delta = (mel_high_freq - mel_low_freq) / (num_bins + 1) + if vtln_high < 0.0: + vtln_high += nyquist + assert ( + vtln_warp_factor == 1.0 + or low_freq < vtln_low < high_freq + and 0.0 < vtln_high < high_freq + and vtln_low < vtln_high + ), "Bad values in options: vtln-low {} and vtln-high {}, versus low-freq {} and high-freq {}".format( + vtln_low, vtln_high, low_freq, high_freq + ) + bin = paddle.arange(num_bins).unsqueeze(1) + left_mel = mel_low_freq + bin * mel_freq_delta + center_mel = mel_low_freq + (bin + 1.0) * mel_freq_delta + right_mel = mel_low_freq + (bin + 2.0) * mel_freq_delta + if vtln_warp_factor != 1.0: + left_mel = vtln_warp_mel_freq( + vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, left_mel + ) + center_mel = vtln_warp_mel_freq( + vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, center_mel + ) + right_mel = vtln_warp_mel_freq( + vtln_low, vtln_high, low_freq, high_freq, vtln_warp_factor, right_mel + ) + center_freqs = inverse_mel_scale(center_mel) + mel = mel_scale(fft_bin_width * paddle.arange(num_fft_bins)).unsqueeze(0) + up_slope = (mel - left_mel) / (center_mel - left_mel) + down_slope = (right_mel - mel) / (right_mel - center_mel) + if vtln_warp_factor == 1.0: + bins = paddle.maximum( + paddle.zeros(1), paddle.minimum(up_slope, down_slope) + ) + else: + bins = paddle.zeros_like(up_slope) + up_idx = paddle.gt(mel, left_mel) & paddle.le(mel, center_mel) + down_idx = paddle.gt(mel, center_mel) & paddle.lt(mel, right_mel) + bins[up_idx] = up_slope[up_idx] + bins[down_idx] = down_slope[down_idx] + return bins, center_freqs + + +def fbank( + waveform: paddle.Tensor, + blackman_coeff: float = 0.42, + channel: int = -1, + dither: float = 0.0, + energy_floor: float = 1.0, + frame_length: float = 25.0, + frame_shift: float = 10.0, + high_freq: float = 0.0, + htk_compat: bool = False, + low_freq: float = 20.0, + min_duration: float = 0.0, + num_mel_bins: int = 23, + preemphasis_coefficient: float = 0.97, + raw_energy: bool = True, + remove_dc_offset: bool = True, + round_to_power_of_two: bool = True, + sample_frequency: float = 16000.0, + snip_edges: bool = True, + subtract_mean: bool = False, + use_energy: bool = False, + use_log_fbank: bool = True, + use_power: bool = True, + vtln_high: float = -500.0, + vtln_low: float = 100.0, + vtln_warp: float = 1.0, + window_type: str = POVEY, +) -> paddle.Tensor: + device, dtype = waveform.place, waveform.dtype + ( + waveform, + window_shift, + window_size, + padded_window_size, + ) = _get_waveform_and_window_properties( + waveform, + channel, + sample_frequency, + frame_shift, + frame_length, + round_to_power_of_two, + preemphasis_coefficient, + ) + if len(waveform) < min_duration * sample_frequency: + return paddle.empty(0, place=place, dtype=dtype) + strided_input, signal_log_energy = _get_window( + waveform, + padded_window_size, + window_size, + window_shift, + window_type, + blackman_coeff, + snip_edges, + raw_energy, + energy_floor, + dither, + remove_dc_offset, + preemphasis_coefficient, + ) + spectrum = paddle.fft.rfft(strided_input).abs() + if use_power: + spectrum = spectrum.pow(2.0) + mel_energies, _ = get_mel_banks( + num_mel_bins, + padded_window_size, + sample_frequency, + low_freq, + high_freq, + vtln_low, + vtln_high, + vtln_warp, + ) + mel_energies = mel_energies.to(device=device, dtype=dtype) + mel_energies = paddle.nn.functional.pad( + mel_energies, (0, 1), mode="constant", value=0 + ) + mel_energies = paddle.mm(input=spectrum, mat2=mel_energies.T) + if use_log_fbank: + mel_energies = paddle.maximum( + mel_energies, _get_epsilon(device, dtype) + ).log() + if use_energy: + signal_log_energy = signal_log_energy.unsqueeze(1) + if htk_compat: + mel_energies = paddle.cat((mel_energies, signal_log_energy), axis=1) + else: + mel_energies = paddle.cat((signal_log_energy, mel_energies), axis=1) + mel_energies = _subtract_column_mean(mel_energies, subtract_mean) + return mel_energies + +def _get_dct_matrix(num_ceps: int, num_mel_bins: int) -> paddle.Tensor: + dct_matrix = paddle.audio.functional.create_dct( + n_mfcc=num_mel_bins, + n_mels=num_mel_bins, + norm='ortho' + ) + first_col = paddle.full( + shape=[num_mel_bins, 1], + fill_value=math.sqrt(1 / float(num_mel_bins)) + ) + if num_ceps > 1: + other_cols = dct_matrix[:, 1:num_ceps] + dct_matrix = paddle.concat([first_col, other_cols], axis=1) + else: + dct_matrix = first_col + return dct_matrix + + +def _get_lifter_coeffs(num_ceps: int, cepstral_lifter: float) -> paddle.Tensor: + i = paddle.arange(num_ceps) + return 1.0 + 0.5 * cepstral_lifter * paddle.sin(math.pi * i / cepstral_lifter) + + +def mfcc( + waveform: paddle.Tensor, + blackman_coeff: float = 0.42, + cepstral_lifter: float = 22.0, + channel: int = -1, + dither: float = 0.0, + energy_floor: float = 1.0, + frame_length: float = 25.0, + frame_shift: float = 10.0, + high_freq: float = 0.0, + htk_compat: bool = False, + low_freq: float = 20.0, + num_ceps: int = 13, + min_duration: float = 0.0, + num_mel_bins: int = 23, + preemphasis_coefficient: float = 0.97, + raw_energy: bool = True, + remove_dc_offset: bool = True, + round_to_power_of_two: bool = True, + sample_frequency: float = 16000.0, + snip_edges: bool = True, + subtract_mean: bool = False, + use_energy: bool = False, + vtln_high: float = -500.0, + vtln_low: float = 100.0, + vtln_warp: float = 1.0, + window_type: str = POVEY, +) -> paddle.Tensor: + """Create a mfcc from a raw audio signal. This matches the input/output of Kaldi's + compute-mfcc-feats. + + Args: + waveform (Tensor): Tensor of audio of size (c, n) where c is in the range [0,2) + blackman_coeff (float, optional): Constant coefficient for generalized Blackman window. (Default: ``0.42``) + cepstral_lifter (float, optional): Constant that controls scaling of MFCCs (Default: ``22.0``) + channel (int, optional): Channel to extract (-1 -> expect mono, 0 -> left, 1 -> right) (Default: ``-1``) + dither (float, optional): Dithering constant (0.0 means no dither). If you turn this off, you should set + the energy_floor option, e.g. to 1.0 or 0.1 (Default: ``0.0``) + energy_floor (float, optional): Floor on energy (absolute, not relative) in Spectrogram computation. Caution: + this floor is applied to the zeroth component, representing the total signal energy. The floor on the + individual spectrogram elements is fixed at std::numeric_limits::epsilon(). (Default: ``1.0``) + frame_length (float, optional): Frame length in milliseconds (Default: ``25.0``) + frame_shift (float, optional): Frame shift in milliseconds (Default: ``10.0``) + high_freq (float, optional): High cutoff frequency for mel bins (if <= 0, offset from Nyquist) + (Default: ``0.0``) + htk_compat (bool, optional): If true, put energy last. Warning: not sufficient to get HTK compatible + features (need to change other parameters). (Default: ``False``) + low_freq (float, optional): Low cutoff frequency for mel bins (Default: ``20.0``) + num_ceps (int, optional): Number of cepstra in MFCC computation (including C0) (Default: ``13``) + min_duration (float, optional): Minimum duration of segments to process (in seconds). (Default: ``0.0``) + num_mel_bins (int, optional): Number of triangular mel-frequency bins (Default: ``23``) + preemphasis_coefficient (float, optional): Coefficient for use in signal preemphasis (Default: ``0.97``) + raw_energy (bool, optional): If True, compute energy before preemphasis and windowing (Default: ``True``) + remove_dc_offset (bool, optional): Subtract mean from waveform on each frame (Default: ``True``) + round_to_power_of_two (bool, optional): If True, round window size to power of two by zero-padding input + to FFT. (Default: ``True``) + sample_frequency (float, optional): Waveform data sample frequency (must match the waveform file, if + specified there) (Default: ``16000.0``) + snip_edges (bool, optional): If True, end effects will be handled by outputting only frames that completely fit + in the file, and the number of frames depends on the frame_length. If False, the number of frames + depends only on the frame_shift, and we reflect the data at the ends. (Default: ``True``) + subtract_mean (bool, optional): Subtract mean of each feature file [CMS]; not recommended to do + it this way. (Default: ``False``) + use_energy (bool, optional): Add an extra dimension with energy to the FBANK output. (Default: ``False``) + vtln_high (float, optional): High inflection point in piecewise linear VTLN warping function (if + negative, offset from high-mel-freq (Default: ``-500.0``) + vtln_low (float, optional): Low inflection point in piecewise linear VTLN warping function (Default: ``100.0``) + vtln_warp (float, optional): Vtln warp factor (only applicable if vtln_map not specified) (Default: ``1.0``) + window_type (str, optional): Type of window ('hamming'|'hanning'|'povey'|'rectangular'|'blackman') + (Default: ``"povey"``) + + Returns: + Tensor: A mfcc identical to what Kaldi would output. The shape is (m, ``num_ceps``) + where m is calculated in _get_strided + """ + assert ( + num_ceps <= num_mel_bins + ), "num_ceps cannot be larger than num_mel_bins: %d vs %d" % ( + num_ceps, + num_mel_bins, + ) + device, dtype = waveform.device, waveform.dtype + feature = fbank( + waveform=waveform, + blackman_coeff=blackman_coeff, + channel=channel, + dither=dither, + energy_floor=energy_floor, + frame_length=frame_length, + frame_shift=frame_shift, + high_freq=high_freq, + htk_compat=htk_compat, + low_freq=low_freq, + min_duration=min_duration, + num_mel_bins=num_mel_bins, + preemphasis_coefficient=preemphasis_coefficient, + raw_energy=raw_energy, + remove_dc_offset=remove_dc_offset, + round_to_power_of_two=round_to_power_of_two, + sample_frequency=sample_frequency, + snip_edges=snip_edges, + subtract_mean=False, + use_energy=use_energy, + use_log_fbank=True, + use_power=True, + vtln_high=vtln_high, + vtln_low=vtln_low, + vtln_warp=vtln_warp, + window_type=window_type, + ) + if use_energy: + signal_log_energy = feature[:, num_mel_bins if htk_compat else 0] + mel_offset = int(not htk_compat) + feature = feature[:, mel_offset : num_mel_bins + mel_offset] + dct_matrix = _get_dct_matrix(num_ceps, num_mel_bins).to(dtype=dtype, device=device) + feature = feature.matmul(dct_matrix) + if cepstral_lifter != 0.0: + lifter_coeffs = _get_lifter_coeffs(num_ceps, cepstral_lifter).unsqueeze(0) + feature *= lifter_coeffs.to(device=device, dtype=dtype) + if use_energy: + feature[:, 0] = signal_log_energy + if htk_compat: + energy = feature[:, 0].unsqueeze(1) + feature = feature[:, 1:] + if not use_energy: + energy *= math.sqrt(2) + feature = paddle.cat((feature, energy), axis=1) + feature = _subtract_column_mean(feature, subtract_mean) + return feature diff --git a/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/tokenizer.py b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/tokenizer.py new file mode 100644 index 000000000..779ede8d9 --- /dev/null +++ b/paddlespeech/t2s/frontend/CosyVoiceFrontEnd/tokenizer.py @@ -0,0 +1,578 @@ +import base64 +import os +from functools import lru_cache +from typing import Optional + +import paddle + +import tiktoken +from whisper.tokenizer import Tokenizer +from paddlenlp.transformers import AutoTokenizer +LANGUAGES = { + "en": "english", + "zh": "chinese", + "de": "german", + "es": "spanish", + "ru": "russian", + "ko": "korean", + "fr": "french", + "ja": "japanese", + "pt": "portuguese", + "tr": "turkish", + "pl": "polish", + "ca": "catalan", + "nl": "dutch", + "ar": "arabic", + "sv": "swedish", + "it": "italian", + "id": "indonesian", + "hi": "hindi", + "fi": "finnish", + "vi": "vietnamese", + "he": "hebrew", + "uk": "ukrainian", + "el": "greek", + "ms": "malay", + "cs": "czech", + "ro": "romanian", + "da": "danish", + "hu": "hungarian", + "ta": "tamil", + "no": "norwegian", + "th": "thai", + "ur": "urdu", + "hr": "croatian", + "bg": "bulgarian", + "lt": "lithuanian", + "la": "latin", + "mi": "maori", + "ml": "malayalam", + "cy": "welsh", + "sk": "slovak", + "te": "telugu", + "fa": "persian", + "lv": "latvian", + "bn": "bengali", + "sr": "serbian", + "az": "azerbaijani", + "sl": "slovenian", + "kn": "kannada", + "et": "estonian", + "mk": "macedonian", + "br": "breton", + "eu": "basque", + "is": "icelandic", + "hy": "armenian", + "ne": "nepali", + "mn": "mongolian", + "bs": "bosnian", + "kk": "kazakh", + "sq": "albanian", + "sw": "swahili", + "gl": "galician", + "mr": "marathi", + "pa": "punjabi", + "si": "sinhala", + "km": "khmer", + "sn": "shona", + "yo": "yoruba", + "so": "somali", + "af": "afrikaans", + "oc": "occitan", + "ka": "georgian", + "be": "belarusian", + "tg": "tajik", + "sd": "sindhi", + "gu": "gujarati", + "am": "amharic", + "yi": "yiddish", + "lo": "lao", + "uz": "uzbek", + "fo": "faroese", + "ht": "haitian creole", + "ps": "pashto", + "tk": "turkmen", + "nn": "nynorsk", + "mt": "maltese", + "sa": "sanskrit", + "lb": "luxembourgish", + "my": "myanmar", + "bo": "tibetan", + "tl": "tagalog", + "mg": "malagasy", + "as": "assamese", + "tt": "tatar", + "haw": "hawaiian", + "ln": "lingala", + "ha": "hausa", + "ba": "bashkir", + "jw": "javanese", + "su": "sundanese", + "yue": "cantonese", + "minnan": "minnan", + "wuyu": "wuyu", + "dialect": "dialect", + "zh/en": "zh/en", + "en/zh": "en/zh", +} +TO_LANGUAGE_CODE = { + **{language: code for code, language in LANGUAGES.items()}, + "burmese": "my", + "valencian": "ca", + "flemish": "nl", + "haitian": "ht", + "letzeburgesch": "lb", + "pushto": "ps", + "panjabi": "pa", + "moldavian": "ro", + "moldovan": "ro", + "sinhalese": "si", + "castilian": "es", + "mandarin": "zh", +} +AUDIO_EVENT = { + "ASR": "ASR", + "AED": "AED", + "SER": "SER", + "Speech": "Speech", + "/Speech": "/Speech", + "BGM": "BGM", + "/BGM": "/BGM", + "Laughter": "Laughter", + "/Laughter": "/Laughter", + "Applause": "Applause", + "/Applause": "/Applause", +} +EMOTION = {"HAPPY": "HAPPY", "SAD": "SAD", "ANGRY": "ANGRY", "NEUTRAL": "NEUTRAL"} +TTS_Vocal_Token = { + "TTS/B": "TTS/B", + "TTS/O": "TTS/O", + "TTS/Q": "TTS/Q", + "TTS/A": "TTS/A", + "TTS/CO": "TTS/CO", + "TTS/CL": "TTS/CL", + "TTS/H": "TTS/H", + **{f"TTS/SP{i:02d}": f"TTS/SP{i:02d}" for i in range(1, 14)}, +} + + +@lru_cache(maxsize=None) +def get_encoding(name: str = "gpt2", num_languages: int = 99): + vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken") + ranks = { + base64.b64decode(token): int(rank) + for token, rank in (line.split() for line in open(vocab_path) if line) + } + n_vocab = len(ranks) + special_tokens = {} + specials = [ + "<|endoftext|>", + "<|startoftranscript|>", + *[f"<|{lang}|>" for lang in list(LANGUAGES.keys())[:num_languages]], + *[f"<|{audio_event}|>" for audio_event in list(AUDIO_EVENT.keys())], + *[f"<|{emotion}|>" for emotion in list(EMOTION.keys())], + "<|translate|>", + "<|transcribe|>", + "<|startoflm|>", + "<|startofprev|>", + "<|nospeech|>", + "<|notimestamps|>", + *[f"<|SPECIAL_TOKEN_{i}|>" for i in range(1, 31)], + *[f"<|{tts}|>" for tts in list(TTS_Vocal_Token.keys())], + *[f"<|{i * 0.02:.2f}|>" for i in range(1501)], + ] + for token in specials: + special_tokens[token] = n_vocab + n_vocab += 1 + return tiktoken.Encoding( + name=os.path.basename(vocab_path), + explicit_n_vocab=n_vocab, + pat_str="'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+", + mergeable_ranks=ranks, + special_tokens=special_tokens, + ) + + +@lru_cache(maxsize=None) +def get_tokenizer( + multilingual: bool, + *, + num_languages: int = 99, + language: Optional[str] = None, + task: Optional[str] = None, +) -> Tokenizer: + if language is not None: + language = language.lower() + if language not in LANGUAGES: + if language in TO_LANGUAGE_CODE: + language = TO_LANGUAGE_CODE[language] + else: + raise ValueError(f"Unsupported language: {language}") + if multilingual: + encoding_name = "multilingual_zh_ja_yue_char_del" + language = language or "en" + task = task or "transcribe" + else: + encoding_name = "gpt2" + language = None + task = None + encoding = get_encoding(name=encoding_name, num_languages=num_languages) + return Tokenizer( + encoding=encoding, num_languages=num_languages, language=language, task=task + ) + + +class CosyVoice2Tokenizer: + def __init__(self, token_path, skip_special_tokens=True): + super().__init__() + special_tokens = { + "eos_token": "<|endoftext|>", + "pad_token": "<|endoftext|>", + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|endofprompt|>", + "[breath]", + "", + "", + "[noise]", + "[laughter]", + "[cough]", + "[clucking]", + "[accent]", + "[quick_breath]", + "", + "", + "[hissing]", + "[sigh]", + "[vocalized-noise]", + "[lipsmack]", + "[mn]", + ], + } + self.special_tokens = special_tokens + self.tokenizer = AutoTokenizer.from_pretrained(token_path) + self.tokenizer.add_special_tokens(special_tokens) + self.skip_special_tokens = skip_special_tokens + + def encode(self, text, **kwargs): + tokens = self.tokenizer(text) + tokens = tokens["input_ids"][0] + return tokens + + def decode(self, tokens): + tokens = paddle.tensor(tokens, dtype=paddle.int64) + text = self.tokenizer.batch_decode( + [tokens], skip_special_tokens=self.skip_special_tokens + )[0] + return text + + +class CosyVoice3Tokenizer(CosyVoice2Tokenizer): + def __init__(self, token_path, skip_special_tokens=True): + special_tokens = { + "eos_token": "<|endoftext|>", + "pad_token": "<|endoftext|>", + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|endofprompt|>", + "[breath]", + "", + "", + "[noise]", + "[laughter]", + "[cough]", + "[clucking]", + "[accent]", + "[quick_breath]", + "", + "", + "[hissing]", + "[sigh]", + "[vocalized-noise]", + "[lipsmack]", + "[mn]", + "<|endofsystem|>", + "[AA]", + "[AA0]", + "[AA1]", + "[AA2]", + "[AE]", + "[AE0]", + "[AE1]", + "[AE2]", + "[AH]", + "[AH0]", + "[AH1]", + "[AH2]", + "[AO]", + "[AO0]", + "[AO1]", + "[AO2]", + "[AW]", + "[AW0]", + "[AW1]", + "[AW2]", + "[AY]", + "[AY0]", + "[AY1]", + "[AY2]", + "[B]", + "[CH]", + "[D]", + "[DH]", + "[EH]", + "[EH0]", + "[EH1]", + "[EH2]", + "[ER]", + "[ER0]", + "[ER1]", + "[ER2]", + "[EY]", + "[EY0]", + "[EY1]", + "[EY2]", + "[F]", + "[G]", + "[HH]", + "[IH]", + "[IH0]", + "[IH1]", + "[IH2]", + "[IY]", + "[IY0]", + "[IY1]", + "[IY2]", + "[JH]", + "[K]", + "[L]", + "[M]", + "[N]", + "[NG]", + "[OW]", + "[OW0]", + "[OW1]", + "[OW2]", + "[OY]", + "[OY0]", + "[OY1]", + "[OY2]", + "[P]", + "[R]", + "[S]", + "[SH]", + "[T]", + "[TH]", + "[UH]", + "[UH0]", + "[UH1]", + "[UH2]", + "[UW]", + "[UW0]", + "[UW1]", + "[UW2]", + "[V]", + "[W]", + "[Y]", + "[Z]", + "[ZH]", + "[a]", + "[ai]", + "[an]", + "[ang]", + "[ao]", + "[b]", + "[c]", + "[ch]", + "[d]", + "[e]", + "[ei]", + "[en]", + "[eng]", + "[f]", + "[g]", + "[h]", + "[i]", + "[ian]", + "[in]", + "[ing]", + "[iu]", + "[ià]", + "[iàn]", + "[iàng]", + "[iào]", + "[iá]", + "[ián]", + "[iáng]", + "[iáo]", + "[iè]", + "[ié]", + "[iòng]", + "[ióng]", + "[iù]", + "[iú]", + "[iā]", + "[iān]", + "[iāng]", + "[iāo]", + "[iē]", + "[iě]", + "[iōng]", + "[iū]", + "[iǎ]", + "[iǎn]", + "[iǎng]", + "[iǎo]", + "[iǒng]", + "[iǔ]", + "[j]", + "[k]", + "[l]", + "[m]", + "[n]", + "[o]", + "[ong]", + "[ou]", + "[p]", + "[q]", + "[r]", + "[s]", + "[sh]", + "[t]", + "[u]", + "[uang]", + "[ue]", + "[un]", + "[uo]", + "[uà]", + "[uài]", + "[uàn]", + "[uàng]", + "[uá]", + "[uái]", + "[uán]", + "[uáng]", + "[uè]", + "[ué]", + "[uì]", + "[uí]", + "[uò]", + "[uó]", + "[uā]", + "[uāi]", + "[uān]", + "[uāng]", + "[uē]", + "[uě]", + "[uī]", + "[uō]", + "[uǎ]", + "[uǎi]", + "[uǎn]", + "[uǎng]", + "[uǐ]", + "[uǒ]", + "[vè]", + "[w]", + "[x]", + "[y]", + "[z]", + "[zh]", + "[à]", + "[ài]", + "[àn]", + "[àng]", + "[ào]", + "[á]", + "[ái]", + "[án]", + "[áng]", + "[áo]", + "[è]", + "[èi]", + "[èn]", + "[èng]", + "[èr]", + "[é]", + "[éi]", + "[én]", + "[éng]", + "[ér]", + "[ì]", + "[ìn]", + "[ìng]", + "[í]", + "[ín]", + "[íng]", + "[ò]", + "[òng]", + "[òu]", + "[ó]", + "[óng]", + "[óu]", + "[ù]", + "[ùn]", + "[ú]", + "[ún]", + "[ā]", + "[āi]", + "[ān]", + "[āng]", + "[āo]", + "[ē]", + "[ēi]", + "[ēn]", + "[ēng]", + "[ě]", + "[ěi]", + "[ěn]", + "[ěng]", + "[ěr]", + "[ī]", + "[īn]", + "[īng]", + "[ō]", + "[ōng]", + "[ōu]", + "[ū]", + "[ūn]", + "[ǎ]", + "[ǎi]", + "[ǎn]", + "[ǎng]", + "[ǎo]", + "[ǐ]", + "[ǐn]", + "[ǐng]", + "[ǒ]", + "[ǒng]", + "[ǒu]", + "[ǔ]", + "[ǔn]", + "[ǘ]", + "[ǚ]", + "[ǜ]", + ], + } + self.special_tokens = special_tokens + self.tokenizer = transformers.AutoTokenizer.from_pretrained(token_path) + self.tokenizer.add_special_tokens(special_tokens) + self.skip_special_tokens = skip_special_tokens + + +@lru_cache(maxsize=None) +def get_qwen_tokenizer( + token_path: str, skip_special_tokens: bool, version: str = "cosyvoice2" +): + if version == "cosyvoice2": + return CosyVoice2Tokenizer( + token_path=token_path, skip_special_tokens=skip_special_tokens + ) + elif version == "cosyvoice3": + return CosyVoice3Tokenizer( + token_path=token_path, skip_special_tokens=skip_special_tokens + ) + else: + raise ValueError diff --git a/paddlespeech/t2s/models/CosyVoice/llm.py b/paddlespeech/t2s/models/CosyVoice/llm.py index 2e903c9bb..ae35ff0af 100644 --- a/paddlespeech/t2s/models/CosyVoice/llm.py +++ b/paddlespeech/t2s/models/CosyVoice/llm.py @@ -579,6 +579,7 @@ class Qwen2LM(TransformerLM): cache=cache, idx = i ) + logp = F.log_softmax(self.llm_decoder(y_pred[:, -1]), axis = -1) top_ids = self.sampling_ids( logp.squeeze(axis=0), diff --git a/paddlespeech/t2s/models/hifigan/cosy_hifigan.py b/paddlespeech/t2s/models/hifigan/cosy_hifigan.py index 90471c61e..d722bf2eb 100644 --- a/paddlespeech/t2s/models/hifigan/cosy_hifigan.py +++ b/paddlespeech/t2s/models/hifigan/cosy_hifigan.py @@ -26,12 +26,8 @@ class ResBlock(paddle.nn.Layer): self.convs1 = paddle.nn.LayerList() self.convs2 = paddle.nn.LayerList() for dilation in dilations: - self.convs1.append(paddle.nn.utils.weight_norm(layer=paddle.nn. - Conv1D(channels, channels, kernel_size, 1, dilation= - dilation, padding=get_padding(kernel_size, dilation)))) - self.convs2.append(paddle.nn.utils.weight_norm(layer=paddle.nn. - Conv1D(channels, channels, kernel_size, 1, dilation=1, - padding=get_padding(kernel_size, 1)))) + self.convs1.append(paddle.nn.Conv1D(channels, channels, kernel_size, 1, dilation=dilation, padding=get_padding(kernel_size, dilation))) + self.convs2.append(paddle.nn.Conv1D(channels, channels, kernel_size, 1, dilation=1,padding=get_padding(kernel_size, 1))) self.convs1.apply(init_weights) self.convs2.apply(init_weights) self.activations1 = paddle.nn.LayerList(sublayers=[Snake(channels, @@ -105,7 +101,6 @@ class SineGen(paddle.nn.Layer): sine_waves = sine_waves * uv + noise return sine_waves, uv, noise - class SourceModuleHnNSF(paddle.nn.Layer): """ SourceModule for hn-nsf SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, @@ -124,16 +119,24 @@ class SourceModuleHnNSF(paddle.nn.Layer): uv (batchsize, length, 1) """ - def __init__(self, sampling_rate, upsample_scale, harmonic_num=0, - sine_amp=0.1, add_noise_std=0.003, voiced_threshod=0): + def __init__(self, sampling_rate, upsample_scale, harmonic_num=0, sine_amp=0.1, + add_noise_std=0.003, voiced_threshod=0, sinegen_type='1', causal=False): super(SourceModuleHnNSF, self).__init__() + self.sine_amp = sine_amp self.noise_std = add_noise_std - self.l_sin_gen = SineGen(sampling_rate, harmonic_num, sine_amp, - add_noise_std, voiced_threshod) - self.l_linear = paddle.nn.Linear(in_features=harmonic_num + 1, - out_features=1) + if sinegen_type == '1': + self.l_sin_gen = SineGen(sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod) + else: + self.l_sin_gen = SineGen2(sampling_rate, upsample_scale, harmonic_num, sine_amp, add_noise_std, voiced_threshod, causal=causal) + self.l_linear = paddle.nn.Linear(harmonic_num + 1, 1) self.l_tanh = paddle.nn.Tanh() + self.causal = causal + paddle.seed(1986) + + if causal is True: + self.uv = paddle.rand(shape=[1, 300 * 24000, 1]) + self.register_buffer('uv_buffer', self.uv) def forward(self, x): """ @@ -143,16 +146,143 @@ class SourceModuleHnNSF(paddle.nn.Layer): noise_source (batchsize, length 1) """ with paddle.no_grad(): - sine_wavs, uv, _ = self.l_sin_gen(paddle.transpose(x,perm=[0,2,1])) - sine_wavs = paddle.transpose(sine_wavs,perm=[0,2,1]) - uv = paddle.transpose(uv,perm=[0,2,1]) + sine_wavs, uv, _ = self.l_sin_gen(x) + sine_merge = self.l_tanh(self.l_linear(sine_wavs)) - - noise = paddle.randn(shape=uv.shape, dtype=uv.dtype - ) * self.sine_amp / 3 + + if not self.training and self.causal: + noise = self.uv_buffer[:, :uv.shape[1]] * self.sine_amp / 3 + else: + noise = paddle.randn(shape=uv.shape, dtype=uv.dtype) * self.sine_amp / 3 return sine_merge, noise, uv - - +# class SourceModuleHnNSF(paddle.nn.Layer): +# """ SourceModule for hn-nsf +# SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, +# add_noise_std=0.003, voiced_threshod=0) +# sampling_rate: sampling_rate in Hz +# harmonic_num: number of harmonic above F0 (default: 0) +# sine_amp: amplitude of sine source signal (default: 0.1) +# add_noise_std: std of additive Gaussian noise (default: 0.003) +# note that amplitude of noise in unvoiced is decided +# by sine_amp +# voiced_threshold: threhold to set U/V given F0 (default: 0) +# Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) +# F0_sampled (batchsize, length, 1) +# Sine_source (batchsize, length, 1) +# noise_source (batchsize, length 1) +# uv (batchsize, length, 1) +# """ + +# def __init__(self, sampling_rate, upsample_scale, harmonic_num=0, +# sine_amp=0.1, add_noise_std=0.003, voiced_threshod=0): +# super(SourceModuleHnNSF, self).__init__() +# self.sine_amp = sine_amp +# self.noise_std = add_noise_std +# self.l_sin_gen = SineGen(sampling_rate, harmonic_num, sine_amp, +# add_noise_std, voiced_threshod) +# self.l_linear = paddle.nn.Linear(in_features=harmonic_num + 1, +# out_features=1) +# self.l_tanh = paddle.nn.Tanh() + +# def forward(self, x): +# """ +# Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) +# F0_sampled (batchsize, length, 1) +# Sine_source (batchsize, length, 1) +# noise_source (batchsize, length 1) +# """ +# with paddle.no_grad(): +# sine_wavs, uv, _ = self.l_sin_gen(paddle.transpose(x,perm=[0,2,1])) +# sine_wavs = paddle.transpose(sine_wavs,perm=[0,2,1]) +# uv = paddle.transpose(uv,perm=[0,2,1]) +# sine_merge = self.l_tanh(self.l_linear(sine_wavs)) + +# noise = paddle.randn(shape=uv.shape, dtype=uv.dtype +# ) * self.sine_amp / 3 +# return sine_merge, noise, uv + + +# class SineGen2(paddle.nn.Layer): +# """ Definition of sine generator +# SineGen(samp_rate, harmonic_num = 0, +# sine_amp = 0.1, noise_std = 0.003, +# voiced_threshold = 0, +# flag_for_pulse=False) +# samp_rate: sampling rate in Hz +# harmonic_num: number of harmonic overtones (default 0) +# sine_amp: amplitude of sine-wavefrom (default 0.1) +# noise_std: std of Gaussian noise (default 0.003) +# voiced_thoreshold: F0 threshold for U/V classification (default 0) +# flag_for_pulse: this SinGen is used inside PulseGen (default False) +# Note: when flag_for_pulse is True, the first time step of a voiced +# segment is always sin(np.pi) or cos(0) +# """ + +# def __init__(self, samp_rate, upsample_scale, harmonic_num=0, sine_amp= +# 0.1, noise_std=0.003, voiced_threshold=0, flag_for_pulse=False): +# super(SineGen2, self).__init__() +# self.sine_amp = sine_amp +# self.noise_std = noise_std +# self.harmonic_num = harmonic_num +# self.axis = self.harmonic_num + 1 +# self.sampling_rate = samp_rate +# self.voiced_threshold = voiced_threshold +# self.flag_for_pulse = flag_for_pulse +# self.upsample_scale = upsample_scale + +# def _f02uv(self, f0): +# uv = (f0 > self.voiced_threshold).astype(paddle.float32) +# return uv + +# def _f02sine(self, f0_values): +# """ f0_values: (batchsize, length, axis) +# where axis indicates fundamental tone and overtones +# """ +# rad_values = f0_values / self.sampling_rate % 1 +# rand_ini = paddle.rand(shape=[f0_values.shape[0], f0_values.shape[2]]) +# rand_ini[:, 0] = 0 +# rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini +# if not self.flag_for_pulse: +# x = paddle.transpose(rad_values,perm = [0,2,1]) + +# rad_values = paddle.transpose(paddle.nn.functional.interpolate(x=x, scale_factor=1 / self.upsample_scale, mode='linear'),perm = [0,2,1]) +# phase = paddle.cumsum(rad_values, axis=1) * 2 * np.pi +# phase = paddle.transpose(paddle.nn.functional.interpolate(x=paddle.transpose(phase,perm = [0,2,1]) * self.upsample_scale, scale_factor=int(self.upsample_scale),mode='linear'),perm = [0,2,1]) +# sines = paddle.sin(phase) +# else: +# uv = self._f02uv(f0_values) +# uv_1 = paddle.roll(uv, shifts=-1, axis=1) +# uv_1[:, -1, :] = 1 +# u_loc = (uv < 1) * (uv_1 > 0) +# tmp_cumsum = paddle.cumsum(rad_values, axis=1) +# for idx in range(f0_values.shape[0]): +# temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :] +# temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :] +# tmp_cumsum[idx, :, :] = 0 +# tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum +# i_phase = paddle.cumsum(rad_values - tmp_cumsum, axis=1) +# sines = paddle.cos(i_phase * 2 * np.pi) +# return sines + +# def forward(self, f0): +# """ sine_tensor, uv = forward(f0) +# input F0: tensor(batchsize=1, length, axis=1) +# f0 for unvoiced steps should be 0 +# output sine_tensor: tensor(batchsize=1, length, axis) +# output uv: tensor(batchsize=1, length, 1) +# """ +# paddle.seed(1986) +# fn = paddle.multiply(f0, paddle.to_tensor([[range(1, self.harmonic_num + +# 2)]],dtype='float32',place=f0.place)) + +# sine_waves = self._f02sine(fn) * self.sine_amp +# uv = self._f02uv(f0) +# noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 +# noise = noise_amp * paddle.randn(shape=sine_waves.shape, dtype= +# sine_waves.dtype) +# sine_waves = sine_waves * uv + noise + +# return sine_waves, uv, noise class SineGen2(paddle.nn.Layer): """ Definition of sine generator SineGen(samp_rate, harmonic_num = 0, @@ -169,73 +299,105 @@ class SineGen2(paddle.nn.Layer): segment is always sin(np.pi) or cos(0) """ - def __init__(self, samp_rate, upsample_scale, harmonic_num=0, sine_amp= - 0.1, noise_std=0.003, voiced_threshold=0, flag_for_pulse=False): + def __init__(self, samp_rate, upsample_scale, harmonic_num=0, + sine_amp=0.1, noise_std=0.003, + voiced_threshold=0, + flag_for_pulse=False, + causal=False): super(SineGen2, self).__init__() self.sine_amp = sine_amp self.noise_std = noise_std self.harmonic_num = harmonic_num - self.axis = self.harmonic_num + 1 + self.dim = self.harmonic_num + 1 self.sampling_rate = samp_rate self.voiced_threshold = voiced_threshold self.flag_for_pulse = flag_for_pulse self.upsample_scale = upsample_scale + self.causal = causal + paddle.seed(1986) + if causal is True: + self.rand_ini = paddle.rand(shape=[1, 9]) + self.rand_ini[:, 0] = 0 + self.sine_waves = paddle.rand(shape=[1, 300 * 24000, 9]) + self.register_buffer('rand_ini_buffer', self.rand_ini) + self.register_buffer('sine_waves_buffer', self.sine_waves) def _f02uv(self, f0): - uv = (f0 > self.voiced_threshold).astype(paddle.float32) + uv = (f0 > self.voiced_threshold).astype('float32') return uv def _f02sine(self, f0_values): - """ f0_values: (batchsize, length, axis) - where axis indicates fundamental tone and overtones - """ - rad_values = f0_values / self.sampling_rate % 1 - rand_ini = paddle.rand(shape=[f0_values.shape[0], f0_values.shape[2]]) - rand_ini[:, 0] = 0 - rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini + """ f0_values: (batchsize, length, dim) """ + rad_values = (f0_values / self.sampling_rate) % 1 + if not self.training and self.causal: + rad_values[:, 0, :] = rad_values[:, 0, :] + self.rand_ini_buffer + else: + rand_ini = paddle.rand(shape=[f0_values.shape[0], f0_values.shape[2]]) + rand_ini[:, 0] = 0 + rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini + if not self.flag_for_pulse: - x = paddle.transpose(rad_values,perm = [0,2,1]) + scale_factor_down = 1.0 / self.upsample_scale - rad_values = paddle.transpose(paddle.nn.functional.interpolate(x=x, scale_factor=1 / self.upsample_scale, mode='linear'),perm = [0,2,1]) + rad_values = paddle.nn.functional.interpolate( + rad_values.transpose([0, 2, 1]), + scale_factor=scale_factor_down, + mode="linear" + ).transpose([0, 2, 1]) phase = paddle.cumsum(rad_values, axis=1) * 2 * np.pi - phase = paddle.transpose(paddle.nn.functional.interpolate(x=paddle.transpose(phase,perm = [0,2,1]) * self.upsample_scale, scale_factor=int(self.upsample_scale),mode='linear'),perm = [0,2,1]) + + interpolate_mode = "nearest" if self.causal else 'linear' + phase = paddle.transpose(paddle.nn.functional.interpolate( + paddle.transpose(phase,perm=[0, 2, 1])*self.upsample_scale, + scale_factor=float(self.upsample_scale), + mode=interpolate_mode + ),perm = [0, 2, 1]) + sines = paddle.sin(phase) else: uv = self._f02uv(f0_values) + uv_1 = paddle.roll(uv, shifts=-1, axis=1) uv_1[:, -1, :] = 1 u_loc = (uv < 1) * (uv_1 > 0) + tmp_cumsum = paddle.cumsum(rad_values, axis=1) for idx in range(f0_values.shape[0]): temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :] temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :] tmp_cumsum[idx, :, :] = 0 tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum + i_phase = paddle.cumsum(rad_values - tmp_cumsum, axis=1) sines = paddle.cos(i_phase * 2 * np.pi) + return sines def forward(self, f0): - """ sine_tensor, uv = forward(f0) - input F0: tensor(batchsize=1, length, axis=1) - f0 for unvoiced steps should be 0 - output sine_tensor: tensor(batchsize=1, length, axis) - output uv: tensor(batchsize=1, length, 1) - """ + """ sine_tensor, uv = forward(f0) """ paddle.seed(1986) - fn = paddle.multiply(f0, paddle.to_tensor([[range(1, self.harmonic_num + - 2)]],dtype='float32',place=f0.place)) - + harmonic_coeffs = paddle.to_tensor( + [list(range(1, self.harmonic_num + 2))], + dtype='float32' + ).reshape([1, 1, -1]) + + fn = f0 * harmonic_coeffs + sine_waves = self._f02sine(fn) * self.sine_amp uv = self._f02uv(f0) + noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 - noise = noise_amp * paddle.randn(shape=sine_waves.shape, dtype= - sine_waves.dtype) + + if not self.training and self.causal: + noise = noise_amp * self.sine_waves_buffer[:, :sine_waves.shape[1]] + else: + noise = noise_amp * paddle.randn( + shape=sine_waves.shape, + dtype=sine_waves.dtype + ) sine_waves = sine_waves * uv + noise - return sine_waves, uv, noise - class SourceModuleHnNSF2(paddle.nn.Layer): """ SourceModule for hn-nsf SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, @@ -307,20 +469,20 @@ class HiFTGenerator(paddle.nn.Layer): self.audio_limit = audio_limit self.num_kernels = len(resblock_kernel_sizes) self.num_upsamples = len(upsample_rates) - this_SourceModuleHnNSF = (SourceModuleHnNSF if self.sampling_rate == - 22050 else SourceModuleHnNSF2) - self.m_source = this_SourceModuleHnNSF(sampling_rate=sampling_rate, - upsample_scale=np.prod(upsample_rates) * istft_params['hop_len' - ], harmonic_num=nb_harmonics, sine_amp=nsf_alpha, add_noise_std - =nsf_sigma, voiced_threshod=nsf_voiced_threshold) + self.m_source = SourceModuleHnNSF( + sampling_rate=sampling_rate, + upsample_scale=np.prod(upsample_rates) * istft_params["hop_len"], + harmonic_num=nb_harmonics, + sine_amp=nsf_alpha, + add_noise_std=nsf_sigma, + voiced_threshod=nsf_voiced_threshold, + sinegen_type='1' if self.sampling_rate == 22050 else '2', + causal=False) self.f0_upsamp = paddle.nn.Upsample(scale_factor=(1,int(np.prod( upsample_rates) * istft_params['hop_len']))) - self.conv_pre = paddle.nn.utils.weight_norm(layer=paddle.nn.Conv1D(in_channels = in_channels, out_channels = base_channels, kernel_size=7, stride=1, padding=3)) + self.conv_pre = paddle.nn.Conv1D(in_channels = in_channels, out_channels = base_channels, kernel_size=7, stride=1, padding=3) self.ups = paddle.nn.LayerList() for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - self.ups.append(paddle.nn.utils.weight_norm(layer=paddle.nn. - Conv1DTranspose(in_channels=base_channels // 2 ** i, - out_channels=base_channels // 2 ** (i + 1), kernel_size=k, - stride=u, padding=(k - u) // 2))) + self.ups.append(paddle.nn.Conv1DTranspose(in_channels=base_channels // 2 ** i,out_channels=base_channels // 2 ** (i + 1), kernel_size=k,stride=u, padding=(k - u) // 2)) self.source_downs = paddle.nn.LayerList() self.source_resblocks = paddle.nn.LayerList() downsample_rates = [1] + upsample_rates[::-1][:-1] @@ -346,8 +508,7 @@ class HiFTGenerator(paddle.nn.Layer): for _, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): self.resblocks.append(ResBlock(ch, k, d)) - self.conv_post = paddle.nn.utils.weight_norm(layer=paddle.nn.Conv1D - (ch, istft_params['n_fft'] + 2, 7, 1, padding=3)) + self.conv_post = paddle.nn.Conv1D(ch, istft_params['n_fft'] + 2, 7, 1, padding=3) self.ups.apply(init_weights) self.conv_post.apply(init_weights) self.reflection_pad = paddle.nn.Pad1D(padding=(1, 0), mode='reflect') @@ -435,9 +596,12 @@ class HiFTGenerator(paddle.nn.Layer): s_4d = self.f0_upsamp(f0_4d) s_3d = s_4d.squeeze(2) s = paddle.transpose(s_3d, perm=[0, 2, 1]) + s, _, _ = self.m_source(s) s = paddle.transpose(s,perm = [0,2,1]) + if cache_source.shape[2] != 0: s[:, :, :cache_source.shape[2]] = cache_source generated_speech = self.decode(x=speech_feat, s=s) + return generated_speech, s diff --git a/paddlespeech/t2s/models/hifigan/f0_predictor.py b/paddlespeech/t2s/models/hifigan/f0_predictor.py index 8cf686181..f5e922c1a 100644 --- a/paddlespeech/t2s/models/hifigan/f0_predictor.py +++ b/paddlespeech/t2s/models/hifigan/f0_predictor.py @@ -5,21 +5,23 @@ class ConvRNNF0Predictor(paddle.nn.Layer): int=512): super().__init__() self.num_class = num_class - self.condnet = paddle.nn.Sequential(paddle.nn.utils.weight_norm( - layer=paddle.nn.Conv1D(in_channels, cond_channels, kernel_size= - 3, padding=1)), paddle.nn.ELU(), paddle.nn.utils.weight_norm( - layer=paddle.nn.Conv1D(cond_channels, cond_channels, - kernel_size=3, padding=1)), paddle.nn.ELU(), paddle.nn.utils. - weight_norm(layer=paddle.nn.Conv1D(cond_channels, cond_channels, - kernel_size=3, padding=1)), paddle.nn.ELU(), paddle.nn.utils. - weight_norm(layer=paddle.nn.Conv1D(cond_channels, cond_channels, - kernel_size=3, padding=1)), paddle.nn.ELU(), paddle.nn.utils. - weight_norm(layer=paddle.nn.Conv1D(cond_channels, cond_channels, - kernel_size=3, padding=1)), paddle.nn.ELU()) + self.condnet = paddle.nn.Sequential( + paddle.nn.Conv1D(in_channels, cond_channels, kernel_size=3, padding=1), + paddle.nn.ELU(), + paddle.nn.Conv1D(cond_channels, cond_channels,kernel_size=3, padding=1), + paddle.nn.ELU(), + paddle.nn.Conv1D(cond_channels, cond_channels,kernel_size=3, padding=1), + paddle.nn.ELU(), + paddle.nn.Conv1D(cond_channels, cond_channels,kernel_size=3, padding=1), + paddle.nn.ELU(), + paddle.nn.Conv1D(cond_channels, cond_channels,kernel_size=3, padding=1), + paddle.nn.ELU() + ) self.classifier = paddle.nn.Linear(in_features=cond_channels, out_features=self.num_class) def forward(self, x: paddle.Tensor) ->paddle.Tensor: - x = self.condnet(x) + for idx,layer in enumerate(self.condnet): + x = layer(x) x = paddle.transpose(x, perm=[0, 2, 1]) return paddle.abs(x=self.classifier(x).squeeze(-1)) diff --git a/paddlespeech/t2s/modules/flow/decoder.py b/paddlespeech/t2s/modules/flow/decoder.py index 7744656b1..0489317e7 100644 --- a/paddlespeech/t2s/modules/flow/decoder.py +++ b/paddlespeech/t2s/modules/flow/decoder.py @@ -110,15 +110,11 @@ class TimestepEmbedding(nn.Layer): if condition is not None and self.cond_proj is not None: sample = sample + self.cond_proj(condition) sample = self.linear_1(sample) - # print("sample2:",sample) if self.act is not None: sample = self.act(sample) - # print("sample3:",sample) sample = self.linear_2(sample) - # print("sample4:",sample) if self.post_act is not None: sample = self.post_act(sample) - # print("sample5:",sample) return sample class Upsample1D(nn.Layer): @@ -705,6 +701,7 @@ class CausalConditionalDecoder(nn.Layer): for resnet, transformer_blocks, downsample in self.down_blocks: mask_down = masks[-1] x = resnet(x, mask_down, t) + x = rearrange(x, "b c t -> b t c").contiguous() # 假设 rearrange 函数已实现 if streaming is True: attn_mask = add_optional_chunk_mask(x, mask_down.astype('bool'), False, False, 0, self.static_chunk_size, -1) # 使用 astype('bool') @@ -717,6 +714,7 @@ class CausalConditionalDecoder(nn.Layer): attention_mask=attn_mask, timestep=t, ) + x = rearrange(x, "b t c -> b c t").contiguous() hiddens.append(x) # Save hidden states for skip connections x = downsample(x * mask_down) diff --git a/paddlespeech/t2s/modules/flow/flow.py b/paddlespeech/t2s/modules/flow/flow.py index 2ca92c535..2c1517f78 100644 --- a/paddlespeech/t2s/modules/flow/flow.py +++ b/paddlespeech/t2s/modules/flow/flow.py @@ -285,7 +285,7 @@ class CausalMaskedDiffWithXvec(paddle.nn.Layer): assert token.shape[0] == 1 embedding = paddle.nn.functional.normalize(x=embedding, axis=1) embedding = self.spk_embed_affine_layer(embedding) - + token, token_len = ( paddle.cat([prompt_token, token], dim=1), prompt_token_len + token_len, @@ -302,6 +302,8 @@ class CausalMaskedDiffWithXvec(paddle.nn.Layer): h, h_lengths = self.encoder( token, token_len, context=context, streaming=streaming ) + + mel_len1, mel_len2 = prompt_feat.shape[1], h.shape[1] - prompt_feat.shape[1] h = self.encoder_proj(h) conds = paddle.zeros( @@ -318,7 +320,7 @@ class CausalMaskedDiffWithXvec(paddle.nn.Layer): n_timesteps=10, streaming=streaming, ) - paddle.save(feat,'/root/paddlejob/workspace/zhangjinghong/CosyVoice/feat.pdparams') + feat = feat[:, :, mel_len1:] assert feat.shape[2] == mel_len2 return feat.float(), None diff --git a/paddlespeech/t2s/modules/flow/flow_matching.py b/paddlespeech/t2s/modules/flow/flow_matching.py index 9c68deb48..98443535c 100644 --- a/paddlespeech/t2s/modules/flow/flow_matching.py +++ b/paddlespeech/t2s/modules/flow/flow_matching.py @@ -180,6 +180,7 @@ class ConditionalCFM(BASECFM): shape: (batch_size, spk_emb_dim) cond: Not used but kept for future purposes """ + t, _, dt = t_span[0], t_span[-1], t_span[1] - t_span[0] t = t.unsqueeze(axis=0) sol = [] @@ -323,10 +324,10 @@ class CausalConditionalCFM(ConditionalCFM): """ z = self.rand_noise[:, :, : mu.shape[2]].to(mu.place).to(mu.dtype) * temperature - t_span = paddle.linspace(start=0, stop=1, num=n_timesteps + 1, dtype=mu.dtype) if self.t_scheduler == "cosine": t_span = 1 - paddle.cos(t_span * 0.5 * paddle.pi) + return ( self.solve_euler( z, diff --git a/paddlespeech/t2s/modules/flow/matcha_transformer.py b/paddlespeech/t2s/modules/flow/matcha_transformer.py index 3be13dd10..db2214729 100644 --- a/paddlespeech/t2s/modules/flow/matcha_transformer.py +++ b/paddlespeech/t2s/modules/flow/matcha_transformer.py @@ -134,16 +134,6 @@ class FeedForward(paddle.nn.Layer): inner_dim = int(dim * mult) dim_out = dim_out if dim_out is not None else dim act_fn = GELU(dim, inner_dim) -# if activation_fn == "gelu": -# >>>>>> act_fn = diffusers.models.attention.GELU(dim, inner_dim) -# if activation_fn == "gelu-approximate": -# >>>>>> act_fn = diffusers.models.attention.GELU(dim, inner_dim, approximate="tanh") -# elif activation_fn == "geglu": -# >>>>>> act_fn = diffusers.models.attention.GEGLU(dim, inner_dim) -# elif activation_fn == "geglu-approximate": -# act_fn = diffusers.models.attention.ApproximateGELU(dim, inner_dim) -# elif activation_fn == "snakebeta": - # act_fn = SnakeBeta(dim, inner_dim) self.net = paddle.nn.LayerList(sublayers=[]) self.net.append(act_fn) self.net.append(paddle.nn.Dropout(p=dropout)) @@ -274,9 +264,11 @@ class BasicTransformerBlock(paddle.nn.Layer): else attention_mask, **cross_attention_kwargs, ) + if self.use_ada_layer_norm_zero: attn_output = gate_msa.unsqueeze(1) * attn_output hidden_states = attn_output + hidden_states + if self.attn2 is not None: norm_hidden_states = ( self.norm2(hidden_states, timestep) @@ -291,10 +283,12 @@ class BasicTransformerBlock(paddle.nn.Layer): ) hidden_states = attn_output + hidden_states norm_hidden_states = self.norm3(hidden_states) + if self.use_ada_layer_norm_zero: norm_hidden_states = ( norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] ) + if self._chunk_size is not None: if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0: raise ValueError( @@ -312,6 +306,7 @@ class BasicTransformerBlock(paddle.nn.Layer): ) else: ff_output = self.ff(norm_hidden_states) + if self.use_ada_layer_norm_zero: ff_output = gate_mlp.unsqueeze(1) * ff_output hidden_states = ff_output + hidden_states diff --git a/paddlespeech/t2s/modules/transformer/subsampling.py b/paddlespeech/t2s/modules/transformer/subsampling.py index ca46569bb..20220e8d6 100644 --- a/paddlespeech/t2s/modules/transformer/subsampling.py +++ b/paddlespeech/t2s/modules/transformer/subsampling.py @@ -71,7 +71,6 @@ class LinearNoSubsampling(BaseSubsampling): where time' = time . """ - x = self.out(x) x, pos_emb = self.pos_enc(x, offset) return x, pos_emb, x_mask diff --git a/paddlespeech/t2s/modules/transformer/upsample_encoder.py b/paddlespeech/t2s/modules/transformer/upsample_encoder.py index dfdcb1620..ea7550b27 100644 --- a/paddlespeech/t2s/modules/transformer/upsample_encoder.py +++ b/paddlespeech/t2s/modules/transformer/upsample_encoder.py @@ -69,6 +69,7 @@ class PreLookaheadLayer(paddle.nn.Layer): """ outputs = paddle.transpose(inputs, perm=[0, 2, 1]).contiguous() context = paddle.transpose(context, perm=[0, 2, 1]).contiguous() + if context.shape[2] == 0: outputs = F.pad( outputs, [0, self.pre_lookahead_len], mode="constant", value=0.0 @@ -84,13 +85,15 @@ class PreLookaheadLayer(paddle.nn.Layer): mode="constant", value=0.0, ) + outputs = paddle.nn.functional.leaky_relu(x=self.conv1(outputs)) + outputs = F.pad( outputs, [self.conv2._kernel_size[0] - 1, 0], mode="constant", value=0.0 ) outputs = self.conv2(outputs) outputs = paddle.transpose(outputs, perm=[0, 2, 1]).contiguous() - + outputs = outputs + inputs return outputs @@ -276,10 +279,12 @@ class UpsampleConformerEncoder(paddle.nn.Layer): https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2 """ T = xs.shape[1] + masks = ~make_pad_mask(xs_lens, T).unsqueeze(1) if self.global_cmvn is not None: xs = self.global_cmvn(xs) xs, pos_emb, masks = self.embed(xs, masks) + if context.shape[1] != 0: assert ( self.training is False @@ -297,8 +302,11 @@ class UpsampleConformerEncoder(paddle.nn.Layer): -1, ) xs = self.pre_lookahead_layer(xs, context=context) + + xs = self.forward_layers(xs, chunk_masks, pos_emb, mask_pad) # + xs = paddle.transpose(xs, perm=[0, 2, 1]).contiguous() xs, xs_lens = self.up_layer(xs, xs_lens) xs = paddle.transpose(xs, perm=[0, 2, 1]).contiguous() @@ -318,6 +326,7 @@ class UpsampleConformerEncoder(paddle.nn.Layer): xs = self.forward_up_layers(xs, chunk_masks, pos_emb, mask_pad) if self.normalize_before: xs = self.after_norm(xs) + return xs, masks def forward_layers(