# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import os import re from pathlib import Path from typing import Dict from typing import List from typing import Union DICT_EN = 'tools/aligner/cmudict-0.7b' DICT_ZH = 'tools/aligner/simple.lexicon' MODEL_DIR_EN = 'tools/aligner/vctk_model.zip' MODEL_DIR_ZH = 'tools/aligner/aishell3_model.zip' MFA_PHONE_EN = 'tools/aligner/vctk_model/meta.yaml' MFA_PHONE_ZH = 'tools/aligner/aishell3_model/meta.yaml' MFA_PATH = 'tools/montreal-forced-aligner/bin' os.environ['PATH'] = MFA_PATH + '/:' + os.environ['PATH'] def check_phone(label_file: Union[str, Path], pronunciation_phones: Dict[str, str], mfa_phones: List[str], am_phones: List[str], oov_record: str="./oov_info.txt", lang: str="zh"): """Check whether the phoneme corresponding to the audio text content is in the phoneme list of the pretrained mfa model to ensure that the alignment is normal. Check whether the phoneme corresponding to the audio text content is in the phoneme list of the pretrained am model to ensure finetune (normalize) is normal. Args: label_file (Union[str, Path]): label file, format: utt_id|phone seq pronunciation_phones (dict): pronunciation to phones map dict mfa_phones (list): the phone list of pretrained mfa model am_phones (list): the phone list of pretrained mfa model Returns: oov_words (list): oov words oov_files (list): utt id list that exist oov oov_file_words (dict): the oov file and oov phone in this file """ oov_words = [] oov_files = [] oov_file_words = {} with open(label_file, "r") as f: for line in f.readlines(): utt_id = line.split("|")[0] transcription = line.strip().split("|")[1] transcription = re.sub( r'[：、，；。？！,.:;"?!”’《》【】<=>{}()（）#&@“”^_|…\\]', '', transcription) if lang == "en": transcription = transcription.upper() flag = 0 temp_oov_words = [] for word in transcription.split(" "): if word not in pronunciation_phones.keys(): temp_oov_words.append(word) flag = 1 if word not in oov_words: oov_words.append(word) else: for p in pronunciation_phones[word]: if p not in mfa_phones or p not in am_phones: temp_oov_words.append(word) flag = 1 if word not in oov_words: oov_words.append(word) if flag == 1: oov_files.append(utt_id) oov_file_words[utt_id] = temp_oov_words if oov_record is not None: with open(oov_record, "w") as fw: fw.write("oov_words: " + str(oov_words) + "\n") fw.write("oov_files: " + str(oov_files) + "\n") fw.write("oov_file_words: " + str(oov_file_words) + "\n") return oov_words, oov_files, oov_file_words def get_pronunciation_phones(lexicon_file: Union[str, Path]): # pronunciation to phones pronunciation_phones = {} with open(lexicon_file, "r") as f2: for line in f2.readlines(): line_list = line.strip().split(" ") pronunciation = line_list[0] if line_list[1] == '': phones = line_list[2:] else: phones = line_list[1:] pronunciation_phones[pronunciation] = phones return pronunciation_phones def get_mfa_phone(mfa_phone_file: Union[str, Path]): # get phones from pretrained mfa model (meta.yaml) mfa_phones = [] with open(mfa_phone_file, "r") as f: for line in f.readlines(): if line.startswith("-"): phone = line.strip().split(" ")[-1] mfa_phones.append(phone) return mfa_phones def get_am_phone(am_phone_file: Union[str, Path]): # get phones from pretrained am model (phone_id_map.txt) am_phones = [] with open(am_phone_file, "r") as f: for line in f.readlines(): phone = line.strip().split(" ")[0] am_phones.append(phone) return am_phones def get_check_result(label_file: Union[str, Path], am_phone_file: Union[str, Path], input_dir: Union[str, Path], newdir_name: str="newdir", lang: str="zh"): """Check if there is any audio in the input that contains the oov word according to label_file. Copy audio that does not contain oov word to input_dir / newdir_name. Generate label file and save to input_dir / newdir_name. Args: label_file (Union[str, Path]): input audio label file, format: utt|pronunciation am_phone_file (Union[str, Path]): pretrained am model phone file input_dir (Union[str, Path]): input dir newdir_name (str): directory name saved after checking oov lang (str): input audio language """ if lang == 'en': lexicon_file = DICT_EN mfa_phone_file = MFA_PHONE_EN elif lang == 'zh': lexicon_file = DICT_ZH mfa_phone_file = MFA_PHONE_ZH else: print('please input right lang!!') pronunciation_phones = get_pronunciation_phones(lexicon_file) mfa_phones = get_mfa_phone(mfa_phone_file) am_phones = get_am_phone(am_phone_file) oov_words, oov_files, oov_file_words = check_phone( label_file=label_file, pronunciation_phones=pronunciation_phones, mfa_phones=mfa_phones, am_phones=am_phones, oov_record="./oov_info.txt", lang=lang) input_dir = Path(input_dir).expanduser() new_dir = input_dir / newdir_name new_dir.mkdir(parents=True, exist_ok=True) with open(label_file, "r") as f: for line in f.readlines(): utt_id = line.split("|")[0] if utt_id not in oov_files: transcription = line.split("|")[1].strip() wav_file = str(input_dir) + "/" + utt_id + ".wav" new_wav_file = str(new_dir) + "/" + utt_id + ".wav" os.system("cp %s %s" % (wav_file, new_wav_file)) single_file = str(new_dir) + "/" + utt_id + ".txt" with open(single_file, "w") as fw: fw.write(transcription) if __name__ == '__main__': # parse config and args parser = argparse.ArgumentParser( description="Preprocess audio and then extract features.") parser.add_argument( "--input_dir", type=str, default="./input/csmsc_mini", help="directory containing audio and label file") parser.add_argument( "--pretrained_model_dir", type=str, default="./pretrained_models/fastspeech2_aishell3_ckpt_1.1.0", help="Path to pretrained model") parser.add_argument( "--newdir_name", type=str, default="newdir", help="directory name saved after checking oov") parser.add_argument( '--lang', type=str, default='zh', choices=['zh', 'en'], help='Choose input audio language. zh or en') args = parser.parse_args() # if args.lang == 'en': # lexicon_file = DICT_EN # mfa_phone_file = MFA_PHONE_EN # elif args.lang == 'zh': # lexicon_file = DICT_ZH # mfa_phone_file = MFA_PHONE_ZH # else: # print('please input right lang!!') assert args.lang == "zh" or args.lang == "en", "please input right lang! zh or en" input_dir = Path(args.input_dir).expanduser() pretrained_model_dir = Path(args.pretrained_model_dir).expanduser() am_phone_file = pretrained_model_dir / "phone_id_map.txt" label_file = input_dir / "labels.txt" get_check_result( label_file=label_file, am_phone_file=am_phone_file, input_dir=input_dir, newdir_name=args.newdir_name, lang=args.lang)