From 8891621e2ca8a44635c8936700c8ac10d3f7c117 Mon Sep 17 00:00:00 2001 From: Honei <915574300@qq.com> Date: Fri, 28 Jan 2022 16:34:48 +0800 Subject: [PATCH] [vector]voxceleb1 data prepare: compatible kaldi trial and data split dir (#1412) * compatible kaldi trial and data split dir * remove unuseless code --- dataset/voxceleb/voxceleb1.py | 59 ++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/dataset/voxceleb/voxceleb1.py b/dataset/voxceleb/voxceleb1.py index 2c6dff222..ce7447516 100644 --- a/dataset/voxceleb/voxceleb1.py +++ b/dataset/voxceleb/voxceleb1.py @@ -45,15 +45,26 @@ DATA_HOME = os.path.expanduser('.') # you need use --no-check-certificate to connect the target download url BASE_URL = "https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a" -DATA_LIST = { + +# dev data +DEV_LIST = { "vox1_dev_wav_partaa": "e395d020928bc15670b570a21695ed96", "vox1_dev_wav_partab": "bbfaaccefab65d82b21903e81a8a8020", "vox1_dev_wav_partac": "017d579a2a96a077f40042ec33e51512", "vox1_dev_wav_partad": "7bb1e9f70fddc7a678fa998ea8b3ba19", - "vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102", } +DEV_TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f532ba230b" + +# test data +TEST_LIST = {"vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102"} +TEST_TARGET_DATA = "vox1_test_wav.zip vox1_test_wav.zip 185fdc63c3c739954633d50379a3d102" -TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f532ba230b" +# kaldi trial +# this trial file is organized by kaldi according the official file, +# which is a little different with the official trial veri_test2.txt +KALDI_BASE_URL = "http://www.openslr.org/resources/49/" +TRIAL_LIST = {"voxceleb1_test_v2.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7"} +TRIAL_TARGET_DATA = "voxceleb1_test_v2.txt voxceleb1_test_v2.txt 29fc7cc1c5d59f0816dc15d6e8be60f7" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -69,7 +80,6 @@ parser.add_argument( args = parser.parse_args() - def create_manifest(data_dir, manifest_path_prefix): print("Creating manifest %s ..." % manifest_path_prefix) json_lines = [] @@ -79,7 +89,7 @@ def create_manifest(data_dir, manifest_path_prefix): total_num = 0 speakers = set() for audio_path in glob.glob(data_path, recursive=True): - audio_id = "/".join(audio_path.split("/")[-3:]) + audio_id = "-".join(audio_path.split("/")[-3:]) utt2spk = audio_path.split("/")[-3] duration = soundfile.info(audio_path).duration text = "" @@ -99,14 +109,17 @@ def create_manifest(data_dir, manifest_path_prefix): total_num += 1 speakers.add(utt2spk) + # data_dir_name refer to dev or test + # voxceleb1 is given explicit in the path + data_dir_name = Path(data_dir).name + manifest_path_prefix = manifest_path_prefix + "." + data_dir_name with codecs.open(manifest_path_prefix, 'w', encoding='utf-8') as f: for line in json_lines: f.write(line + "\n") manifest_dir = os.path.dirname(manifest_path_prefix) - # data_dir_name refer to voxceleb1, which is used to distingush the voxceleb2 dataset info - data_dir_name = Path(data_dir).name - meta_path = os.path.join(manifest_dir, data_dir_name) + ".meta" + meta_path = os.path.join(manifest_dir, "voxceleb1." + + data_dir_name) + ".meta" with codecs.open(meta_path, 'w', encoding='utf-8') as f: print(f"{total_num} utts", file=f) print(f"{len(speakers)} speakers", file=f) @@ -115,10 +128,8 @@ def create_manifest(data_dir, manifest_path_prefix): print(f"{total_text / total_sec} text/sec", file=f) print(f"{total_sec / total_num} sec/utt", file=f) - def prepare_dataset(base_url, data_list, target_dir, manifest_path, target_data): - data_dir = os.path.join(target_dir, "voxceleb1") if not os.path.exists(target_dir): os.mkdir(target_dir) @@ -126,7 +137,7 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path, if not os.path.exists(os.path.join(target_dir, "wav")): # download all dataset part for zip_part in data_list.keys(): - download_url = base_url + "/" + zip_part + " --no-check-certificate " + download_url = " --no-check-certificate " + base_url + "/" + zip_part download( url=download_url, md5sum=data_list[zip_part], @@ -136,8 +147,8 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path, all_target_part, target_name, target_md5sum = target_data.split() target_name = os.path.join(target_dir, target_name) if not os.path.exists(target_name): - pack_part_cmd = "cat {}/{} > {}/{}".format( - target_dir, all_target_part, target_dir, target_name) + pack_part_cmd = "cat {}/{} > {}".format(target_dir, all_target_part, + target_name) subprocess.call(pack_part_cmd, shell=True) # check the target zip file md5sum @@ -147,13 +158,11 @@ def prepare_dataset(base_url, data_list, target_dir, manifest_path, print("Check {} md5sum successfully".format(target_name)) # unzip the all zip file - unzip(target_name, target_dir) - unzip(os.path.join(target_dir, "vox1_test_wav.zip"), target_dir) + if target_name.endswith(".zip"): + unzip(target_name, target_dir) # create the manifest file - create_manifest( - data_dir=args.target_dir, manifest_path_prefix=args.manifest_prefix) - + create_manifest(data_dir=target_dir, manifest_path_prefix=manifest_path) def main(): if args.target_dir.startswith('~'): @@ -161,13 +170,19 @@ def main(): prepare_dataset( base_url=BASE_URL, - data_list=DATA_LIST, - target_dir=args.target_dir, + data_list=DEV_LIST, + target_dir=os.path.join(args.target_dir, "dev"), manifest_path=args.manifest_prefix, - target_data=TARGET_DATA) + target_data=DEV_TARGET_DATA) - print("Manifest prepare done!") + prepare_dataset( + base_url=BASE_URL, + data_list=TEST_LIST, + target_dir=os.path.join(args.target_dir, "test"), + manifest_path=args.manifest_prefix, + target_data=TEST_TARGET_DATA) + print("Manifest prepare done!") if __name__ == '__main__': main()