Merge pull request #1630 from Honei/vox12
[vec]voxceleb convert dataset format to paddlespeechpull/1690/head
Honei
3 years ago
committed by
GitHub
commit
48e0177767
@ -0,0 +1,53 @@
|
|||||||
|
###########################################
|
||||||
|
# Data #
|
||||||
|
###########################################
|
||||||
|
augment: True
|
||||||
|
batch_size: 16
|
||||||
|
num_workers: 2
|
||||||
|
num_speakers: 1211 # 1211 vox1, 5994 vox2, 7205 vox1+2, test speakers: 41
|
||||||
|
shuffle: True
|
||||||
|
skip_prep: False
|
||||||
|
split_ratio: 0.9
|
||||||
|
chunk_duration: 3.0 # seconds
|
||||||
|
random_chunk: True
|
||||||
|
verification_file: data/vox1/veri_test2.txt
|
||||||
|
|
||||||
|
###########################################################
|
||||||
|
# FEATURE EXTRACTION SETTING #
|
||||||
|
###########################################################
|
||||||
|
# currently, we only support fbank
|
||||||
|
sr: 16000 # sample rate
|
||||||
|
n_mels: 80
|
||||||
|
window_size: 400 #25ms, sample rate 16000, 25 * 16000 / 1000 = 400
|
||||||
|
hop_size: 160 #10ms, sample rate 16000, 10 * 16000 / 1000 = 160
|
||||||
|
|
||||||
|
###########################################################
|
||||||
|
# MODEL SETTING #
|
||||||
|
###########################################################
|
||||||
|
# currently, we only support ecapa-tdnn in the ecapa_tdnn.yaml
|
||||||
|
# if we want use another model, please choose another configuration yaml file
|
||||||
|
model:
|
||||||
|
input_size: 80
|
||||||
|
channels: [512, 512, 512, 512, 1536]
|
||||||
|
kernel_sizes: [5, 3, 3, 3, 1]
|
||||||
|
dilations: [1, 2, 3, 4, 1]
|
||||||
|
attention_channels: 128
|
||||||
|
lin_neurons: 192
|
||||||
|
|
||||||
|
###########################################
|
||||||
|
# Training #
|
||||||
|
###########################################
|
||||||
|
seed: 1986 # according from speechbrain configuration
|
||||||
|
epochs: 100
|
||||||
|
save_interval: 10
|
||||||
|
log_interval: 10
|
||||||
|
learning_rate: 1e-8
|
||||||
|
|
||||||
|
|
||||||
|
###########################################
|
||||||
|
# Testing #
|
||||||
|
###########################################
|
||||||
|
global_embedding_norm: True
|
||||||
|
embedding_mean_norm: True
|
||||||
|
embedding_std_norm: False
|
||||||
|
|
||||||
@ -0,0 +1,167 @@
|
|||||||
|
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""
|
||||||
|
Convert the PaddleSpeech jsonline format data to csv format data in voxceleb experiment.
|
||||||
|
Currently, Speaker Identificaton Training process use csv format.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import os
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
import tqdm
|
||||||
|
from yacs.config import CfgNode
|
||||||
|
|
||||||
|
from paddleaudio import load as load_audio
|
||||||
|
from paddlespeech.s2t.utils.log import Log
|
||||||
|
from paddlespeech.vector.utils.vector_utils import get_chunks
|
||||||
|
|
||||||
|
logger = Log(__name__).getlog()
|
||||||
|
|
||||||
|
|
||||||
|
def get_chunks_list(wav_file: str,
|
||||||
|
split_chunks: bool,
|
||||||
|
base_path: str,
|
||||||
|
chunk_duration: float=3.0) -> List[List[str]]:
|
||||||
|
"""Get the single audio file info
|
||||||
|
|
||||||
|
Args:
|
||||||
|
wav_file (list): the wav audio file and get this audio segment info list
|
||||||
|
split_chunks (bool): audio split flag
|
||||||
|
base_path (str): the audio base path
|
||||||
|
chunk_duration (float): the chunk duration.
|
||||||
|
if set the split_chunks, we split the audio into multi-chunks segment.
|
||||||
|
"""
|
||||||
|
waveform, sr = load_audio(wav_file)
|
||||||
|
audio_id = wav_file.split("/rir_noise/")[-1].split(".")[0]
|
||||||
|
audio_duration = waveform.shape[0] / sr
|
||||||
|
|
||||||
|
ret = []
|
||||||
|
if split_chunks and audio_duration > chunk_duration: # Split into pieces of self.chunk_duration seconds.
|
||||||
|
uniq_chunks_list = get_chunks(chunk_duration, audio_id, audio_duration)
|
||||||
|
|
||||||
|
for idx, chunk in enumerate(uniq_chunks_list):
|
||||||
|
s, e = chunk.split("_")[-2:] # Timestamps of start and end
|
||||||
|
start_sample = int(float(s) * sr)
|
||||||
|
end_sample = int(float(e) * sr)
|
||||||
|
|
||||||
|
# currently, all vector csv data format use one representation
|
||||||
|
# id, duration, wav, start, stop, label
|
||||||
|
# in rirs noise, all the label name is 'noise'
|
||||||
|
# the label is string type and we will convert it to integer type in training
|
||||||
|
ret.append([
|
||||||
|
chunk, audio_duration, wav_file, start_sample, end_sample,
|
||||||
|
"noise"
|
||||||
|
])
|
||||||
|
else: # Keep whole audio.
|
||||||
|
ret.append(
|
||||||
|
[audio_id, audio_duration, wav_file, 0, waveform.shape[0], "noise"])
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
def generate_csv(wav_files,
|
||||||
|
output_file: str,
|
||||||
|
base_path: str,
|
||||||
|
split_chunks: bool=True):
|
||||||
|
"""Prepare the csv file according the wav files
|
||||||
|
|
||||||
|
Args:
|
||||||
|
wav_files (list): all the audio list to prepare the csv file
|
||||||
|
output_file (str): the output csv file
|
||||||
|
config (CfgNode): yaml configuration content
|
||||||
|
split_chunks (bool): audio split flag
|
||||||
|
"""
|
||||||
|
logger.info(f'Generating csv: {output_file}')
|
||||||
|
header = ["utt_id", "duration", "wav", "start", "stop", "label"]
|
||||||
|
csv_lines = []
|
||||||
|
for item in tqdm.tqdm(wav_files):
|
||||||
|
csv_lines.extend(
|
||||||
|
get_chunks_list(
|
||||||
|
item, base_path=base_path, split_chunks=split_chunks))
|
||||||
|
|
||||||
|
if not os.path.exists(os.path.dirname(output_file)):
|
||||||
|
os.makedirs(os.path.dirname(output_file))
|
||||||
|
|
||||||
|
with open(output_file, mode="w") as csv_f:
|
||||||
|
csv_writer = csv.writer(
|
||||||
|
csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
|
||||||
|
csv_writer.writerow(header)
|
||||||
|
for line in csv_lines:
|
||||||
|
csv_writer.writerow(line)
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_data(args, config):
|
||||||
|
"""Convert the jsonline format to csv format
|
||||||
|
|
||||||
|
Args:
|
||||||
|
args (argparse.Namespace): scripts args
|
||||||
|
config (CfgNode): yaml configuration content
|
||||||
|
"""
|
||||||
|
# if external config set the skip_prep flat, we will do nothing
|
||||||
|
if config.skip_prep:
|
||||||
|
return
|
||||||
|
|
||||||
|
base_path = args.noise_dir
|
||||||
|
wav_path = os.path.join(base_path, "RIRS_NOISES")
|
||||||
|
logger.info(f"base path: {base_path}")
|
||||||
|
logger.info(f"wav path: {wav_path}")
|
||||||
|
rir_list = os.path.join(wav_path, "real_rirs_isotropic_noises", "rir_list")
|
||||||
|
rir_files = []
|
||||||
|
with open(rir_list, 'r') as f:
|
||||||
|
for line in f.readlines():
|
||||||
|
rir_file = line.strip().split(' ')[-1]
|
||||||
|
rir_files.append(os.path.join(base_path, rir_file))
|
||||||
|
|
||||||
|
noise_list = os.path.join(wav_path, "pointsource_noises", "noise_list")
|
||||||
|
noise_files = []
|
||||||
|
with open(noise_list, 'r') as f:
|
||||||
|
for line in f.readlines():
|
||||||
|
noise_file = line.strip().split(' ')[-1]
|
||||||
|
noise_files.append(os.path.join(base_path, noise_file))
|
||||||
|
|
||||||
|
csv_path = os.path.join(args.data_dir, 'csv')
|
||||||
|
logger.info(f"csv path: {csv_path}")
|
||||||
|
generate_csv(
|
||||||
|
rir_files, os.path.join(csv_path, 'rir.csv'), base_path=base_path)
|
||||||
|
generate_csv(
|
||||||
|
noise_files, os.path.join(csv_path, 'noise.csv'), base_path=base_path)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument(
|
||||||
|
"--noise_dir",
|
||||||
|
default=None,
|
||||||
|
required=True,
|
||||||
|
help="The noise dataset dataset directory.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--data_dir",
|
||||||
|
default=None,
|
||||||
|
required=True,
|
||||||
|
help="The target directory stores the csv files")
|
||||||
|
parser.add_argument(
|
||||||
|
"--config",
|
||||||
|
default=None,
|
||||||
|
required=True,
|
||||||
|
type=str,
|
||||||
|
help="configuration file")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# parse the yaml config file
|
||||||
|
config = CfgNode(new_allowed=True)
|
||||||
|
if args.config:
|
||||||
|
config.merge_from_file(args.config)
|
||||||
|
|
||||||
|
# prepare the csv file from jsonlines files
|
||||||
|
prepare_data(args, config)
|
||||||
@ -0,0 +1,251 @@
|
|||||||
|
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""
|
||||||
|
Convert the PaddleSpeech jsonline format data to csv format data in voxceleb experiment.
|
||||||
|
Currently, Speaker Identificaton Training process use csv format.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
|
||||||
|
import tqdm
|
||||||
|
from yacs.config import CfgNode
|
||||||
|
|
||||||
|
from paddleaudio import load as load_audio
|
||||||
|
from paddlespeech.s2t.utils.log import Log
|
||||||
|
from paddlespeech.vector.utils.vector_utils import get_chunks
|
||||||
|
|
||||||
|
logger = Log(__name__).getlog()
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_csv(wav_files, output_file, config, split_chunks=True):
|
||||||
|
"""Prepare the csv file according the wav files
|
||||||
|
|
||||||
|
Args:
|
||||||
|
wav_files (list): all the audio list to prepare the csv file
|
||||||
|
output_file (str): the output csv file
|
||||||
|
config (CfgNode): yaml configuration content
|
||||||
|
split_chunks (bool, optional): audio split flag. Defaults to True.
|
||||||
|
"""
|
||||||
|
if not os.path.exists(os.path.dirname(output_file)):
|
||||||
|
os.makedirs(os.path.dirname(output_file))
|
||||||
|
csv_lines = []
|
||||||
|
header = ["utt_id", "duration", "wav", "start", "stop", "label"]
|
||||||
|
# voxceleb meta info for each training utterance segment
|
||||||
|
# we extract a segment from a utterance to train
|
||||||
|
# and the segment' period is between start and stop time point in the original wav file
|
||||||
|
# each field in the meta info means as follows:
|
||||||
|
# utt_id: the utterance segment name, which is uniq in training dataset
|
||||||
|
# duration: the total utterance time
|
||||||
|
# wav: utterance file path, which should be absoulute path
|
||||||
|
# start: start point in the original wav file sample point range
|
||||||
|
# stop: stop point in the original wav file sample point range
|
||||||
|
# label: the utterance segment's label name,
|
||||||
|
# which is speaker name in speaker verification domain
|
||||||
|
for item in tqdm.tqdm(wav_files, total=len(wav_files)):
|
||||||
|
item = json.loads(item.strip())
|
||||||
|
audio_id = item['utt'].replace(".wav",
|
||||||
|
"") # we remove the wav suffix name
|
||||||
|
audio_duration = item['feat_shape'][0]
|
||||||
|
wav_file = item['feat']
|
||||||
|
label = audio_id.split('-')[
|
||||||
|
0] # speaker name in speaker verification domain
|
||||||
|
waveform, sr = load_audio(wav_file)
|
||||||
|
if split_chunks:
|
||||||
|
uniq_chunks_list = get_chunks(config.chunk_duration, audio_id,
|
||||||
|
audio_duration)
|
||||||
|
for chunk in uniq_chunks_list:
|
||||||
|
s, e = chunk.split("_")[-2:] # Timestamps of start and end
|
||||||
|
start_sample = int(float(s) * sr)
|
||||||
|
end_sample = int(float(e) * sr)
|
||||||
|
# id, duration, wav, start, stop, label
|
||||||
|
# in vector, the label in speaker id
|
||||||
|
csv_lines.append([
|
||||||
|
chunk, audio_duration, wav_file, start_sample, end_sample,
|
||||||
|
label
|
||||||
|
])
|
||||||
|
else:
|
||||||
|
csv_lines.append([
|
||||||
|
audio_id, audio_duration, wav_file, 0, waveform.shape[0], label
|
||||||
|
])
|
||||||
|
|
||||||
|
with open(output_file, mode="w") as csv_f:
|
||||||
|
csv_writer = csv.writer(
|
||||||
|
csv_f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
|
||||||
|
csv_writer.writerow(header)
|
||||||
|
for line in csv_lines:
|
||||||
|
csv_writer.writerow(line)
|
||||||
|
|
||||||
|
|
||||||
|
def get_enroll_test_list(dataset_list, verification_file):
|
||||||
|
"""Get the enroll and test utterance list from all the voxceleb1 test utterance dataset.
|
||||||
|
Generally, we get the enroll and test utterances from the verfification file.
|
||||||
|
The verification file format as follows:
|
||||||
|
target/nontarget enroll-utt test-utt,
|
||||||
|
we set 0 as nontarget and 1 as target, eg:
|
||||||
|
0 a.wav b.wav
|
||||||
|
1 a.wav a.wav
|
||||||
|
|
||||||
|
Args:
|
||||||
|
dataset_list (list): all the dataset to get the test utterances
|
||||||
|
verification_file (str): voxceleb1 trial file
|
||||||
|
"""
|
||||||
|
logger.info(f"verification file: {verification_file}")
|
||||||
|
enroll_audios = set()
|
||||||
|
test_audios = set()
|
||||||
|
with open(verification_file, 'r') as f:
|
||||||
|
for line in f:
|
||||||
|
_, enroll_file, test_file = line.strip().split(' ')
|
||||||
|
enroll_audios.add('-'.join(enroll_file.split('/')))
|
||||||
|
test_audios.add('-'.join(test_file.split('/')))
|
||||||
|
|
||||||
|
enroll_files = []
|
||||||
|
test_files = []
|
||||||
|
for dataset in dataset_list:
|
||||||
|
with open(dataset, 'r') as f:
|
||||||
|
for line in f:
|
||||||
|
# audio_id may be in enroll and test at the same time
|
||||||
|
# eg: 1 a.wav a.wav
|
||||||
|
# the audio a.wav is enroll and test file at the same time
|
||||||
|
audio_id = json.loads(line.strip())['utt']
|
||||||
|
if audio_id in enroll_audios:
|
||||||
|
enroll_files.append(line)
|
||||||
|
if audio_id in test_audios:
|
||||||
|
test_files.append(line)
|
||||||
|
|
||||||
|
enroll_files = sorted(enroll_files)
|
||||||
|
test_files = sorted(test_files)
|
||||||
|
|
||||||
|
return enroll_files, test_files
|
||||||
|
|
||||||
|
|
||||||
|
def get_train_dev_list(dataset_list, target_dir, split_ratio):
|
||||||
|
"""Get the train and dev utterance list from all the training utterance dataset.
|
||||||
|
Generally, we use the split_ratio as the train dataset ratio,
|
||||||
|
and the remaining utterance (ratio is 1 - split_ratio) is the dev dataset
|
||||||
|
|
||||||
|
Args:
|
||||||
|
dataset_list (list): all the dataset to get the all utterances
|
||||||
|
target_dir (str): the target train and dev directory,
|
||||||
|
we will create the csv directory to store the {train,dev}.csv file
|
||||||
|
split_ratio (float): train dataset ratio in all utterance list
|
||||||
|
"""
|
||||||
|
logger.info("start to get train and dev utt list")
|
||||||
|
if not os.path.exists(os.path.join(target_dir, "meta")):
|
||||||
|
os.makedirs(os.path.join(target_dir, "meta"))
|
||||||
|
|
||||||
|
audio_files = []
|
||||||
|
speakers = set()
|
||||||
|
for dataset in dataset_list:
|
||||||
|
with open(dataset, 'r') as f:
|
||||||
|
for line in f:
|
||||||
|
# the label is speaker name
|
||||||
|
label_name = json.loads(line.strip())['utt2spk']
|
||||||
|
speakers.add(label_name)
|
||||||
|
audio_files.append(line.strip())
|
||||||
|
speakers = sorted(speakers)
|
||||||
|
logger.info(f"we get {len(speakers)} speakers from all the train dataset")
|
||||||
|
|
||||||
|
with open(os.path.join(target_dir, "meta", "label2id.txt"), 'w') as f:
|
||||||
|
for label_id, label_name in enumerate(speakers):
|
||||||
|
f.write(f'{label_name} {label_id}\n')
|
||||||
|
logger.info(
|
||||||
|
f'we store the speakers to {os.path.join(target_dir, "meta", "label2id.txt")}'
|
||||||
|
)
|
||||||
|
|
||||||
|
# the split_ratio is for train dataset
|
||||||
|
# the remaining is for dev dataset
|
||||||
|
split_idx = int(split_ratio * len(audio_files))
|
||||||
|
audio_files = sorted(audio_files)
|
||||||
|
random.shuffle(audio_files)
|
||||||
|
train_files, dev_files = audio_files[:split_idx], audio_files[split_idx:]
|
||||||
|
logger.info(
|
||||||
|
f"we get train utterances: {len(train_files)}, dev utterance: {len(dev_files)}"
|
||||||
|
)
|
||||||
|
return train_files, dev_files
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_data(args, config):
|
||||||
|
"""Convert the jsonline format to csv format
|
||||||
|
|
||||||
|
Args:
|
||||||
|
args (argparse.Namespace): scripts args
|
||||||
|
config (CfgNode): yaml configuration content
|
||||||
|
"""
|
||||||
|
# stage0: set the random seed
|
||||||
|
random.seed(config.seed)
|
||||||
|
|
||||||
|
# if external config set the skip_prep flat, we will do nothing
|
||||||
|
if config.skip_prep:
|
||||||
|
return
|
||||||
|
|
||||||
|
# stage 1: prepare the enroll and test csv file
|
||||||
|
# And we generate the speaker to label file label2id.txt
|
||||||
|
logger.info("start to prepare the data csv file")
|
||||||
|
enroll_files, test_files = get_enroll_test_list(
|
||||||
|
[args.test], verification_file=config.verification_file)
|
||||||
|
prepare_csv(
|
||||||
|
enroll_files,
|
||||||
|
os.path.join(args.target_dir, "csv", "enroll.csv"),
|
||||||
|
config,
|
||||||
|
split_chunks=False)
|
||||||
|
prepare_csv(
|
||||||
|
test_files,
|
||||||
|
os.path.join(args.target_dir, "csv", "test.csv"),
|
||||||
|
config,
|
||||||
|
split_chunks=False)
|
||||||
|
|
||||||
|
# stage 2: prepare the train and dev csv file
|
||||||
|
# we get the train dataset ratio as config.split_ratio
|
||||||
|
# and the remaining is dev dataset
|
||||||
|
logger.info("start to prepare the data csv file")
|
||||||
|
train_files, dev_files = get_train_dev_list(
|
||||||
|
args.train, target_dir=args.target_dir, split_ratio=config.split_ratio)
|
||||||
|
prepare_csv(train_files,
|
||||||
|
os.path.join(args.target_dir, "csv", "train.csv"), config)
|
||||||
|
prepare_csv(dev_files,
|
||||||
|
os.path.join(args.target_dir, "csv", "dev.csv"), config)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument(
|
||||||
|
"--train",
|
||||||
|
required=True,
|
||||||
|
nargs='+',
|
||||||
|
help="The jsonline files list for train.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--test", required=True, help="The jsonline file for test")
|
||||||
|
parser.add_argument(
|
||||||
|
"--target_dir",
|
||||||
|
default=None,
|
||||||
|
required=True,
|
||||||
|
help="The target directory stores the csv files and meta file.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--config",
|
||||||
|
default=None,
|
||||||
|
required=True,
|
||||||
|
type=str,
|
||||||
|
help="configuration file")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# parse the yaml config file
|
||||||
|
config = CfgNode(new_allowed=True)
|
||||||
|
if args.config:
|
||||||
|
config.merge_from_file(args.config)
|
||||||
|
|
||||||
|
# prepare the csv file from jsonlines files
|
||||||
|
prepare_data(args, config)
|
||||||
@ -0,0 +1,192 @@
|
|||||||
|
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from dataclasses import fields
|
||||||
|
from paddle.io import Dataset
|
||||||
|
|
||||||
|
from paddleaudio import load as load_audio
|
||||||
|
from paddleaudio.compliance.librosa import melspectrogram
|
||||||
|
from paddlespeech.s2t.utils.log import Log
|
||||||
|
logger = Log(__name__).getlog()
|
||||||
|
|
||||||
|
# the audio meta info in the vector CSVDataset
|
||||||
|
# utt_id: the utterance segment name
|
||||||
|
# duration: utterance segment time
|
||||||
|
# wav: utterance file path
|
||||||
|
# start: start point in the original wav file
|
||||||
|
# stop: stop point in the original wav file
|
||||||
|
# label: the utterance segment's label id
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class meta_info:
|
||||||
|
"""the audio meta info in the vector CSVDataset
|
||||||
|
|
||||||
|
Args:
|
||||||
|
utt_id (str): the utterance segment name
|
||||||
|
duration (float): utterance segment time
|
||||||
|
wav (str): utterance file path
|
||||||
|
start (int): start point in the original wav file
|
||||||
|
stop (int): stop point in the original wav file
|
||||||
|
lab_id (str): the utterance segment's label id
|
||||||
|
"""
|
||||||
|
utt_id: str
|
||||||
|
duration: float
|
||||||
|
wav: str
|
||||||
|
start: int
|
||||||
|
stop: int
|
||||||
|
label: str
|
||||||
|
|
||||||
|
|
||||||
|
# csv dataset support feature type
|
||||||
|
# raw: return the pcm data sample point
|
||||||
|
# melspectrogram: fbank feature
|
||||||
|
feat_funcs = {
|
||||||
|
'raw': None,
|
||||||
|
'melspectrogram': melspectrogram,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CSVDataset(Dataset):
|
||||||
|
def __init__(self,
|
||||||
|
csv_path,
|
||||||
|
label2id_path=None,
|
||||||
|
config=None,
|
||||||
|
random_chunk=True,
|
||||||
|
feat_type: str="raw",
|
||||||
|
n_train_snts: int=-1,
|
||||||
|
**kwargs):
|
||||||
|
"""Implement the CSV Dataset
|
||||||
|
|
||||||
|
Args:
|
||||||
|
csv_path (str): csv dataset file path
|
||||||
|
label2id_path (str): the utterance label to integer id map file path
|
||||||
|
config (CfgNode): yaml config
|
||||||
|
feat_type (str): dataset feature type. if it is raw, it return pcm data.
|
||||||
|
n_train_snts (int): select the n_train_snts sample from the dataset.
|
||||||
|
if n_train_snts = -1, dataset will load all the sample.
|
||||||
|
Default value is -1.
|
||||||
|
kwargs : feature type args
|
||||||
|
"""
|
||||||
|
super().__init__()
|
||||||
|
self.csv_path = csv_path
|
||||||
|
self.label2id_path = label2id_path
|
||||||
|
self.config = config
|
||||||
|
self.random_chunk = random_chunk
|
||||||
|
self.feat_type = feat_type
|
||||||
|
self.n_train_snts = n_train_snts
|
||||||
|
self.feat_config = kwargs
|
||||||
|
self.id2label = {}
|
||||||
|
self.label2id = {}
|
||||||
|
self.data = self.load_data_csv()
|
||||||
|
self.load_speaker_to_label()
|
||||||
|
|
||||||
|
def load_data_csv(self):
|
||||||
|
"""Load the csv dataset content and store them in the data property
|
||||||
|
the csv dataset's format has six fields,
|
||||||
|
that is audio_id or utt_id, audio duration, segment start point, segment stop point
|
||||||
|
and utterance label.
|
||||||
|
Note in training period, the utterance label must has a map to integer id in label2id_path
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list: the csv data with meta_info type
|
||||||
|
"""
|
||||||
|
data = []
|
||||||
|
|
||||||
|
with open(self.csv_path, 'r') as rf:
|
||||||
|
for line in rf.readlines()[1:]:
|
||||||
|
audio_id, duration, wav, start, stop, spk_id = line.strip(
|
||||||
|
).split(',')
|
||||||
|
data.append(
|
||||||
|
meta_info(audio_id,
|
||||||
|
float(duration), wav,
|
||||||
|
int(start), int(stop), spk_id))
|
||||||
|
if self.n_train_snts > 0:
|
||||||
|
sample_num = min(self.n_train_snts, len(data))
|
||||||
|
data = data[0:sample_num]
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
def load_speaker_to_label(self):
|
||||||
|
"""Load the utterance label map content.
|
||||||
|
In vector domain, we call the utterance label as speaker label.
|
||||||
|
The speaker label is real speaker label in speaker verification domain,
|
||||||
|
and in language identification is language label.
|
||||||
|
"""
|
||||||
|
if not self.label2id_path:
|
||||||
|
logger.warning("No speaker id to label file")
|
||||||
|
return
|
||||||
|
|
||||||
|
with open(self.label2id_path, 'r') as f:
|
||||||
|
for line in f.readlines():
|
||||||
|
label_name, label_id = line.strip().split(' ')
|
||||||
|
self.label2id[label_name] = int(label_id)
|
||||||
|
self.id2label[int(label_id)] = label_name
|
||||||
|
|
||||||
|
def convert_to_record(self, idx: int):
|
||||||
|
"""convert the dataset sample to training record the CSV Dataset
|
||||||
|
|
||||||
|
Args:
|
||||||
|
idx (int) : the request index in all the dataset
|
||||||
|
"""
|
||||||
|
sample = self.data[idx]
|
||||||
|
|
||||||
|
record = {}
|
||||||
|
# To show all fields in a namedtuple: `type(sample)._fields`
|
||||||
|
for field in fields(sample):
|
||||||
|
record[field.name] = getattr(sample, field.name)
|
||||||
|
|
||||||
|
waveform, sr = load_audio(record['wav'])
|
||||||
|
|
||||||
|
# random select a chunk audio samples from the audio
|
||||||
|
if self.config and self.config.random_chunk:
|
||||||
|
num_wav_samples = waveform.shape[0]
|
||||||
|
num_chunk_samples = int(self.config.chunk_duration * sr)
|
||||||
|
start = random.randint(0, num_wav_samples - num_chunk_samples - 1)
|
||||||
|
stop = start + num_chunk_samples
|
||||||
|
else:
|
||||||
|
start = record['start']
|
||||||
|
stop = record['stop']
|
||||||
|
|
||||||
|
# we only return the waveform as feat
|
||||||
|
waveform = waveform[start:stop]
|
||||||
|
|
||||||
|
# all availabel feature type is in feat_funcs
|
||||||
|
assert self.feat_type in feat_funcs.keys(), \
|
||||||
|
f"Unknown feat_type: {self.feat_type}, it must be one in {list(feat_funcs.keys())}"
|
||||||
|
feat_func = feat_funcs[self.feat_type]
|
||||||
|
feat = feat_func(
|
||||||
|
waveform, sr=sr, **self.feat_config) if feat_func else waveform
|
||||||
|
|
||||||
|
record.update({'feat': feat})
|
||||||
|
if self.label2id:
|
||||||
|
record.update({'label': self.label2id[record['label']]})
|
||||||
|
|
||||||
|
return record
|
||||||
|
|
||||||
|
def __getitem__(self, idx):
|
||||||
|
"""Return the specific index sample
|
||||||
|
|
||||||
|
Args:
|
||||||
|
idx (int) : the request index in all the dataset
|
||||||
|
"""
|
||||||
|
return self.convert_to_record(idx)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
"""Return the dataset length
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: the length num of the dataset
|
||||||
|
"""
|
||||||
|
return len(self.data)
|
||||||
@ -0,0 +1,214 @@
|
|||||||
|
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
import paddle
|
||||||
|
|
||||||
|
|
||||||
|
class InputNormalization:
|
||||||
|
spk_dict_mean: Dict[int, paddle.Tensor]
|
||||||
|
spk_dict_std: Dict[int, paddle.Tensor]
|
||||||
|
spk_dict_count: Dict[int, int]
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
mean_norm=True,
|
||||||
|
std_norm=True,
|
||||||
|
norm_type="global", ):
|
||||||
|
"""Do feature or embedding mean and std norm
|
||||||
|
|
||||||
|
Args:
|
||||||
|
mean_norm (bool, optional): mean norm flag. Defaults to True.
|
||||||
|
std_norm (bool, optional): std norm flag. Defaults to True.
|
||||||
|
norm_type (str, optional): norm type. Defaults to "global".
|
||||||
|
"""
|
||||||
|
super().__init__()
|
||||||
|
self.training = True
|
||||||
|
self.mean_norm = mean_norm
|
||||||
|
self.std_norm = std_norm
|
||||||
|
self.norm_type = norm_type
|
||||||
|
self.glob_mean = paddle.to_tensor([0], dtype="float32")
|
||||||
|
self.glob_std = paddle.to_tensor([0], dtype="float32")
|
||||||
|
self.spk_dict_mean = {}
|
||||||
|
self.spk_dict_std = {}
|
||||||
|
self.spk_dict_count = {}
|
||||||
|
self.weight = 1.0
|
||||||
|
self.count = 0
|
||||||
|
self.eps = 1e-10
|
||||||
|
|
||||||
|
def __call__(self,
|
||||||
|
x,
|
||||||
|
lengths,
|
||||||
|
spk_ids=paddle.to_tensor([], dtype="float32")):
|
||||||
|
"""Returns the tensor with the surrounding context.
|
||||||
|
Args:
|
||||||
|
x (paddle.Tensor): A batch of tensors.
|
||||||
|
lengths (paddle.Tensor): A batch of tensors containing the relative length of each
|
||||||
|
sentence (e.g, [0.7, 0.9, 1.0]). It is used to avoid
|
||||||
|
computing stats on zero-padded steps.
|
||||||
|
spk_ids (_type_, optional): tensor containing the ids of each speaker (e.g, [0 10 6]).
|
||||||
|
It is used to perform per-speaker normalization when
|
||||||
|
norm_type='speaker'. Defaults to paddle.to_tensor([], dtype="float32").
|
||||||
|
Returns:
|
||||||
|
paddle.Tensor: The normalized feature or embedding
|
||||||
|
"""
|
||||||
|
N_batches = x.shape[0]
|
||||||
|
# print(f"x shape: {x.shape[1]}")
|
||||||
|
current_means = []
|
||||||
|
current_stds = []
|
||||||
|
|
||||||
|
for snt_id in range(N_batches):
|
||||||
|
|
||||||
|
# Avoiding padded time steps
|
||||||
|
# actual size is the actual time data length
|
||||||
|
actual_size = paddle.round(lengths[snt_id] *
|
||||||
|
x.shape[1]).astype("int32")
|
||||||
|
# computing actual time data statistics
|
||||||
|
current_mean, current_std = self._compute_current_stats(
|
||||||
|
x[snt_id, 0:actual_size, ...].unsqueeze(0))
|
||||||
|
current_means.append(current_mean)
|
||||||
|
current_stds.append(current_std)
|
||||||
|
|
||||||
|
if self.norm_type == "global":
|
||||||
|
current_mean = paddle.mean(paddle.stack(current_means), axis=0)
|
||||||
|
current_std = paddle.mean(paddle.stack(current_stds), axis=0)
|
||||||
|
|
||||||
|
if self.norm_type == "global":
|
||||||
|
|
||||||
|
if self.training:
|
||||||
|
if self.count == 0:
|
||||||
|
self.glob_mean = current_mean
|
||||||
|
self.glob_std = current_std
|
||||||
|
|
||||||
|
else:
|
||||||
|
self.weight = 1 / (self.count + 1)
|
||||||
|
|
||||||
|
self.glob_mean = (
|
||||||
|
1 - self.weight
|
||||||
|
) * self.glob_mean + self.weight * current_mean
|
||||||
|
|
||||||
|
self.glob_std = (
|
||||||
|
1 - self.weight
|
||||||
|
) * self.glob_std + self.weight * current_std
|
||||||
|
|
||||||
|
self.glob_mean.detach()
|
||||||
|
self.glob_std.detach()
|
||||||
|
|
||||||
|
self.count = self.count + 1
|
||||||
|
x = (x - self.glob_mean) / (self.glob_std)
|
||||||
|
return x
|
||||||
|
|
||||||
|
def _compute_current_stats(self, x):
|
||||||
|
"""Returns the tensor with the surrounding context.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
x (paddle.Tensor): A batch of tensors.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
the statistics of the data
|
||||||
|
"""
|
||||||
|
# Compute current mean
|
||||||
|
if self.mean_norm:
|
||||||
|
current_mean = paddle.mean(x, axis=0).detach()
|
||||||
|
else:
|
||||||
|
current_mean = paddle.to_tensor([0.0], dtype="float32")
|
||||||
|
|
||||||
|
# Compute current std
|
||||||
|
if self.std_norm:
|
||||||
|
current_std = paddle.std(x, axis=0).detach()
|
||||||
|
else:
|
||||||
|
current_std = paddle.to_tensor([1.0], dtype="float32")
|
||||||
|
|
||||||
|
# Improving numerical stability of std
|
||||||
|
current_std = paddle.maximum(current_std,
|
||||||
|
self.eps * paddle.ones_like(current_std))
|
||||||
|
|
||||||
|
return current_mean, current_std
|
||||||
|
|
||||||
|
def _statistics_dict(self):
|
||||||
|
"""Fills the dictionary containing the normalization statistics.
|
||||||
|
"""
|
||||||
|
state = {}
|
||||||
|
state["count"] = self.count
|
||||||
|
state["glob_mean"] = self.glob_mean
|
||||||
|
state["glob_std"] = self.glob_std
|
||||||
|
state["spk_dict_mean"] = self.spk_dict_mean
|
||||||
|
state["spk_dict_std"] = self.spk_dict_std
|
||||||
|
state["spk_dict_count"] = self.spk_dict_count
|
||||||
|
|
||||||
|
return state
|
||||||
|
|
||||||
|
def _load_statistics_dict(self, state):
|
||||||
|
"""Loads the dictionary containing the statistics.
|
||||||
|
|
||||||
|
Arguments
|
||||||
|
---------
|
||||||
|
state : dict
|
||||||
|
A dictionary containing the normalization statistics.
|
||||||
|
"""
|
||||||
|
self.count = state["count"]
|
||||||
|
if isinstance(state["glob_mean"], int):
|
||||||
|
self.glob_mean = state["glob_mean"]
|
||||||
|
self.glob_std = state["glob_std"]
|
||||||
|
else:
|
||||||
|
self.glob_mean = state["glob_mean"] # .to(self.device_inp)
|
||||||
|
self.glob_std = state["glob_std"] # .to(self.device_inp)
|
||||||
|
|
||||||
|
# Loading the spk_dict_mean in the right device
|
||||||
|
self.spk_dict_mean = {}
|
||||||
|
for spk in state["spk_dict_mean"]:
|
||||||
|
self.spk_dict_mean[spk] = state["spk_dict_mean"][spk]
|
||||||
|
|
||||||
|
# Loading the spk_dict_std in the right device
|
||||||
|
self.spk_dict_std = {}
|
||||||
|
for spk in state["spk_dict_std"]:
|
||||||
|
self.spk_dict_std[spk] = state["spk_dict_std"][spk]
|
||||||
|
|
||||||
|
self.spk_dict_count = state["spk_dict_count"]
|
||||||
|
|
||||||
|
return state
|
||||||
|
|
||||||
|
def to(self, device):
|
||||||
|
"""Puts the needed tensors in the right device.
|
||||||
|
"""
|
||||||
|
self = super(InputNormalization, self).to(device)
|
||||||
|
self.glob_mean = self.glob_mean.to(device)
|
||||||
|
self.glob_std = self.glob_std.to(device)
|
||||||
|
for spk in self.spk_dict_mean:
|
||||||
|
self.spk_dict_mean[spk] = self.spk_dict_mean[spk].to(device)
|
||||||
|
self.spk_dict_std[spk] = self.spk_dict_std[spk].to(device)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def save(self, path):
|
||||||
|
"""Save statistic dictionary.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path (str): A path where to save the dictionary.
|
||||||
|
"""
|
||||||
|
stats = self._statistics_dict()
|
||||||
|
paddle.save(stats, path)
|
||||||
|
|
||||||
|
def _load(self, path, end_of_epoch=False, device=None):
|
||||||
|
"""Load statistic dictionary.
|
||||||
|
|
||||||
|
Arguments
|
||||||
|
---------
|
||||||
|
path : str
|
||||||
|
The path of the statistic dictionary
|
||||||
|
device : str, None
|
||||||
|
Passed to paddle.load(..., map_location=device)
|
||||||
|
"""
|
||||||
|
del end_of_epoch # Unused here.
|
||||||
|
stats = paddle.load(path, map_location=device)
|
||||||
|
self._load_statistics_dict(stats)
|
||||||
@ -0,0 +1,32 @@
|
|||||||
|
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
def get_chunks(seg_dur, audio_id, audio_duration):
|
||||||
|
"""Get all chunk segments from a utterance
|
||||||
|
|
||||||
|
Args:
|
||||||
|
seg_dur (float): segment chunk duration, seconds
|
||||||
|
audio_id (str): utterance name,
|
||||||
|
audio_duration (float): utterance duration, seconds
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List: all the chunk segments
|
||||||
|
"""
|
||||||
|
num_chunks = int(audio_duration / seg_dur) # all in seconds
|
||||||
|
chunk_lst = [
|
||||||
|
audio_id + "_" + str(i * seg_dur) + "_" + str(i * seg_dur + seg_dur)
|
||||||
|
for i in range(num_chunks)
|
||||||
|
]
|
||||||
|
return chunk_lst
|
||||||
Loading…
Reference in new issue