You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
102 lines
3.1 KiB
102 lines
3.1 KiB
2 years ago
|
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
"""Normalize feature files and dump them."""
|
||
|
|
import argparse
|
||
|
|
import logging
|
||
|
|
from operator import itemgetter
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import jsonlines
|
||
|
|
import numpy as np
|
||
|
|
import tqdm
|
||
|
|
|
||
|
|
from paddlespeech.t2s.datasets.data_table import DataTable
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
"""Run preprocessing process."""
|
||
|
|
parser = argparse.ArgumentParser(
|
||
|
|
description="Normalize dumped raw features (See detail in parallel_wavegan/bin/normalize.py)."
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"--metadata",
|
||
|
|
type=str,
|
||
|
|
required=True,
|
||
|
|
help="directory including feature files to be normalized. "
|
||
|
|
"you need to specify either *-scp or rootdir.")
|
||
|
|
|
||
|
|
parser.add_argument(
|
||
|
|
"--dumpdir",
|
||
|
|
type=str,
|
||
|
|
required=True,
|
||
|
|
help="directory to dump normalized feature files.")
|
||
|
|
|
||
|
|
parser.add_argument(
|
||
|
|
"--speaker-dict", type=str, default=None, help="speaker id map file.")
|
||
|
|
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
dumpdir = Path(args.dumpdir).expanduser()
|
||
|
|
# use absolute path
|
||
|
|
dumpdir = dumpdir.resolve()
|
||
|
|
dumpdir.mkdir(parents=True, exist_ok=True)
|
||
|
|
|
||
|
|
# get dataset
|
||
|
|
with jsonlines.open(args.metadata, 'r') as reader:
|
||
|
|
metadata = list(reader)
|
||
|
|
dataset = DataTable(
|
||
|
|
metadata, converters={
|
||
|
|
"speech": np.load,
|
||
|
|
})
|
||
|
|
logging.info(f"The number of files = {len(dataset)}.")
|
||
|
|
|
||
|
|
vocab_speaker = {}
|
||
|
|
with open(args.speaker_dict, 'rt') as f:
|
||
|
|
spk_id = [line.strip().split() for line in f.readlines()]
|
||
|
|
for spk, id in spk_id:
|
||
|
|
vocab_speaker[spk] = int(id)
|
||
|
|
|
||
|
|
# process each file
|
||
|
|
output_metadata = []
|
||
|
|
|
||
|
|
for item in tqdm.tqdm(dataset):
|
||
|
|
utt_id = item['utt_id']
|
||
|
|
speech = item['speech']
|
||
|
|
|
||
|
|
# normalize
|
||
|
|
# 这里暂时写死
|
||
|
|
mean, std = -4, 4
|
||
|
|
speech = (speech - mean) / std
|
||
|
|
speech_path = dumpdir / f"{utt_id}_speech.npy"
|
||
|
|
np.save(speech_path, speech.astype(np.float32), allow_pickle=False)
|
||
|
|
|
||
|
|
spk_id = vocab_speaker[item["speaker"]]
|
||
|
|
record = {
|
||
|
|
"utt_id": item['utt_id'],
|
||
|
|
"spk_id": spk_id,
|
||
|
|
"speech": str(speech_path),
|
||
|
|
}
|
||
|
|
|
||
|
|
output_metadata.append(record)
|
||
|
|
output_metadata.sort(key=itemgetter('utt_id'))
|
||
|
|
output_metadata_path = Path(args.dumpdir) / "metadata.jsonl"
|
||
|
|
with jsonlines.open(output_metadata_path, 'w') as writer:
|
||
|
|
for item in output_metadata:
|
||
|
|
writer.write(item)
|
||
|
|
logging.info(f"metadata dumped into {output_metadata_path}")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|