You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
203 lines
7.3 KiB
203 lines
7.3 KiB
"""
|
|
Download, unpack and create manifest file for the Librespeech dataset.
|
|
|
|
A manifest file is a dataset summarization, with each line a json format
|
|
string containing meta data for one audio clip, including its filepath,
|
|
transcription string, and duration. It serves as a unified interface for
|
|
different data sets.
|
|
"""
|
|
|
|
import paddle.v2 as paddle
|
|
from paddle.v2.dataset.common import md5file
|
|
import distutils.util
|
|
import os
|
|
import wget
|
|
import tarfile
|
|
import argparse
|
|
import soundfile
|
|
import json
|
|
|
|
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')
|
|
|
|
URL_ROOT = "http://www.openslr.org/resources/12"
|
|
URL_TEST_CLEAN = URL_ROOT + "/test-clean.tar.gz"
|
|
URL_TEST_OTHER = URL_ROOT + "/test-other.tar.gz"
|
|
URL_DEV_CLEAN = URL_ROOT + "/dev-clean.tar.gz"
|
|
URL_DEV_OTHER = URL_ROOT + "/dev-other.tar.gz"
|
|
URL_TRAIN_CLEAN_100 = URL_ROOT + "/train-clean-100.tar.gz"
|
|
URL_TRAIN_CLEAN_360 = URL_ROOT + "/train-clean-360.tar.gz"
|
|
URL_TRAIN_OTHER_500 = URL_ROOT + "/train-other-500.tar.gz"
|
|
|
|
MD5_TEST_CLEAN = "32fa31d27d2e1cad72775fee3f4849a9"
|
|
MD5_TEST_OTHER = "fb5a50374b501bb3bac4815ee91d3135"
|
|
MD5_DEV_CLEAN = "42e2234ba48799c1f50f24a7926300a1"
|
|
MD5_DEV_OTHER = "c8d0bcc9cca99d4f8b62fcc847357931"
|
|
MD5_TRAIN_CLEAN_100 = "2a93770f6d5c6c964bc36631d331a522"
|
|
MD5_TRAIN_CLEAN_360 = "c0e676e450a7ff2f54aeade5171606fa"
|
|
MD5_TRAIN_OTHER_500 = "d1a0fd59409feb2c614ce4d30c387708"
|
|
|
|
NUM_LINES_TEST_CLEAN = 2620
|
|
NUM_LINES_TEST_OTHER = 2939
|
|
NUM_LINES_DEV_CLEAN = 2703
|
|
NUM_LINES_DEV_OTHER = 2864
|
|
NUM_LINES_TRAIN_CLEAN_100 = 28539
|
|
NUM_LINES_TRAIN_CLEAN_360 = 104014
|
|
NUM_LINES_TRAIN_OTHER_500 = 148688
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description='Downloads and prepare LibriSpeech dataset.')
|
|
parser.add_argument(
|
|
"--target_dir",
|
|
default=DATA_HOME + "/Libri",
|
|
type=str,
|
|
help="Directory to save the dataset. (default: %(default)s)")
|
|
parser.add_argument(
|
|
"--manifest_prefix",
|
|
default="manifest.libri",
|
|
type=str,
|
|
help="Filepath prefix for output manifests. (default: %(default)s)")
|
|
parser.add_argument(
|
|
"--full_download",
|
|
default="True",
|
|
type=distutils.util.strtobool,
|
|
help="Download all datasets for Librispeech."
|
|
" If False, only download a minimal requirement (test-clean, dev-clean"
|
|
" train-clean-100). (default: %(default)s)")
|
|
args = parser.parse_args()
|
|
|
|
|
|
def download(url, md5sum, target_dir):
|
|
"""
|
|
Download file from url to target_dir, and check md5sum.
|
|
"""
|
|
if not os.path.exists(target_dir): os.makedirs(target_dir)
|
|
filepath = os.path.join(target_dir, url.split("/")[-1])
|
|
if not (os.path.exists(filepath) and md5file(filepath) == md5sum):
|
|
print("Downloading %s ..." % url)
|
|
wget.download(url, target_dir)
|
|
print("\nMD5 Chesksum %s ..." % filepath)
|
|
if not md5file(filepath) == md5sum:
|
|
raise RuntimeError("MD5 checksum failed.")
|
|
else:
|
|
print("File exists, skip downloading. (%s)" % filepath)
|
|
return filepath
|
|
|
|
|
|
def unpack(filepath, target_dir):
|
|
"""
|
|
Unpack the file to the target_dir.
|
|
"""
|
|
print("Unpacking %s ..." % filepath)
|
|
tar = tarfile.open(filepath)
|
|
tar.extractall(target_dir)
|
|
tar.close()
|
|
|
|
|
|
def create_manifest(data_dir, manifest_path):
|
|
"""
|
|
Create a manifest file summarizing the dataset (list of filepath and meta
|
|
data).
|
|
|
|
Each line of the manifest contains one audio clip filepath, its
|
|
transcription text string, and its duration. Manifest file servers as a
|
|
unified interfance to organize data sets.
|
|
"""
|
|
print("Creating manifest %s ..." % manifest_path)
|
|
json_lines = []
|
|
for subfolder, _, filelist in sorted(os.walk(data_dir)):
|
|
text_filelist = [
|
|
filename for filename in filelist if filename.endswith('trans.txt')
|
|
]
|
|
if len(text_filelist) > 0:
|
|
text_filepath = os.path.join(data_dir, subfolder, text_filelist[0])
|
|
for line in open(text_filepath):
|
|
segments = line.strip().split()
|
|
text = ' '.join(segments[1:]).lower()
|
|
audio_filepath = os.path.join(data_dir, subfolder,
|
|
segments[0] + '.flac')
|
|
audio_data, samplerate = soundfile.read(audio_filepath)
|
|
duration = float(len(audio_data)) / samplerate
|
|
json_lines.append(
|
|
json.dumps({
|
|
'audio_filepath': audio_filepath,
|
|
'duration': duration,
|
|
'text': text
|
|
}))
|
|
with open(manifest_path, 'w') as out_file:
|
|
for line in json_lines:
|
|
out_file.write(line + '\n')
|
|
|
|
|
|
def verify_file_line_number(filepath, num_lines):
|
|
with open(filepath, 'r') as file:
|
|
return len(file.readlines()) == num_lines
|
|
|
|
|
|
def prepare_dataset(url, md5sum, target_dir, manifest_path, num_lines):
|
|
"""
|
|
Download, unpack and create summmary manifest file.
|
|
"""
|
|
# download
|
|
filepath = download(url, md5sum, target_dir)
|
|
# unpack
|
|
if not os.path.exists(os.path.join(target_dir, "LibriSpeech")):
|
|
unpack(filepath, target_dir)
|
|
else:
|
|
print("Unpacked data exists, skip unpacking.")
|
|
# create manifest and verify line number
|
|
create_manifest(target_dir, manifest_path)
|
|
if not verify_file_line_number(manifest_path, num_lines):
|
|
raise RuntimeError("Manifest line number check failed. "
|
|
"Please remove directory and try running the script "
|
|
"again.")
|
|
|
|
|
|
def main():
|
|
prepare_dataset(
|
|
url=URL_TEST_CLEAN,
|
|
md5sum=MD5_TEST_CLEAN,
|
|
target_dir=os.path.join(args.target_dir, "test-clean"),
|
|
manifest_path=args.manifest_prefix + ".test-clean",
|
|
num_lines=NUM_LINES_TEST_CLEAN)
|
|
prepare_dataset(
|
|
url=URL_DEV_CLEAN,
|
|
md5sum=MD5_DEV_CLEAN,
|
|
target_dir=os.path.join(args.target_dir, "dev-clean"),
|
|
manifest_path=args.manifest_prefix + ".dev-clean",
|
|
num_lines=NUM_LINES_DEV_CLEAN)
|
|
prepare_dataset(
|
|
url=URL_TRAIN_CLEAN_100,
|
|
md5sum=MD5_TRAIN_CLEAN_100,
|
|
target_dir=os.path.join(args.target_dir, "train-clean-100"),
|
|
manifest_path=args.manifest_prefix + ".train-clean-100",
|
|
num_lines=NUM_LINES_TRAIN_CLEAN_100)
|
|
if args.full_download:
|
|
prepare_dataset(
|
|
url=URL_TEST_OTHER,
|
|
md5sum=MD5_TEST_OTHER,
|
|
target_dir=os.path.join(args.target_dir, "test-other"),
|
|
manifest_path=args.manifest_prefix + ".test-other",
|
|
num_lines=NUM_LINES_TEST_OTHER)
|
|
prepare_dataset(
|
|
url=URL_DEV_OTHER,
|
|
md5sum=MD5_DEV_OTHER,
|
|
target_dir=os.path.join(args.target_dir, "dev-other"),
|
|
manifest_path=args.manifest_prefix + ".dev-other",
|
|
num_lines=NUM_LINES_DEV_OTHER)
|
|
prepare_dataset(
|
|
url=URL_TRAIN_CLEAN_360,
|
|
md5sum=MD5_TRAIN_CLEAN_360,
|
|
target_dir=os.path.join(args.target_dir, "train-clean-360"),
|
|
manifest_path=args.manifest_prefix + ".train-clean-360",
|
|
num_lines=NUM_LINES_TRAIN_CLEAN_360)
|
|
prepare_dataset(
|
|
url=URL_TRAIN_OTHER_500,
|
|
md5sum=MD5_TRAIN_OTHER_500,
|
|
target_dir=os.path.join(args.target_dir, "train-other-500"),
|
|
manifest_path=args.manifest_prefix + ".train-other-500",
|
|
num_lines=NUM_LINES_TRAIN_OTHER_500)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|