Xinghai Sun
7 years ago
parent
a00a436b52
commit
ae7ef7929a
@ -0,0 +1,126 @@
|
||||
"""Prepare Librispeech ASR datasets.
|
||||
|
||||
Download, unpack and create manifest files.
|
||||
Manifest file is a json-format file with each line containing the
|
||||
meta data (i.e. audio filepath, transcript and audio duration)
|
||||
of each audio file in the data set.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import distutils.util
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
import argparse
|
||||
import soundfile
|
||||
import json
|
||||
import codecs
|
||||
from paddle.v2.dataset.common import md5file
|
||||
|
||||
DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset/speech')
|
||||
|
||||
URL_ROOT = "http://www.openslr.org/resources/12"
|
||||
URL_DEV_CLEAN = URL_ROOT + "/dev-clean.tar.gz"
|
||||
MD5_DEV_CLEAN = "42e2234ba48799c1f50f24a7926300a1"
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--target_dir",
|
||||
default=DATA_HOME + "/tiny",
|
||||
type=str,
|
||||
help="Directory to save the dataset. (default: %(default)s)")
|
||||
parser.add_argument(
|
||||
"--manifest_prefix",
|
||||
default="manifest",
|
||||
type=str,
|
||||
help="Filepath prefix for output manifests. (default: %(default)s)")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def download(url, md5sum, target_dir):
|
||||
"""
|
||||
Download file from url to target_dir, and check md5sum.
|
||||
"""
|
||||
if not os.path.exists(target_dir): os.makedirs(target_dir)
|
||||
filepath = os.path.join(target_dir, url.split("/")[-1])
|
||||
if not (os.path.exists(filepath) and md5file(filepath) == md5sum):
|
||||
print("Downloading %s ..." % url)
|
||||
os.system("wget -c " + url + " -P " + target_dir)
|
||||
print("\nMD5 Chesksum %s ..." % filepath)
|
||||
if not md5file(filepath) == md5sum:
|
||||
raise RuntimeError("MD5 checksum failed.")
|
||||
else:
|
||||
print("File exists, skip downloading. (%s)" % filepath)
|
||||
return filepath
|
||||
|
||||
|
||||
def unpack(filepath, target_dir):
|
||||
"""
|
||||
Unpack the file to the target_dir.
|
||||
"""
|
||||
print("Unpacking %s ..." % filepath)
|
||||
tar = tarfile.open(filepath)
|
||||
tar.extractall(target_dir)
|
||||
tar.close()
|
||||
|
||||
|
||||
def create_manifest(data_dir, manifest_path):
|
||||
"""
|
||||
Create a manifest json file summarizing the data set, with each line
|
||||
containing the meta data (i.e. audio filepath, transcription text, audio
|
||||
duration) of each audio file within the data set.
|
||||
"""
|
||||
print("Creating manifest %s ..." % manifest_path)
|
||||
json_lines = []
|
||||
for subfolder, _, filelist in sorted(os.walk(data_dir)):
|
||||
text_filelist = [
|
||||
filename for filename in filelist if filename.endswith('trans.txt')
|
||||
]
|
||||
if len(text_filelist) > 0:
|
||||
text_filepath = os.path.join(data_dir, subfolder, text_filelist[0])
|
||||
for line in open(text_filepath):
|
||||
segments = line.strip().split()
|
||||
text = ' '.join(segments[1:]).lower()
|
||||
audio_filepath = os.path.join(data_dir, subfolder,
|
||||
segments[0] + '.flac')
|
||||
audio_data, samplerate = soundfile.read(audio_filepath)
|
||||
duration = float(len(audio_data)) / samplerate
|
||||
json_lines.append(
|
||||
json.dumps({
|
||||
'audio_filepath': audio_filepath,
|
||||
'duration': duration,
|
||||
'text': text
|
||||
}))
|
||||
with codecs.open(manifest_path, 'w', 'utf-8') as out_file:
|
||||
for line in json_lines:
|
||||
out_file.write(line + '\n')
|
||||
|
||||
|
||||
def prepare_dataset(url, md5sum, target_dir, manifest_path):
|
||||
"""
|
||||
Download, unpack and create summmary manifest file.
|
||||
"""
|
||||
if not os.path.exists(os.path.join(target_dir, "LibriSpeech")):
|
||||
# download
|
||||
filepath = download(url, md5sum, target_dir)
|
||||
# unpack
|
||||
unpack(filepath, target_dir)
|
||||
else:
|
||||
print("Skip downloading and unpacking. Data already exists in %s." %
|
||||
target_dir)
|
||||
# create manifest json file
|
||||
create_manifest(target_dir, manifest_path)
|
||||
|
||||
|
||||
def main():
|
||||
prepare_dataset(
|
||||
url=URL_DEV_CLEAN,
|
||||
md5sum=MD5_DEV_CLEAN,
|
||||
target_dir=os.path.join(args.target_dir, "dev-clean"),
|
||||
manifest_path=args.manifest_prefix + ".dev-clean")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,39 +0,0 @@
|
||||
#! /usr/bin/bash
|
||||
|
||||
pushd ../..
|
||||
|
||||
# download data, generate manifests
|
||||
python data/librispeech/librispeech.py \
|
||||
--manifest_prefix='data/librispeech/manifest' \
|
||||
--full_download='True' \
|
||||
--target_dir='~/.cache/paddle/dataset/speech/Libri'
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Prepare LibriSpeech failed. Terminated."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat data/librispeech/manifest.train* | shuf > data/librispeech/manifest.train
|
||||
|
||||
|
||||
# build vocabulary (for English data, we can just skip this)
|
||||
# python tools/build_vocab.py \
|
||||
# --count_threshold=0 \
|
||||
# --vocab_path='data/librispeech/eng_vocab.txt' \
|
||||
# --manifest_paths='data/librispeech/manifeset.train'
|
||||
|
||||
|
||||
# compute mean and stddev for normalizer
|
||||
python tools/compute_mean_std.py \
|
||||
--manifest_path='data/librispeech/manifest.train' \
|
||||
--num_samples=2000 \
|
||||
--specgram_type='linear' \
|
||||
--output_path='data/librispeech/mean_std.npz'
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compute mean and stddev failed. Terminated."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
echo "LibriSpeech Data preparation done."
|
||||
@ -0,0 +1,45 @@
|
||||
#! /usr/bin/bash
|
||||
|
||||
pushd ../..
|
||||
|
||||
# download data, generate manifests
|
||||
python data/tiny/tiny.py \
|
||||
--manifest_prefix='data/tiny/manifest' \
|
||||
--target_dir=$HOME'/.cache/paddle/dataset/speech/tiny'
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Prepare LibriSpeech failed. Terminated."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat data/tiny/manifest.dev-clean | head -n 32 > data/tiny/manifest.train
|
||||
cat data/tiny/manifest.dev-clean | head -n 48 | tail -n 16 > data/tiny/manifest.dev
|
||||
cat data/tiny/manifest.dev-clean | head -n 64 | tail -n 16 > data/tiny/manifest.test
|
||||
|
||||
|
||||
# build vocabulary
|
||||
python tools/build_vocab.py \
|
||||
--count_threshold=0 \
|
||||
--vocab_path='data/tiny/vocab.txt' \
|
||||
--manifest_paths='data/tiny/manifest.train'
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Build vocabulary failed. Terminated."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# compute mean and stddev for normalizer
|
||||
python tools/compute_mean_std.py \
|
||||
--manifest_path='data/tiny/manifest.train' \
|
||||
--num_samples=32 \
|
||||
--specgram_type='linear' \
|
||||
--output_path='data/tiny/mean_std.npz'
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Compute mean and stddev failed. Terminated."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
echo "Tiny data preparation done."
|
||||
Loading…
Reference in new issue