Merge branch 'develop' of https://github.com/PaddlePaddle/models into fix-81
yangyaming
8 years ago
commit
86eeb52c06
@ -1,411 +0,0 @@
|
||||
"""
|
||||
Providing basic audio data preprocessing pipeline, and offering
|
||||
both instance-level and batch-level data reader interfaces.
|
||||
"""
|
||||
import paddle.v2 as paddle
|
||||
import logging
|
||||
import json
|
||||
import random
|
||||
import soundfile
|
||||
import numpy as np
|
||||
import itertools
|
||||
import os
|
||||
|
||||
RANDOM_SEED = 0
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DataGenerator(object):
|
||||
"""
|
||||
DataGenerator provides basic audio data preprocessing pipeline, and offers
|
||||
both instance-level and batch-level data reader interfaces.
|
||||
Normalized FFT are used as audio features here.
|
||||
|
||||
:param vocab_filepath: Vocabulary file path for indexing tokenized
|
||||
transcriptions.
|
||||
:type vocab_filepath: basestring
|
||||
:param normalizer_manifest_path: Manifest filepath for collecting feature
|
||||
normalization statistics, e.g. mean, std.
|
||||
:type normalizer_manifest_path: basestring
|
||||
:param normalizer_num_samples: Number of instances sampled for collecting
|
||||
feature normalization statistics.
|
||||
Default is 100.
|
||||
:type normalizer_num_samples: int
|
||||
:param max_duration: Audio clips with duration (in seconds) greater than
|
||||
this will be discarded. Default is 20.0.
|
||||
:type max_duration: float
|
||||
:param min_duration: Audio clips with duration (in seconds) smaller than
|
||||
this will be discarded. Default is 0.0.
|
||||
:type min_duration: float
|
||||
:param stride_ms: Striding size (in milliseconds) for generating frames.
|
||||
Default is 10.0.
|
||||
:type stride_ms: float
|
||||
:param window_ms: Window size (in milliseconds) for frames. Default is 20.0.
|
||||
:type window_ms: float
|
||||
:param max_frequency: Maximun frequency for FFT features. FFT features of
|
||||
frequency larger than this will be discarded.
|
||||
If set None, all features will be kept.
|
||||
Default is None.
|
||||
:type max_frequency: float
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
vocab_filepath,
|
||||
normalizer_manifest_path,
|
||||
normalizer_num_samples=100,
|
||||
max_duration=20.0,
|
||||
min_duration=0.0,
|
||||
stride_ms=10.0,
|
||||
window_ms=20.0,
|
||||
max_frequency=None):
|
||||
self.__max_duration__ = max_duration
|
||||
self.__min_duration__ = min_duration
|
||||
self.__stride_ms__ = stride_ms
|
||||
self.__window_ms__ = window_ms
|
||||
self.__max_frequency__ = max_frequency
|
||||
self.__epoc__ = 0
|
||||
self.__random__ = random.Random(RANDOM_SEED)
|
||||
# load vocabulary (dictionary)
|
||||
self.__vocab_dict__, self.__vocab_list__ = \
|
||||
self.__load_vocabulary_from_file__(vocab_filepath)
|
||||
# collect normalizer statistics
|
||||
self.__mean__, self.__std__ = self.__collect_normalizer_statistics__(
|
||||
manifest_path=normalizer_manifest_path,
|
||||
num_samples=normalizer_num_samples)
|
||||
|
||||
def __audio_featurize__(self, audio_filename):
|
||||
"""
|
||||
Preprocess audio data, including feature extraction, normalization etc..
|
||||
"""
|
||||
features = self.__audio_basic_featurize__(audio_filename)
|
||||
return self.__normalize__(features)
|
||||
|
||||
def __text_featurize__(self, text):
|
||||
"""
|
||||
Preprocess text data, including tokenizing and token indexing etc..
|
||||
"""
|
||||
return self.__convert_text_to_char_index__(
|
||||
text=text, vocabulary=self.__vocab_dict__)
|
||||
|
||||
def __audio_basic_featurize__(self, audio_filename):
|
||||
"""
|
||||
Compute basic (without normalization etc.) features for audio data.
|
||||
"""
|
||||
return self.__spectrogram_from_file__(
|
||||
filename=audio_filename,
|
||||
stride_ms=self.__stride_ms__,
|
||||
window_ms=self.__window_ms__,
|
||||
max_freq=self.__max_frequency__)
|
||||
|
||||
def __collect_normalizer_statistics__(self, manifest_path, num_samples=100):
|
||||
"""
|
||||
Compute feature normalization statistics, i.e. mean and stddev.
|
||||
"""
|
||||
# read manifest
|
||||
manifest = self.__read_manifest__(
|
||||
manifest_path=manifest_path,
|
||||
max_duration=self.__max_duration__,
|
||||
min_duration=self.__min_duration__)
|
||||
# sample for statistics
|
||||
sampled_manifest = self.__random__.sample(manifest, num_samples)
|
||||
# extract spectrogram feature
|
||||
features = []
|
||||
for instance in sampled_manifest:
|
||||
spectrogram = self.__audio_basic_featurize__(
|
||||
instance["audio_filepath"])
|
||||
features.append(spectrogram)
|
||||
features = np.hstack(features)
|
||||
mean = np.mean(features, axis=1).reshape([-1, 1])
|
||||
std = np.std(features, axis=1).reshape([-1, 1])
|
||||
return mean, std
|
||||
|
||||
def __normalize__(self, features, eps=1e-14):
|
||||
"""
|
||||
Normalize features to be of zero mean and unit stddev.
|
||||
"""
|
||||
return (features - self.__mean__) / (self.__std__ + eps)
|
||||
|
||||
def __spectrogram_from_file__(self,
|
||||
filename,
|
||||
stride_ms=10.0,
|
||||
window_ms=20.0,
|
||||
max_freq=None,
|
||||
eps=1e-14):
|
||||
"""
|
||||
Laod audio data and calculate the log of spectrogram by FFT.
|
||||
Refer to utils.py in https://github.com/baidu-research/ba-dls-deepspeech
|
||||
"""
|
||||
audio, sample_rate = soundfile.read(filename)
|
||||
if audio.ndim >= 2:
|
||||
audio = np.mean(audio, 1)
|
||||
if max_freq is None:
|
||||
max_freq = sample_rate / 2
|
||||
if max_freq > sample_rate / 2:
|
||||
raise ValueError("max_freq must be greater than half of "
|
||||
"sample rate.")
|
||||
if stride_ms > window_ms:
|
||||
raise ValueError("Stride size must not be greater than "
|
||||
"window size.")
|
||||
stride_size = int(0.001 * sample_rate * stride_ms)
|
||||
window_size = int(0.001 * sample_rate * window_ms)
|
||||
spectrogram, freqs = self.__extract_spectrogram__(
|
||||
audio,
|
||||
window_size=window_size,
|
||||
stride_size=stride_size,
|
||||
sample_rate=sample_rate)
|
||||
ind = np.where(freqs <= max_freq)[0][-1] + 1
|
||||
return np.log(spectrogram[:ind, :] + eps)
|
||||
|
||||
def __extract_spectrogram__(self, samples, window_size, stride_size,
|
||||
sample_rate):
|
||||
"""
|
||||
Compute the spectrogram by FFT for a discrete real signal.
|
||||
Refer to utils.py in https://github.com/baidu-research/ba-dls-deepspeech
|
||||
"""
|
||||
# extract strided windows
|
||||
truncate_size = (len(samples) - window_size) % stride_size
|
||||
samples = samples[:len(samples) - truncate_size]
|
||||
nshape = (window_size, (len(samples) - window_size) // stride_size + 1)
|
||||
nstrides = (samples.strides[0], samples.strides[0] * stride_size)
|
||||
windows = np.lib.stride_tricks.as_strided(
|
||||
samples, shape=nshape, strides=nstrides)
|
||||
assert np.all(
|
||||
windows[:, 1] == samples[stride_size:(stride_size + window_size)])
|
||||
# window weighting, squared Fast Fourier Transform (fft), scaling
|
||||
weighting = np.hanning(window_size)[:, None]
|
||||
fft = np.fft.rfft(windows * weighting, axis=0)
|
||||
fft = np.absolute(fft)**2
|
||||
scale = np.sum(weighting**2) * sample_rate
|
||||
fft[1:-1, :] *= (2.0 / scale)
|
||||
fft[(0, -1), :] /= scale
|
||||
# prepare fft frequency list
|
||||
freqs = float(sample_rate) / window_size * np.arange(fft.shape[0])
|
||||
return fft, freqs
|
||||
|
||||
def __load_vocabulary_from_file__(self, vocabulary_path):
|
||||
"""
|
||||
Load vocabulary from file.
|
||||
"""
|
||||
if not os.path.exists(vocabulary_path):
|
||||
raise ValueError("Vocabulary file %s not found.", vocabulary_path)
|
||||
vocab_lines = []
|
||||
with open(vocabulary_path, 'r') as file:
|
||||
vocab_lines.extend(file.readlines())
|
||||
vocab_list = [line[:-1] for line in vocab_lines]
|
||||
vocab_dict = dict(
|
||||
[(token, id) for (id, token) in enumerate(vocab_list)])
|
||||
return vocab_dict, vocab_list
|
||||
|
||||
def __convert_text_to_char_index__(self, text, vocabulary):
|
||||
"""
|
||||
Convert text string to a list of character index integers.
|
||||
"""
|
||||
return [vocabulary[w] for w in text]
|
||||
|
||||
def __read_manifest__(self, manifest_path, max_duration, min_duration):
|
||||
"""
|
||||
Load and parse manifest file.
|
||||
"""
|
||||
manifest = []
|
||||
for json_line in open(manifest_path):
|
||||
try:
|
||||
json_data = json.loads(json_line)
|
||||
except Exception as e:
|
||||
raise ValueError("Error reading manifest: %s" % str(e))
|
||||
if (json_data["duration"] <= max_duration and
|
||||
json_data["duration"] >= min_duration):
|
||||
manifest.append(json_data)
|
||||
return manifest
|
||||
|
||||
def __padding_batch__(self, batch, padding_to=-1, flatten=False):
|
||||
"""
|
||||
Padding audio part of features (only in the time axis -- column axis)
|
||||
with zeros, to make each instance in the batch share the same
|
||||
audio feature shape.
|
||||
|
||||
If `padding_to` is set -1, the maximun column numbers in the batch will
|
||||
be used as the target size. Otherwise, `padding_to` will be the target
|
||||
size. Default is -1.
|
||||
|
||||
If `flatten` is set True, audio data will be flatten to be a 1-dim
|
||||
ndarray. Default is False.
|
||||
"""
|
||||
new_batch = []
|
||||
# get target shape
|
||||
max_length = max([audio.shape[1] for audio, text in batch])
|
||||
if padding_to != -1:
|
||||
if padding_to < max_length:
|
||||
raise ValueError("If padding_to is not -1, it should be greater"
|
||||
" or equal to the original instance length.")
|
||||
max_length = padding_to
|
||||
# padding
|
||||
for audio, text in batch:
|
||||
padded_audio = np.zeros([audio.shape[0], max_length])
|
||||
padded_audio[:, :audio.shape[1]] = audio
|
||||
if flatten:
|
||||
padded_audio = padded_audio.flatten()
|
||||
new_batch.append((padded_audio, text))
|
||||
return new_batch
|
||||
|
||||
def __batch_shuffle__(self, manifest, batch_size):
|
||||
"""
|
||||
The instances have different lengths and they cannot be
|
||||
combined into a single matrix multiplication. It usually
|
||||
sorts the training examples by length and combines only
|
||||
similarly-sized instances into minibatches, pads with
|
||||
silence when necessary so that all instances in a batch
|
||||
have the same length. This batch shuffle fuction is used
|
||||
to make similarly-sized instances into minibatches and
|
||||
make a batch-wise shuffle.
|
||||
|
||||
1. Sort the audio clips by duration.
|
||||
2. Generate a random number `k`, k in [0, batch_size).
|
||||
3. Randomly remove `k` instances in order to make different mini-batches,
|
||||
then make minibatches and each minibatch size is batch_size.
|
||||
4. Shuffle the minibatches.
|
||||
|
||||
:param manifest: manifest file.
|
||||
:type manifest: list
|
||||
:param batch_size: Batch size. This size is also used for generate
|
||||
a random number for batch shuffle.
|
||||
:type batch_size: int
|
||||
:return: batch shuffled mainifest.
|
||||
:rtype: list
|
||||
"""
|
||||
manifest.sort(key=lambda x: x["duration"])
|
||||
shift_len = self.__random__.randint(0, batch_size - 1)
|
||||
batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)
|
||||
self.__random__.shuffle(batch_manifest)
|
||||
batch_manifest = list(sum(batch_manifest, ()))
|
||||
res_len = len(manifest) - shift_len - len(batch_manifest)
|
||||
batch_manifest.extend(manifest[-res_len:])
|
||||
batch_manifest.extend(manifest[0:shift_len])
|
||||
return batch_manifest
|
||||
|
||||
def instance_reader_creator(self, manifest):
|
||||
"""
|
||||
Instance reader creator for audio data. Creat a callable function to
|
||||
produce instances of data.
|
||||
|
||||
Instance: a tuple of a numpy ndarray of audio spectrogram and a list of
|
||||
tokenized and indexed transcription text.
|
||||
|
||||
:param manifest: Filepath of manifest for audio clip files.
|
||||
:type manifest: basestring
|
||||
:return: Data reader function.
|
||||
:rtype: callable
|
||||
"""
|
||||
|
||||
def reader():
|
||||
# extract spectrogram feature
|
||||
for instance in manifest:
|
||||
spectrogram = self.__audio_featurize__(
|
||||
instance["audio_filepath"])
|
||||
transcript = self.__text_featurize__(instance["text"])
|
||||
yield (spectrogram, transcript)
|
||||
|
||||
return reader
|
||||
|
||||
def batch_reader_creator(self,
|
||||
manifest_path,
|
||||
batch_size,
|
||||
padding_to=-1,
|
||||
flatten=False,
|
||||
sortagrad=False,
|
||||
batch_shuffle=False):
|
||||
"""
|
||||
Batch data reader creator for audio data. Creat a callable function to
|
||||
produce batches of data.
|
||||
|
||||
Audio features will be padded with zeros to make each instance in the
|
||||
batch to share the same audio feature shape.
|
||||
|
||||
:param manifest_path: Filepath of manifest for audio clip files.
|
||||
:type manifest_path: basestring
|
||||
:param batch_size: Instance number in a batch.
|
||||
:type batch_size: int
|
||||
:param padding_to: If set -1, the maximun column numbers in the batch
|
||||
will be used as the target size for padding.
|
||||
Otherwise, `padding_to` will be the target size.
|
||||
Default is -1.
|
||||
:type padding_to: int
|
||||
:param flatten: If set True, audio data will be flatten to be a 1-dim
|
||||
ndarray. Otherwise, 2-dim ndarray. Default is False.
|
||||
:type flatten: bool
|
||||
:param sortagrad: Sort the audio clips by duration in the first epoc
|
||||
if set True.
|
||||
:type sortagrad: bool
|
||||
:param batch_shuffle: Shuffle the audio clips if set True. It is
|
||||
not a thorough instance-wise shuffle, but a
|
||||
specific batch-wise shuffle. For more details,
|
||||
please see `__batch_shuffle__` function.
|
||||
:type batch_shuffle: bool
|
||||
:return: Batch reader function, producing batches of data when called.
|
||||
:rtype: callable
|
||||
"""
|
||||
|
||||
def batch_reader():
|
||||
# read manifest
|
||||
manifest = self.__read_manifest__(
|
||||
manifest_path=manifest_path,
|
||||
max_duration=self.__max_duration__,
|
||||
min_duration=self.__min_duration__)
|
||||
|
||||
# sort (by duration) or shuffle manifest
|
||||
if self.__epoc__ == 0 and sortagrad:
|
||||
manifest.sort(key=lambda x: x["duration"])
|
||||
elif batch_shuffle:
|
||||
manifest = self.__batch_shuffle__(manifest, batch_size)
|
||||
|
||||
instance_reader = self.instance_reader_creator(manifest)
|
||||
batch = []
|
||||
for instance in instance_reader():
|
||||
batch.append(instance)
|
||||
if len(batch) == batch_size:
|
||||
yield self.__padding_batch__(batch, padding_to, flatten)
|
||||
batch = []
|
||||
if len(batch) > 0:
|
||||
yield self.__padding_batch__(batch, padding_to, flatten)
|
||||
self.__epoc__ += 1
|
||||
|
||||
return batch_reader
|
||||
|
||||
def vocabulary_size(self):
|
||||
"""
|
||||
Get vocabulary size.
|
||||
|
||||
:return: Vocabulary size.
|
||||
:rtype: int
|
||||
"""
|
||||
return len(self.__vocab_list__)
|
||||
|
||||
def vocabulary_dict(self):
|
||||
"""
|
||||
Get vocabulary in dict.
|
||||
|
||||
:return: Vocabulary in dict.
|
||||
:rtype: dict
|
||||
"""
|
||||
return self.__vocab_dict__
|
||||
|
||||
def vocabulary_list(self):
|
||||
"""
|
||||
Get vocabulary in list.
|
||||
|
||||
:return: Vocabulary in list
|
||||
:rtype: list
|
||||
"""
|
||||
return self.__vocab_list__
|
||||
|
||||
def data_name_feeding(self):
|
||||
"""
|
||||
Get feeddings (data field name and corresponding field id).
|
||||
|
||||
:return: Feeding dict.
|
||||
:rtype: dict
|
||||
"""
|
||||
feeding = {
|
||||
"audio_spectrogram": 0,
|
||||
"transcript_text": 1,
|
||||
}
|
||||
return feeding
|
||||
@ -0,0 +1,57 @@
|
||||
"""Compute mean and std for feature normalizer, and save to file."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
from data_utils.normalizer import FeatureNormalizer
|
||||
from data_utils.augmentor.augmentation import AugmentationPipeline
|
||||
from data_utils.featurizer.audio_featurizer import AudioFeaturizer
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Computing mean and stddev for feature normalizer.')
|
||||
parser.add_argument(
|
||||
"--manifest_path",
|
||||
default='datasets/manifest.train',
|
||||
type=str,
|
||||
help="Manifest path for computing normalizer's mean and stddev."
|
||||
"(default: %(default)s)")
|
||||
parser.add_argument(
|
||||
"--num_samples",
|
||||
default=2000,
|
||||
type=int,
|
||||
help="Number of samples for computing mean and stddev. "
|
||||
"(default: %(default)s)")
|
||||
parser.add_argument(
|
||||
"--augmentation_config",
|
||||
default='{}',
|
||||
type=str,
|
||||
help="Augmentation configuration in json-format. "
|
||||
"(default: %(default)s)")
|
||||
parser.add_argument(
|
||||
"--output_file",
|
||||
default='mean_std.npz',
|
||||
type=str,
|
||||
help="Filepath to write mean and std to (.npz)."
|
||||
"(default: %(default)s)")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
augmentation_pipeline = AugmentationPipeline(args.augmentation_config)
|
||||
audio_featurizer = AudioFeaturizer()
|
||||
|
||||
def augment_and_featurize(audio_segment):
|
||||
augmentation_pipeline.transform_audio(audio_segment)
|
||||
return audio_featurizer.featurize(audio_segment)
|
||||
|
||||
normalizer = FeatureNormalizer(
|
||||
mean_std_filepath=None,
|
||||
manifest_path=args.manifest_path,
|
||||
featurize_func=augment_and_featurize,
|
||||
num_samples=args.num_samples)
|
||||
normalizer.write_to_file(args.output_file)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,252 @@
|
||||
"""Contains the audio segment class."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import io
|
||||
import soundfile
|
||||
|
||||
|
||||
class AudioSegment(object):
|
||||
"""Monaural audio segment abstraction.
|
||||
|
||||
:param samples: Audio samples [num_samples x num_channels].
|
||||
:type samples: ndarray.float32
|
||||
:param sample_rate: Audio sample rate.
|
||||
:type sample_rate: int
|
||||
:raises TypeError: If the sample data type is not float or int.
|
||||
"""
|
||||
|
||||
def __init__(self, samples, sample_rate):
|
||||
"""Create audio segment from samples.
|
||||
|
||||
Samples are convert float32 internally, with int scaled to [-1, 1].
|
||||
"""
|
||||
self._samples = self._convert_samples_to_float32(samples)
|
||||
self._sample_rate = sample_rate
|
||||
if self._samples.ndim >= 2:
|
||||
self._samples = np.mean(self._samples, 1)
|
||||
|
||||
def __eq__(self, other):
|
||||
"""Return whether two objects are equal."""
|
||||
if type(other) is not type(self):
|
||||
return False
|
||||
if self._sample_rate != other._sample_rate:
|
||||
return False
|
||||
if self._samples.shape != other._samples.shape:
|
||||
return False
|
||||
if np.any(self.samples != other._samples):
|
||||
return False
|
||||
return True
|
||||
|
||||
def __ne__(self, other):
|
||||
"""Return whether two objects are unequal."""
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __str__(self):
|
||||
"""Return human-readable representation of segment."""
|
||||
return ("%s: num_samples=%d, sample_rate=%d, duration=%.2fsec, "
|
||||
"rms=%.2fdB" % (type(self), self.num_samples, self.sample_rate,
|
||||
self.duration, self.rms_db))
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, file):
|
||||
"""Create audio segment from audio file.
|
||||
|
||||
:param filepath: Filepath or file object to audio file.
|
||||
:type filepath: basestring|file
|
||||
:return: Audio segment instance.
|
||||
:rtype: AudioSegment
|
||||
"""
|
||||
samples, sample_rate = soundfile.read(file, dtype='float32')
|
||||
return cls(samples, sample_rate)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, bytes):
|
||||
"""Create audio segment from a byte string containing audio samples.
|
||||
|
||||
:param bytes: Byte string containing audio samples.
|
||||
:type bytes: str
|
||||
:return: Audio segment instance.
|
||||
:rtype: AudioSegment
|
||||
"""
|
||||
samples, sample_rate = soundfile.read(
|
||||
io.BytesIO(bytes), dtype='float32')
|
||||
return cls(samples, sample_rate)
|
||||
|
||||
def to_wav_file(self, filepath, dtype='float32'):
|
||||
"""Save audio segment to disk as wav file.
|
||||
|
||||
:param filepath: WAV filepath or file object to save the
|
||||
audio segment.
|
||||
:type filepath: basestring|file
|
||||
:param dtype: Subtype for audio file. Options: 'int16', 'int32',
|
||||
'float32', 'float64'. Default is 'float32'.
|
||||
:type dtype: str
|
||||
:raises TypeError: If dtype is not supported.
|
||||
"""
|
||||
samples = self._convert_samples_from_float32(self._samples, dtype)
|
||||
subtype_map = {
|
||||
'int16': 'PCM_16',
|
||||
'int32': 'PCM_32',
|
||||
'float32': 'FLOAT',
|
||||
'float64': 'DOUBLE'
|
||||
}
|
||||
soundfile.write(
|
||||
filepath,
|
||||
samples,
|
||||
self._sample_rate,
|
||||
format='WAV',
|
||||
subtype=subtype_map[dtype])
|
||||
|
||||
def to_bytes(self, dtype='float32'):
|
||||
"""Create a byte string containing the audio content.
|
||||
|
||||
:param dtype: Data type for export samples. Options: 'int16', 'int32',
|
||||
'float32', 'float64'. Default is 'float32'.
|
||||
:type dtype: str
|
||||
:return: Byte string containing audio content.
|
||||
:rtype: str
|
||||
"""
|
||||
samples = self._convert_samples_from_float32(self._samples, dtype)
|
||||
return samples.tostring()
|
||||
|
||||
def apply_gain(self, gain):
|
||||
"""Apply gain in decibels to samples.
|
||||
|
||||
Note that this is an in-place transformation.
|
||||
|
||||
:param gain: Gain in decibels to apply to samples.
|
||||
:type gain: float
|
||||
"""
|
||||
self._samples *= 10.**(gain / 20.)
|
||||
|
||||
def change_speed(self, speed_rate):
|
||||
"""Change the audio speed by linear interpolation.
|
||||
|
||||
Note that this is an in-place transformation.
|
||||
|
||||
:param speed_rate: Rate of speed change:
|
||||
speed_rate > 1.0, speed up the audio;
|
||||
speed_rate = 1.0, unchanged;
|
||||
speed_rate < 1.0, slow down the audio;
|
||||
speed_rate <= 0.0, not allowed, raise ValueError.
|
||||
:type speed_rate: float
|
||||
:raises ValueError: If speed_rate <= 0.0.
|
||||
"""
|
||||
if speed_rate <= 0:
|
||||
raise ValueError("speed_rate should be greater than zero.")
|
||||
old_length = self._samples.shape[0]
|
||||
new_length = int(old_length / speed_rate)
|
||||
old_indices = np.arange(old_length)
|
||||
new_indices = np.linspace(start=0, stop=old_length, num=new_length)
|
||||
self._samples = np.interp(new_indices, old_indices, self._samples)
|
||||
|
||||
def normalize(self, target_sample_rate):
|
||||
raise NotImplementedError()
|
||||
|
||||
def resample(self, target_sample_rate):
|
||||
raise NotImplementedError()
|
||||
|
||||
def pad_silence(self, duration, sides='both'):
|
||||
raise NotImplementedError()
|
||||
|
||||
def subsegment(self, start_sec=None, end_sec=None):
|
||||
raise NotImplementedError()
|
||||
|
||||
def convolve(self, filter, allow_resample=False):
|
||||
raise NotImplementedError()
|
||||
|
||||
def convolve_and_normalize(self, filter, allow_resample=False):
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
def samples(self):
|
||||
"""Return audio samples.
|
||||
|
||||
:return: Audio samples.
|
||||
:rtype: ndarray
|
||||
"""
|
||||
return self._samples.copy()
|
||||
|
||||
@property
|
||||
def sample_rate(self):
|
||||
"""Return audio sample rate.
|
||||
|
||||
:return: Audio sample rate.
|
||||
:rtype: int
|
||||
"""
|
||||
return self._sample_rate
|
||||
|
||||
@property
|
||||
def num_samples(self):
|
||||
"""Return number of samples.
|
||||
|
||||
:return: Number of samples.
|
||||
:rtype: int
|
||||
"""
|
||||
return self._samples.shape(0)
|
||||
|
||||
@property
|
||||
def duration(self):
|
||||
"""Return audio duration.
|
||||
|
||||
:return: Audio duration in seconds.
|
||||
:rtype: float
|
||||
"""
|
||||
return self._samples.shape[0] / float(self._sample_rate)
|
||||
|
||||
@property
|
||||
def rms_db(self):
|
||||
"""Return root mean square energy of the audio in decibels.
|
||||
|
||||
:return: Root mean square energy in decibels.
|
||||
:rtype: float
|
||||
"""
|
||||
# square root => multiply by 10 instead of 20 for dBs
|
||||
mean_square = np.mean(self._samples**2)
|
||||
return 10 * np.log10(mean_square)
|
||||
|
||||
def _convert_samples_to_float32(self, samples):
|
||||
"""Convert sample type to float32.
|
||||
|
||||
Audio sample type is usually integer or float-point.
|
||||
Integers will be scaled to [-1, 1] in float32.
|
||||
"""
|
||||
float32_samples = samples.astype('float32')
|
||||
if samples.dtype in np.sctypes['int']:
|
||||
bits = np.iinfo(samples.dtype).bits
|
||||
float32_samples *= (1. / 2**(bits - 1))
|
||||
elif samples.dtype in np.sctypes['float']:
|
||||
pass
|
||||
else:
|
||||
raise TypeError("Unsupported sample type: %s." % samples.dtype)
|
||||
return float32_samples
|
||||
|
||||
def _convert_samples_from_float32(self, samples, dtype):
|
||||
"""Convert sample type from float32 to dtype.
|
||||
|
||||
Audio sample type is usually integer or float-point. For integer
|
||||
type, float32 will be rescaled from [-1, 1] to the maximum range
|
||||
supported by the integer type.
|
||||
|
||||
This is for writing a audio file.
|
||||
"""
|
||||
dtype = np.dtype(dtype)
|
||||
output_samples = samples.copy()
|
||||
if dtype in np.sctypes['int']:
|
||||
bits = np.iinfo(dtype).bits
|
||||
output_samples *= (2**(bits - 1) / 1.)
|
||||
min_val = np.iinfo(dtype).min
|
||||
max_val = np.iinfo(dtype).max
|
||||
output_samples[output_samples > max_val] = max_val
|
||||
output_samples[output_samples < min_val] = min_val
|
||||
elif samples.dtype in np.sctypes['float']:
|
||||
min_val = np.finfo(dtype).min
|
||||
max_val = np.finfo(dtype).max
|
||||
output_samples[output_samples > max_val] = max_val
|
||||
output_samples[output_samples < min_val] = min_val
|
||||
else:
|
||||
raise TypeError("Unsupported sample type: %s." % samples.dtype)
|
||||
return output_samples.astype(dtype)
|
||||
@ -0,0 +1,80 @@
|
||||
"""Contains the data augmentation pipeline."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import json
|
||||
import random
|
||||
from data_utils.augmentor.volume_perturb import VolumePerturbAugmentor
|
||||
|
||||
|
||||
class AugmentationPipeline(object):
|
||||
"""Build a pre-processing pipeline with various augmentation models.Such a
|
||||
data augmentation pipeline is oftern leveraged to augment the training
|
||||
samples to make the model invariant to certain types of perturbations in the
|
||||
real world, improving model's generalization ability.
|
||||
|
||||
The pipeline is built according the the augmentation configuration in json
|
||||
string, e.g.
|
||||
|
||||
.. code-block::
|
||||
|
||||
'[{"type": "volume",
|
||||
"params": {"min_gain_dBFS": -15,
|
||||
"max_gain_dBFS": 15},
|
||||
"prob": 0.5},
|
||||
{"type": "speed",
|
||||
"params": {"min_speed_rate": 0.8,
|
||||
"max_speed_rate": 1.2},
|
||||
"prob": 0.5}
|
||||
]'
|
||||
|
||||
This augmentation configuration inserts two augmentation models
|
||||
into the pipeline, with one is VolumePerturbAugmentor and the other
|
||||
SpeedPerturbAugmentor. "prob" indicates the probability of the current
|
||||
augmentor to take effect.
|
||||
|
||||
:param augmentation_config: Augmentation configuration in json string.
|
||||
:type augmentation_config: str
|
||||
:param random_seed: Random seed.
|
||||
:type random_seed: int
|
||||
:raises ValueError: If the augmentation json config is in incorrect format".
|
||||
"""
|
||||
|
||||
def __init__(self, augmentation_config, random_seed=0):
|
||||
self._rng = random.Random(random_seed)
|
||||
self._augmentors, self._rates = self._parse_pipeline_from(
|
||||
augmentation_config)
|
||||
|
||||
def transform_audio(self, audio_segment):
|
||||
"""Run the pre-processing pipeline for data augmentation.
|
||||
|
||||
Note that this is an in-place transformation.
|
||||
|
||||
:param audio_segment: Audio segment to process.
|
||||
:type audio_segment: AudioSegmenet|SpeechSegment
|
||||
"""
|
||||
for augmentor, rate in zip(self._augmentors, self._rates):
|
||||
if self._rng.uniform(0., 1.) <= rate:
|
||||
augmentor.transform_audio(audio_segment)
|
||||
|
||||
def _parse_pipeline_from(self, config_json):
|
||||
"""Parse the config json to build a augmentation pipelien."""
|
||||
try:
|
||||
configs = json.loads(config_json)
|
||||
augmentors = [
|
||||
self._get_augmentor(config["type"], config["params"])
|
||||
for config in configs
|
||||
]
|
||||
rates = [config["prob"] for config in configs]
|
||||
except Exception as e:
|
||||
raise ValueError("Failed to parse the augmentation config json: "
|
||||
"%s" % str(e))
|
||||
return augmentors, rates
|
||||
|
||||
def _get_augmentor(self, augmentor_type, params):
|
||||
"""Return an augmentation model by the type name, and pass in params."""
|
||||
if augmentor_type == "volume":
|
||||
return VolumePerturbAugmentor(self._rng, **params)
|
||||
else:
|
||||
raise ValueError("Unknown augmentor type [%s]." % augmentor_type)
|
||||
@ -0,0 +1,33 @@
|
||||
"""Contains the abstract base class for augmentation models."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
|
||||
class AugmentorBase(object):
|
||||
"""Abstract base class for augmentation model (augmentor) class.
|
||||
All augmentor classes should inherit from this class, and implement the
|
||||
following abstract methods.
|
||||
"""
|
||||
|
||||
__metaclass__ = ABCMeta
|
||||
|
||||
@abstractmethod
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_audio(self, audio_segment):
|
||||
"""Adds various effects to the input audio segment. Such effects
|
||||
will augment the training data to make the model invariant to certain
|
||||
types of perturbations in the real world, improving model's
|
||||
generalization ability.
|
||||
|
||||
Note that this is an in-place transformation.
|
||||
|
||||
:param audio_segment: Audio segment to add effects to.
|
||||
:type audio_segment: AudioSegmenet|SpeechSegment
|
||||
"""
|
||||
pass
|
||||
@ -0,0 +1,40 @@
|
||||
"""Contains the volume perturb augmentation model."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from data_utils.augmentor.base import AugmentorBase
|
||||
|
||||
|
||||
class VolumePerturbAugmentor(AugmentorBase):
|
||||
"""Augmentation model for adding random volume perturbation.
|
||||
|
||||
This is used for multi-loudness training of PCEN. See
|
||||
|
||||
https://arxiv.org/pdf/1607.05666v1.pdf
|
||||
|
||||
for more details.
|
||||
|
||||
:param rng: Random generator object.
|
||||
:type rng: random.Random
|
||||
:param min_gain_dBFS: Minimal gain in dBFS.
|
||||
:type min_gain_dBFS: float
|
||||
:param max_gain_dBFS: Maximal gain in dBFS.
|
||||
:type max_gain_dBFS: float
|
||||
"""
|
||||
|
||||
def __init__(self, rng, min_gain_dBFS, max_gain_dBFS):
|
||||
self._min_gain_dBFS = min_gain_dBFS
|
||||
self._max_gain_dBFS = max_gain_dBFS
|
||||
self._rng = rng
|
||||
|
||||
def transform_audio(self, audio_segment):
|
||||
"""Change audio loadness.
|
||||
|
||||
Note that this is an in-place transformation.
|
||||
|
||||
:param audio_segment: Audio segment to add effects to.
|
||||
:type audio_segment: AudioSegmenet|SpeechSegment
|
||||
"""
|
||||
gain = self._rng.uniform(min_gain_dBFS, max_gain_dBFS)
|
||||
audio_segment.apply_gain(gain)
|
||||
@ -0,0 +1,273 @@
|
||||
"""Contains data generator for orgnaizing various audio data preprocessing
|
||||
pipeline and offering data reader interface of PaddlePaddle requirements.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import random
|
||||
import numpy as np
|
||||
import paddle.v2 as paddle
|
||||
from data_utils import utils
|
||||
from data_utils.augmentor.augmentation import AugmentationPipeline
|
||||
from data_utils.featurizer.speech_featurizer import SpeechFeaturizer
|
||||
from data_utils.speech import SpeechSegment
|
||||
from data_utils.normalizer import FeatureNormalizer
|
||||
|
||||
|
||||
class DataGenerator(object):
|
||||
"""
|
||||
DataGenerator provides basic audio data preprocessing pipeline, and offers
|
||||
data reader interfaces of PaddlePaddle requirements.
|
||||
|
||||
:param vocab_filepath: Vocabulary filepath for indexing tokenized
|
||||
transcripts.
|
||||
:type vocab_filepath: basestring
|
||||
:param mean_std_filepath: File containing the pre-computed mean and stddev.
|
||||
:type mean_std_filepath: None|basestring
|
||||
:param augmentation_config: Augmentation configuration in json string.
|
||||
Details see AugmentationPipeline.__doc__.
|
||||
:type augmentation_config: str
|
||||
:param max_duration: Audio with duration (in seconds) greater than
|
||||
this will be discarded.
|
||||
:type max_duration: float
|
||||
:param min_duration: Audio with duration (in seconds) smaller than
|
||||
this will be discarded.
|
||||
:type min_duration: float
|
||||
:param stride_ms: Striding size (in milliseconds) for generating frames.
|
||||
:type stride_ms: float
|
||||
:param window_ms: Window size (in milliseconds) for generating frames.
|
||||
:type window_ms: float
|
||||
:param max_freq: Used when specgram_type is 'linear', only FFT bins
|
||||
corresponding to frequencies between [0, max_freq] are
|
||||
returned.
|
||||
:types max_freq: None|float
|
||||
:param specgram_type: Specgram feature type. Options: 'linear'.
|
||||
:type specgram_type: str
|
||||
:param random_seed: Random seed.
|
||||
:type random_seed: int
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
vocab_filepath,
|
||||
mean_std_filepath,
|
||||
augmentation_config='{}',
|
||||
max_duration=float('inf'),
|
||||
min_duration=0.0,
|
||||
stride_ms=10.0,
|
||||
window_ms=20.0,
|
||||
max_freq=None,
|
||||
specgram_type='linear',
|
||||
random_seed=0):
|
||||
self._max_duration = max_duration
|
||||
self._min_duration = min_duration
|
||||
self._normalizer = FeatureNormalizer(mean_std_filepath)
|
||||
self._augmentation_pipeline = AugmentationPipeline(
|
||||
augmentation_config=augmentation_config, random_seed=random_seed)
|
||||
self._speech_featurizer = SpeechFeaturizer(
|
||||
vocab_filepath=vocab_filepath,
|
||||
specgram_type=specgram_type,
|
||||
stride_ms=stride_ms,
|
||||
window_ms=window_ms,
|
||||
max_freq=max_freq)
|
||||
self._rng = random.Random(random_seed)
|
||||
self._epoch = 0
|
||||
|
||||
def batch_reader_creator(self,
|
||||
manifest_path,
|
||||
batch_size,
|
||||
min_batch_size=1,
|
||||
padding_to=-1,
|
||||
flatten=False,
|
||||
sortagrad=False,
|
||||
shuffle_method="batch_shuffle"):
|
||||
"""
|
||||
Batch data reader creator for audio data. Return a callable generator
|
||||
function to produce batches of data.
|
||||
|
||||
Audio features within one batch will be padded with zeros to have the
|
||||
same shape, or a user-defined shape.
|
||||
|
||||
:param manifest_path: Filepath of manifest for audio files.
|
||||
:type manifest_path: basestring
|
||||
:param batch_size: Number of instances in a batch.
|
||||
:type batch_size: int
|
||||
:param min_batch_size: Any batch with batch size smaller than this will
|
||||
be discarded. (To be deprecated in the future.)
|
||||
:type min_batch_size: int
|
||||
:param padding_to: If set -1, the maximun shape in the batch
|
||||
will be used as the target shape for padding.
|
||||
Otherwise, `padding_to` will be the target shape.
|
||||
:type padding_to: int
|
||||
:param flatten: If set True, audio features will be flatten to 1darray.
|
||||
:type flatten: bool
|
||||
:param sortagrad: If set True, sort the instances by audio duration
|
||||
in the first epoch for speed up training.
|
||||
:type sortagrad: bool
|
||||
:param shuffle_method: Shuffle method. Options:
|
||||
'' or None: no shuffle.
|
||||
'instance_shuffle': instance-wise shuffle.
|
||||
'batch_shuffle': similarly-sized instances are
|
||||
put into batches, and then
|
||||
batch-wise shuffle the batches.
|
||||
For more details, please see
|
||||
``_batch_shuffle.__doc__``.
|
||||
'batch_shuffle_clipped': 'batch_shuffle' with
|
||||
head shift and tail
|
||||
clipping. For more
|
||||
details, please see
|
||||
``_batch_shuffle``.
|
||||
If sortagrad is True, shuffle is disabled
|
||||
for the first epoch.
|
||||
:type shuffle_method: None|str
|
||||
:return: Batch reader function, producing batches of data when called.
|
||||
:rtype: callable
|
||||
"""
|
||||
|
||||
def batch_reader():
|
||||
# read manifest
|
||||
manifest = utils.read_manifest(
|
||||
manifest_path=manifest_path,
|
||||
max_duration=self._max_duration,
|
||||
min_duration=self._min_duration)
|
||||
# sort (by duration) or batch-wise shuffle the manifest
|
||||
if self._epoch == 0 and sortagrad:
|
||||
manifest.sort(key=lambda x: x["duration"])
|
||||
else:
|
||||
if shuffle_method == "batch_shuffle":
|
||||
manifest = self._batch_shuffle(
|
||||
manifest, batch_size, clipped=False)
|
||||
elif shuffle_method == "batch_shuffle_clipped":
|
||||
manifest = self._batch_shuffle(
|
||||
manifest, batch_size, clipped=True)
|
||||
elif shuffle_method == "instance_shuffle":
|
||||
self._rng.shuffle(manifest)
|
||||
elif not shuffle_method:
|
||||
pass
|
||||
else:
|
||||
raise ValueError("Unknown shuffle method %s." %
|
||||
shuffle_method)
|
||||
# prepare batches
|
||||
instance_reader = self._instance_reader_creator(manifest)
|
||||
batch = []
|
||||
for instance in instance_reader():
|
||||
batch.append(instance)
|
||||
if len(batch) == batch_size:
|
||||
yield self._padding_batch(batch, padding_to, flatten)
|
||||
batch = []
|
||||
if len(batch) >= min_batch_size:
|
||||
yield self._padding_batch(batch, padding_to, flatten)
|
||||
self._epoch += 1
|
||||
|
||||
return batch_reader
|
||||
|
||||
@property
|
||||
def feeding(self):
|
||||
"""Returns data reader's feeding dict.
|
||||
|
||||
:return: Data feeding dict.
|
||||
:rtype: dict
|
||||
"""
|
||||
return {"audio_spectrogram": 0, "transcript_text": 1}
|
||||
|
||||
@property
|
||||
def vocab_size(self):
|
||||
"""Return the vocabulary size.
|
||||
|
||||
:return: Vocabulary size.
|
||||
:rtype: int
|
||||
"""
|
||||
return self._speech_featurizer.vocab_size
|
||||
|
||||
@property
|
||||
def vocab_list(self):
|
||||
"""Return the vocabulary in list.
|
||||
|
||||
:return: Vocabulary in list.
|
||||
:rtype: list
|
||||
"""
|
||||
return self._speech_featurizer.vocab_list
|
||||
|
||||
def _process_utterance(self, filename, transcript):
|
||||
"""Load, augment, featurize and normalize for speech data."""
|
||||
speech_segment = SpeechSegment.from_file(filename, transcript)
|
||||
self._augmentation_pipeline.transform_audio(speech_segment)
|
||||
specgram, text_ids = self._speech_featurizer.featurize(speech_segment)
|
||||
specgram = self._normalizer.apply(specgram)
|
||||
return specgram, text_ids
|
||||
|
||||
def _instance_reader_creator(self, manifest):
|
||||
"""
|
||||
Instance reader creator. Create a callable function to produce
|
||||
instances of data.
|
||||
|
||||
Instance: a tuple of ndarray of audio spectrogram and a list of
|
||||
token indices for transcript.
|
||||
"""
|
||||
|
||||
def reader():
|
||||
for instance in manifest:
|
||||
yield self._process_utterance(instance["audio_filepath"],
|
||||
instance["text"])
|
||||
|
||||
return reader
|
||||
|
||||
def _padding_batch(self, batch, padding_to=-1, flatten=False):
|
||||
"""
|
||||
Padding audio features with zeros to make them have the same shape (or
|
||||
a user-defined shape) within one bach.
|
||||
|
||||
If ``padding_to`` is -1, the maximun shape in the batch will be used
|
||||
as the target shape for padding. Otherwise, `padding_to` will be the
|
||||
target shape (only refers to the second axis).
|
||||
|
||||
If `flatten` is True, features will be flatten to 1darray.
|
||||
"""
|
||||
new_batch = []
|
||||
# get target shape
|
||||
max_length = max([audio.shape[1] for audio, text in batch])
|
||||
if padding_to != -1:
|
||||
if padding_to < max_length:
|
||||
raise ValueError("If padding_to is not -1, it should be larger "
|
||||
"than any instance's shape in the batch")
|
||||
max_length = padding_to
|
||||
# padding
|
||||
for audio, text in batch:
|
||||
padded_audio = np.zeros([audio.shape[0], max_length])
|
||||
padded_audio[:, :audio.shape[1]] = audio
|
||||
if flatten:
|
||||
padded_audio = padded_audio.flatten()
|
||||
new_batch.append((padded_audio, text))
|
||||
return new_batch
|
||||
|
||||
def _batch_shuffle(self, manifest, batch_size, clipped=False):
|
||||
"""Put similarly-sized instances into minibatches for better efficiency
|
||||
and make a batch-wise shuffle.
|
||||
|
||||
1. Sort the audio clips by duration.
|
||||
2. Generate a random number `k`, k in [0, batch_size).
|
||||
3. Randomly shift `k` instances in order to create different batches
|
||||
for different epochs. Create minibatches.
|
||||
4. Shuffle the minibatches.
|
||||
|
||||
:param manifest: Manifest contents. List of dict.
|
||||
:type manifest: list
|
||||
:param batch_size: Batch size. This size is also used for generate
|
||||
a random number for batch shuffle.
|
||||
:type batch_size: int
|
||||
:param clipped: Whether to clip the heading (small shift) and trailing
|
||||
(incomplete batch) instances.
|
||||
:type clipped: bool
|
||||
:return: Batch shuffled mainifest.
|
||||
:rtype: list
|
||||
"""
|
||||
manifest.sort(key=lambda x: x["duration"])
|
||||
shift_len = self._rng.randint(0, batch_size - 1)
|
||||
batch_manifest = zip(*[iter(manifest[shift_len:])] * batch_size)
|
||||
self._rng.shuffle(batch_manifest)
|
||||
batch_manifest = list(sum(batch_manifest, ()))
|
||||
if not clipped:
|
||||
res_len = len(manifest) - shift_len - len(batch_manifest)
|
||||
batch_manifest.extend(manifest[-res_len:])
|
||||
batch_manifest.extend(manifest[0:shift_len])
|
||||
return batch_manifest
|
||||
@ -0,0 +1,106 @@
|
||||
"""Contains the audio featurizer class."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
from data_utils import utils
|
||||
from data_utils.audio import AudioSegment
|
||||
|
||||
|
||||
class AudioFeaturizer(object):
|
||||
"""Audio featurizer, for extracting features from audio contents of
|
||||
AudioSegment or SpeechSegment.
|
||||
|
||||
Currently, it only supports feature type of linear spectrogram.
|
||||
|
||||
:param specgram_type: Specgram feature type. Options: 'linear'.
|
||||
:type specgram_type: str
|
||||
:param stride_ms: Striding size (in milliseconds) for generating frames.
|
||||
:type stride_ms: float
|
||||
:param window_ms: Window size (in milliseconds) for generating frames.
|
||||
:type window_ms: float
|
||||
:param max_freq: Used when specgram_type is 'linear', only FFT bins
|
||||
corresponding to frequencies between [0, max_freq] are
|
||||
returned.
|
||||
:types max_freq: None|float
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
specgram_type='linear',
|
||||
stride_ms=10.0,
|
||||
window_ms=20.0,
|
||||
max_freq=None):
|
||||
self._specgram_type = specgram_type
|
||||
self._stride_ms = stride_ms
|
||||
self._window_ms = window_ms
|
||||
self._max_freq = max_freq
|
||||
|
||||
def featurize(self, audio_segment):
|
||||
"""Extract audio features from AudioSegment or SpeechSegment.
|
||||
|
||||
:param audio_segment: Audio/speech segment to extract features from.
|
||||
:type audio_segment: AudioSegment|SpeechSegment
|
||||
:return: Spectrogram audio feature in 2darray.
|
||||
:rtype: ndarray
|
||||
"""
|
||||
return self._compute_specgram(audio_segment.samples,
|
||||
audio_segment.sample_rate)
|
||||
|
||||
def _compute_specgram(self, samples, sample_rate):
|
||||
"""Extract various audio features."""
|
||||
if self._specgram_type == 'linear':
|
||||
return self._compute_linear_specgram(
|
||||
samples, sample_rate, self._stride_ms, self._window_ms,
|
||||
self._max_freq)
|
||||
else:
|
||||
raise ValueError("Unknown specgram_type %s. "
|
||||
"Supported values: linear." % self._specgram_type)
|
||||
|
||||
def _compute_linear_specgram(self,
|
||||
samples,
|
||||
sample_rate,
|
||||
stride_ms=10.0,
|
||||
window_ms=20.0,
|
||||
max_freq=None,
|
||||
eps=1e-14):
|
||||
"""Compute the linear spectrogram from FFT energy."""
|
||||
if max_freq is None:
|
||||
max_freq = sample_rate / 2
|
||||
if max_freq > sample_rate / 2:
|
||||
raise ValueError("max_freq must be greater than half of "
|
||||
"sample rate.")
|
||||
if stride_ms > window_ms:
|
||||
raise ValueError("Stride size must not be greater than "
|
||||
"window size.")
|
||||
stride_size = int(0.001 * sample_rate * stride_ms)
|
||||
window_size = int(0.001 * sample_rate * window_ms)
|
||||
specgram, freqs = self._specgram_real(
|
||||
samples,
|
||||
window_size=window_size,
|
||||
stride_size=stride_size,
|
||||
sample_rate=sample_rate)
|
||||
ind = np.where(freqs <= max_freq)[0][-1] + 1
|
||||
return np.log(specgram[:ind, :] + eps)
|
||||
|
||||
def _specgram_real(self, samples, window_size, stride_size, sample_rate):
|
||||
"""Compute the spectrogram for samples from a real signal."""
|
||||
# extract strided windows
|
||||
truncate_size = (len(samples) - window_size) % stride_size
|
||||
samples = samples[:len(samples) - truncate_size]
|
||||
nshape = (window_size, (len(samples) - window_size) // stride_size + 1)
|
||||
nstrides = (samples.strides[0], samples.strides[0] * stride_size)
|
||||
windows = np.lib.stride_tricks.as_strided(
|
||||
samples, shape=nshape, strides=nstrides)
|
||||
assert np.all(
|
||||
windows[:, 1] == samples[stride_size:(stride_size + window_size)])
|
||||
# window weighting, squared Fast Fourier Transform (fft), scaling
|
||||
weighting = np.hanning(window_size)[:, None]
|
||||
fft = np.fft.rfft(windows * weighting, axis=0)
|
||||
fft = np.absolute(fft)**2
|
||||
scale = np.sum(weighting**2) * sample_rate
|
||||
fft[1:-1, :] *= (2.0 / scale)
|
||||
fft[(0, -1), :] /= scale
|
||||
# prepare fft frequency list
|
||||
freqs = float(sample_rate) / window_size * np.arange(fft.shape[0])
|
||||
return fft, freqs
|
||||
@ -0,0 +1,77 @@
|
||||
"""Contains the speech featurizer class."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from data_utils.featurizer.audio_featurizer import AudioFeaturizer
|
||||
from data_utils.featurizer.text_featurizer import TextFeaturizer
|
||||
|
||||
|
||||
class SpeechFeaturizer(object):
|
||||
"""Speech featurizer, for extracting features from both audio and transcript
|
||||
contents of SpeechSegment.
|
||||
|
||||
Currently, for audio parts, it only supports feature type of linear
|
||||
spectrogram; for transcript parts, it only supports char-level tokenizing
|
||||
and conversion into a list of token indices. Note that the token indexing
|
||||
order follows the given vocabulary file.
|
||||
|
||||
:param vocab_filepath: Filepath to load vocabulary for token indices
|
||||
conversion.
|
||||
:type specgram_type: basestring
|
||||
:param specgram_type: Specgram feature type. Options: 'linear'.
|
||||
:type specgram_type: str
|
||||
:param stride_ms: Striding size (in milliseconds) for generating frames.
|
||||
:type stride_ms: float
|
||||
:param window_ms: Window size (in milliseconds) for generating frames.
|
||||
:type window_ms: float
|
||||
:param max_freq: Used when specgram_type is 'linear', only FFT bins
|
||||
corresponding to frequencies between [0, max_freq] are
|
||||
returned.
|
||||
:types max_freq: None|float
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
vocab_filepath,
|
||||
specgram_type='linear',
|
||||
stride_ms=10.0,
|
||||
window_ms=20.0,
|
||||
max_freq=None):
|
||||
self._audio_featurizer = AudioFeaturizer(specgram_type, stride_ms,
|
||||
window_ms, max_freq)
|
||||
self._text_featurizer = TextFeaturizer(vocab_filepath)
|
||||
|
||||
def featurize(self, speech_segment):
|
||||
"""Extract features for speech segment.
|
||||
|
||||
1. For audio parts, extract the audio features.
|
||||
2. For transcript parts, convert text string to a list of token indices
|
||||
in char-level.
|
||||
|
||||
:param audio_segment: Speech segment to extract features from.
|
||||
:type audio_segment: SpeechSegment
|
||||
:return: A tuple of 1) spectrogram audio feature in 2darray, 2) list of
|
||||
char-level token indices.
|
||||
:rtype: tuple
|
||||
"""
|
||||
audio_feature = self._audio_featurizer.featurize(speech_segment)
|
||||
text_ids = self._text_featurizer.featurize(speech_segment.transcript)
|
||||
return audio_feature, text_ids
|
||||
|
||||
@property
|
||||
def vocab_size(self):
|
||||
"""Return the vocabulary size.
|
||||
|
||||
:return: Vocabulary size.
|
||||
:rtype: int
|
||||
"""
|
||||
return self._text_featurizer.vocab_size
|
||||
|
||||
@property
|
||||
def vocab_list(self):
|
||||
"""Return the vocabulary in list.
|
||||
|
||||
:return: Vocabulary in list.
|
||||
:rtype: list
|
||||
"""
|
||||
return self._text_featurizer.vocab_list
|
||||
@ -0,0 +1,67 @@
|
||||
"""Contains the text featurizer class."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
|
||||
|
||||
class TextFeaturizer(object):
|
||||
"""Text featurizer, for processing or extracting features from text.
|
||||
|
||||
Currently, it only supports char-level tokenizing and conversion into
|
||||
a list of token indices. Note that the token indexing order follows the
|
||||
given vocabulary file.
|
||||
|
||||
:param vocab_filepath: Filepath to load vocabulary for token indices
|
||||
conversion.
|
||||
:type specgram_type: basestring
|
||||
"""
|
||||
|
||||
def __init__(self, vocab_filepath):
|
||||
self._vocab_dict, self._vocab_list = self._load_vocabulary_from_file(
|
||||
vocab_filepath)
|
||||
|
||||
def featurize(self, text):
|
||||
"""Convert text string to a list of token indices in char-level.Note
|
||||
that the token indexing order follows the given vocabulary file.
|
||||
|
||||
:param text: Text to process.
|
||||
:type text: basestring
|
||||
:return: List of char-level token indices.
|
||||
:rtype: list
|
||||
"""
|
||||
tokens = self._char_tokenize(text)
|
||||
return [self._vocab_dict[token] for token in tokens]
|
||||
|
||||
@property
|
||||
def vocab_size(self):
|
||||
"""Return the vocabulary size.
|
||||
|
||||
:return: Vocabulary size.
|
||||
:rtype: int
|
||||
"""
|
||||
return len(self._vocab_list)
|
||||
|
||||
@property
|
||||
def vocab_list(self):
|
||||
"""Return the vocabulary in list.
|
||||
|
||||
:return: Vocabulary in list.
|
||||
:rtype: list
|
||||
"""
|
||||
return self._vocab_list
|
||||
|
||||
def _char_tokenize(self, text):
|
||||
"""Character tokenizer."""
|
||||
return list(text.strip())
|
||||
|
||||
def _load_vocabulary_from_file(self, vocab_filepath):
|
||||
"""Load vocabulary from file."""
|
||||
vocab_lines = []
|
||||
with open(vocab_filepath, 'r') as file:
|
||||
vocab_lines.extend(file.readlines())
|
||||
vocab_list = [line[:-1] for line in vocab_lines]
|
||||
vocab_dict = dict(
|
||||
[(token, id) for (id, token) in enumerate(vocab_list)])
|
||||
return vocab_dict, vocab_list
|
||||
@ -0,0 +1,87 @@
|
||||
"""Contains feature normalizers."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import random
|
||||
import data_utils.utils as utils
|
||||
from data_utils.audio import AudioSegment
|
||||
|
||||
|
||||
class FeatureNormalizer(object):
|
||||
"""Feature normalizer. Normalize features to be of zero mean and unit
|
||||
stddev.
|
||||
|
||||
if mean_std_filepath is provided (not None), the normalizer will directly
|
||||
initilize from the file. Otherwise, both manifest_path and featurize_func
|
||||
should be given for on-the-fly mean and stddev computing.
|
||||
|
||||
:param mean_std_filepath: File containing the pre-computed mean and stddev.
|
||||
:type mean_std_filepath: None|basestring
|
||||
:param manifest_path: Manifest of instances for computing mean and stddev.
|
||||
:type meanifest_path: None|basestring
|
||||
:param featurize_func: Function to extract features. It should be callable
|
||||
with ``featurize_func(audio_segment)``.
|
||||
:type featurize_func: None|callable
|
||||
:param num_samples: Number of random samples for computing mean and stddev.
|
||||
:type num_samples: int
|
||||
:param random_seed: Random seed for sampling instances.
|
||||
:type random_seed: int
|
||||
:raises ValueError: If both mean_std_filepath and manifest_path
|
||||
(or both mean_std_filepath and featurize_func) are None.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
mean_std_filepath,
|
||||
manifest_path=None,
|
||||
featurize_func=None,
|
||||
num_samples=500,
|
||||
random_seed=0):
|
||||
if not mean_std_filepath:
|
||||
if not (manifest_path and featurize_func):
|
||||
raise ValueError("If mean_std_filepath is None, meanifest_path "
|
||||
"and featurize_func should not be None.")
|
||||
self._rng = random.Random(random_seed)
|
||||
self._compute_mean_std(manifest_path, featurize_func, num_samples)
|
||||
else:
|
||||
self._read_mean_std_from_file(mean_std_filepath)
|
||||
|
||||
def apply(self, features, eps=1e-14):
|
||||
"""Normalize features to be of zero mean and unit stddev.
|
||||
|
||||
:param features: Input features to be normalized.
|
||||
:type features: ndarray
|
||||
:param eps: added to stddev to provide numerical stablibity.
|
||||
:type eps: float
|
||||
:return: Normalized features.
|
||||
:rtype: ndarray
|
||||
"""
|
||||
return (features - self._mean) / (self._std + eps)
|
||||
|
||||
def write_to_file(self, filepath):
|
||||
"""Write the mean and stddev to the file.
|
||||
|
||||
:param filepath: File to write mean and stddev.
|
||||
:type filepath: basestring
|
||||
"""
|
||||
np.savez(filepath, mean=self._mean, std=self._std)
|
||||
|
||||
def _read_mean_std_from_file(self, filepath):
|
||||
"""Load mean and std from file."""
|
||||
npzfile = np.load(filepath)
|
||||
self._mean = npzfile["mean"]
|
||||
self._std = npzfile["std"]
|
||||
|
||||
def _compute_mean_std(self, manifest_path, featurize_func, num_samples):
|
||||
"""Compute mean and std from randomly sampled instances."""
|
||||
manifest = utils.read_manifest(manifest_path)
|
||||
sampled_manifest = self._rng.sample(manifest, num_samples)
|
||||
features = []
|
||||
for instance in sampled_manifest:
|
||||
features.append(
|
||||
featurize_func(
|
||||
AudioSegment.from_file(instance["audio_filepath"])))
|
||||
features = np.hstack(features)
|
||||
self._mean = np.mean(features, axis=1).reshape([-1, 1])
|
||||
self._std = np.std(features, axis=1).reshape([-1, 1])
|
||||
@ -0,0 +1,75 @@
|
||||
"""Contains the speech segment class."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from data_utils.audio import AudioSegment
|
||||
|
||||
|
||||
class SpeechSegment(AudioSegment):
|
||||
"""Speech segment abstraction, a subclass of AudioSegment,
|
||||
with an additional transcript.
|
||||
|
||||
:param samples: Audio samples [num_samples x num_channels].
|
||||
:type samples: ndarray.float32
|
||||
:param sample_rate: Audio sample rate.
|
||||
:type sample_rate: int
|
||||
:param transcript: Transcript text for the speech.
|
||||
:type transript: basestring
|
||||
:raises TypeError: If the sample data type is not float or int.
|
||||
"""
|
||||
|
||||
def __init__(self, samples, sample_rate, transcript):
|
||||
AudioSegment.__init__(self, samples, sample_rate)
|
||||
self._transcript = transcript
|
||||
|
||||
def __eq__(self, other):
|
||||
"""Return whether two objects are equal.
|
||||
"""
|
||||
if not AudioSegment.__eq__(self, other):
|
||||
return False
|
||||
if self._transcript != other._transcript:
|
||||
return False
|
||||
return True
|
||||
|
||||
def __ne__(self, other):
|
||||
"""Return whether two objects are unequal."""
|
||||
return not self.__eq__(other)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, filepath, transcript):
|
||||
"""Create speech segment from audio file and corresponding transcript.
|
||||
|
||||
:param filepath: Filepath or file object to audio file.
|
||||
:type filepath: basestring|file
|
||||
:param transcript: Transcript text for the speech.
|
||||
:type transript: basestring
|
||||
:return: Audio segment instance.
|
||||
:rtype: AudioSegment
|
||||
"""
|
||||
audio = AudioSegment.from_file(filepath)
|
||||
return cls(audio.samples, audio.sample_rate, transcript)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, bytes, transcript):
|
||||
"""Create speech segment from a byte string and corresponding
|
||||
transcript.
|
||||
|
||||
:param bytes: Byte string containing audio samples.
|
||||
:type bytes: str
|
||||
:param transcript: Transcript text for the speech.
|
||||
:type transript: basestring
|
||||
:return: Audio segment instance.
|
||||
:rtype: AudioSegment
|
||||
"""
|
||||
audio = AudioSegment.from_bytes(bytes)
|
||||
return cls(audio.samples, audio.sample_rate, transcript)
|
||||
|
||||
@property
|
||||
def transcript(self):
|
||||
"""Return the transcript text.
|
||||
|
||||
:return: Transcript text for the speech.
|
||||
:rtype: basestring
|
||||
"""
|
||||
return self._transcript
|
||||
@ -0,0 +1,34 @@
|
||||
"""Contains data helper functions."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import json
|
||||
|
||||
|
||||
def read_manifest(manifest_path, max_duration=float('inf'), min_duration=0.0):
|
||||
"""Load and parse manifest file.
|
||||
|
||||
Instances with durations outside [min_duration, max_duration] will be
|
||||
filtered out.
|
||||
|
||||
:param manifest_path: Manifest file to load and parse.
|
||||
:type manifest_path: basestring
|
||||
:param max_duration: Maximal duration in seconds for instance filter.
|
||||
:type max_duration: float
|
||||
:param min_duration: Minimal duration in seconds for instance filter.
|
||||
:type min_duration: float
|
||||
:return: Manifest parsing results. List of dict.
|
||||
:rtype: list
|
||||
:raises IOError: If failed to parse the manifest.
|
||||
"""
|
||||
manifest = []
|
||||
for json_line in open(manifest_path):
|
||||
try:
|
||||
json_data = json.loads(json_line)
|
||||
except Exception as e:
|
||||
raise IOError("Error reading manifest: %s" % str(e))
|
||||
if (json_data["duration"] <= max_duration and
|
||||
json_data["duration"] >= min_duration):
|
||||
manifest.append(json_data)
|
||||
return manifest
|
||||
@ -0,0 +1,13 @@
|
||||
cd librispeech
|
||||
python librispeech.py
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Prepare LibriSpeech failed. Terminated."
|
||||
exit 1
|
||||
fi
|
||||
cd -
|
||||
|
||||
cat librispeech/manifest.train* | shuf > manifest.train
|
||||
cat librispeech/manifest.dev-clean > manifest.dev
|
||||
cat librispeech/manifest.test-clean > manifest.test
|
||||
|
||||
echo "All done."
|
||||
@ -0,0 +1,25 @@
|
||||
"""Contains common utility functions."""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
|
||||
def print_arguments(args):
|
||||
"""Print argparse's arguments.
|
||||
|
||||
Usage:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("name", default="Jonh", type=str, help="User name.")
|
||||
args = parser.parse_args()
|
||||
print_arguments(args)
|
||||
|
||||
:param args: Input argparse.Namespace for printing.
|
||||
:type args: argparse.Namespace
|
||||
"""
|
||||
print("----- Configuration Arguments -----")
|
||||
for arg, value in vars(args).iteritems():
|
||||
print("%s: %s" % (arg, value))
|
||||
print("------------------------------------")
|
||||
Loading…
Reference in new issue