Code clean up according to comments in https://github.com/PaddlePaddle/PaddleSpeech/pull/3242
jiamingkong
2 years ago
parent
3b6651ba7c
commit
60bd7f202e
@ -1,9 +1,9 @@
|
||||
# LibriSpeech
|
||||
|
||||
## hubertASR
|
||||
## WavLMASR
|
||||
Fintuning on train-clean-100
|
||||
train: Epoch 3, 1*V100-32G, batchsize: 4, accum_grad: 8
|
||||
train: Epoch 16, 4*A800-80G, batchsize: 16, accum_grad: 8
|
||||
|
||||
| Model | Params | Config | Augmentation| Test set | Decode method | WER |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| hubertASR | 326.16M | conf/hubertASR.yaml | spec_aug | test-clean | greedy search | 0.05868 |
|
||||
| WavLMASR | 326.16M | conf/wavlmasr.yaml | spec_aug | test-clean | greedy search | 0.0561 |
|
||||
|
||||
@ -1,18 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from paddlespeech.dataset.s2t import avg_ckpts_main
|
||||
|
||||
if __name__ == '__main__':
|
||||
avg_ckpts_main()
|
||||
Binary file not shown.
@ -1,251 +0,0 @@
|
||||
# Copyright (c) 2023 speechbrain Authors. All Rights Reserved.
|
||||
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Modified from speechbrain 2023 (https://github.com/speechbrain/speechbrain/blob/develop/speechbrain/processing/signal_processing.py)
|
||||
"""
|
||||
Low level signal processing utilities
|
||||
Authors
|
||||
* Peter Plantinga 2020
|
||||
* Francois Grondin 2020
|
||||
* William Aris 2020
|
||||
* Samuele Cornell 2020
|
||||
* Sarthak Yadav 2022
|
||||
"""
|
||||
import numpy as np
|
||||
import paddle
|
||||
|
||||
|
||||
def blackman_window(window_length, periodic=True):
|
||||
"""Blackman window function.
|
||||
Arguments
|
||||
---------
|
||||
window_length : int
|
||||
Controlling the returned window size.
|
||||
periodic : bool
|
||||
Determines whether the returned window trims off the
|
||||
last duplicate value from the symmetric window
|
||||
|
||||
Returns
|
||||
-------
|
||||
A 1-D tensor of size (window_length) containing the window
|
||||
"""
|
||||
if window_length == 0:
|
||||
return []
|
||||
if window_length == 1:
|
||||
return paddle.ones([1])
|
||||
if periodic:
|
||||
window_length += 1
|
||||
window = paddle.arange(window_length) * (np.pi / (window_length - 1))
|
||||
window = 0.08 * paddle.cos(window * 4) - 0.5 * paddle.cos(window * 2) + 0.42
|
||||
return window[:-1] if periodic else window
|
||||
|
||||
|
||||
def compute_amplitude(waveforms, lengths=None, amp_type="avg", scale="linear"):
|
||||
"""Compute amplitude of a batch of waveforms.
|
||||
Arguments
|
||||
---------
|
||||
waveform : tensor
|
||||
The waveforms used for computing amplitude.
|
||||
Shape should be `[time]` or `[batch, time]` or
|
||||
`[batch, time, channels]`.
|
||||
lengths : tensor
|
||||
The lengths of the waveforms excluding the padding.
|
||||
Shape should be a single dimension, `[batch]`.
|
||||
amp_type : str
|
||||
Whether to compute "avg" average or "peak" amplitude.
|
||||
Choose between ["avg", "peak"].
|
||||
scale : str
|
||||
Whether to compute amplitude in "dB" or "linear" scale.
|
||||
Choose between ["linear", "dB"].
|
||||
Returns
|
||||
-------
|
||||
The average amplitude of the waveforms.
|
||||
Example
|
||||
-------
|
||||
>>> signal = paddle.sin(paddle.arange(16000.0)).unsqueeze(0)
|
||||
>>> compute_amplitude(signal, signal.size(1))
|
||||
tensor([[0.6366]])
|
||||
"""
|
||||
if len(waveforms.shape) == 1:
|
||||
waveforms = waveforms.unsqueeze(0)
|
||||
|
||||
assert amp_type in ["avg", "peak"]
|
||||
assert scale in ["linear", "dB"]
|
||||
|
||||
if amp_type == "avg":
|
||||
if lengths is None:
|
||||
out = paddle.mean(paddle.abs(waveforms), axis=1, keepdim=True)
|
||||
else:
|
||||
wav_sum = paddle.sum(paddle.abs(waveforms), axis=1, keepdim=True)
|
||||
out = wav_sum / lengths
|
||||
elif amp_type == "peak":
|
||||
out = paddle.max(paddle.abs(waveforms), axis=1, keepdim=True)[0]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
if scale == "linear":
|
||||
return out
|
||||
elif scale == "dB":
|
||||
return paddle.clip(20 * paddle.log10(out), min=-80) # clamp zeros
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def convolve1d(
|
||||
waveform,
|
||||
kernel,
|
||||
padding=0,
|
||||
pad_type="constant",
|
||||
stride=1,
|
||||
groups=1,
|
||||
use_fft=False,
|
||||
rotation_index=0, ):
|
||||
"""Use paddle.nn.functional to perform 1d padding and conv.
|
||||
Arguments
|
||||
---------
|
||||
waveform : tensor
|
||||
The tensor to perform operations on.
|
||||
kernel : tensor
|
||||
The filter to apply during convolution.
|
||||
padding : int or tuple
|
||||
The padding (pad_left, pad_right) to apply.
|
||||
If an integer is passed instead, this is passed
|
||||
to the conv1d function and pad_type is ignored.
|
||||
pad_type : str
|
||||
The type of padding to use. Passed directly to
|
||||
`paddle.nn.functional.pad`, see Paddle documentation
|
||||
for available options.
|
||||
stride : int
|
||||
The number of units to move each time convolution is applied.
|
||||
Passed to conv1d. Has no effect if `use_fft` is True.
|
||||
groups : int
|
||||
This option is passed to `conv1d` to split the input into groups for
|
||||
convolution. Input channels should be divisible by the number of groups.
|
||||
use_fft : bool
|
||||
When `use_fft` is passed `True`, then compute the convolution in the
|
||||
spectral domain using complex multiply. This is more efficient on CPU
|
||||
when the size of the kernel is large (e.g. reverberation). WARNING:
|
||||
Without padding, circular convolution occurs. This makes little
|
||||
difference in the case of reverberation, but may make more difference
|
||||
with different kernels.
|
||||
rotation_index : int
|
||||
This option only applies if `use_fft` is true. If so, the kernel is
|
||||
rolled by this amount before convolution to shift the output location.
|
||||
Returns
|
||||
-------
|
||||
The convolved waveform.
|
||||
Example
|
||||
-------
|
||||
>>> from speechbrain.dataio.dataio import read_audio
|
||||
>>> signal = read_audio('tests/samples/single-mic/example1.wav')
|
||||
>>> signal = signal.unsqueeze(0).unsqueeze(2)
|
||||
>>> kernel = paddle.rand([1, 10, 1])
|
||||
>>> signal = convolve1d(signal, kernel, padding=(9, 0))
|
||||
"""
|
||||
if len(waveform.shape) != 3:
|
||||
raise ValueError("Convolve1D expects a 3-dimensional tensor")
|
||||
|
||||
# Move time dimension last, which pad and fft and conv expect.
|
||||
waveform = waveform.transpose([0, 2, 1])
|
||||
kernel = kernel.transpose([0, 2, 1])
|
||||
# Padding can be a tuple (left_pad, right_pad) or an int
|
||||
if isinstance(padding, tuple):
|
||||
waveform = paddle.nn.functional.pad(
|
||||
x=waveform, pad=padding, mode=pad_type, data_format='NCL')
|
||||
|
||||
# This approach uses FFT, which is more efficient if the kernel is large
|
||||
if use_fft:
|
||||
# Pad kernel to same length as signal, ensuring correct alignment
|
||||
zero_length = waveform.shape[-1] - kernel.shape[-1]
|
||||
|
||||
# Handle case where signal is shorter
|
||||
if zero_length < 0:
|
||||
kernel = kernel[..., :zero_length]
|
||||
zero_length = 0
|
||||
|
||||
# Perform rotation to ensure alignment
|
||||
zeros = paddle.zeros(
|
||||
[kernel.shape[0], kernel.shape[1], zero_length], dtype=kernel.dtype)
|
||||
after_index = kernel[..., rotation_index:]
|
||||
before_index = kernel[..., :rotation_index]
|
||||
kernel = paddle.concat((after_index, zeros, before_index), axis=-1)
|
||||
|
||||
# Multiply in frequency domain to convolve in time domain
|
||||
import paddle.fft as fft
|
||||
|
||||
result = fft.rfft(waveform) * fft.rfft(kernel)
|
||||
convolved = fft.irfft(result, n=waveform.shape[-1])
|
||||
|
||||
# Use the implementation given by paddle, which should be efficient on GPU
|
||||
else:
|
||||
convolved = paddle.nn.functional.conv1d(
|
||||
x=waveform,
|
||||
weight=kernel,
|
||||
stride=stride,
|
||||
groups=groups,
|
||||
padding=padding if not isinstance(padding, tuple) else 0, )
|
||||
|
||||
# Return time dimension to the second dimension.
|
||||
return convolved.transpose([0, 2, 1])
|
||||
|
||||
|
||||
def notch_filter(notch_freq, filter_width=101, notch_width=0.05):
|
||||
"""Returns a notch filter constructed from a high-pass and low-pass filter.
|
||||
(from https://tomroelandts.com/articles/
|
||||
how-to-create-simple-band-pass-and-band-reject-filters)
|
||||
Arguments
|
||||
---------
|
||||
notch_freq : float
|
||||
frequency to put notch as a fraction of the
|
||||
sampling rate / 2. The range of possible inputs is 0 to 1.
|
||||
filter_width : int
|
||||
Filter width in samples. Longer filters have
|
||||
smaller transition bands, but are more inefficient.
|
||||
notch_width : float
|
||||
Width of the notch, as a fraction of the sampling_rate / 2.
|
||||
"""
|
||||
|
||||
# Check inputs
|
||||
assert 0 < notch_freq <= 1
|
||||
assert filter_width % 2 != 0
|
||||
pad = filter_width // 2
|
||||
inputs = paddle.arange(filter_width) - pad
|
||||
|
||||
# Avoid frequencies that are too low
|
||||
notch_freq += notch_width
|
||||
|
||||
# Define sinc function, avoiding division by zero
|
||||
def sinc(x):
|
||||
"Computes the sinc function."
|
||||
|
||||
def _sinc(x):
|
||||
return paddle.sin(x) / x
|
||||
|
||||
# The zero is at the middle index
|
||||
return paddle.concat(
|
||||
[_sinc(x[:pad]), paddle.ones([1]), _sinc(x[pad + 1:])])
|
||||
|
||||
# Compute a low-pass filter with cutoff frequency notch_freq.
|
||||
hlpf = sinc(3 * (notch_freq - notch_width) * inputs)
|
||||
hlpf *= blackman_window(filter_width)
|
||||
hlpf /= paddle.sum(hlpf)
|
||||
|
||||
# Compute a high-pass filter with cutoff frequency notch_freq.
|
||||
hhpf = sinc(3 * (notch_freq + notch_width) * inputs)
|
||||
hhpf *= blackman_window(filter_width)
|
||||
hhpf /= -paddle.sum(hhpf)
|
||||
hhpf[pad] += 1
|
||||
|
||||
# Adding filters creates notch filter
|
||||
return (hlpf + hhpf).view(1, -1, 1)
|
||||
@ -1,901 +0,0 @@
|
||||
# Copyright (c) 2023 speechbrain Authors. All Rights Reserved.
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Modified from speechbrain(https://github.com/speechbrain/speechbrain/blob/develop/speechbrain/processing/speech_augmentation.py)
|
||||
"""Classes for mutating speech data for data augmentation.
|
||||
This module provides classes that produce realistic distortions of speech
|
||||
data for the purpose of training speech processing models. The list of
|
||||
distortions includes adding noise, adding reverberation, changing speed,
|
||||
and more. All the classes are of type `torch.nn.Module`. This gives the
|
||||
possibility to have end-to-end differentiability and
|
||||
backpropagate the gradient through them. In addition, all operations
|
||||
are expected to be performed on the GPU (where available) for efficiency.
|
||||
|
||||
Authors
|
||||
* Peter Plantinga 2020
|
||||
"""
|
||||
import math
|
||||
|
||||
import paddle
|
||||
import paddle.nn as nn
|
||||
|
||||
from .signal_processing import compute_amplitude
|
||||
from .signal_processing import convolve1d
|
||||
from .signal_processing import notch_filter
|
||||
|
||||
|
||||
class SpeedPerturb(nn.Layer):
|
||||
"""Slightly speed up or slow down an audio signal.
|
||||
Resample the audio signal at a rate that is similar to the original rate,
|
||||
to achieve a slightly slower or slightly faster signal. This technique is
|
||||
outlined in the paper: "Audio Augmentation for Speech Recognition"
|
||||
Arguments
|
||||
---------
|
||||
orig_freq : int
|
||||
The frequency of the original signal.
|
||||
speeds : list
|
||||
The speeds that the signal should be changed to, as a percentage of the
|
||||
original signal (i.e. `speeds` is divided by 100 to get a ratio).
|
||||
perturb_prob : float
|
||||
The chance that the batch will be speed-
|
||||
perturbed. By default, every batch is perturbed.
|
||||
Example
|
||||
-------
|
||||
>>> from speechbrain.dataio.dataio import read_audio
|
||||
>>> signal = read_audio('tests/samples/single-mic/example1.wav')
|
||||
>>> perturbator = SpeedPerturb(orig_freq=16000, speeds=[90])
|
||||
>>> clean = signal.unsqueeze(0)
|
||||
>>> perturbed = perturbator(clean)
|
||||
>>> clean.shape
|
||||
paddle.shape([1, 52173])
|
||||
>>> perturbed.shape
|
||||
paddle.shape([1, 46956])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
orig_freq,
|
||||
speeds=[90, 100, 110],
|
||||
perturb_prob=1.0, ):
|
||||
super().__init__()
|
||||
self.orig_freq = orig_freq
|
||||
self.speeds = speeds
|
||||
self.perturb_prob = perturb_prob
|
||||
|
||||
# Initialize index of perturbation
|
||||
self.samp_index = 0
|
||||
# Initialize resamplers
|
||||
self.resamplers = []
|
||||
for speed in self.speeds:
|
||||
config = {
|
||||
"orig_freq": self.orig_freq,
|
||||
"new_freq": self.orig_freq * speed // 100,
|
||||
}
|
||||
self.resamplers.append(Resample(**config))
|
||||
|
||||
def forward(self, waveform):
|
||||
"""
|
||||
Arguments
|
||||
---------
|
||||
waveforms : tensor
|
||||
Shape should be `[batch, time]` or `[batch, time, channels]`.
|
||||
lengths : tensor
|
||||
Shape should be a single dimension, `[batch]`.
|
||||
Returns
|
||||
-------
|
||||
Tensor of shape `[batch, time]` or `[batch, time, channels]`.
|
||||
"""
|
||||
|
||||
# Don't perturb (return early) 1-`perturb_prob` portion of the batches
|
||||
if paddle.rand([1]) > self.perturb_prob:
|
||||
return waveform.clone()
|
||||
# Perform a random perturbation
|
||||
self.samp_index = paddle.randint(len(self.speeds), shape=(1, ))[0]
|
||||
perturbed_waveform = self.resamplers[self.samp_index](waveform)
|
||||
|
||||
return perturbed_waveform
|
||||
|
||||
|
||||
class Resample(nn.Layer):
|
||||
"""This class resamples an audio signal using sinc-based interpolation.
|
||||
|
||||
It is a modification of the `resample` function from torchaudio
|
||||
(https://pytorch.org/audio/stable/tutorials/audio_resampling_tutorial.html)
|
||||
|
||||
Arguments
|
||||
---------
|
||||
orig_freq : int
|
||||
the sampling frequency of the input signal.
|
||||
new_freq : int
|
||||
the new sampling frequency after this operation is performed.
|
||||
lowpass_filter_width : int
|
||||
Controls the sharpness of the filter, larger numbers result in a
|
||||
sharper filter, but they are less efficient. Values from 4 to 10 are allowed.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
orig_freq=16000,
|
||||
new_freq=16000,
|
||||
lowpass_filter_width=6, ):
|
||||
super().__init__()
|
||||
self.orig_freq = orig_freq
|
||||
self.new_freq = new_freq
|
||||
self.lowpass_filter_width = lowpass_filter_width
|
||||
|
||||
# Compute rate for striding
|
||||
self._compute_strides()
|
||||
assert self.orig_freq % self.conv_stride == 0
|
||||
assert self.new_freq % self.conv_transpose_stride == 0
|
||||
|
||||
def _compute_strides(self):
|
||||
"""Compute the phases in polyphase filter.
|
||||
|
||||
(almost directly from torchaudio.compliance.kaldi)
|
||||
"""
|
||||
|
||||
# Compute new unit based on ratio of in/out frequencies
|
||||
base_freq = math.gcd(self.orig_freq, self.new_freq)
|
||||
input_samples_in_unit = self.orig_freq // base_freq
|
||||
self.output_samples = self.new_freq // base_freq
|
||||
|
||||
# Store the appropriate stride based on the new units
|
||||
self.conv_stride = input_samples_in_unit
|
||||
self.conv_transpose_stride = self.output_samples
|
||||
|
||||
def forward(self, waveforms):
|
||||
"""
|
||||
Arguments
|
||||
---------
|
||||
waveforms : tensor
|
||||
Shape should be `[batch, time]` or `[batch, time, channels]`.
|
||||
lengths : tensor
|
||||
Shape should be a single dimension, `[batch]`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Tensor of shape `[batch, time]` or `[batch, time, channels]`.
|
||||
"""
|
||||
|
||||
if not hasattr(self, "first_indices"):
|
||||
self._indices_and_weights(waveforms)
|
||||
|
||||
# Don't do anything if the frequencies are the same
|
||||
if self.orig_freq == self.new_freq:
|
||||
return waveforms
|
||||
unsqueezed = False
|
||||
if len(waveforms.shape) == 2:
|
||||
waveforms = waveforms.unsqueeze(1)
|
||||
unsqueezed = True
|
||||
elif len(waveforms.shape) == 3:
|
||||
waveforms = waveforms.transpose([0, 2, 1])
|
||||
else:
|
||||
raise ValueError("Input must be 2 or 3 dimensions")
|
||||
|
||||
# Do resampling
|
||||
resampled_waveform = self._perform_resample(waveforms)
|
||||
|
||||
if unsqueezed:
|
||||
resampled_waveform = resampled_waveform.squeeze(1)
|
||||
else:
|
||||
resampled_waveform = resampled_waveform.transpose([0, 2, 1])
|
||||
|
||||
return resampled_waveform
|
||||
|
||||
def _perform_resample(self, waveforms):
|
||||
"""Resamples the waveform at the new frequency.
|
||||
|
||||
This matches Kaldi's OfflineFeatureTpl ResampleWaveform which uses a
|
||||
LinearResample (resample a signal at linearly spaced intervals to
|
||||
up/downsample a signal). LinearResample (LR) means that the output
|
||||
signal is at linearly spaced intervals (i.e the output signal has a
|
||||
frequency of `new_freq`). It uses sinc/bandlimited interpolation to
|
||||
upsample/downsample the signal.
|
||||
|
||||
(almost directly from torchaudio.compliance.kaldi)
|
||||
|
||||
https://ccrma.stanford.edu/~jos/resample/
|
||||
Theory_Ideal_Bandlimited_Interpolation.html
|
||||
|
||||
https://github.com/kaldi-asr/kaldi/blob/master/src/feat/resample.h#L56
|
||||
|
||||
Arguments
|
||||
---------
|
||||
waveforms : tensor
|
||||
The batch of audio signals to resample.
|
||||
|
||||
Returns
|
||||
-------
|
||||
The waveforms at the new frequency.
|
||||
"""
|
||||
|
||||
# Compute output size and initialize
|
||||
batch_size, num_channels, wave_len = waveforms.shape
|
||||
window_size = self.weights.shape[1]
|
||||
tot_output_samp = self._output_samples(wave_len)
|
||||
resampled_waveform = paddle.zeros(
|
||||
(batch_size, num_channels, tot_output_samp))
|
||||
# self.weights = self.weights.to(waveforms.device)
|
||||
|
||||
# Check weights are on correct device
|
||||
# if waveforms.device != self.weights.device:
|
||||
# self.weights = self.weights.to(waveforms.device)
|
||||
|
||||
# eye size: (num_channels, num_channels, 1)
|
||||
eye = paddle.eye(num_channels).unsqueeze(2)
|
||||
|
||||
# Iterate over the phases in the polyphase filter
|
||||
for i in range(self.first_indices.shape[0]):
|
||||
wave_to_conv = waveforms
|
||||
first_index = int(self.first_indices[i].item())
|
||||
if first_index >= 0:
|
||||
# trim the signal as the filter will not be applied
|
||||
# before the first_index
|
||||
wave_to_conv = wave_to_conv[..., first_index:]
|
||||
|
||||
# pad the right of the signal to allow partial convolutions
|
||||
# meaning compute values for partial windows (e.g. end of the
|
||||
# window is outside the signal length)
|
||||
max_index = (tot_output_samp - 1) // self.output_samples
|
||||
end_index = max_index * self.conv_stride + window_size
|
||||
current_wave_len = wave_len - first_index
|
||||
right_padding = max(0, end_index + 1 - current_wave_len)
|
||||
left_padding = max(0, -first_index)
|
||||
wave_to_conv = paddle.nn.functional.pad(
|
||||
wave_to_conv, (left_padding, right_padding), data_format='NCL')
|
||||
conv_wave = paddle.nn.functional.conv1d(
|
||||
x=wave_to_conv,
|
||||
weight=self.weights[i].repeat(num_channels, 1, 1),
|
||||
stride=self.conv_stride,
|
||||
groups=num_channels, )
|
||||
|
||||
# we want conv_wave[:, i] to be at
|
||||
# output[:, i + n*conv_transpose_stride]
|
||||
dilated_conv_wave = paddle.nn.functional.conv1d_transpose(
|
||||
conv_wave, eye, stride=self.conv_transpose_stride)
|
||||
|
||||
# pad dilated_conv_wave so it reaches the output length if needed.
|
||||
left_padding = i
|
||||
previous_padding = left_padding + dilated_conv_wave.shape[-1]
|
||||
right_padding = max(0, tot_output_samp - previous_padding)
|
||||
dilated_conv_wave = paddle.nn.functional.pad(
|
||||
dilated_conv_wave, (left_padding, right_padding),
|
||||
data_format='NCL')
|
||||
dilated_conv_wave = dilated_conv_wave[..., :tot_output_samp]
|
||||
|
||||
resampled_waveform += dilated_conv_wave
|
||||
|
||||
return resampled_waveform
|
||||
|
||||
def _output_samples(self, input_num_samp):
|
||||
"""Based on LinearResample::GetNumOutputSamples.
|
||||
|
||||
LinearResample (LR) means that the output signal is at
|
||||
linearly spaced intervals (i.e the output signal has a
|
||||
frequency of ``new_freq``). It uses sinc/bandlimited
|
||||
interpolation to upsample/downsample the signal.
|
||||
|
||||
(almost directly from torchaudio.compliance.kaldi)
|
||||
|
||||
Arguments
|
||||
---------
|
||||
input_num_samp : int
|
||||
The number of samples in each example in the batch.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Number of samples in the output waveform.
|
||||
"""
|
||||
|
||||
# For exact computation, we measure time in "ticks" of 1.0 / tick_freq,
|
||||
# where tick_freq is the least common multiple of samp_in and
|
||||
# samp_out.
|
||||
samp_in = int(self.orig_freq)
|
||||
samp_out = int(self.new_freq)
|
||||
|
||||
tick_freq = abs(samp_in * samp_out) // math.gcd(samp_in, samp_out)
|
||||
ticks_per_input_period = tick_freq // samp_in
|
||||
|
||||
# work out the number of ticks in the time interval
|
||||
# [ 0, input_num_samp/samp_in ).
|
||||
interval_length = input_num_samp * ticks_per_input_period
|
||||
if interval_length <= 0:
|
||||
return 0
|
||||
ticks_per_output_period = tick_freq // samp_out
|
||||
|
||||
# Get the last output-sample in the closed interval,
|
||||
# i.e. replacing [ ) with [ ]. Note: integer division rounds down.
|
||||
# See http://en.wikipedia.org/wiki/Interval_(mathematics) for an
|
||||
# explanation of the notation.
|
||||
last_output_samp = interval_length // ticks_per_output_period
|
||||
|
||||
# We need the last output-sample in the open interval, so if it
|
||||
# takes us to the end of the interval exactly, subtract one.
|
||||
if last_output_samp * ticks_per_output_period == interval_length:
|
||||
last_output_samp -= 1
|
||||
|
||||
# First output-sample index is zero, so the number of output samples
|
||||
# is the last output-sample plus one.
|
||||
num_output_samp = last_output_samp + 1
|
||||
|
||||
return num_output_samp
|
||||
|
||||
def _indices_and_weights(self, waveforms):
|
||||
"""Based on LinearResample::SetIndexesAndWeights
|
||||
|
||||
Retrieves the weights for resampling as well as the indices in which
|
||||
they are valid. LinearResample (LR) means that the output signal is at
|
||||
linearly spaced intervals (i.e the output signal has a frequency
|
||||
of ``new_freq``). It uses sinc/bandlimited interpolation to
|
||||
upsample/downsample the signal.
|
||||
|
||||
Returns
|
||||
-------
|
||||
- the place where each filter should start being applied
|
||||
- the filters to be applied to the signal for resampling
|
||||
"""
|
||||
|
||||
# Lowpass filter frequency depends on smaller of two frequencies
|
||||
min_freq = min(self.orig_freq, self.new_freq)
|
||||
lowpass_cutoff = 0.99 * 0.5 * min_freq
|
||||
|
||||
assert lowpass_cutoff * 2 <= min_freq
|
||||
window_width = self.lowpass_filter_width / (2.0 * lowpass_cutoff)
|
||||
|
||||
assert lowpass_cutoff < min(self.orig_freq, self.new_freq) / 2
|
||||
output_t = paddle.arange(start=0.0, end=self.output_samples)
|
||||
output_t /= self.new_freq
|
||||
min_t = output_t - window_width
|
||||
max_t = output_t + window_width
|
||||
|
||||
min_input_index = paddle.ceil(min_t * self.orig_freq)
|
||||
max_input_index = paddle.floor(max_t * self.orig_freq)
|
||||
num_indices = max_input_index - min_input_index + 1
|
||||
|
||||
max_weight_width = num_indices.max()
|
||||
j = paddle.arange(max_weight_width)
|
||||
input_index = min_input_index.unsqueeze(1) + j.unsqueeze(0)
|
||||
delta_t = (input_index / self.orig_freq) - output_t.unsqueeze(1)
|
||||
|
||||
weights = paddle.zeros_like(delta_t)
|
||||
|
||||
inside_window_indices = delta_t.abs() < (window_width)
|
||||
# raised-cosine (Hanning) window with width `window_width`
|
||||
weights[inside_window_indices] = 0.5 * (1 + paddle.cos(
|
||||
2 * math.pi * lowpass_cutoff / self.lowpass_filter_width *
|
||||
delta_t[inside_window_indices]))
|
||||
t_eq_zero_indices = delta_t == 0.0
|
||||
t_not_eq_zero_indices = ~t_eq_zero_indices
|
||||
|
||||
# sinc filter function
|
||||
weights[t_not_eq_zero_indices] *= paddle.sin(
|
||||
2 * math.pi * lowpass_cutoff * delta_t[t_not_eq_zero_indices]) / (
|
||||
math.pi * delta_t[t_not_eq_zero_indices])
|
||||
|
||||
# limit of the function at t = 0
|
||||
weights[t_eq_zero_indices] *= 2 * lowpass_cutoff
|
||||
|
||||
# size (output_samples, max_weight_width)
|
||||
weights /= self.orig_freq
|
||||
|
||||
self.first_indices = min_input_index
|
||||
self.weights = weights
|
||||
|
||||
|
||||
class DropFreq(nn.Layer):
|
||||
"""This class drops a random frequency from the signal.
|
||||
The purpose of this class is to teach models to learn to rely on all parts
|
||||
of the signal, not just a few frequency bands.
|
||||
Arguments
|
||||
---------
|
||||
drop_freq_low : float
|
||||
The low end of frequencies that can be dropped,
|
||||
as a fraction of the sampling rate / 2.
|
||||
drop_freq_high : float
|
||||
The high end of frequencies that can be
|
||||
dropped, as a fraction of the sampling rate / 2.
|
||||
drop_count_low : int
|
||||
The low end of number of frequencies that could be dropped.
|
||||
drop_count_high : int
|
||||
The high end of number of frequencies that could be dropped.
|
||||
drop_width : float
|
||||
The width of the frequency band to drop, as
|
||||
a fraction of the sampling_rate / 2.
|
||||
drop_prob : float
|
||||
The probability that the batch of signals will have a frequency
|
||||
dropped. By default, every batch has frequencies dropped.
|
||||
Example
|
||||
-------
|
||||
>>> from speechbrain.dataio.dataio import read_audio
|
||||
>>> dropper = DropFreq()
|
||||
>>> signal = read_audio('tests/samples/single-mic/example1.wav')
|
||||
>>> dropped_signal = dropper(signal.unsqueeze(0))
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
drop_freq_low=1e-14,
|
||||
drop_freq_high=1,
|
||||
drop_count_low=1,
|
||||
drop_count_high=2,
|
||||
drop_width=0.05,
|
||||
drop_prob=1, ):
|
||||
super().__init__()
|
||||
self.drop_freq_low = drop_freq_low
|
||||
self.drop_freq_high = drop_freq_high
|
||||
self.drop_count_low = drop_count_low
|
||||
self.drop_count_high = drop_count_high
|
||||
self.drop_width = drop_width
|
||||
self.drop_prob = drop_prob
|
||||
|
||||
def forward(self, waveforms):
|
||||
"""
|
||||
Arguments
|
||||
---------
|
||||
waveforms : tensor
|
||||
Shape should be `[batch, time]` or `[batch, time, channels]`.
|
||||
Returns
|
||||
-------
|
||||
Tensor of shape `[batch, time]` or `[batch, time, channels]`.
|
||||
"""
|
||||
|
||||
# Don't drop (return early) 1-`drop_prob` portion of the batches
|
||||
dropped_waveform = waveforms.clone()
|
||||
if paddle.rand([1]) > self.drop_prob:
|
||||
return dropped_waveform
|
||||
|
||||
# Add channels dimension
|
||||
if len(waveforms.shape) == 2:
|
||||
dropped_waveform = dropped_waveform.unsqueeze(-1)
|
||||
|
||||
# Pick number of frequencies to drop
|
||||
drop_count = paddle.randint(
|
||||
low=self.drop_count_low,
|
||||
high=self.drop_count_high + 1,
|
||||
shape=(1, ), )
|
||||
|
||||
# Filter parameters
|
||||
filter_length = 101
|
||||
pad = filter_length // 2
|
||||
|
||||
# Start with delta function
|
||||
drop_filter = paddle.zeros([1, filter_length, 1])
|
||||
drop_filter[0, pad, 0] = 1
|
||||
|
||||
if drop_count.shape == 0:
|
||||
# Pick a frequency to drop
|
||||
drop_range = self.drop_freq_high - self.drop_freq_low
|
||||
drop_frequency = (
|
||||
paddle.rand(drop_count) * drop_range + self.drop_freq_low)
|
||||
# Subtract each frequency
|
||||
for frequency in drop_frequency:
|
||||
notch_kernel = notch_filter(
|
||||
frequency,
|
||||
filter_length,
|
||||
self.drop_width, )
|
||||
drop_filter = convolve1d(drop_filter, notch_kernel, pad)
|
||||
|
||||
# Apply filter
|
||||
dropped_waveform = convolve1d(dropped_waveform, drop_filter, pad)
|
||||
|
||||
# Remove channels dimension if added
|
||||
return dropped_waveform.squeeze(-1)
|
||||
|
||||
|
||||
class DropChunk(nn.Layer):
|
||||
"""This class drops portions of the input signal.
|
||||
Using `DropChunk` as an augmentation strategy helps a models learn to rely
|
||||
on all parts of the signal, since it can't expect a given part to be
|
||||
present.
|
||||
Arguments
|
||||
---------
|
||||
drop_length_low : int
|
||||
The low end of lengths for which to set the
|
||||
signal to zero, in samples.
|
||||
drop_length_high : int
|
||||
The high end of lengths for which to set the
|
||||
signal to zero, in samples.
|
||||
drop_count_low : int
|
||||
The low end of number of times that the signal
|
||||
can be dropped to zero.
|
||||
drop_count_high : int
|
||||
The high end of number of times that the signal
|
||||
can be dropped to zero.
|
||||
drop_start : int
|
||||
The first index for which dropping will be allowed.
|
||||
drop_end : int
|
||||
The last index for which dropping will be allowed.
|
||||
drop_prob : float
|
||||
The probability that the batch of signals will
|
||||
have a portion dropped. By default, every batch
|
||||
has portions dropped.
|
||||
noise_factor : float
|
||||
The factor relative to average amplitude of an utterance
|
||||
to use for scaling the white noise inserted. 1 keeps
|
||||
the average amplitude the same, while 0 inserts all 0's.
|
||||
Example
|
||||
-------
|
||||
>>> from speechbrain.dataio.dataio import read_audio
|
||||
>>> dropper = DropChunk(drop_start=100, drop_end=200, noise_factor=0.)
|
||||
>>> signal = read_audio('tests/samples/single-mic/example1.wav')
|
||||
>>> signal = signal.unsqueeze(0) # [batch, time, channels]
|
||||
>>> length = paddle.ones([1])
|
||||
>>> dropped_signal = dropper(signal, length)
|
||||
>>> float(dropped_signal[:, 150])
|
||||
0.0
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
drop_length_low=100,
|
||||
drop_length_high=1000,
|
||||
drop_count_low=1,
|
||||
drop_count_high=10,
|
||||
drop_start=0,
|
||||
drop_end=None,
|
||||
drop_prob=1,
|
||||
noise_factor=0.0, ):
|
||||
super().__init__()
|
||||
self.drop_length_low = drop_length_low
|
||||
self.drop_length_high = drop_length_high
|
||||
self.drop_count_low = drop_count_low
|
||||
self.drop_count_high = drop_count_high
|
||||
self.drop_start = drop_start
|
||||
self.drop_end = drop_end
|
||||
self.drop_prob = drop_prob
|
||||
self.noise_factor = noise_factor
|
||||
|
||||
# Validate low < high
|
||||
if drop_length_low > drop_length_high:
|
||||
raise ValueError("Low limit must not be more than high limit")
|
||||
if drop_count_low > drop_count_high:
|
||||
raise ValueError("Low limit must not be more than high limit")
|
||||
|
||||
# Make sure the length doesn't exceed end - start
|
||||
if drop_end is not None and drop_end >= 0:
|
||||
if drop_start > drop_end:
|
||||
raise ValueError("Low limit must not be more than high limit")
|
||||
|
||||
drop_range = drop_end - drop_start
|
||||
self.drop_length_low = min(drop_length_low, drop_range)
|
||||
self.drop_length_high = min(drop_length_high, drop_range)
|
||||
|
||||
def forward(self, waveforms, lengths):
|
||||
"""
|
||||
Arguments
|
||||
---------
|
||||
waveforms : tensor
|
||||
Shape should be `[batch, time]` or `[batch, time, channels]`.
|
||||
lengths : tensor
|
||||
Shape should be a single dimension, `[batch]`.
|
||||
Returns
|
||||
-------
|
||||
Tensor of shape `[batch, time]` or
|
||||
`[batch, time, channels]`
|
||||
"""
|
||||
|
||||
# Reading input list
|
||||
lengths = (lengths * waveforms.shape[1]).long()
|
||||
batch_size = waveforms.shape[0]
|
||||
dropped_waveform = waveforms.clone()
|
||||
|
||||
# Don't drop (return early) 1-`drop_prob` portion of the batches
|
||||
if paddle.rand([1]) > self.drop_prob:
|
||||
return dropped_waveform
|
||||
|
||||
# Store original amplitude for computing white noise amplitude
|
||||
clean_amplitude = compute_amplitude(waveforms, lengths.unsqueeze(1))
|
||||
|
||||
# Pick a number of times to drop
|
||||
drop_times = paddle.randint(
|
||||
low=self.drop_count_low,
|
||||
high=self.drop_count_high + 1,
|
||||
shape=(batch_size, ), )
|
||||
|
||||
# Iterate batch to set mask
|
||||
for i in range(batch_size):
|
||||
if drop_times[i] == 0:
|
||||
continue
|
||||
|
||||
# Pick lengths
|
||||
length = paddle.randint(
|
||||
low=self.drop_length_low,
|
||||
high=self.drop_length_high + 1,
|
||||
shape=(drop_times[i], ), )
|
||||
|
||||
# Compute range of starting locations
|
||||
start_min = self.drop_start
|
||||
if start_min < 0:
|
||||
start_min += lengths[i]
|
||||
start_max = self.drop_end
|
||||
if start_max is None:
|
||||
start_max = lengths[i]
|
||||
if start_max < 0:
|
||||
start_max += lengths[i]
|
||||
start_max = max(0, start_max - length.max())
|
||||
|
||||
# Pick starting locations
|
||||
start = paddle.randint(
|
||||
low=start_min,
|
||||
high=start_max + 1,
|
||||
shape=(drop_times[i], ), )
|
||||
|
||||
end = start + length
|
||||
|
||||
# Update waveform
|
||||
if not self.noise_factor:
|
||||
for j in range(drop_times[i]):
|
||||
dropped_waveform[i, start[j]:end[j]] = 0.0
|
||||
else:
|
||||
# Uniform distribution of -2 to +2 * avg amplitude should
|
||||
# preserve the average for normalization
|
||||
noise_max = 2 * clean_amplitude[i] * self.noise_factor
|
||||
for j in range(drop_times[i]):
|
||||
# zero-center the noise distribution
|
||||
noise_vec = paddle.rand([length[j]])
|
||||
noise_vec = 2 * noise_max * noise_vec - noise_max
|
||||
dropped_waveform[i, start[j]:end[j]] = noise_vec
|
||||
|
||||
return dropped_waveform
|
||||
|
||||
|
||||
class SpecAugment(paddle.nn.Layer):
|
||||
"""An implementation of the SpecAugment algorithm.
|
||||
Reference:
|
||||
https://arxiv.org/abs/1904.08779
|
||||
Arguments
|
||||
---------
|
||||
time_warp : bool
|
||||
Whether applying time warping.
|
||||
time_warp_window : int
|
||||
Time warp window.
|
||||
time_warp_mode : str
|
||||
Interpolation mode for time warping (default "bicubic").
|
||||
freq_mask : bool
|
||||
Whether applying freq mask.
|
||||
freq_mask_width : int or tuple
|
||||
Freq mask width range.
|
||||
n_freq_mask : int
|
||||
Number of freq mask.
|
||||
time_mask : bool
|
||||
Whether applying time mask.
|
||||
time_mask_width : int or tuple
|
||||
Time mask width range.
|
||||
n_time_mask : int
|
||||
Number of time mask.
|
||||
replace_with_zero : bool
|
||||
If True, replace masked value with 0, else replace masked value with mean of the input tensor.
|
||||
Example
|
||||
-------
|
||||
>>> aug = SpecAugment()
|
||||
>>> a = paddle.rand([8, 120, 80])
|
||||
>>> a = aug(a)
|
||||
>>> print(a.shape)
|
||||
paddle.Size([8, 120, 80])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
time_warp=True,
|
||||
time_warp_window=5,
|
||||
time_warp_mode="bicubic",
|
||||
freq_mask=True,
|
||||
freq_mask_width=(0, 20),
|
||||
n_freq_mask=2,
|
||||
time_mask=True,
|
||||
time_mask_width=(0, 100),
|
||||
n_time_mask=2,
|
||||
replace_with_zero=True, ):
|
||||
super().__init__()
|
||||
assert (
|
||||
time_warp or freq_mask or time_mask
|
||||
), "at least one of time_warp, time_mask, or freq_mask should be applied"
|
||||
|
||||
self.apply_time_warp = time_warp
|
||||
self.time_warp_window = time_warp_window
|
||||
self.time_warp_mode = time_warp_mode
|
||||
|
||||
self.freq_mask = freq_mask
|
||||
if isinstance(freq_mask_width, int):
|
||||
freq_mask_width = (0, freq_mask_width)
|
||||
self.freq_mask_width = freq_mask_width
|
||||
self.n_freq_mask = n_freq_mask
|
||||
|
||||
self.time_mask = time_mask
|
||||
if isinstance(time_mask_width, int):
|
||||
time_mask_width = (0, time_mask_width)
|
||||
self.time_mask_width = time_mask_width
|
||||
self.n_time_mask = n_time_mask
|
||||
|
||||
self.replace_with_zero = replace_with_zero
|
||||
|
||||
def forward(self, x):
|
||||
"""Takes in input a tensors and returns an augmented one."""
|
||||
if self.apply_time_warp:
|
||||
x = self.time_warp(x)
|
||||
if self.freq_mask:
|
||||
x = self.mask_along_axis(x, dim=2)
|
||||
if self.time_mask:
|
||||
x = self.mask_along_axis(x, dim=1)
|
||||
return x
|
||||
|
||||
def time_warp(self, x):
|
||||
"""Time warping with paddle.nn.functional.interpolate"""
|
||||
original_size = x.shape
|
||||
window = self.time_warp_window
|
||||
|
||||
# 2d interpolation requires 4D or higher dimension tensors
|
||||
# x: (Batch, Time, Freq) -> (Batch, 1, Time, Freq)
|
||||
if x.dim() == 3:
|
||||
x = x.unsqueeze(1)
|
||||
|
||||
time = x.shape[2]
|
||||
if time - window <= window:
|
||||
return x.view(*original_size)
|
||||
|
||||
# compute center and corresponding window
|
||||
c = paddle.randint(window, time - window, (1, ))[0]
|
||||
w = paddle.randint(c - window, c + window, (1, ))[0] + 1
|
||||
|
||||
left = paddle.nn.functional.interpolate(
|
||||
x[:, :, :c],
|
||||
(w, x.shape[3]),
|
||||
mode=self.time_warp_mode,
|
||||
align_corners=True, )
|
||||
right = paddle.nn.functional.interpolate(
|
||||
x[:, :, c:],
|
||||
(time - w, x.shape[3]),
|
||||
mode=self.time_warp_mode,
|
||||
align_corners=True, )
|
||||
|
||||
x[:, :, :w] = left
|
||||
x[:, :, w:] = right
|
||||
return x.view(*original_size)
|
||||
|
||||
def mask_along_axis(self, x, dim):
|
||||
"""Mask along time or frequency axis.
|
||||
Arguments
|
||||
---------
|
||||
x : tensor
|
||||
Input tensor.
|
||||
dim : int
|
||||
Corresponding dimension to mask.
|
||||
"""
|
||||
original_size = x.shape
|
||||
if x.dim() == 4:
|
||||
x = x.view(-1, x.shape[2], x.shape[3])
|
||||
|
||||
batch, time, fea = x.shape
|
||||
|
||||
if dim == 1:
|
||||
D = time
|
||||
n_mask = self.n_time_mask
|
||||
width_range = self.time_mask_width
|
||||
else:
|
||||
D = fea
|
||||
n_mask = self.n_freq_mask
|
||||
width_range = self.freq_mask_width
|
||||
|
||||
mask_len = paddle.randint(width_range[0], width_range[1],
|
||||
(batch, n_mask)).unsqueeze(2)
|
||||
|
||||
mask_pos = paddle.randint(0, max(1, D - mask_len.max()),
|
||||
(batch, n_mask)).unsqueeze(2)
|
||||
|
||||
# compute masks
|
||||
arange = paddle.arange(end=D).view(1, 1, -1)
|
||||
mask = (mask_pos <= arange) * (arange < (mask_pos + mask_len))
|
||||
mask = mask.any(axis=1)
|
||||
|
||||
if dim == 1:
|
||||
mask = mask.unsqueeze(2)
|
||||
else:
|
||||
mask = mask.unsqueeze(1)
|
||||
|
||||
if self.replace_with_zero:
|
||||
val = 0.0
|
||||
else:
|
||||
val = x.mean()
|
||||
# same to x.masked_fill_(mask, val)
|
||||
y = paddle.full(x.shape, val, x.dtype)
|
||||
x = paddle.where(mask, y, x)
|
||||
return x.view(*original_size)
|
||||
|
||||
|
||||
class TimeDomainSpecAugment(nn.Layer):
|
||||
"""A time-domain approximation of the SpecAugment algorithm.
|
||||
This augmentation module implements three augmentations in
|
||||
the time-domain.
|
||||
1. Drop chunks of the audio (zero amplitude or white noise)
|
||||
2. Drop frequency bands (with band-drop filters)
|
||||
3. Speed peturbation (via resampling to slightly different rate)
|
||||
Arguments
|
||||
---------
|
||||
perturb_prob : float from 0 to 1
|
||||
The probability that a batch will have speed perturbation applied.
|
||||
drop_freq_prob : float from 0 to 1
|
||||
The probability that a batch will have frequencies dropped.
|
||||
drop_chunk_prob : float from 0 to 1
|
||||
The probability that a batch will have chunks dropped.
|
||||
speeds : list of ints
|
||||
A set of different speeds to use to perturb each batch.
|
||||
See ``speechbrain.processing.speech_augmentation.SpeedPerturb``
|
||||
sample_rate : int
|
||||
Sampling rate of the input waveforms.
|
||||
drop_freq_count_low : int
|
||||
Lowest number of frequencies that could be dropped.
|
||||
drop_freq_count_high : int
|
||||
Highest number of frequencies that could be dropped.
|
||||
drop_chunk_count_low : int
|
||||
Lowest number of chunks that could be dropped.
|
||||
drop_chunk_count_high : int
|
||||
Highest number of chunks that could be dropped.
|
||||
drop_chunk_length_low : int
|
||||
Lowest length of chunks that could be dropped.
|
||||
drop_chunk_length_high : int
|
||||
Highest length of chunks that could be dropped.
|
||||
drop_chunk_noise_factor : float
|
||||
The noise factor used to scale the white noise inserted, relative to
|
||||
the average amplitude of the utterance. Default 0 (no noise inserted).
|
||||
Example
|
||||
-------
|
||||
>>> inputs = paddle.randn([10, 16000])
|
||||
>>> feature_maker = TimeDomainSpecAugment(speeds=[80])
|
||||
>>> feats = feature_maker(inputs, paddle.ones(10))
|
||||
>>> feats.shape
|
||||
paddle.shape([10, 12800])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
perturb_prob=1.0,
|
||||
drop_freq_prob=1.0,
|
||||
drop_chunk_prob=1.0,
|
||||
speeds=[95, 100, 105],
|
||||
sample_rate=16000,
|
||||
drop_freq_count_low=0,
|
||||
drop_freq_count_high=3,
|
||||
drop_chunk_count_low=0,
|
||||
drop_chunk_count_high=5,
|
||||
drop_chunk_length_low=1000,
|
||||
drop_chunk_length_high=2000,
|
||||
drop_chunk_noise_factor=0, ):
|
||||
super().__init__()
|
||||
self.speed_perturb = SpeedPerturb(
|
||||
perturb_prob=perturb_prob, orig_freq=sample_rate, speeds=speeds)
|
||||
self.drop_freq = DropFreq(
|
||||
drop_prob=drop_freq_prob,
|
||||
drop_count_low=drop_freq_count_low,
|
||||
drop_count_high=drop_freq_count_high, )
|
||||
self.drop_chunk = DropChunk(
|
||||
drop_prob=drop_chunk_prob,
|
||||
drop_count_low=drop_chunk_count_low,
|
||||
drop_count_high=drop_chunk_count_high,
|
||||
drop_length_low=drop_chunk_length_low,
|
||||
drop_length_high=drop_chunk_length_high,
|
||||
noise_factor=drop_chunk_noise_factor, )
|
||||
|
||||
def forward(self, waveforms, lengths):
|
||||
"""Returns the distorted waveforms.
|
||||
Arguments
|
||||
---------
|
||||
waveforms : tensor
|
||||
The waveforms to distort
|
||||
"""
|
||||
# Augmentation
|
||||
with paddle.no_grad():
|
||||
waveforms = self.speed_perturb(waveforms)
|
||||
waveforms = self.drop_freq(waveforms)
|
||||
waveforms = self.drop_chunk(waveforms, lengths)
|
||||
return waveforms
|
||||
Loading…
Reference in new issue