diff --git a/speechx/examples/feat/linear_spectrogram_main.cc b/speechx/examples/feat/linear_spectrogram_main.cc index ca76d85c..2d75bb5d 100644 --- a/speechx/examples/feat/linear_spectrogram_main.cc +++ b/speechx/examples/feat/linear_spectrogram_main.cc @@ -181,6 +181,10 @@ int main(int argc, char* argv[]) { ppspeech::LinearSpectrogramOptions opt; opt.frame_opts.frame_length_ms = 20; opt.frame_opts.frame_shift_ms = 10; + opt.frame_opts.dither = 0.0; + opt.frame_opts.remove_dc_offset = false; + opt.frame_opts.window_type = "hanning"; + opt.frame_opts.preemph_coeff = 0.0; LOG(INFO) << "frame length (ms): " << opt.frame_opts.frame_length_ms; LOG(INFO) << "frame shift (ms): " << opt.frame_opts.frame_shift_ms; diff --git a/speechx/speechx/frontend/audio/linear_spectrogram.cc b/speechx/speechx/frontend/audio/linear_spectrogram.cc index 827b8ecc..d6ae3d01 100644 --- a/speechx/speechx/frontend/audio/linear_spectrogram.cc +++ b/speechx/speechx/frontend/audio/linear_spectrogram.cc @@ -14,6 +14,8 @@ #include "frontend/audio/linear_spectrogram.h" #include "kaldi/base/kaldi-math.h" +#include "kaldi/feat/feature-common.h" +#include "kaldi/feat/feature-functions.h" #include "kaldi/matrix/matrix-functions.h" namespace ppspeech { @@ -21,30 +23,23 @@ namespace ppspeech { using kaldi::int32; using kaldi::BaseFloat; using kaldi::Vector; +using kaldi::SubVector; using kaldi::VectorBase; using kaldi::Matrix; using std::vector; LinearSpectrogram::LinearSpectrogram( const LinearSpectrogramOptions& opts, - std::unique_ptr base_extractor) { - opts_ = opts; + std::unique_ptr base_extractor) + : opts_(opts), feature_window_funtion_(opts.frame_opts) { base_extractor_ = std::move(base_extractor); int32 window_size = opts.frame_opts.WindowSize(); int32 window_shift = opts.frame_opts.WindowShift(); - fft_points_ = window_size; + dim_ = window_size / 2 + 1; chunk_sample_size_ = static_cast(opts.streaming_chunk * opts.frame_opts.samp_freq); - hanning_window_.resize(window_size); - - double a = M_2PI / (window_size - 1); - hanning_window_energy_ = 0; - for (int i = 0; i < window_size; ++i) { - hanning_window_[i] = 0.5 - 0.5 * cos(a * i); - hanning_window_energy_ += hanning_window_[i] * hanning_window_[i]; - } - - dim_ = fft_points_ / 2 + 1; // the dimension is Fs/2 Hz + hanning_window_energy_ = kaldi::VecVec(feature_window_funtion_.window, + feature_window_funtion_.window); } void LinearSpectrogram::Accept(const VectorBase& inputs) { @@ -56,99 +51,57 @@ bool LinearSpectrogram::Read(Vector* feats) { bool flag = base_extractor_->Read(&input_feats); if (flag == false || input_feats.Dim() == 0) return false; - vector input_feats_vec(input_feats.Dim()); - std::memcpy(input_feats_vec.data(), - input_feats.Data(), - input_feats.Dim() * sizeof(BaseFloat)); - vector> result; - Compute(input_feats_vec, result); - int32 feat_size = 0; - if (result.size() != 0) { - feat_size = result.size() * result[0].size(); - } - feats->Resize(feat_size); - // todo refactor (SimleGoat) - for (size_t idx = 0; idx < feat_size; ++idx) { - (*feats)(idx) = result[idx / dim_][idx % dim_]; - } - return true; -} - -void LinearSpectrogram::Hanning(vector* data) const { - CHECK_GE(data->size(), hanning_window_.size()); - - for (size_t i = 0; i < hanning_window_.size(); ++i) { - data->at(i) *= hanning_window_[i]; - } -} - -bool LinearSpectrogram::NumpyFft(vector* v, - vector* real, - vector* img) const { - Vector v_tmp; - v_tmp.Resize(v->size()); - std::memcpy(v_tmp.Data(), v->data(), sizeof(BaseFloat) * (v->size())); - RealFft(&v_tmp, true); - v->resize(v_tmp.Dim()); - std::memcpy(v->data(), v_tmp.Data(), sizeof(BaseFloat) * (v->size())); - - real->push_back(v->at(0)); - img->push_back(0); - for (int i = 1; i < v->size() / 2; i++) { - real->push_back(v->at(2 * i)); - img->push_back(v->at(2 * i + 1)); - } - real->push_back(v->at(1)); - img->push_back(0); - + int32 feat_len = input_feats.Dim(); + int32 left_len = reminded_wav_.Dim(); + Vector waves(feat_len + left_len); + waves.Range(0, left_len).CopyFromVec(reminded_wav_); + waves.Range(left_len, feat_len).CopyFromVec(input_feats); + Compute(waves, feats); + int32 frame_shift = opts_.frame_opts.WindowShift(); + int32 num_frames = kaldi::NumFrames(waves.Dim(), opts_.frame_opts); + int32 left_samples = waves.Dim() - frame_shift * num_frames; + reminded_wav_.Resize(left_samples); + reminded_wav_.CopyFromVec( + waves.Range(frame_shift * num_frames, left_samples)); return true; } // Compute spectrogram feat -// todo: refactor later (SmileGoat) -bool LinearSpectrogram::Compute(const vector& waves, - vector>& feats) { - int num_samples = waves.size(); - const int& frame_length = opts_.frame_opts.WindowSize(); - const int& sample_rate = opts_.frame_opts.samp_freq; - const int& frame_shift = opts_.frame_opts.WindowShift(); - const int& fft_points = fft_points_; - const float scale = hanning_window_energy_ * sample_rate; +bool LinearSpectrogram::Compute(const Vector& waves, + Vector* feats) { + int32 num_samples = waves.Dim(); + int32 frame_length = opts_.frame_opts.WindowSize(); + int32 sample_rate = opts_.frame_opts.samp_freq; + BaseFloat scale = 2.0 / (hanning_window_energy_ * sample_rate); if (num_samples < frame_length) { return true; } - int num_frames = 1 + ((num_samples - frame_length) / frame_shift); - feats.resize(num_frames); - vector fft_real((fft_points_ / 2 + 1), 0); - vector fft_img((fft_points_ / 2 + 1), 0); - vector v(frame_length, 0); - vector power((fft_points / 2 + 1)); - - for (int i = 0; i < num_frames; ++i) { - vector data(waves.data() + i * frame_shift, - waves.data() + i * frame_shift + frame_length); - Hanning(&data); - fft_img.clear(); - fft_real.clear(); - v.assign(data.begin(), data.end()); - NumpyFft(&v, &fft_real, &fft_img); - - feats[i].resize(fft_points / 2 + 1); // the last dimension is Fs/2 Hz - for (int j = 0; j < (fft_points / 2 + 1); ++j) { - power[j] = fft_real[j] * fft_real[j] + fft_img[j] * fft_img[j]; - feats[i][j] = power[j]; - - if (j == 0 || j == feats[0].size() - 1) { - feats[i][j] /= scale; - } else { - feats[i][j] *= (2.0 / scale); - } - - // log added eps=1e-14 - feats[i][j] = std::log(feats[i][j] + 1e-14); - } + int32 num_frames = kaldi::NumFrames(num_samples, opts_.frame_opts); + feats->Resize(num_frames * dim_); + Vector window; + + for (int frame_idx = 0; frame_idx < num_frames; ++frame_idx) { + kaldi::ExtractWindow(0, + waves, + frame_idx, + opts_.frame_opts, + feature_window_funtion_, + &window, + NULL); + + SubVector output_row(feats->Data() + frame_idx * dim_, dim_); + window.Resize(frame_length, kaldi::kCopyData); + RealFft(&window, true); + kaldi::ComputePowerSpectrum(&window); + SubVector power_spectrum(window, 0, dim_); + power_spectrum.Scale(scale); + power_spectrum(0) = power_spectrum(0) / 2; + power_spectrum(dim_ - 1) = power_spectrum(dim_ - 1) / 2; + power_spectrum.Add(1e-14); + power_spectrum.ApplyLog(); + output_row.CopyFromVec(power_spectrum); } return true; } diff --git a/speechx/speechx/frontend/audio/linear_spectrogram.h b/speechx/speechx/frontend/audio/linear_spectrogram.h index bbf8d685..896c494d 100644 --- a/speechx/speechx/frontend/audio/linear_spectrogram.h +++ b/speechx/speechx/frontend/audio/linear_spectrogram.h @@ -49,19 +49,15 @@ class LinearSpectrogram : public FrontendInterface { virtual void Reset() { base_extractor_->Reset(); } private: - void Hanning(std::vector* data) const; - bool Compute(const std::vector& waves, - std::vector>& feats); - bool NumpyFft(std::vector* v, - std::vector* real, - std::vector* img) const; + bool Compute(const kaldi::Vector& waves, + kaldi::Vector* feats); - kaldi::int32 fft_points_; size_t dim_; - std::vector hanning_window_; + kaldi::FeatureWindowFunction feature_window_funtion_; kaldi::BaseFloat hanning_window_energy_; LinearSpectrogramOptions opts_; std::unique_ptr base_extractor_; + kaldi::Vector reminded_wav_; int chunk_sample_size_; DISALLOW_COPY_AND_ASSIGN(LinearSpectrogram); };