From bbcb1ea2971e6c2e3a6984f485a1ffc62bd3553b Mon Sep 17 00:00:00 2001 From: Yang Zhou Date: Mon, 6 Jun 2022 22:44:13 +0800 Subject: [PATCH] add nnet_decoder_chunk opt --- .../ctc_prefix_beam_search_decoder_main.cc | 6 +- speechx/speechx/decoder/param.h | 8 +- speechx/speechx/decoder/tlg_decoder_main.cc | 7 +- speechx/speechx/frontend/audio/assembler.cc | 19 +- speechx/speechx/frontend/audio/assembler.h | 13 +- speechx/speechx/nnet/CMakeLists.txt | 9 +- speechx/speechx/nnet/nnet_forward_main.cc | 162 ++++++++++++++++++ 7 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 speechx/speechx/nnet/nnet_forward_main.cc diff --git a/speechx/speechx/decoder/ctc_prefix_beam_search_decoder_main.cc b/speechx/speechx/decoder/ctc_prefix_beam_search_decoder_main.cc index eaec41b7..305449cd 100644 --- a/speechx/speechx/decoder/ctc_prefix_beam_search_decoder_main.cc +++ b/speechx/speechx/decoder/ctc_prefix_beam_search_decoder_main.cc @@ -45,6 +45,7 @@ DEFINE_string(model_cache_names, "chunk_state_h_box,chunk_state_c_box", "model cache names"); DEFINE_string(model_cache_shapes, "5-1-1024,5-1-1024", "model cache shapes"); +DEFINE_int32(nnet_decoder_chunk, 1, "paddle nnet forward chunk"); using kaldi::BaseFloat; using kaldi::Matrix; @@ -90,8 +91,9 @@ int main(int argc, char* argv[]) { std::shared_ptr decodable( new ppspeech::Decodable(nnet, raw_data)); - int32 chunk_size = FLAGS_receptive_field_length; - int32 chunk_stride = FLAGS_downsampling_rate; + int32 chunk_size = FLAGS_receptive_field_length + + (FLAGS_nnet_decoder_chunk - 1) * FLAGS_downsampling_rate; + int32 chunk_stride = FLAGS_downsampling_rate * FLAGS_nnet_decoder_chunk; int32 receptive_field_length = FLAGS_receptive_field_length; LOG(INFO) << "chunk size (frame): " << chunk_size; LOG(INFO) << "chunk stride (frame): " << chunk_stride; diff --git a/speechx/speechx/decoder/param.h b/speechx/speechx/decoder/param.h index f3560343..780a1d6e 100644 --- a/speechx/speechx/decoder/param.h +++ b/speechx/speechx/decoder/param.h @@ -32,7 +32,7 @@ DEFINE_int32(receptive_field_length, DEFINE_int32(downsampling_rate, 4, "two CNN(kernel=5) module downsampling rate."); - +DEFINE_int32(nnet_decoder_chunk, 1, "paddle nnet forward chunk"); // nnet DEFINE_string(model_path, "avg_1.jit.pdmodel", "paddle nnet model"); DEFINE_string(param_path, "avg_1.jit.pdiparams", "paddle nnet model param"); @@ -79,8 +79,10 @@ FeaturePipelineOptions InitFeaturePipelineOptions() { frame_opts.preemph_coeff = 0.0; opts.linear_spectrogram_opts.frame_opts = frame_opts; } - opts.assembler_opts.frame_chunk_size = FLAGS_receptive_field_length; - opts.assembler_opts.frame_chunk_stride = FLAGS_downsampling_rate; + opts.assembler_opts.subsampling_rate = FLAGS_downsampling_rate; + opts.assembler_opts.receptive_filed_length = FLAGS_receptive_field_length; + opts.assembler_opts.nnet_decoder_chunk = FLAGS_nnet_decoder_chunk; + return opts; } diff --git a/speechx/speechx/decoder/tlg_decoder_main.cc b/speechx/speechx/decoder/tlg_decoder_main.cc index fefc16d2..9cf10a18 100644 --- a/speechx/speechx/decoder/tlg_decoder_main.cc +++ b/speechx/speechx/decoder/tlg_decoder_main.cc @@ -28,9 +28,9 @@ DEFINE_string(model_path, "avg_1.jit.pdmodel", "paddle nnet model"); DEFINE_string(param_path, "avg_1.jit.pdiparams", "paddle nnet model param"); DEFINE_string(word_symbol_table, "words.txt", "word symbol table"); DEFINE_string(graph_path, "TLG", "decoder graph"); - DEFINE_double(acoustic_scale, 1.0, "acoustic scale"); DEFINE_int32(max_active, 7500, "decoder graph"); +DEFINE_int32(nnet_decoder_chunk, 1, "paddle nnet forward chunk"); DEFINE_int32(receptive_field_length, 7, "receptive field of two CNN(kernel=5) downsampling module."); @@ -93,8 +93,9 @@ int main(int argc, char* argv[]) { std::shared_ptr decodable( new ppspeech::Decodable(nnet, raw_data, FLAGS_acoustic_scale)); - int32 chunk_size = FLAGS_receptive_field_length; - int32 chunk_stride = FLAGS_downsampling_rate; + int32 chunk_size = FLAGS_receptive_field_length + + (FLAGS_nnet_decoder_chunk - 1) * FLAGS_downsampling_rate; + int32 chunk_stride = FLAGS_downsampling_rate * FLAGS_nnet_decoder_chunk; int32 receptive_field_length = FLAGS_receptive_field_length; LOG(INFO) << "chunk size (frame): " << chunk_size; LOG(INFO) << "chunk stride (frame): " << chunk_stride; diff --git a/speechx/speechx/frontend/audio/assembler.cc b/speechx/speechx/frontend/audio/assembler.cc index 47e0705b..721d2ad0 100644 --- a/speechx/speechx/frontend/audio/assembler.cc +++ b/speechx/speechx/frontend/audio/assembler.cc @@ -23,8 +23,9 @@ using std::unique_ptr; Assembler::Assembler(AssemblerOptions opts, unique_ptr base_extractor) { - frame_chunk_stride_ = opts.frame_chunk_stride; - frame_chunk_size_ = opts.frame_chunk_size; + frame_chunk_stride_ = opts.subsampling_rate * opts.nnet_decoder_chunk; + frame_chunk_size_ = (opts.nnet_decoder_chunk - 1) * opts.subsampling_rate + opts.receptive_filed_length; + receptive_filed_length_ = opts.receptive_filed_length; base_extractor_ = std::move(base_extractor); dim_ = base_extractor_->Dim(); } @@ -48,10 +49,22 @@ bool Assembler::Compute(Vector* feats) { while (feature_cache_.size() < frame_chunk_size_) { Vector feature; result = base_extractor_->Read(&feature); - if (result == false || feature.Dim() == 0) return false; + if (result == false || feature.Dim() == 0) { + if (IsFinished() == false) return false; + break; + } feature_cache_.push(feature); } + if (feature_cache_.size() < receptive_filed_length_) { + return false; + } + + while (feature_cache_.size() < frame_chunk_size_) { + Vector feature(dim_, kaldi::kSetZero); + feature_cache_.push(feature); + } + int32 counter = 0; int32 cache_size = frame_chunk_size_ - frame_chunk_stride_; int32 elem_dim = base_extractor_->Dim(); diff --git a/speechx/speechx/frontend/audio/assembler.h b/speechx/speechx/frontend/audio/assembler.h index 4397d3f6..a477df99 100644 --- a/speechx/speechx/frontend/audio/assembler.h +++ b/speechx/speechx/frontend/audio/assembler.h @@ -20,12 +20,16 @@ namespace ppspeech { struct AssemblerOptions { - int32 frame_chunk_size; - int32 frame_chunk_stride; + // refer:https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/paddlespeech/s2t/exps/deepspeech2/model.py + // the nnet batch forward + int32 receptive_filed_length; + int32 subsampling_rate; + int32 nnet_decoder_chunk; AssemblerOptions() - : frame_chunk_size(1), - frame_chunk_stride(1) {} + : receptive_filed_length(1), + subsampling_rate(1), + nnet_decoder_chunk(1) {} }; class Assembler : public FrontendInterface { @@ -59,6 +63,7 @@ class Assembler : public FrontendInterface { int32 dim_; int32 frame_chunk_size_; // window int32 frame_chunk_stride_; // stride + int32 receptive_filed_length_; std::queue> feature_cache_; std::unique_ptr base_extractor_; DISALLOW_COPY_AND_ASSIGN(Assembler); diff --git a/speechx/speechx/nnet/CMakeLists.txt b/speechx/speechx/nnet/CMakeLists.txt index cee881de..c325ce75 100644 --- a/speechx/speechx/nnet/CMakeLists.txt +++ b/speechx/speechx/nnet/CMakeLists.txt @@ -4,4 +4,11 @@ add_library(nnet STATIC decodable.cc paddle_nnet.cc ) -target_link_libraries(nnet absl::strings) \ No newline at end of file +target_link_libraries(nnet absl::strings) + +set(bin_name nnet_forward_main) +add_executable(${bin_name} ${CMAKE_CURRENT_SOURCE_DIR}/${bin_name}.cc) +target_include_directories(${bin_name} PRIVATE ${SPEECHX_ROOT} ${SPEECHX_ROOT}/kaldi) +target_link_libraries(${bin_name} utils kaldi-util kaldi-matrix gflags glog nnet ${DEPS}) + + diff --git a/speechx/speechx/nnet/nnet_forward_main.cc b/speechx/speechx/nnet/nnet_forward_main.cc new file mode 100644 index 00000000..170b74a5 --- /dev/null +++ b/speechx/speechx/nnet/nnet_forward_main.cc @@ -0,0 +1,162 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "base/flags.h" +#include "base/log.h" +#include "frontend/audio/data_cache.h" +#include "frontend/audio/assembler.h" +#include "kaldi/util/table-types.h" +#include "nnet/decodable.h" +#include "nnet/paddle_nnet.h" + +DEFINE_string(feature_rspecifier, "", "test feature rspecifier"); +DEFINE_string(nnet_prob_wspecifier, "", "nnet porb wspecifier"); +DEFINE_string(model_path, "avg_1.jit.pdmodel", "paddle nnet model"); +DEFINE_string(param_path, "avg_1.jit.pdiparams", "paddle nnet model param"); +DEFINE_int32(nnet_decoder_chunk, 1, "paddle nnet forward chunk"); +DEFINE_int32(receptive_field_length, + 7, + "receptive field of two CNN(kernel=5) downsampling module."); +DEFINE_int32(downsampling_rate, + 4, + "two CNN(kernel=5) module downsampling rate."); +DEFINE_string( + model_input_names, + "audio_chunk,audio_chunk_lens,chunk_state_h_box,chunk_state_c_box", + "model input names"); +DEFINE_string(model_output_names, + "softmax_0.tmp_0,tmp_5,concat_0.tmp_0,concat_1.tmp_0", + "model output names"); +DEFINE_string(model_cache_names, + "chunk_state_h_box,chunk_state_c_box", + "model cache names"); +DEFINE_string(model_cache_shapes, "5-1-1024,5-1-1024", "model cache shapes"); +DEFINE_double(acoustic_scale, 1.0, "acoustic scale"); + +using kaldi::BaseFloat; +using kaldi::Matrix; +using std::vector; + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, false); + google::InitGoogleLogging(argv[0]); + + kaldi::SequentialBaseFloatMatrixReader feature_reader( + FLAGS_feature_rspecifier); + kaldi::BaseFloatMatrixWriter nnet_writer(FLAGS_nnet_prob_wspecifier); + std::string model_graph = FLAGS_model_path; + std::string model_params = FLAGS_param_path; + LOG(INFO) << "model path: " << model_graph; + LOG(INFO) << "model param: " << model_params; + + int32 num_done = 0, num_err = 0; + + ppspeech::ModelOptions model_opts; + model_opts.model_path = model_graph; + model_opts.param_path = model_params; + model_opts.cache_names = FLAGS_model_cache_names; + model_opts.cache_shape = FLAGS_model_cache_shapes; + model_opts.input_names = FLAGS_model_input_names; + model_opts.output_names = FLAGS_model_output_names; + std::shared_ptr nnet( + new ppspeech::PaddleNnet(model_opts)); + std::shared_ptr raw_data(new ppspeech::DataCache()); + std::shared_ptr decodable( + new ppspeech::Decodable(nnet, raw_data, FLAGS_acoustic_scale)); + + int32 chunk_size = FLAGS_receptive_field_length + + (FLAGS_nnet_decoder_chunk - 1) * FLAGS_downsampling_rate; + int32 chunk_stride = FLAGS_downsampling_rate * FLAGS_nnet_decoder_chunk; + int32 receptive_field_length = FLAGS_receptive_field_length; + LOG(INFO) << "chunk size (frame): " << chunk_size; + LOG(INFO) << "chunk stride (frame): " << chunk_stride; + LOG(INFO) << "receptive field (frame): " << receptive_field_length; + kaldi::Timer timer; + for (; !feature_reader.Done(); feature_reader.Next()) { + string utt = feature_reader.Key(); + kaldi::Matrix feature = feature_reader.Value(); + raw_data->SetDim(feature.NumCols()); + LOG(INFO) << "process utt: " << utt; + LOG(INFO) << "rows: " << feature.NumRows(); + LOG(INFO) << "cols: " << feature.NumCols(); + + int32 row_idx = 0; + int32 padding_len = 0; + int32 ori_feature_len = feature.NumRows(); + if ((feature.NumRows() - chunk_size) % chunk_stride != 0) { + padding_len = + chunk_stride - (feature.NumRows() - chunk_size) % chunk_stride; + feature.Resize(feature.NumRows() + padding_len, + feature.NumCols(), + kaldi::kCopyData); + } + int32 num_chunks = (feature.NumRows() - chunk_size) / chunk_stride + 1; + int32 frame_idx = 0; + std::vector> prob_vec; + for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { + kaldi::Vector feature_chunk(chunk_size * + feature.NumCols()); + int32 feature_chunk_size = 0; + if (ori_feature_len > chunk_idx * chunk_stride) { + feature_chunk_size = std::min( + ori_feature_len - chunk_idx * chunk_stride, chunk_size); + } + if (feature_chunk_size < receptive_field_length) break; + + int32 start = chunk_idx * chunk_stride; + for (int row_id = 0; row_id < chunk_size; ++row_id) { + kaldi::SubVector tmp(feature, start); + kaldi::SubVector f_chunk_tmp( + feature_chunk.Data() + row_id * feature.NumCols(), + feature.NumCols()); + f_chunk_tmp.CopyFromVec(tmp); + ++start; + } + raw_data->Accept(feature_chunk); + if (chunk_idx == num_chunks - 1) { + raw_data->SetFinished(); + } + vector prob; + while (decodable->FrameLikelihood(frame_idx, &prob)) { + kaldi::Vector vec_tmp(prob.size()); + std::memcpy(vec_tmp.Data(), prob.data(), sizeof(kaldi::BaseFloat)*prob.size()); + prob_vec.push_back(vec_tmp); + frame_idx++; + } + } + decodable->Reset(); + if (prob_vec.size() == 0) { + // the TokenWriter can not write empty string. + ++num_err; + KALDI_LOG << " the nnet prob of " << utt << " is empty"; + continue; + } + kaldi::Matrix result(prob_vec.size(),prob_vec[0].Dim()); + for (int32 row_idx = 0; row_idx < prob_vec.size(); ++row_idx) { + for (int32 col_idx = 0; col_idx < prob_vec[0].Dim(); ++col_idx) { + result(row_idx, col_idx) = prob_vec[row_idx](col_idx); + } + } + + nnet_writer.Write(utt, result); + ++num_done; + } + + double elapsed = timer.Elapsed(); + KALDI_LOG << " cost:" << elapsed << " s"; + + KALDI_LOG << "Done " << num_done << " utterances, " << num_err + << " with errors."; + return (num_done != 0 ? 0 : 1); +} \ No newline at end of file