Yang Zhou
2 years ago
parent
b78bc6375b
commit
1133540682
@ -0,0 +1,10 @@
|
||||
cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
|
||||
|
||||
add_executable(websocket_server_main ${CMAKE_CURRENT_SOURCE_DIR}/websocket_server_main.cc)
|
||||
target_include_directories(websocket_server_main PRIVATE ${SPEECHX_ROOT} ${SPEECHX_ROOT}/kaldi)
|
||||
target_link_libraries(websocket_server_main PUBLIC frontend kaldi-feat-common nnet decoder fst utils gflags glog kaldi-base kaldi-matrix kaldi-util kaldi-decoder websocket ${DEPS})
|
||||
|
||||
add_executable(websocket_client_main ${CMAKE_CURRENT_SOURCE_DIR}/websocket_client_main.cc)
|
||||
target_include_directories(websocket_client_main PRIVATE ${SPEECHX_ROOT} ${SPEECHX_ROOT}/kaldi)
|
||||
target_link_libraries(websocket_client_main PUBLIC frontend kaldi-feat-common nnet decoder fst utils gflags glog kaldi-base kaldi-matrix kaldi-util kaldi-decoder websocket ${DEPS})
|
||||
|
||||
@ -0,0 +1,82 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "websocket/websocket_client.h"
|
||||
#include "kaldi/feat/wave-reader.h"
|
||||
#include "kaldi/util/kaldi-io.h"
|
||||
#include "kaldi/util/table-types.h"
|
||||
|
||||
DEFINE_string(host, "127.0.0.1", "host of websocket server");
|
||||
DEFINE_int32(port, 201314, "port of websocket server");
|
||||
DEFINE_string(wav_rspecifier, "", "test wav scp path");
|
||||
DEFINE_double(streaming_chunk, 0.1, "streaming feature chunk size");
|
||||
|
||||
using kaldi::int16;
|
||||
int main(int argc, char* argv[]) {
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, false);
|
||||
google::InitGoogleLogging(argv[0]);
|
||||
ppspeech::WebSocketClient client(FLAGS_host, FLAGS_port);
|
||||
|
||||
kaldi::SequentialTableReader<kaldi::WaveHolder> wav_reader(
|
||||
FLAGS_wav_rspecifier);
|
||||
|
||||
const int sample_rate = 16000;
|
||||
const float streaming_chunk = FLAGS_streaming_chunk;
|
||||
const int chunk_sample_size = streaming_chunk * sample_rate;
|
||||
|
||||
for (; !wav_reader.Done(); wav_reader.Next()) {
|
||||
client.SendStartSignal();
|
||||
std::string utt = wav_reader.Key();
|
||||
const kaldi::WaveData& wave_data = wav_reader.Value();
|
||||
CHECK_EQ(wave_data.SampFreq(), sample_rate);
|
||||
|
||||
int32 this_channel = 0;
|
||||
kaldi::SubVector<kaldi::BaseFloat> waveform(wave_data.Data(),
|
||||
this_channel);
|
||||
const int tot_samples = waveform.Dim();
|
||||
int sample_offset = 0;
|
||||
|
||||
while (sample_offset < tot_samples) {
|
||||
int cur_chunk_size =
|
||||
std::min(chunk_sample_size, tot_samples - sample_offset);
|
||||
|
||||
std::vector<int16> wav_chunk(cur_chunk_size);
|
||||
for (int i = 0; i < cur_chunk_size; ++i) {
|
||||
wav_chunk[i] = static_cast<int16>(waveform(sample_offset + i));
|
||||
}
|
||||
client.SendBinaryData(wav_chunk.data(),
|
||||
wav_chunk.size() * sizeof(int16));
|
||||
|
||||
|
||||
sample_offset += cur_chunk_size;
|
||||
LOG(INFO) << "Send " << cur_chunk_size << " samples";
|
||||
std::this_thread::sleep_for(
|
||||
std::chrono::milliseconds(static_cast<int>(1 * 1000)));
|
||||
|
||||
if (cur_chunk_size < chunk_sample_size) {
|
||||
client.SendEndSignal();
|
||||
}
|
||||
}
|
||||
|
||||
while (!client.Done()) {
|
||||
}
|
||||
std::string result = client.GetResult();
|
||||
LOG(INFO) << "utt: " << utt << " " << result;
|
||||
|
||||
|
||||
client.Join();
|
||||
return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "websocket/websocket_server.h"
|
||||
#include "decoder/param.h"
|
||||
|
||||
DEFINE_int32(port, 201314, "websocket listening port");
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, false);
|
||||
google::InitGoogleLogging(argv[0]);
|
||||
|
||||
ppspeech::RecognizerResource resource = ppspeech::InitRecognizerResoure();
|
||||
|
||||
ppspeech::WebSocketServer server(FLAGS_port, resource);
|
||||
LOG(INFO) << "Listening at port " << FLAGS_port;
|
||||
server.Start();
|
||||
return 0;
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "base/common.h"
|
||||
|
||||
#include "decoder/ctc_beam_search_decoder.h"
|
||||
#include "decoder/ctc_tlg_decoder.h"
|
||||
#include "frontend/audio/feature_pipeline.h"
|
||||
|
||||
DEFINE_string(cmvn_file, "", "read cmvn");
|
||||
DEFINE_double(streaming_chunk, 0.1, "streaming feature chunk size");
|
||||
DEFINE_bool(convert2PCM32, true, "audio convert to pcm32");
|
||||
DEFINE_string(model_path, "avg_1.jit.pdmodel", "paddle nnet model");
|
||||
DEFINE_string(params_path, "avg_1.jit.pdiparams", "paddle nnet model param");
|
||||
DEFINE_string(word_symbol_table, "words.txt", "word symbol table");
|
||||
DEFINE_string(graph_path, "TLG", "decoder graph");
|
||||
DEFINE_double(acoustic_scale, 1.0, "acoustic scale");
|
||||
DEFINE_int32(max_active, 7500, "max active");
|
||||
DEFINE_double(beam, 15.0, "decoder beam");
|
||||
DEFINE_double(lattice_beam, 7.5, "decoder beam");
|
||||
DEFINE_int32(receptive_field_length,
|
||||
7,
|
||||
"receptive field of two CNN(kernel=5) downsampling module.");
|
||||
DEFINE_int32(downsampling_rate,
|
||||
4,
|
||||
"two CNN(kernel=5) module downsampling rate.");
|
||||
DEFINE_string(model_output_names,
|
||||
"save_infer_model/scale_0.tmp_1,save_infer_model/"
|
||||
"scale_1.tmp_1,save_infer_model/scale_2.tmp_1,save_infer_model/"
|
||||
"scale_3.tmp_1",
|
||||
"model output names");
|
||||
DEFINE_string(model_cache_names, "5-1-1024,5-1-1024", "model cache names");
|
||||
|
||||
namespace ppspeech {
|
||||
// todo refactor later
|
||||
FeaturePipelineOptions InitFeaturePipelineOptions() {
|
||||
FeaturePipelineOptions opts;
|
||||
opts.cmvn_file = FLAGS_cmvn_file;
|
||||
opts.linear_spectrogram_opts.streaming_chunk = FLAGS_streaming_chunk;
|
||||
opts.convert2PCM32 = FLAGS_convert2PCM32;
|
||||
kaldi::FrameExtractionOptions frame_opts;
|
||||
frame_opts.frame_length_ms = 20;
|
||||
frame_opts.frame_shift_ms = 10;
|
||||
frame_opts.remove_dc_offset = false;
|
||||
frame_opts.window_type = "hanning";
|
||||
frame_opts.preemph_coeff = 0.0;
|
||||
frame_opts.dither = 0.0;
|
||||
opts.linear_spectrogram_opts.frame_opts = frame_opts;
|
||||
opts.feature_cache_opts.frame_chunk_size = FLAGS_receptive_field_length;
|
||||
opts.feature_cache_opts.frame_chunk_stride = FLAGS_downsampling_rate;
|
||||
return opts;
|
||||
}
|
||||
|
||||
ModelOptions InitModelOptions() {
|
||||
ModelOptions model_opts;
|
||||
model_opts.model_path = FLAGS_model_path;
|
||||
model_opts.params_path = FLAGS_params_path;
|
||||
model_opts.cache_shape = FLAGS_model_cache_names;
|
||||
model_opts.output_names = FLAGS_model_output_names;
|
||||
return model_opts;
|
||||
}
|
||||
|
||||
TLGDecoderOptions InitDecoderOptions() {
|
||||
TLGDecoderOptions decoder_opts;
|
||||
decoder_opts.word_symbol_table = FLAGS_word_symbol_table;
|
||||
decoder_opts.fst_path = FLAGS_graph_path;
|
||||
decoder_opts.opts.max_active = FLAGS_max_active;
|
||||
decoder_opts.opts.beam = FLAGS_beam;
|
||||
decoder_opts.opts.lattice_beam = FLAGS_lattice_beam;
|
||||
return decoder_opts;
|
||||
}
|
||||
|
||||
RecognizerResource InitRecognizerResoure() {
|
||||
RecognizerResource resource;
|
||||
resource.acoustic_scale = FLAGS_acoustic_scale;
|
||||
resource.feature_pipeline_opts = InitFeaturePipelineOptions();
|
||||
resource.model_opts = InitModelOptions();
|
||||
resource.tlg_opts = InitDecoderOptions();
|
||||
return resource;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "decoder/recognizer.h"
|
||||
|
||||
namespace ppspeech {
|
||||
|
||||
using kaldi::Vector;
|
||||
using kaldi::VectorBase;
|
||||
using kaldi::BaseFloat;
|
||||
using std::vector;
|
||||
using kaldi::SubVector;
|
||||
using std::unique_ptr;
|
||||
|
||||
Recognizer::Recognizer(const RecognizerResource& resource) {
|
||||
// resource_ = resource;
|
||||
const FeaturePipelineOptions& feature_opts = resource.feature_pipeline_opts;
|
||||
feature_pipeline_.reset(new FeaturePipeline(feature_opts));
|
||||
std::shared_ptr<PaddleNnet> nnet(new PaddleNnet(resource.model_opts));
|
||||
BaseFloat ac_scale = resource.acoustic_scale;
|
||||
decodable_.reset(new Decodable(nnet, feature_pipeline_, ac_scale));
|
||||
decoder_.reset(new TLGDecoder(resource.tlg_opts));
|
||||
input_finished_ = false;
|
||||
}
|
||||
|
||||
void Recognizer::Accept(const Vector<BaseFloat>& waves) {
|
||||
feature_pipeline_->Accept(waves);
|
||||
}
|
||||
|
||||
void Recognizer::Decode() { decoder_->AdvanceDecode(decodable_); }
|
||||
|
||||
std::string Recognizer::GetFinalResult() {
|
||||
return decoder_->GetFinalBestPath();
|
||||
}
|
||||
|
||||
void Recognizer::SetFinished() {
|
||||
feature_pipeline_->SetFinished();
|
||||
input_finished_ = true;
|
||||
}
|
||||
|
||||
bool Recognizer::IsFinished() { return input_finished_; }
|
||||
|
||||
void Recognizer::Reset() {
|
||||
feature_pipeline_->Reset();
|
||||
decodable_->Reset();
|
||||
decoder_->Reset();
|
||||
}
|
||||
|
||||
} // namespace ppspeech
|
||||
@ -0,0 +1,59 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// todo refactor later (SGoat)
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "decoder/ctc_beam_search_decoder.h"
|
||||
#include "decoder/ctc_tlg_decoder.h"
|
||||
#include "frontend/audio/feature_pipeline.h"
|
||||
#include "nnet/decodable.h"
|
||||
#include "nnet/paddle_nnet.h"
|
||||
|
||||
namespace ppspeech {
|
||||
|
||||
struct RecognizerResource {
|
||||
FeaturePipelineOptions feature_pipeline_opts;
|
||||
ModelOptions model_opts;
|
||||
TLGDecoderOptions tlg_opts;
|
||||
// CTCBeamSearchOptions beam_search_opts;
|
||||
kaldi::BaseFloat acoustic_scale;
|
||||
RecognizerResource()
|
||||
: acoustic_scale(1.0),
|
||||
feature_pipeline_opts(),
|
||||
model_opts(),
|
||||
tlg_opts() {}
|
||||
};
|
||||
|
||||
class Recognizer {
|
||||
public:
|
||||
explicit Recognizer(const RecognizerResource& resouce);
|
||||
void Accept(const kaldi::Vector<kaldi::BaseFloat>& waves);
|
||||
void Decode();
|
||||
std::string GetFinalResult();
|
||||
void SetFinished();
|
||||
bool IsFinished();
|
||||
void Reset();
|
||||
|
||||
private:
|
||||
// std::shared_ptr<RecognizerResource> resource_;
|
||||
// RecognizerResource resource_;
|
||||
std::shared_ptr<FeaturePipeline> feature_pipeline_;
|
||||
std::shared_ptr<Decodable> decodable_;
|
||||
std::unique_ptr<TLGDecoder> decoder_;
|
||||
bool input_finished_;
|
||||
};
|
||||
|
||||
} // namespace ppspeech
|
||||
@ -0,0 +1,36 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "frontend/audio/feature_pipeline.h"
|
||||
|
||||
namespace ppspeech {
|
||||
|
||||
using std::unique_ptr;
|
||||
|
||||
FeaturePipeline::FeaturePipeline(const FeaturePipelineOptions& opts) {
|
||||
unique_ptr<FrontendInterface> data_source(
|
||||
new ppspeech::AudioCache(1000 * kint16max, opts.convert2PCM32));
|
||||
|
||||
unique_ptr<FrontendInterface> linear_spectrogram(
|
||||
new ppspeech::LinearSpectrogram(opts.linear_spectrogram_opts,
|
||||
std::move(data_source)));
|
||||
|
||||
unique_ptr<FrontendInterface> cmvn(
|
||||
new ppspeech::CMVN(opts.cmvn_file, std::move(linear_spectrogram)));
|
||||
|
||||
base_extractor_.reset(
|
||||
new ppspeech::FeatureCache(opts.feature_cache_opts, std::move(cmvn)));
|
||||
}
|
||||
|
||||
} // ppspeech
|
||||
@ -0,0 +1,57 @@
|
||||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// todo refactor later (SGoat)
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "frontend/audio/audio_cache.h"
|
||||
#include "frontend/audio/data_cache.h"
|
||||
#include "frontend/audio/feature_cache.h"
|
||||
#include "frontend/audio/frontend_itf.h"
|
||||
#include "frontend/audio/linear_spectrogram.h"
|
||||
#include "frontend/audio/normalizer.h"
|
||||
|
||||
namespace ppspeech {
|
||||
|
||||
struct FeaturePipelineOptions {
|
||||
std::string cmvn_file;
|
||||
bool convert2PCM32;
|
||||
LinearSpectrogramOptions linear_spectrogram_opts;
|
||||
FeatureCacheOptions feature_cache_opts;
|
||||
FeaturePipelineOptions()
|
||||
: cmvn_file(""),
|
||||
convert2PCM32(false),
|
||||
linear_spectrogram_opts(),
|
||||
feature_cache_opts() {}
|
||||
};
|
||||
|
||||
class FeaturePipeline : public FrontendInterface {
|
||||
public:
|
||||
explicit FeaturePipeline(const FeaturePipelineOptions& opts);
|
||||
virtual void Accept(const kaldi::VectorBase<kaldi::BaseFloat>& waves) {
|
||||
base_extractor_->Accept(waves);
|
||||
}
|
||||
virtual bool Read(kaldi::Vector<kaldi::BaseFloat>* feats) {
|
||||
return base_extractor_->Read(feats);
|
||||
}
|
||||
virtual size_t Dim() const { return base_extractor_->Dim(); }
|
||||
virtual void SetFinished() { base_extractor_->SetFinished(); }
|
||||
virtual bool IsFinished() const { return base_extractor_->IsFinished(); }
|
||||
virtual void Reset() { base_extractor_->Reset(); }
|
||||
|
||||
private:
|
||||
std::unique_ptr<FrontendInterface> base_extractor_;
|
||||
};
|
||||
}
|
||||
Loading…
Reference in new issue