[runtime] support onnx runtime && support ios compile (#3101)
* support vad ios compile * support onnx model recognize * add build ios shpull/3112/head
masimeng1994
2 years ago
committed by
GitHub
parent
767f6dd4e2
commit
9e5a39cae0
@ -0,0 +1,91 @@
|
||||
# https://www.jianshu.com/p/33672fb819f5
|
||||
|
||||
PATH="/Applications/CMake.app/Contents/bin":"$PATH"
|
||||
tools_dir=$1
|
||||
ios_toolchain_cmake=${tools_dir}/"/ios-cmake-4.2.0/ios.toolchain.cmake"
|
||||
fastdeploy_dir=${tools_dir}"/fastdeploy-ort-mac-build/"
|
||||
build_targets=("OS64")
|
||||
build_type_array=("Release")
|
||||
|
||||
#static_name="libocr"
|
||||
#lib_name="libocr"
|
||||
|
||||
# Switch to workpath
|
||||
current_path=`cd $(dirname $0);pwd`
|
||||
work_path=${current_path}/
|
||||
build_path=${current_path}/build/
|
||||
output_path=${current_path}/output/
|
||||
cd ${work_path}
|
||||
|
||||
# Clean
|
||||
rm -rf ${build_path}
|
||||
rm -rf ${output_path}
|
||||
|
||||
if [ "$1"x = "clean"x ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Build Every Target
|
||||
for target in "${build_targets[@]}"
|
||||
do
|
||||
for build_type in "${build_type_array[@]}"
|
||||
do
|
||||
echo -e "\033[1;36;40mBuilding ${build_type} ${target} ... \033[0m"
|
||||
target_build_path=${build_path}/${target}/${build_type}/
|
||||
mkdir -p ${target_build_path}
|
||||
|
||||
cd ${target_build_path}
|
||||
if [ $? -ne 0 ];then
|
||||
echo -e "\033[1;31;40mcd ${target_build_path} failed \033[0m"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
if [ ${target} == "OS64" ];then
|
||||
fastdeploy_install_dir=${fastdeploy_dir}/arm64
|
||||
else
|
||||
fastdeploy_install_dir=""
|
||||
echo "fastdeploy_install_dir is null"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
cmake -DCMAKE_TOOLCHAIN_FILE=${ios_toolchain_cmake} \
|
||||
-DBUILD_IN_MACOS=ON \
|
||||
-DBUILD_SHARED_LIBS=OFF \
|
||||
-DWITH_ASR=OFF \
|
||||
-DWITH_CLS=OFF \
|
||||
-DWITH_VAD=ON \
|
||||
-DFASTDEPLOY_INSTALL_DIR=${fastdeploy_install_dir} \
|
||||
-DPLATFORM=${target} ../../../
|
||||
|
||||
cmake --build . --config ${build_type}
|
||||
|
||||
mkdir output
|
||||
cp engine/vad/interface/libpps_vad_interface.a output
|
||||
cp engine/vad/interface/vad_interface_main.app/vad_interface_main output
|
||||
cp ${fastdeploy_install_dir}/lib/libfastdeploy.dylib output
|
||||
cp ${fastdeploy_install_dir}/third_libs/install/onnxruntime/lib/libonnxruntime.dylib output
|
||||
|
||||
done
|
||||
done
|
||||
|
||||
## combine all ios libraries
|
||||
#DEVROOT=/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/
|
||||
#LIPO_TOOL=${DEVROOT}/usr/bin/lipo
|
||||
#LIBRARY_PATH=${build_path}
|
||||
#LIBRARY_OUTPUT_PATH=${output_path}/IOS
|
||||
#mkdir -p ${LIBRARY_OUTPUT_PATH}
|
||||
#
|
||||
#${LIPO_TOOL} \
|
||||
# -arch i386 ${LIBRARY_PATH}/ios_x86/Release/${lib_name}.a \
|
||||
# -arch x86_64 ${LIBRARY_PATH}/ios_x86_64/Release/${lib_name}.a \
|
||||
# -arch armv7 ${LIBRARY_PATH}/ios_armv7/Release/${lib_name}.a \
|
||||
# -arch armv7s ${LIBRARY_PATH}/ios_armv7s/Release/${lib_name}.a \
|
||||
# -arch arm64 ${LIBRARY_PATH}/ios_armv8/Release/${lib_name}.a \
|
||||
# -output ${LIBRARY_OUTPUT_PATH}/${lib_name}.a -create
|
||||
#
|
||||
#cp ${work_path}/lib/houyi/lib/ios/libhouyi_score.a ${LIBRARY_OUTPUT_PATH}/
|
||||
#cp ${work_path}/interface/ocr-interface.h ${output_path}
|
||||
#cp ${work_path}/version/release.v ${output_path}
|
||||
#
|
||||
#echo -e "\033[1;36;40mBuild All Target Success At:\n${output_path}\033[0m"
|
||||
#exit 0
|
||||
@ -0,0 +1 @@
|
||||
cmake_policy(SET CMP0077 NEW)
|
||||
@ -0,0 +1,52 @@
|
||||
# Copyright (c) 2020 Mobvoi Inc (Binbin Zhang, Di Wu)
|
||||
# 2022 ZeXuan Li (lizexuan@huya.com)
|
||||
# Xingchen Song(sxc19@mails.tsinghua.edu.cn)
|
||||
# hamddct@gmail.com (Mddct)
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if(WITH_ONNX)
|
||||
set(ONNX_VERSION "1.12.0")
|
||||
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
|
||||
set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-win-x64-${ONNX_VERSION}.zip")
|
||||
set(URL_HASH "SHA256=8b5d61204989350b7904ac277f5fbccd3e6736ddbb6ec001e412723d71c9c176")
|
||||
elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
|
||||
set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-aarch64-${ONNX_VERSION}.tgz")
|
||||
set(URL_HASH "SHA256=5820d9f343df73c63b6b2b174a1ff62575032e171c9564bcf92060f46827d0ac")
|
||||
else()
|
||||
set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz")
|
||||
set(URL_HASH "SHA256=5d503ce8540358b59be26c675e42081be14a3e833a5301926f555451046929c5")
|
||||
endif()
|
||||
elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
|
||||
set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-osx-x86_64-${ONNX_VERSION}.tgz")
|
||||
set(URL_HASH "SHA256=09b17f712f8c6f19bb63da35d508815b443cbb473e16c6192abfaa297c02f600")
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported CMake System Name '${CMAKE_SYSTEM_NAME}' (expected 'Windows', 'Linux' or 'Darwin')")
|
||||
endif()
|
||||
|
||||
FetchContent_Declare(onnxruntime
|
||||
URL ${ONNX_URL}
|
||||
URL_HASH ${URL_HASH}
|
||||
)
|
||||
FetchContent_MakeAvailable(onnxruntime)
|
||||
include_directories(${onnxruntime_SOURCE_DIR}/include)
|
||||
link_directories(${onnxruntime_SOURCE_DIR}/lib)
|
||||
|
||||
if(MSVC)
|
||||
file(GLOB ONNX_DLLS "${onnxruntime_SOURCE_DIR}/lib/*.dll")
|
||||
file(COPY ${ONNX_DLLS} DESTINATION ${CMAKE_BINARY_DIR}/bin/${CMAKE_BUILD_TYPE})
|
||||
endif()
|
||||
|
||||
add_definitions(-DUSE_ONNX)
|
||||
endif()
|
||||
@ -0,0 +1,448 @@
|
||||
// Copyright (c) 2020 Mobvoi Inc (Binbin Zhang, Di Wu)
|
||||
// 2022 ZeXuan Li (lizexuan@huya.com)
|
||||
// Xingchen Song(sxc19@mails.tsinghua.edu.cn)
|
||||
// hamddct@gmail.com (Mddct)
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "nnet/u2_onnx_nnet.h"
|
||||
|
||||
namespace ppspeech {
|
||||
|
||||
Ort::Env U2OnnxNnet::env_ = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "");
|
||||
Ort::SessionOptions U2OnnxNnet::session_options_ = Ort::SessionOptions();
|
||||
|
||||
void U2OnnxNnet::InitEngineThreads(int num_threads) {
|
||||
session_options_.SetIntraOpNumThreads(num_threads);
|
||||
}
|
||||
|
||||
void U2OnnxNnet::LoadModel(const std::string& model_dir) {
|
||||
std::string encoder_onnx_path = model_dir + "/encoder.onnx";
|
||||
std::string rescore_onnx_path = model_dir + "/decoder.onnx";
|
||||
std::string ctc_onnx_path = model_dir + "/ctc.onnx";
|
||||
// 1. Load sessions
|
||||
try {
|
||||
#ifdef _MSC_VER
|
||||
encoder_session_ = std::make_shared<Ort::Session>(
|
||||
env_, ToWString(encoder_onnx_path).c_str(), session_options_);
|
||||
rescore_session_ = std::make_shared<Ort::Session>(
|
||||
env_, ToWString(rescore_onnx_path).c_str(), session_options_);
|
||||
ctc_session_ = std::make_shared<Ort::Session>(
|
||||
env_, ToWString(ctc_onnx_path).c_str(), session_options_);
|
||||
#else
|
||||
encoder_session_ = std::make_shared<Ort::Session>(
|
||||
env_, encoder_onnx_path.c_str(), session_options_);
|
||||
rescore_session_ = std::make_shared<Ort::Session>(
|
||||
env_, rescore_onnx_path.c_str(), session_options_);
|
||||
ctc_session_ = std::make_shared<Ort::Session>(env_, ctc_onnx_path.c_str(),
|
||||
session_options_);
|
||||
#endif
|
||||
} catch (std::exception const& e) {
|
||||
LOG(ERROR) << "error when load onnx model: " << e.what();
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// 2. Read metadata
|
||||
auto model_metadata = encoder_session_->GetModelMetadata();
|
||||
|
||||
Ort::AllocatorWithDefaultOptions allocator;
|
||||
encoder_output_size_ =
|
||||
atoi(model_metadata.LookupCustomMetadataMap("output_size", allocator));
|
||||
num_blocks_ =
|
||||
atoi(model_metadata.LookupCustomMetadataMap("num_blocks", allocator));
|
||||
head_ = atoi(model_metadata.LookupCustomMetadataMap("head", allocator));
|
||||
cnn_module_kernel_ = atoi(
|
||||
model_metadata.LookupCustomMetadataMap("cnn_module_kernel", allocator));
|
||||
subsampling_rate_ = atoi(
|
||||
model_metadata.LookupCustomMetadataMap("subsampling_rate", allocator));
|
||||
right_context_ =
|
||||
atoi(model_metadata.LookupCustomMetadataMap("right_context", allocator));
|
||||
sos_ = atoi(model_metadata.LookupCustomMetadataMap("sos_symbol", allocator));
|
||||
eos_ = atoi(model_metadata.LookupCustomMetadataMap("eos_symbol", allocator));
|
||||
is_bidecoder_ = atoi(model_metadata.LookupCustomMetadataMap(
|
||||
"is_bidirectional_decoder", allocator));
|
||||
chunk_size_ =
|
||||
atoi(model_metadata.LookupCustomMetadataMap("chunk_size", allocator));
|
||||
num_left_chunks_ =
|
||||
atoi(model_metadata.LookupCustomMetadataMap("left_chunks", allocator));
|
||||
|
||||
LOG(INFO) << "Onnx Model Info:";
|
||||
LOG(INFO) << "\tencoder_output_size " << encoder_output_size_;
|
||||
LOG(INFO) << "\tnum_blocks " << num_blocks_;
|
||||
LOG(INFO) << "\thead " << head_;
|
||||
LOG(INFO) << "\tcnn_module_kernel " << cnn_module_kernel_;
|
||||
LOG(INFO) << "\tsubsampling_rate " << subsampling_rate_;
|
||||
LOG(INFO) << "\tright_context " << right_context_;
|
||||
LOG(INFO) << "\tsos " << sos_;
|
||||
LOG(INFO) << "\teos " << eos_;
|
||||
LOG(INFO) << "\tis bidirectional decoder " << is_bidecoder_;
|
||||
LOG(INFO) << "\tchunk_size " << chunk_size_;
|
||||
LOG(INFO) << "\tnum_left_chunks " << num_left_chunks_;
|
||||
|
||||
// 3. Read model nodes
|
||||
LOG(INFO) << "Onnx Encoder:";
|
||||
GetInputOutputInfo(encoder_session_, &encoder_in_names_, &encoder_out_names_);
|
||||
LOG(INFO) << "Onnx CTC:";
|
||||
GetInputOutputInfo(ctc_session_, &ctc_in_names_, &ctc_out_names_);
|
||||
LOG(INFO) << "Onnx Rescore:";
|
||||
GetInputOutputInfo(rescore_session_, &rescore_in_names_, &rescore_out_names_);
|
||||
}
|
||||
|
||||
U2OnnxNnet::U2OnnxNnet(const ModelOptions& opts) : opts_(opts) {
|
||||
LoadModel(opts_.model_path);
|
||||
}
|
||||
|
||||
// shallow copy
|
||||
U2OnnxNnet::U2OnnxNnet(const U2OnnxNnet& other) {
|
||||
// metadatas
|
||||
encoder_output_size_ = other.encoder_output_size_;
|
||||
num_blocks_ = other.num_blocks_;
|
||||
head_ = other.head_;
|
||||
cnn_module_kernel_ = other.cnn_module_kernel_;
|
||||
right_context_ = other.right_context_;
|
||||
subsampling_rate_ = other.subsampling_rate_;
|
||||
sos_ = other.sos_;
|
||||
eos_ = other.eos_;
|
||||
is_bidecoder_ = other.is_bidecoder_;
|
||||
chunk_size_ = other.chunk_size_;
|
||||
num_left_chunks_ = other.num_left_chunks_;
|
||||
offset_ = other.offset_;
|
||||
|
||||
// sessions
|
||||
encoder_session_ = other.encoder_session_;
|
||||
ctc_session_ = other.ctc_session_;
|
||||
rescore_session_ = other.rescore_session_;
|
||||
|
||||
// node names
|
||||
encoder_in_names_ = other.encoder_in_names_;
|
||||
encoder_out_names_ = other.encoder_out_names_;
|
||||
ctc_in_names_ = other.ctc_in_names_;
|
||||
ctc_out_names_ = other.ctc_out_names_;
|
||||
rescore_in_names_ = other.rescore_in_names_;
|
||||
rescore_out_names_ = other.rescore_out_names_;
|
||||
}
|
||||
|
||||
void U2OnnxNnet::GetInputOutputInfo(
|
||||
const std::shared_ptr<Ort::Session>& session,
|
||||
std::vector<const char*>* in_names, std::vector<const char*>* out_names) {
|
||||
Ort::AllocatorWithDefaultOptions allocator;
|
||||
// Input info
|
||||
int num_nodes = session->GetInputCount();
|
||||
in_names->resize(num_nodes);
|
||||
for (int i = 0; i < num_nodes; ++i) {
|
||||
char* name = session->GetInputName(i, allocator);
|
||||
Ort::TypeInfo type_info = session->GetInputTypeInfo(i);
|
||||
auto tensor_info = type_info.GetTensorTypeAndShapeInfo();
|
||||
ONNXTensorElementDataType type = tensor_info.GetElementType();
|
||||
std::vector<int64_t> node_dims = tensor_info.GetShape();
|
||||
std::stringstream shape;
|
||||
for (auto j : node_dims) {
|
||||
shape << j;
|
||||
shape << " ";
|
||||
}
|
||||
LOG(INFO) << "\tInput " << i << " : name=" << name << " type=" << type
|
||||
<< " dims=" << shape.str();
|
||||
(*in_names)[i] = name;
|
||||
}
|
||||
// Output info
|
||||
num_nodes = session->GetOutputCount();
|
||||
out_names->resize(num_nodes);
|
||||
for (int i = 0; i < num_nodes; ++i) {
|
||||
char* name = session->GetOutputName(i, allocator);
|
||||
Ort::TypeInfo type_info = session->GetOutputTypeInfo(i);
|
||||
auto tensor_info = type_info.GetTensorTypeAndShapeInfo();
|
||||
ONNXTensorElementDataType type = tensor_info.GetElementType();
|
||||
std::vector<int64_t> node_dims = tensor_info.GetShape();
|
||||
std::stringstream shape;
|
||||
for (auto j : node_dims) {
|
||||
shape << j;
|
||||
shape << " ";
|
||||
}
|
||||
LOG(INFO) << "\tOutput " << i << " : name=" << name << " type=" << type
|
||||
<< " dims=" << shape.str();
|
||||
(*out_names)[i] = name;
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<NnetBase> U2OnnxNnet::Clone() const {
|
||||
auto asr_model = std::make_shared<U2OnnxNnet>(*this);
|
||||
// reset inner state for new decoding
|
||||
asr_model->Reset();
|
||||
return asr_model;
|
||||
}
|
||||
|
||||
void U2OnnxNnet::Reset() {
|
||||
offset_ = 0;
|
||||
encoder_outs_.clear();
|
||||
cached_feats_.clear();
|
||||
// Reset att_cache
|
||||
Ort::MemoryInfo memory_info =
|
||||
Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
|
||||
if (num_left_chunks_ > 0) {
|
||||
int required_cache_size = chunk_size_ * num_left_chunks_;
|
||||
offset_ = required_cache_size;
|
||||
att_cache_.resize(num_blocks_ * head_ * required_cache_size *
|
||||
encoder_output_size_ / head_ * 2,
|
||||
0.0);
|
||||
const int64_t att_cache_shape[] = {num_blocks_, head_, required_cache_size,
|
||||
encoder_output_size_ / head_ * 2};
|
||||
att_cache_ort_ = Ort::Value::CreateTensor<float>(
|
||||
memory_info, att_cache_.data(), att_cache_.size(), att_cache_shape, 4);
|
||||
} else {
|
||||
att_cache_.resize(0, 0.0);
|
||||
const int64_t att_cache_shape[] = {num_blocks_, head_, 0,
|
||||
encoder_output_size_ / head_ * 2};
|
||||
att_cache_ort_ = Ort::Value::CreateTensor<float>(
|
||||
memory_info, att_cache_.data(), att_cache_.size(), att_cache_shape, 4);
|
||||
}
|
||||
|
||||
// Reset cnn_cache
|
||||
cnn_cache_.resize(
|
||||
num_blocks_ * encoder_output_size_ * (cnn_module_kernel_ - 1), 0.0);
|
||||
const int64_t cnn_cache_shape[] = {num_blocks_, 1, encoder_output_size_,
|
||||
cnn_module_kernel_ - 1};
|
||||
cnn_cache_ort_ = Ort::Value::CreateTensor<float>(
|
||||
memory_info, cnn_cache_.data(), cnn_cache_.size(), cnn_cache_shape, 4);
|
||||
}
|
||||
|
||||
void U2OnnxNnet::FeedForward(const std::vector<BaseFloat>& features,
|
||||
const int32& feature_dim,
|
||||
NnetOut* out) {
|
||||
kaldi::Timer timer;
|
||||
|
||||
std::vector<kaldi::BaseFloat> ctc_probs;
|
||||
ForwardEncoderChunkImpl(
|
||||
features, feature_dim, &out->logprobs, &out->vocab_dim);
|
||||
VLOG(1) << "FeedForward cost: " << timer.Elapsed() << " sec. "
|
||||
<< features.size() / feature_dim << " frames.";
|
||||
}
|
||||
|
||||
void U2OnnxNnet::ForwardEncoderChunkImpl(
|
||||
const std::vector<kaldi::BaseFloat>& chunk_feats,
|
||||
const int32& feat_dim,
|
||||
std::vector<kaldi::BaseFloat>* out_prob,
|
||||
int32* vocab_dim) {
|
||||
|
||||
Ort::MemoryInfo memory_info =
|
||||
Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
|
||||
// 1. Prepare onnx required data, splice cached_feature_ and chunk_feats
|
||||
// chunk
|
||||
int num_frames = chunk_feats.size() / feat_dim;
|
||||
VLOG(3) << "num_frames: " << num_frames;
|
||||
VLOG(3) << "feat_dim: " << feat_dim;
|
||||
const int feature_dim = feat_dim;
|
||||
std::vector<float> feats;
|
||||
feats.insert(feats.end(), chunk_feats.begin(), chunk_feats.end());
|
||||
const int64_t feats_shape[3] = {1, num_frames, feature_dim};
|
||||
Ort::Value feats_ort = Ort::Value::CreateTensor<float>(
|
||||
memory_info, feats.data(), feats.size(), feats_shape, 3);
|
||||
// offset
|
||||
int64_t offset_int64 = static_cast<int64_t>(offset_);
|
||||
Ort::Value offset_ort = Ort::Value::CreateTensor<int64_t>(
|
||||
memory_info, &offset_int64, 1, std::vector<int64_t>{}.data(), 0);
|
||||
// required_cache_size
|
||||
int64_t required_cache_size = chunk_size_ * num_left_chunks_;
|
||||
Ort::Value required_cache_size_ort = Ort::Value::CreateTensor<int64_t>(
|
||||
memory_info, &required_cache_size, 1, std::vector<int64_t>{}.data(), 0);
|
||||
// att_mask
|
||||
Ort::Value att_mask_ort{nullptr};
|
||||
std::vector<uint8_t> att_mask(required_cache_size + chunk_size_, 1);
|
||||
if (num_left_chunks_ > 0) {
|
||||
int chunk_idx = offset_ / chunk_size_ - num_left_chunks_;
|
||||
if (chunk_idx < num_left_chunks_) {
|
||||
for (int i = 0; i < (num_left_chunks_ - chunk_idx) * chunk_size_; ++i) {
|
||||
att_mask[i] = 0;
|
||||
}
|
||||
}
|
||||
const int64_t att_mask_shape[] = {1, 1, required_cache_size + chunk_size_};
|
||||
att_mask_ort = Ort::Value::CreateTensor<bool>(
|
||||
memory_info, reinterpret_cast<bool*>(att_mask.data()), att_mask.size(),
|
||||
att_mask_shape, 3);
|
||||
}
|
||||
|
||||
// 2. Encoder chunk forward
|
||||
std::vector<Ort::Value> inputs;
|
||||
for (auto name : encoder_in_names_) {
|
||||
if (!strcmp(name, "chunk")) {
|
||||
inputs.emplace_back(std::move(feats_ort));
|
||||
} else if (!strcmp(name, "offset")) {
|
||||
inputs.emplace_back(std::move(offset_ort));
|
||||
} else if (!strcmp(name, "required_cache_size")) {
|
||||
inputs.emplace_back(std::move(required_cache_size_ort));
|
||||
} else if (!strcmp(name, "att_cache")) {
|
||||
inputs.emplace_back(std::move(att_cache_ort_));
|
||||
} else if (!strcmp(name, "cnn_cache")) {
|
||||
inputs.emplace_back(std::move(cnn_cache_ort_));
|
||||
} else if (!strcmp(name, "att_mask")) {
|
||||
inputs.emplace_back(std::move(att_mask_ort));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Ort::Value> ort_outputs = encoder_session_->Run(
|
||||
Ort::RunOptions{nullptr}, encoder_in_names_.data(), inputs.data(),
|
||||
inputs.size(), encoder_out_names_.data(), encoder_out_names_.size());
|
||||
|
||||
offset_ += static_cast<int>(
|
||||
ort_outputs[0].GetTensorTypeAndShapeInfo().GetShape()[1]);
|
||||
att_cache_ort_ = std::move(ort_outputs[1]);
|
||||
cnn_cache_ort_ = std::move(ort_outputs[2]);
|
||||
|
||||
std::vector<Ort::Value> ctc_inputs;
|
||||
ctc_inputs.emplace_back(std::move(ort_outputs[0]));
|
||||
|
||||
std::vector<Ort::Value> ctc_ort_outputs = ctc_session_->Run(
|
||||
Ort::RunOptions{nullptr}, ctc_in_names_.data(), ctc_inputs.data(),
|
||||
ctc_inputs.size(), ctc_out_names_.data(), ctc_out_names_.size());
|
||||
encoder_outs_.push_back(std::move(ctc_inputs[0]));
|
||||
|
||||
float* logp_data = ctc_ort_outputs[0].GetTensorMutableData<float>();
|
||||
auto type_info = ctc_ort_outputs[0].GetTensorTypeAndShapeInfo();
|
||||
|
||||
// Copy to output, (B=1,T,D)
|
||||
std::vector<int64_t> ctc_log_probs_shape = type_info.GetShape();
|
||||
CHECK_EQ(ctc_log_probs_shape.size(), 3);
|
||||
int B = ctc_log_probs_shape[0];
|
||||
CHECK_EQ(B, 1);
|
||||
int T = ctc_log_probs_shape[1];
|
||||
int D = ctc_log_probs_shape[2];
|
||||
*vocab_dim = D;
|
||||
|
||||
out_prob->resize(T * D);
|
||||
std::memcpy(
|
||||
out_prob->data(), logp_data, T * D * sizeof(kaldi::BaseFloat));
|
||||
return;
|
||||
}
|
||||
|
||||
float U2OnnxNnet::ComputeAttentionScore(const float* prob,
|
||||
const std::vector<int>& hyp, int eos,
|
||||
int decode_out_len) {
|
||||
float score = 0.0f;
|
||||
for (size_t j = 0; j < hyp.size(); ++j) {
|
||||
score += *(prob + j * decode_out_len + hyp[j]);
|
||||
}
|
||||
score += *(prob + hyp.size() * decode_out_len + eos);
|
||||
return score;
|
||||
}
|
||||
|
||||
void U2OnnxNnet::AttentionRescoring(const std::vector<std::vector<int>>& hyps,
|
||||
float reverse_weight,
|
||||
std::vector<float>* rescoring_score) {
|
||||
Ort::MemoryInfo memory_info =
|
||||
Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
|
||||
CHECK(rescoring_score != nullptr);
|
||||
int num_hyps = hyps.size();
|
||||
rescoring_score->resize(num_hyps, 0.0f);
|
||||
|
||||
if (num_hyps == 0) {
|
||||
return;
|
||||
}
|
||||
// No encoder output
|
||||
if (encoder_outs_.size() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<int64_t> hyps_lens;
|
||||
int max_hyps_len = 0;
|
||||
for (size_t i = 0; i < num_hyps; ++i) {
|
||||
int length = hyps[i].size() + 1;
|
||||
max_hyps_len = std::max(length, max_hyps_len);
|
||||
hyps_lens.emplace_back(static_cast<int64_t>(length));
|
||||
}
|
||||
|
||||
std::vector<float> rescore_input;
|
||||
int encoder_len = 0;
|
||||
for (int i = 0; i < encoder_outs_.size(); i++) {
|
||||
float* encoder_outs_data = encoder_outs_[i].GetTensorMutableData<float>();
|
||||
auto type_info = encoder_outs_[i].GetTensorTypeAndShapeInfo();
|
||||
for (int j = 0; j < type_info.GetElementCount(); j++) {
|
||||
rescore_input.emplace_back(encoder_outs_data[j]);
|
||||
}
|
||||
encoder_len += type_info.GetShape()[1];
|
||||
}
|
||||
|
||||
const int64_t decode_input_shape[] = {1, encoder_len, encoder_output_size_};
|
||||
|
||||
std::vector<int64_t> hyps_pad;
|
||||
|
||||
for (size_t i = 0; i < num_hyps; ++i) {
|
||||
const std::vector<int>& hyp = hyps[i];
|
||||
hyps_pad.emplace_back(sos_);
|
||||
size_t j = 0;
|
||||
for (; j < hyp.size(); ++j) {
|
||||
hyps_pad.emplace_back(hyp[j]);
|
||||
}
|
||||
if (j == max_hyps_len - 1) {
|
||||
continue;
|
||||
}
|
||||
for (; j < max_hyps_len - 1; ++j) {
|
||||
hyps_pad.emplace_back(0);
|
||||
}
|
||||
}
|
||||
|
||||
const int64_t hyps_pad_shape[] = {num_hyps, max_hyps_len};
|
||||
|
||||
const int64_t hyps_lens_shape[] = {num_hyps};
|
||||
|
||||
Ort::Value decode_input_tensor_ = Ort::Value::CreateTensor<float>(
|
||||
memory_info, rescore_input.data(), rescore_input.size(),
|
||||
decode_input_shape, 3);
|
||||
Ort::Value hyps_pad_tensor_ = Ort::Value::CreateTensor<int64_t>(
|
||||
memory_info, hyps_pad.data(), hyps_pad.size(), hyps_pad_shape, 2);
|
||||
Ort::Value hyps_lens_tensor_ = Ort::Value::CreateTensor<int64_t>(
|
||||
memory_info, hyps_lens.data(), hyps_lens.size(), hyps_lens_shape, 1);
|
||||
|
||||
std::vector<Ort::Value> rescore_inputs;
|
||||
|
||||
rescore_inputs.emplace_back(std::move(hyps_pad_tensor_));
|
||||
rescore_inputs.emplace_back(std::move(hyps_lens_tensor_));
|
||||
rescore_inputs.emplace_back(std::move(decode_input_tensor_));
|
||||
|
||||
std::vector<Ort::Value> rescore_outputs = rescore_session_->Run(
|
||||
Ort::RunOptions{nullptr}, rescore_in_names_.data(), rescore_inputs.data(),
|
||||
rescore_inputs.size(), rescore_out_names_.data(),
|
||||
rescore_out_names_.size());
|
||||
|
||||
float* decoder_outs_data = rescore_outputs[0].GetTensorMutableData<float>();
|
||||
float* r_decoder_outs_data = rescore_outputs[1].GetTensorMutableData<float>();
|
||||
|
||||
auto type_info = rescore_outputs[0].GetTensorTypeAndShapeInfo();
|
||||
int decode_out_len = type_info.GetShape()[2];
|
||||
|
||||
for (size_t i = 0; i < num_hyps; ++i) {
|
||||
const std::vector<int>& hyp = hyps[i];
|
||||
float score = 0.0f;
|
||||
// left to right decoder score
|
||||
score = ComputeAttentionScore(
|
||||
decoder_outs_data + max_hyps_len * decode_out_len * i, hyp, eos_,
|
||||
decode_out_len);
|
||||
// Optional: Used for right to left score
|
||||
float r_score = 0.0f;
|
||||
if (is_bidecoder_ && reverse_weight > 0) {
|
||||
std::vector<int> r_hyp(hyp.size());
|
||||
std::reverse_copy(hyp.begin(), hyp.end(), r_hyp.begin());
|
||||
// right to left decoder score
|
||||
r_score = ComputeAttentionScore(
|
||||
r_decoder_outs_data + max_hyps_len * decode_out_len * i, r_hyp, eos_,
|
||||
decode_out_len);
|
||||
}
|
||||
// combined left-to-right and right-to-left score
|
||||
(*rescoring_score)[i] =
|
||||
score * (1 - reverse_weight) + r_score * reverse_weight;
|
||||
}
|
||||
}
|
||||
|
||||
void U2OnnxNnet::EncoderOuts(
|
||||
std::vector<std::vector<kaldi::BaseFloat>>* encoder_out) const {
|
||||
}
|
||||
|
||||
} //namepace ppspeech
|
||||
@ -0,0 +1,116 @@
|
||||
// Copyright (c) 2020 Mobvoi Inc (Binbin Zhang, Di Wu)
|
||||
// 2022 ZeXuan Li (lizexuan@huya.com)
|
||||
// Xingchen Song(sxc19@mails.tsinghua.edu.cn)
|
||||
// hamddct@gmail.com (Mddct)
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "base/common.h"
|
||||
#include "matrix/kaldi-matrix.h"
|
||||
#include "nnet/nnet_itf.h"
|
||||
#include "nnet/u2_nnet.h"
|
||||
|
||||
#include "onnxruntime_cxx_api.h" // NOLINT
|
||||
|
||||
|
||||
namespace ppspeech {
|
||||
|
||||
class U2OnnxNnet : public U2NnetBase {
|
||||
public:
|
||||
static void InitEngineThreads(int num_threads = 1);
|
||||
|
||||
public:
|
||||
explicit U2OnnxNnet(const ModelOptions& opts);
|
||||
U2OnnxNnet(const U2OnnxNnet& other);
|
||||
|
||||
void FeedForward(const std::vector<kaldi::BaseFloat>& features,
|
||||
const int32& feature_dim,
|
||||
NnetOut* out) override;
|
||||
|
||||
void Reset() override;
|
||||
|
||||
bool IsLogProb() override { return true; }
|
||||
|
||||
void Dim();
|
||||
|
||||
void LoadModel(const std::string& model_dir);
|
||||
// void Warmup();
|
||||
|
||||
// std::shared_ptr<paddle::jit::Layer> model() const { return model_; }
|
||||
|
||||
std::shared_ptr<NnetBase> Clone() const override;
|
||||
|
||||
void ForwardEncoderChunkImpl(
|
||||
const std::vector<kaldi::BaseFloat>& chunk_feats,
|
||||
const int32& feat_dim,
|
||||
std::vector<kaldi::BaseFloat>* ctc_probs,
|
||||
int32* vocab_dim) override;
|
||||
|
||||
// float ComputePathScore(const paddle::Tensor& prob,
|
||||
// const std::vector<int>& hyp,
|
||||
// int eos);
|
||||
float ComputeAttentionScore(const float* prob, const std::vector<int>& hyp,
|
||||
int eos, int decode_out_len);
|
||||
|
||||
void AttentionRescoring(const std::vector<std::vector<int>>& hyps,
|
||||
float reverse_weight,
|
||||
std::vector<float>* rescoring_score) override;
|
||||
|
||||
// debug
|
||||
// void FeedEncoderOuts(const paddle::Tensor& encoder_out);
|
||||
|
||||
void EncoderOuts(
|
||||
std::vector<std::vector<kaldi::BaseFloat>>* encoder_out) const;
|
||||
|
||||
// copy from wenet
|
||||
void GetInputOutputInfo(const std::shared_ptr<Ort::Session>& session,
|
||||
std::vector<const char*>* in_names,
|
||||
std::vector<const char*>* out_names);
|
||||
private:
|
||||
ModelOptions opts_;
|
||||
|
||||
int encoder_output_size_ = 0;
|
||||
int num_blocks_ = 0;
|
||||
int cnn_module_kernel_ = 0;
|
||||
int head_ = 0;
|
||||
|
||||
// sessions
|
||||
// NOTE(Mddct): The Env holds the logging state used by all other objects.
|
||||
// One Env must be created before using any other Onnxruntime functionality.
|
||||
static Ort::Env env_; // shared environment across threads.
|
||||
static Ort::SessionOptions session_options_;
|
||||
std::shared_ptr<Ort::Session> encoder_session_ = nullptr;
|
||||
std::shared_ptr<Ort::Session> rescore_session_ = nullptr;
|
||||
std::shared_ptr<Ort::Session> ctc_session_ = nullptr;
|
||||
|
||||
// node names
|
||||
std::vector<const char*> encoder_in_names_, encoder_out_names_;
|
||||
std::vector<const char*> ctc_in_names_, ctc_out_names_;
|
||||
std::vector<const char*> rescore_in_names_, rescore_out_names_;
|
||||
|
||||
// caches
|
||||
Ort::Value att_cache_ort_{nullptr};
|
||||
Ort::Value cnn_cache_ort_{nullptr};
|
||||
std::vector<Ort::Value> encoder_outs_;
|
||||
// NOTE: Instead of making a copy of the xx_cache, ONNX only maintains
|
||||
// its data pointer when initializing xx_cache_ort (see https://github.com/
|
||||
// microsoft/onnxruntime/blob/master/onnxruntime/core/framework
|
||||
// /tensor.cc#L102-L129), so we need the following variables to keep
|
||||
// our data "alive" during the lifetime of decoder.
|
||||
std::vector<float> att_cache_;
|
||||
std::vector<float> cnn_cache_;
|
||||
};
|
||||
|
||||
} // namespace ppspeech
|
||||
@ -1,6 +1,4 @@
|
||||
include_directories(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../
|
||||
)
|
||||
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
add_subdirectory(nnet)
|
||||
|
||||
|
||||
Loading…
Reference in new issue