|
|
|
@ -1,18 +1,17 @@
|
|
|
|
|
#include <iostream>
|
|
|
|
|
#include <map>
|
|
|
|
|
#include "ctc_decoders.h"
|
|
|
|
|
#include <algorithm>
|
|
|
|
|
#include <utility>
|
|
|
|
|
#include <cmath>
|
|
|
|
|
#include <iostream>
|
|
|
|
|
#include <limits>
|
|
|
|
|
#include "fst/fstlib.h"
|
|
|
|
|
#include "ctc_decoders.h"
|
|
|
|
|
#include <map>
|
|
|
|
|
#include <utility>
|
|
|
|
|
#include "ThreadPool.h"
|
|
|
|
|
#include "decoder_utils.h"
|
|
|
|
|
#include "fst/fstlib.h"
|
|
|
|
|
#include "path_trie.h"
|
|
|
|
|
#include "ThreadPool.h"
|
|
|
|
|
|
|
|
|
|
std::string ctc_best_path_decoder(std::vector<std::vector<double>> probs_seq,
|
|
|
|
|
std::vector<std::string> vocabulary)
|
|
|
|
|
{
|
|
|
|
|
std::vector<std::string> vocabulary) {
|
|
|
|
|
// dimension check
|
|
|
|
|
int num_time_steps = probs_seq.size();
|
|
|
|
|
for (int i = 0; i < num_time_steps; i++) {
|
|
|
|
@ -56,15 +55,14 @@ std::string ctc_best_path_decoder(std::vector<std::vector<double> > probs_seq,
|
|
|
|
|
return best_path_result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::vector<std::pair<double, std::string> >
|
|
|
|
|
ctc_beam_search_decoder(std::vector<std::vector<double> > probs_seq,
|
|
|
|
|
std::vector<std::pair<double, std::string>> ctc_beam_search_decoder(
|
|
|
|
|
std::vector<std::vector<double>> probs_seq,
|
|
|
|
|
int beam_size,
|
|
|
|
|
std::vector<std::string> vocabulary,
|
|
|
|
|
int blank_id,
|
|
|
|
|
double cutoff_prob,
|
|
|
|
|
int cutoff_top_n,
|
|
|
|
|
Scorer *ext_scorer)
|
|
|
|
|
{
|
|
|
|
|
Scorer *extscorer) {
|
|
|
|
|
// dimension check
|
|
|
|
|
int num_time_steps = probs_seq.size();
|
|
|
|
|
for (int i = 0; i < num_time_steps; i++) {
|
|
|
|
@ -82,8 +80,8 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// assign space ID
|
|
|
|
|
std::vector<std::string>::iterator it = std::find(vocabulary.begin(),
|
|
|
|
|
vocabulary.end(), " ");
|
|
|
|
|
std::vector<std::string>::iterator it =
|
|
|
|
|
std::find(vocabulary.begin(), vocabulary.end(), " ");
|
|
|
|
|
int space_id = it - vocabulary.begin();
|
|
|
|
|
// if no space in vocabulary
|
|
|
|
|
if (space_id >= vocabulary.size()) {
|
|
|
|
@ -92,25 +90,23 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
|
|
|
|
|
// init prefixes' root
|
|
|
|
|
PathTrie root;
|
|
|
|
|
root._score = root._log_prob_b_prev = 0.0;
|
|
|
|
|
root.score = root.log_prob_b_prev = 0.0;
|
|
|
|
|
std::vector<PathTrie *> prefixes;
|
|
|
|
|
prefixes.push_back(&root);
|
|
|
|
|
|
|
|
|
|
if ( ext_scorer != nullptr) {
|
|
|
|
|
if (ext_scorer->is_char_map_empty()) {
|
|
|
|
|
ext_scorer->set_char_map(vocabulary);
|
|
|
|
|
if (extscorer != nullptr) {
|
|
|
|
|
if (extscorer->is_char_map_empty()) {
|
|
|
|
|
extscorer->set_char_map(vocabulary);
|
|
|
|
|
}
|
|
|
|
|
if (!ext_scorer->is_character_based()) {
|
|
|
|
|
if (ext_scorer->dictionary == nullptr) {
|
|
|
|
|
if (!extscorer->is_character_based()) {
|
|
|
|
|
if (extscorer->dictionary == nullptr) {
|
|
|
|
|
// fill dictionary for fst
|
|
|
|
|
ext_scorer->fill_dictionary(true);
|
|
|
|
|
extscorer->fill_dictionary(true);
|
|
|
|
|
}
|
|
|
|
|
auto fst_dict = static_cast<fst::StdVectorFst*>
|
|
|
|
|
(ext_scorer->dictionary);
|
|
|
|
|
auto fst_dict = static_cast<fst::StdVectorFst *>(extscorer->dictionary);
|
|
|
|
|
fst::StdVectorFst *dict_ptr = fst_dict->Copy(true);
|
|
|
|
|
root.set_dictionary(dict_ptr);
|
|
|
|
|
auto matcher = std::make_shared<FSTMATCH>
|
|
|
|
|
(*dict_ptr, fst::MATCH_INPUT);
|
|
|
|
|
auto matcher = std::make_shared<FSTMATCH>(*dict_ptr, fst::MATCH_INPUT);
|
|
|
|
|
root.set_matcher(matcher);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -125,21 +121,20 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
|
|
|
|
|
float min_cutoff = -NUM_FLT_INF;
|
|
|
|
|
bool full_beam = false;
|
|
|
|
|
if (ext_scorer != nullptr) {
|
|
|
|
|
if (extscorer != nullptr) {
|
|
|
|
|
int num_prefixes = std::min((int)prefixes.size(), beam_size);
|
|
|
|
|
std::sort(prefixes.begin(), prefixes.begin() + num_prefixes,
|
|
|
|
|
prefix_compare);
|
|
|
|
|
min_cutoff = prefixes[num_prefixes-1]->_score + log(prob[blank_id])
|
|
|
|
|
- std::max(0.0, ext_scorer->beta);
|
|
|
|
|
std::sort(
|
|
|
|
|
prefixes.begin(), prefixes.begin() + num_prefixes, prefix_compare);
|
|
|
|
|
min_cutoff = prefixes[num_prefixes - 1]->score + log(prob[blank_id]) -
|
|
|
|
|
std::max(0.0, extscorer->beta);
|
|
|
|
|
full_beam = (num_prefixes == beam_size);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// pruning of vacobulary
|
|
|
|
|
int cutoff_len = prob.size();
|
|
|
|
|
if (cutoff_prob < 1.0 || cutoff_top_n < prob.size()) {
|
|
|
|
|
std::sort(prob_idx.begin(),
|
|
|
|
|
prob_idx.end(),
|
|
|
|
|
pair_comp_second_rev<int, double>);
|
|
|
|
|
std::sort(
|
|
|
|
|
prob_idx.begin(), prob_idx.end(), pair_comp_second_rev<int, double>);
|
|
|
|
|
if (cutoff_prob < 1.0) {
|
|
|
|
|
double cum_prob = 0.0;
|
|
|
|
|
cutoff_len = 0;
|
|
|
|
@ -150,13 +145,13 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
cutoff_len = std::min(cutoff_len, cutoff_top_n);
|
|
|
|
|
prob_idx = std::vector<std::pair<int, double> >( prob_idx.begin(),
|
|
|
|
|
prob_idx.begin() + cutoff_len);
|
|
|
|
|
prob_idx = std::vector<std::pair<int, double>>(
|
|
|
|
|
prob_idx.begin(), prob_idx.begin() + cutoff_len);
|
|
|
|
|
}
|
|
|
|
|
std::vector<std::pair<int, float>> log_prob_idx;
|
|
|
|
|
for (int i = 0; i < cutoff_len; i++) {
|
|
|
|
|
log_prob_idx.push_back(std::pair<int, float>
|
|
|
|
|
(prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN)));
|
|
|
|
|
log_prob_idx.push_back(std::pair<int, float>(
|
|
|
|
|
prob_idx[i].first, log(prob_idx[i].second + NUM_FLT_MIN)));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// loop over chars
|
|
|
|
@ -167,21 +162,19 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
for (int i = 0; i < prefixes.size() && i < beam_size; i++) {
|
|
|
|
|
auto prefix = prefixes[i];
|
|
|
|
|
|
|
|
|
|
if (full_beam && log_prob_c + prefix->_score < min_cutoff) {
|
|
|
|
|
if (full_beam && log_prob_c + prefix->score < min_cutoff) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
// blank
|
|
|
|
|
if (c == blank_id) {
|
|
|
|
|
prefix->_log_prob_b_cur = log_sum_exp(
|
|
|
|
|
prefix->_log_prob_b_cur,
|
|
|
|
|
log_prob_c + prefix->_score);
|
|
|
|
|
prefix->log_prob_b_cur =
|
|
|
|
|
log_sum_exp(prefix->log_prob_b_cur, log_prob_c + prefix->score);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
// repeated character
|
|
|
|
|
if (c == prefix->_character) {
|
|
|
|
|
prefix->_log_prob_nb_cur = log_sum_exp(
|
|
|
|
|
prefix->_log_prob_nb_cur,
|
|
|
|
|
log_prob_c + prefix->_log_prob_nb_prev);
|
|
|
|
|
if (c == prefix->character) {
|
|
|
|
|
prefix->log_prob_nb_cur = log_sum_exp(
|
|
|
|
|
prefix->log_prob_nb_cur, log_prob_c + prefix->log_prob_nb_prev);
|
|
|
|
|
}
|
|
|
|
|
// get new prefix
|
|
|
|
|
auto prefix_new = prefix->get_path_trie(c);
|
|
|
|
@ -189,36 +182,35 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
if (prefix_new != nullptr) {
|
|
|
|
|
float log_p = -NUM_FLT_INF;
|
|
|
|
|
|
|
|
|
|
if (c == prefix->_character
|
|
|
|
|
&& prefix->_log_prob_b_prev > -NUM_FLT_INF) {
|
|
|
|
|
log_p = log_prob_c + prefix->_log_prob_b_prev;
|
|
|
|
|
} else if (c != prefix->_character) {
|
|
|
|
|
log_p = log_prob_c + prefix->_score;
|
|
|
|
|
if (c == prefix->character &&
|
|
|
|
|
prefix->log_prob_b_prev > -NUM_FLT_INF) {
|
|
|
|
|
log_p = log_prob_c + prefix->log_prob_b_prev;
|
|
|
|
|
} else if (c != prefix->character) {
|
|
|
|
|
log_p = log_prob_c + prefix->score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// language model scoring
|
|
|
|
|
if (ext_scorer != nullptr &&
|
|
|
|
|
(c == space_id || ext_scorer->is_character_based()) ) {
|
|
|
|
|
PathTrie *prefix_to_score = nullptr;
|
|
|
|
|
if (extscorer != nullptr &&
|
|
|
|
|
(c == space_id || extscorer->is_character_based())) {
|
|
|
|
|
PathTrie *prefix_toscore = nullptr;
|
|
|
|
|
|
|
|
|
|
// skip scoring the space
|
|
|
|
|
if (ext_scorer->is_character_based()) {
|
|
|
|
|
prefix_to_score = prefix_new;
|
|
|
|
|
if (extscorer->is_character_based()) {
|
|
|
|
|
prefix_toscore = prefix_new;
|
|
|
|
|
} else {
|
|
|
|
|
prefix_to_score = prefix;
|
|
|
|
|
prefix_toscore = prefix;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
double score = 0.0;
|
|
|
|
|
std::vector<std::string> ngram;
|
|
|
|
|
ngram = ext_scorer->make_ngram(prefix_to_score);
|
|
|
|
|
score = ext_scorer->get_log_cond_prob(ngram) *
|
|
|
|
|
ext_scorer->alpha;
|
|
|
|
|
ngram = extscorer->make_ngram(prefix_toscore);
|
|
|
|
|
score = extscorer->get_log_cond_prob(ngram) * extscorer->alpha;
|
|
|
|
|
|
|
|
|
|
log_p += score;
|
|
|
|
|
log_p += ext_scorer->beta;
|
|
|
|
|
log_p += extscorer->beta;
|
|
|
|
|
}
|
|
|
|
|
prefix_new->_log_prob_nb_cur = log_sum_exp(
|
|
|
|
|
prefix_new->_log_prob_nb_cur, log_p);
|
|
|
|
|
prefix_new->log_prob_nb_cur =
|
|
|
|
|
log_sum_exp(prefix_new->log_prob_nb_cur, log_p);
|
|
|
|
|
}
|
|
|
|
|
} // end of loop over prefix
|
|
|
|
|
} // end of loop over chars
|
|
|
|
@ -242,21 +234,20 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
|
|
|
|
|
// compute aproximate ctc score as the return score
|
|
|
|
|
for (size_t i = 0; i < beam_size && i < prefixes.size(); i++) {
|
|
|
|
|
double approx_ctc = prefixes[i]->_score;
|
|
|
|
|
double approx_ctc = prefixes[i]->score;
|
|
|
|
|
|
|
|
|
|
if (ext_scorer != nullptr) {
|
|
|
|
|
if (extscorer != nullptr) {
|
|
|
|
|
std::vector<int> output;
|
|
|
|
|
prefixes[i]->get_path_vec(output);
|
|
|
|
|
size_t prefix_length = output.size();
|
|
|
|
|
auto words = ext_scorer->split_labels(output);
|
|
|
|
|
auto words = extscorer->split_labels(output);
|
|
|
|
|
// remove word insert
|
|
|
|
|
approx_ctc = approx_ctc - prefix_length * ext_scorer->beta;
|
|
|
|
|
approx_ctc = approx_ctc - prefix_length * extscorer->beta;
|
|
|
|
|
// remove language model weight:
|
|
|
|
|
approx_ctc -= (ext_scorer->get_sent_log_prob(words))
|
|
|
|
|
* ext_scorer->alpha;
|
|
|
|
|
approx_ctc -= (extscorer->get_sent_log_prob(words)) * extscorer->alpha;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
prefixes[i]->_approx_ctc = approx_ctc;
|
|
|
|
|
prefixes[i]->approx_ctc = approx_ctc;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// allow for the post processing
|
|
|
|
@ -277,15 +268,14 @@ std::vector<std::pair<double, std::string> >
|
|
|
|
|
for (int j = 0; j < output.size(); j++) {
|
|
|
|
|
output_str += vocabulary[output[j]];
|
|
|
|
|
}
|
|
|
|
|
std::pair<double, std::string>
|
|
|
|
|
output_pair(-space_prefixes[i]->_approx_ctc, output_str);
|
|
|
|
|
std::pair<double, std::string> output_pair(-space_prefixes[i]->approx_ctc,
|
|
|
|
|
output_str);
|
|
|
|
|
output_vecs.emplace_back(output_pair);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return output_vecs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
std::vector<std::vector<std::pair<double, std::string>>>
|
|
|
|
|
ctc_beam_search_decoder_batch(
|
|
|
|
|
std::vector<std::vector<std::vector<double>>> probs_split,
|
|
|
|
@ -295,8 +285,7 @@ std::vector<std::vector<std::pair<double, std::string> > >
|
|
|
|
|
int num_processes,
|
|
|
|
|
double cutoff_prob,
|
|
|
|
|
int cutoff_top_n,
|
|
|
|
|
Scorer *ext_scorer
|
|
|
|
|
) {
|
|
|
|
|
Scorer *extscorer) {
|
|
|
|
|
if (num_processes <= 0) {
|
|
|
|
|
std::cout << "num_processes must be nonnegative!" << std::endl;
|
|
|
|
|
exit(1);
|
|
|
|
@ -307,25 +296,28 @@ std::vector<std::vector<std::pair<double, std::string> > >
|
|
|
|
|
int batch_size = probs_split.size();
|
|
|
|
|
|
|
|
|
|
// scorer filling up
|
|
|
|
|
if ( ext_scorer != nullptr) {
|
|
|
|
|
if (ext_scorer->is_char_map_empty()) {
|
|
|
|
|
ext_scorer->set_char_map(vocabulary);
|
|
|
|
|
if (extscorer != nullptr) {
|
|
|
|
|
if (extscorer->is_char_map_empty()) {
|
|
|
|
|
extscorer->set_char_map(vocabulary);
|
|
|
|
|
}
|
|
|
|
|
if(!ext_scorer->is_character_based()
|
|
|
|
|
&& ext_scorer->dictionary == nullptr) {
|
|
|
|
|
if (!extscorer->is_character_based() &&
|
|
|
|
|
extscorer->dictionary == nullptr) {
|
|
|
|
|
// init dictionary
|
|
|
|
|
ext_scorer->fill_dictionary(true);
|
|
|
|
|
extscorer->fill_dictionary(true);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// enqueue the tasks of decoding
|
|
|
|
|
std::vector<std::future<std::vector<std::pair<double, std::string>>>> res;
|
|
|
|
|
for (int i = 0; i < batch_size; i++) {
|
|
|
|
|
res.emplace_back(
|
|
|
|
|
pool.enqueue(ctc_beam_search_decoder, probs_split[i],
|
|
|
|
|
beam_size, vocabulary, blank_id, cutoff_prob,
|
|
|
|
|
cutoff_top_n, ext_scorer)
|
|
|
|
|
);
|
|
|
|
|
res.emplace_back(pool.enqueue(ctc_beam_search_decoder,
|
|
|
|
|
probs_split[i],
|
|
|
|
|
beam_size,
|
|
|
|
|
vocabulary,
|
|
|
|
|
blank_id,
|
|
|
|
|
cutoff_prob,
|
|
|
|
|
cutoff_top_n,
|
|
|
|
|
extscorer));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// get decoding results
|
|
|
|
|