diff --git a/deploy/ctc_decoders.cpp b/deploy/ctc_decoders.cpp index da37708a..9304c780 100644 --- a/deploy/ctc_decoders.cpp +++ b/deploy/ctc_decoders.cpp @@ -96,13 +96,13 @@ std::vector > prefixes.push_back(&root); if ( ext_scorer != nullptr && !ext_scorer->is_character_based()) { - if (ext_scorer->_dictionary == nullptr) { + if (ext_scorer->dictionary == nullptr) { // TODO: init dictionary ext_scorer->set_char_map(vocabulary); // add_space should be true? ext_scorer->fill_dictionary(true); } - auto fst_dict = static_cast(ext_scorer->_dictionary); + auto fst_dict = static_cast(ext_scorer->dictionary); fst::StdVectorFst* dict_ptr = fst_dict->Copy(true); root.set_dictionary(dict_ptr); auto matcher = std::make_shared(*dict_ptr, fst::MATCH_INPUT); @@ -285,7 +285,7 @@ std::vector > > // dictionary init if ( ext_scorer != nullptr && !ext_scorer->is_character_based() - && ext_scorer->_dictionary == nullptr) { + && ext_scorer->dictionary == nullptr) { // init dictionary ext_scorer->set_char_map(vocabulary); ext_scorer->fill_dictionary(true); diff --git a/deploy/scorer.cpp b/deploy/scorer.cpp index 41f3894a..ced71995 100644 --- a/deploy/scorer.cpp +++ b/deploy/scorer.cpp @@ -15,7 +15,7 @@ Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { this->beta = beta; _is_character_based = true; _language_model = nullptr; - _dictionary = nullptr; + dictionary = nullptr; _max_order = 0; _SPACE_ID = -1; // load language model @@ -25,8 +25,8 @@ Scorer::Scorer(double alpha, double beta, const std::string& lm_path) { Scorer::~Scorer() { if (_language_model != nullptr) delete static_cast(_language_model); - if (_dictionary != nullptr) - delete static_cast(_dictionary); + if (dictionary != nullptr) + delete static_cast(dictionary); } void Scorer::load_LM(const char* filename) { @@ -99,87 +99,11 @@ double Scorer::get_log_prob(const std::vector& words) { return score; } -/* Strip a input sentence - * Parameters: - * str: A reference to the objective string - * ch: The character to prune - * Return: - * void - */ -inline void strip(std::string &str, char ch=' ') { - if (str.size() == 0) return; - int start = 0; - int end = str.size()-1; - for (int i=0; i=0; i--) { - if (str[i] == ch) { - end --; - } else { - break; - } - } - - if (start == 0 && end == str.size()-1) return; - if (start > end) { - std::string emp_str; - str = emp_str; - } else { - str = str.substr(start, end-start+1); - } -} - -int Scorer::word_count(std::string sentence) { - strip(sentence); - int cnt = 1; - for (int i=0; i_language_model; - State state, out_state; - lm::FullScoreReturn ret; - model->BeginSentenceWrite(&state); - - for (util::TokenIter it(sentence, ' '); it; ++it){ - lm::WordIndex wid = model->BaseVocabulary().Index(*it); - ret = model->BaseFullScore(&state, wid, &out_state); - state = out_state; - } - //log10 prob - double log_prob = ret.prob; - return log_prob; -} - void Scorer::reset_params(float alpha, float beta) { this->alpha = alpha; this->beta = beta; } -double Scorer::get_score(std::string sentence, bool log) { - double lm_score = get_log_cond_prob(sentence); - int word_cnt = word_count(sentence); - - double final_score = 0.0; - if (log == false) { - final_score = pow(10, alpha * lm_score) * pow(word_cnt, beta); - } else { - final_score = alpha * lm_score * std::log(10) - + beta * std::log(word_cnt); - } - return final_score; -} - std::string Scorer::vec2str(const std::vector& input) { std::string word; for (auto ind : input) { @@ -188,7 +112,6 @@ std::string Scorer::vec2str(const std::vector& input) { return word; } - std::vector Scorer::split_labels(const std::vector &labels) { if (labels.empty()) @@ -291,6 +214,6 @@ void Scorer::fill_dictionary(bool add_space) { // Finds the simplest equivalent fst. This is unnecessary but decreases // memory usage of the dictionary fst::Minimize(new_dict); - _dictionary = new_dict; + this->dictionary = new_dict; } diff --git a/deploy/scorer.h b/deploy/scorer.h index 17a5f1aa..e5bfecaf 100644 --- a/deploy/scorer.h +++ b/deploy/scorer.h @@ -42,15 +42,8 @@ public: double get_sent_log_prob(const std::vector& words); size_t get_max_order() { return _max_order; } bool is_character_based() { return _is_character_based; } - std::vector get_vocab() { return _vocabulary; } - // word insertion term - int word_count(std::string); - // get the log cond prob of the last word - double get_log_cond_prob(std::string); // reset params alpha & beta void reset_params(float alpha, float beta); - // get the final score - double get_score(std::string, bool log=false); // make ngram std::vector make_ngram(PathTrie* prefix); // fill dictionary for fst @@ -61,7 +54,7 @@ public: double alpha; double beta; // fst dictionary - void* _dictionary; + void* dictionary; protected: void load_LM(const char* filename);