You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
387 lines
18 KiB
387 lines
18 KiB
// fstext/fstext-utils.h
|
|
|
|
// Copyright 2009-2011 Microsoft Corporation
|
|
// 2012-2013 Johns Hopkins University (Author: Daniel Povey)
|
|
// 2013 Guoguo Chen
|
|
// 2014 Telepoint Global Hosting Service, LLC. (Author: David
|
|
// Snyder)
|
|
|
|
// See ../../COPYING for clarification regarding multiple authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
|
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
|
|
// MERCHANTABLITY OR NON-INFRINGEMENT.
|
|
// See the Apache 2 License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#ifndef KALDI_FSTEXT_FSTEXT_UTILS_H_
|
|
#define KALDI_FSTEXT_FSTEXT_UTILS_H_
|
|
|
|
#include <fst/fst-decl.h>
|
|
#include <fst/fstlib.h>
|
|
|
|
#include <algorithm>
|
|
#include <map>
|
|
#include <set>
|
|
#include <vector>
|
|
|
|
#include "fstext/determinize-star.h"
|
|
#include "fstext/remove-eps-local.h"
|
|
#include "base/kaldi-common.h" // for error reporting macros.
|
|
#include "util/text-utils.h" // for SplitStringToVector
|
|
#include "fst/script/print-impl.h"
|
|
|
|
namespace fst {
|
|
|
|
/// Returns the highest numbered output symbol id of the FST (or zero
|
|
/// for an empty FST.
|
|
template <class Arc>
|
|
typename Arc::Label HighestNumberedOutputSymbol(const Fst<Arc> &fst);
|
|
|
|
/// Returns the highest numbered input symbol id of the FST (or zero
|
|
/// for an empty FST.
|
|
template <class Arc>
|
|
typename Arc::Label HighestNumberedInputSymbol(const Fst<Arc> &fst);
|
|
|
|
/// Returns the total number of arcs in an FST.
|
|
template <class Arc>
|
|
typename Arc::StateId NumArcs(const ExpandedFst<Arc> &fst);
|
|
|
|
/// GetInputSymbols gets the list of symbols on the input of fst
|
|
/// (including epsilon, if include_eps == true), as a sorted, unique
|
|
/// list.
|
|
template <class Arc, class I>
|
|
void GetInputSymbols(const Fst<Arc> &fst, bool include_eps,
|
|
std::vector<I> *symbols);
|
|
|
|
/// GetOutputSymbols gets the list of symbols on the output of fst
|
|
/// (including epsilon, if include_eps == true)
|
|
template <class Arc, class I>
|
|
void GetOutputSymbols(const Fst<Arc> &fst, bool include_eps,
|
|
std::vector<I> *symbols);
|
|
|
|
/// ClearSymbols sets all the symbols on the input and/or
|
|
/// output side of the FST to zero, as specified.
|
|
/// It does not alter the symbol tables.
|
|
template <class Arc>
|
|
void ClearSymbols(bool clear_input, bool clear_output, MutableFst<Arc> *fst);
|
|
|
|
template <class I>
|
|
void GetSymbols(const SymbolTable &symtab, bool include_eps,
|
|
std::vector<I> *syms_out);
|
|
|
|
inline void DeterminizeStarInLog(VectorFst<StdArc> *fst, float delta = kDelta,
|
|
bool *debug_ptr = NULL, int max_states = -1);
|
|
|
|
// e.g. of using this function: PushInLog<REWEIGHT_TO_INITIAL>(fst,
|
|
// kPushWeights|kPushLabels);
|
|
|
|
template <ReweightType rtype> // == REWEIGHT_TO_{INITIAL, FINAL}
|
|
void PushInLog(VectorFst<StdArc> *fst, uint32 ptype, float delta = kDelta) {
|
|
// PushInLog pushes the FST
|
|
// and returns a new pushed FST (labels and weights pushed to the left).
|
|
VectorFst<LogArc> *fst_log =
|
|
new VectorFst<LogArc>; // Want to determinize in log semiring.
|
|
Cast(*fst, fst_log);
|
|
VectorFst<StdArc> tmp;
|
|
*fst = tmp; // free up memory.
|
|
VectorFst<LogArc> *fst_pushed_log = new VectorFst<LogArc>;
|
|
Push<LogArc, rtype>(*fst_log, fst_pushed_log, ptype, delta);
|
|
Cast(*fst_pushed_log, fst);
|
|
delete fst_log;
|
|
delete fst_pushed_log;
|
|
}
|
|
|
|
// Minimizes after encoding; applicable to all FSTs. It is like what you get
|
|
// from the Minimize() function, except it will not push the weights, or the
|
|
// symbols. This is better for our recipes, as we avoid ever pushing the
|
|
// weights. However, it will only minimize optimally if your graphs are such
|
|
// that the symbols are as far to the left as they can go, and the weights
|
|
// in combinable paths are the same... hard to formalize this, but it's
|
|
// something that is satisified by our normal FSTs.
|
|
template <class Arc>
|
|
void MinimizeEncoded(VectorFst<Arc> *fst, float delta = kDelta) {
|
|
Map(fst, QuantizeMapper<Arc>(delta));
|
|
EncodeMapper<Arc> encoder(kEncodeLabels | kEncodeWeights, ENCODE);
|
|
Encode(fst, &encoder);
|
|
internal::AcceptorMinimize(fst);
|
|
Decode(fst, encoder);
|
|
}
|
|
|
|
/// GetLinearSymbolSequence gets the symbol sequence from a linear FST.
|
|
/// If the FST is not just a linear sequence, it returns false. If it is
|
|
/// a linear sequence (including the empty FST), it returns true. In this
|
|
/// case it outputs the symbol
|
|
/// sequences as "isymbols_out" and "osymbols_out" (removing epsilons), and
|
|
/// the total weight as "tot_weight". The total weight will be Weight::Zero()
|
|
/// if the FST is empty. If any of the output pointers are NULL, it does not
|
|
/// create that output.
|
|
template <class Arc, class I>
|
|
bool GetLinearSymbolSequence(const Fst<Arc> &fst, std::vector<I> *isymbols_out,
|
|
std::vector<I> *osymbols_out,
|
|
typename Arc::Weight *tot_weight_out);
|
|
|
|
/// This function converts an FST with a special structure, which is
|
|
/// output by the OpenFst functions ShortestPath and RandGen, and converts
|
|
/// them into a std::vector of separate FSTs. This special structure is that
|
|
/// the only state that has more than one (arcs-out or final-prob) is the
|
|
/// start state. fsts_out is resized to the appropriate size.
|
|
template <class Arc>
|
|
void ConvertNbestToVector(const Fst<Arc> &fst,
|
|
std::vector<VectorFst<Arc> > *fsts_out);
|
|
|
|
/// Takes the n-shortest-paths (using ShortestPath), but outputs
|
|
/// the result as a vector of up to n fsts. This function will
|
|
/// size the "fsts_out" vector to however many paths it got
|
|
/// (which will not exceed n). n must be >= 1.
|
|
template <class Arc>
|
|
void NbestAsFsts(const Fst<Arc> &fst, size_t n,
|
|
std::vector<VectorFst<Arc> > *fsts_out);
|
|
|
|
/// Creates unweighted linear acceptor from symbol sequence.
|
|
template <class Arc, class I>
|
|
void MakeLinearAcceptor(const std::vector<I> &labels, MutableFst<Arc> *ofst);
|
|
|
|
/// Creates an unweighted acceptor with a linear structure, with alternatives
|
|
/// at each position. Epsilon is treated like a normal symbol here.
|
|
/// Each position in "labels" must have at least one alternative.
|
|
template <class Arc, class I>
|
|
void MakeLinearAcceptorWithAlternatives(
|
|
const std::vector<std::vector<I> > &labels, MutableFst<Arc> *ofst);
|
|
|
|
/// Does PreDeterminize and DeterminizeStar and then removes the disambiguation
|
|
/// symbols. This is a form of determinization that will never blow up. Note
|
|
/// that ifst is non-const and can be considered to be destroyed by this
|
|
/// operation.
|
|
/// Does not do epsilon removal (RemoveEpsLocal)-- this is so it's safe to cast
|
|
/// to log and do this, and maintain equivalence in tropical.
|
|
|
|
template <class Arc>
|
|
void SafeDeterminizeWrapper(MutableFst<Arc> *ifst, MutableFst<Arc> *ofst,
|
|
float delta = kDelta);
|
|
|
|
/// SafeDeterminizeMinimizeWapper is as SafeDeterminizeWrapper except that it
|
|
/// also minimizes (encoded minimization, which is safe). This algorithm will
|
|
/// destroy "ifst".
|
|
template <class Arc>
|
|
void SafeDeterminizeMinimizeWrapper(MutableFst<Arc> *ifst, VectorFst<Arc> *ofst,
|
|
float delta = kDelta);
|
|
|
|
/// SafeDeterminizeMinimizeWapperInLog is as SafeDeterminizeMinimizeWrapper
|
|
/// except it first casts tothe log semiring.
|
|
void SafeDeterminizeMinimizeWrapperInLog(VectorFst<StdArc> *ifst,
|
|
VectorFst<StdArc> *ofst,
|
|
float delta = kDelta);
|
|
|
|
/// RemoveSomeInputSymbols removes any symbol that appears in "to_remove", from
|
|
/// the input side of the FST, replacing them with epsilon.
|
|
template <class Arc, class I>
|
|
void RemoveSomeInputSymbols(const std::vector<I> &to_remove,
|
|
MutableFst<Arc> *fst);
|
|
|
|
// MapInputSymbols will replace any input symbol i that is between 0 and
|
|
// symbol_map.size()-1, with symbol_map[i]. It removes the input symbol
|
|
// table of the FST.
|
|
template <class Arc, class I>
|
|
void MapInputSymbols(const std::vector<I> &symbol_map, MutableFst<Arc> *fst);
|
|
|
|
template <class Arc>
|
|
void RemoveWeights(MutableFst<Arc> *fst);
|
|
|
|
/// Returns true if and only if the FST is such that the input symbols
|
|
/// on arcs entering any given state all have the same value.
|
|
/// if "start_is_epsilon", treat start-state as an epsilon input arc
|
|
/// [i.e. ensure only epsilon can enter start-state].
|
|
template <class Arc>
|
|
bool PrecedingInputSymbolsAreSame(bool start_is_epsilon, const Fst<Arc> &fst);
|
|
|
|
/// This is as PrecedingInputSymbolsAreSame, but with a functor f that maps
|
|
/// labels to classes. The function tests whether the symbols preceding any
|
|
/// given state are in the same class. Formally, f is of a type F that has an
|
|
/// operator of type F::Result F::operator() (F::Arg a) const; where F::Result
|
|
/// is an integer type and F::Arc can be constructed from Arc::Label. this must
|
|
/// apply to valid labels and also to kNoLabel (so we can have a marker for the
|
|
/// invalid labels.
|
|
template <class Arc, class F>
|
|
bool PrecedingInputSymbolsAreSameClass(bool start_is_epsilon,
|
|
const Fst<Arc> &fst, const F &f);
|
|
|
|
/// Returns true if and only if the FST is such that the input symbols
|
|
/// on arcs exiting any given state all have the same value.
|
|
/// If end_is_epsilon, treat end-state as an epsilon output arc [i.e. ensure
|
|
/// end-states cannot have non-epsilon output transitions.]
|
|
template <class Arc>
|
|
bool FollowingInputSymbolsAreSame(bool end_is_epsilon, const Fst<Arc> &fst);
|
|
|
|
template <class Arc, class F>
|
|
bool FollowingInputSymbolsAreSameClass(bool end_is_epsilon, const Fst<Arc> &fst,
|
|
const F &f);
|
|
|
|
/// MakePrecedingInputSymbolsSame ensures that all arcs entering any given fst
|
|
/// state have the same input symbol. It does this by detecting states
|
|
/// that have differing input symbols going in, and inserting, for each of
|
|
/// the preceding arcs with non-epsilon input symbol, a new dummy state that
|
|
/// has an epsilon link to the fst state.
|
|
/// If "start_is_epsilon", ensure that start-state can have only epsilon-links
|
|
/// into it.
|
|
template <class Arc>
|
|
void MakePrecedingInputSymbolsSame(bool start_is_epsilon, MutableFst<Arc> *fst);
|
|
|
|
/// As MakePrecedingInputSymbolsSame, but takes a functor object that maps
|
|
/// labels to classes.
|
|
template <class Arc, class F>
|
|
void MakePrecedingInputSymbolsSameClass(bool start_is_epsilon,
|
|
MutableFst<Arc> *fst, const F &f);
|
|
|
|
/// MakeFollowingInputSymbolsSame ensures that all arcs exiting any given fst
|
|
/// state have the same input symbol. It does this by detecting states that
|
|
/// have differing input symbols on arcs that exit it, and inserting, for each
|
|
/// of the following arcs with non-epsilon input symbol, a new dummy state that
|
|
/// has an input-epsilon link from the fst state. The output symbol and weight
|
|
/// stay on the link to the dummy state (in order to keep the FST
|
|
/// output-deterministic and stochastic, if it already was). If end_is_epsilon,
|
|
/// treat "being a final-state" like having an epsilon output link.
|
|
template <class Arc>
|
|
void MakeFollowingInputSymbolsSame(bool end_is_epsilon, MutableFst<Arc> *fst);
|
|
|
|
/// As MakeFollowingInputSymbolsSame, but takes a functor object that maps
|
|
/// labels to classes.
|
|
template <class Arc, class F>
|
|
void MakeFollowingInputSymbolsSameClass(bool end_is_epsilon,
|
|
MutableFst<Arc> *fst, const F &f);
|
|
|
|
/// MakeLoopFst creates an FST that has a state that is both initial and
|
|
/// final (weight == Weight::One()), and for each non-NULL pointer fsts[i],
|
|
/// it has an arc out whose output-symbol is i and which goes to a
|
|
/// sub-graph whose input language is equivalent to fsts[i], where the
|
|
/// final-state becomes a transition to the loop-state. Each fst in "fsts"
|
|
/// should be an acceptor. The fst MakeLoopFst returns is output-deterministic,
|
|
/// but not output-epsilon free necessarily, and arcs are sorted on output
|
|
/// label. Note: if some of the pointers in the input vector "fsts" have the
|
|
/// same value, "MakeLoopFst" uses this to speed up the computation.
|
|
|
|
/// Formally: suppose I is the set of indexes i such that fsts[i] != NULL.
|
|
/// Let L[i] be the language that the acceptor fsts[i] accepts.
|
|
/// Let the language K be the set of input-output pairs i:l such
|
|
/// that i in I and l in L[i]. Then the FST returned by MakeLoopFst
|
|
/// accepts the language K*, where * is the Kleene closure (CLOSURE_STAR)
|
|
/// of K.
|
|
|
|
/// We could have implemented this via a combination of "project",
|
|
/// "concat", "union" and "closure". But that FST would have been
|
|
/// less well optimized and would have a lot of final-states.
|
|
|
|
template <class Arc>
|
|
VectorFst<Arc> *MakeLoopFst(const std::vector<const ExpandedFst<Arc> *> &fsts);
|
|
|
|
/// ApplyProbabilityScale is applicable to FSTs in the log or tropical semiring.
|
|
/// It multiplies the arc and final weights by "scale" [this is not the Mul
|
|
/// operation of the semiring, it's actual multiplication, which is equivalent
|
|
/// to taking a power in the semiring].
|
|
template <class Arc>
|
|
void ApplyProbabilityScale(float scale, MutableFst<Arc> *fst);
|
|
|
|
/// EqualAlign is similar to RandGen, but it generates a sequence with exactly
|
|
/// "length" input symbols. It returns true on success, false on failure
|
|
/// (failure is partly random but should never happen in practice for normal
|
|
/// speech models.) It generates a random path through the input FST, finds out
|
|
/// which subset of the states it visits along the way have self-loops with
|
|
/// inupt symbols on them, and outputs a path with exactly enough self-loops to
|
|
/// have the requested number of input symbols. Note that EqualAlign does not
|
|
/// use the probabilities on the FST. It just uses equal probabilities in the
|
|
/// first stage of selection (since the output will anyway not be a truly random
|
|
/// sample from the FST). The input fst "ifst" must be connected or this may
|
|
/// enter an infinite loop.
|
|
template <class Arc>
|
|
bool EqualAlign(const Fst<Arc> &ifst, typename Arc::StateId length,
|
|
int rand_seed, MutableFst<Arc> *ofst, int num_retries = 10);
|
|
|
|
// RemoveUselessArcs removes arcs such that there is no input symbol
|
|
// sequence for which the best path through the FST would contain
|
|
// those arcs [for these purposes, epsilon is not treated as a real symbol].
|
|
// This is mainly geared towards decoding-graph FSTs which may contain
|
|
// transitions that have less likely words on them that would never be
|
|
// taken. We do not claim that this algorithm removes all such arcs;
|
|
// it just does the best job it can.
|
|
// Only works for tropical (not log) semiring as it uses
|
|
// NaturalLess.
|
|
template <class Arc>
|
|
void RemoveUselessArcs(MutableFst<Arc> *fst);
|
|
|
|
// PhiCompose is a version of composition where
|
|
// the right hand FST (fst2) is treated as a backoff
|
|
// LM, with the phi symbol (e.g. #0) treated as a
|
|
// "failure transition", only taken when we don't
|
|
// have a match for the requested symbol.
|
|
template <class Arc>
|
|
void PhiCompose(const Fst<Arc> &fst1, const Fst<Arc> &fst2,
|
|
typename Arc::Label phi_label, MutableFst<Arc> *fst);
|
|
|
|
// PropagateFinal propagates final-probs through
|
|
// "phi" transitions (note that here, phi_label may
|
|
// be epsilon if you want). If you have a backoff LM
|
|
// with special symbols ("phi") on the backoff arcs
|
|
// instead of epsilon, you may use PhiCompose to compose
|
|
// with it, but this won't do the right thing w.r.t.
|
|
// final probabilities. You should first call PropagateFinal
|
|
// on the FST with phi's i it (fst2 in PhiCompose above),
|
|
// to fix this. If a state does not have a final-prob,
|
|
// but has a phi transition, it makes the state's final-prob
|
|
// (phi-prob * final-prob-of-dest-state), and does this
|
|
// recursively i.e. follows phi transitions on the dest state
|
|
// first. It behaves as if there were a super-final state
|
|
// with a special symbol leading to it, from each currently
|
|
// final state. Note that this may not behave as desired
|
|
// if there are epsilons in your FST; it might be better
|
|
// to remove those before calling this function.
|
|
|
|
template <class Arc>
|
|
void PropagateFinal(typename Arc::Label phi_label, MutableFst<Arc> *fst);
|
|
|
|
// PhiCompose is a version of composition where
|
|
// the right hand FST (fst2) has speciall "rho transitions"
|
|
// which are taken whenever no normal transition matches; these
|
|
// transitions will be rewritten with whatever symbol was on
|
|
// the first FST.
|
|
template <class Arc>
|
|
void RhoCompose(const Fst<Arc> &fst1, const Fst<Arc> &fst2,
|
|
typename Arc::Label rho_label, MutableFst<Arc> *fst);
|
|
|
|
/** This function returns true if, in the semiring of the FST, the sum (within
|
|
the semiring) of all the arcs out of each state in the FST is one, to within
|
|
delta. After MakeStochasticFst, this should be true (for a connected FST).
|
|
|
|
@param fst [in] the FST that we are testing.
|
|
@param delta [in] the tolerance to within which we test equality to 1.
|
|
@param min_sum [out] if non, NULL, contents will be set to the minimum sum
|
|
of weights.
|
|
@param max_sum [out] if non, NULL, contents will be set to the maximum sum
|
|
of weights.
|
|
@return Returns true if the FST is stochastic, and false otherwise.
|
|
*/
|
|
|
|
template <class Arc>
|
|
bool IsStochasticFst(const Fst<Arc> &fst,
|
|
float delta = kDelta, // kDelta = 1.0/1024.0 by default.
|
|
typename Arc::Weight *min_sum = NULL,
|
|
typename Arc::Weight *max_sum = NULL);
|
|
|
|
// IsStochasticFstInLog makes sure it's stochastic after casting to log.
|
|
inline bool IsStochasticFstInLog(
|
|
const Fst<StdArc> &fst,
|
|
float delta = kDelta, // kDelta = 1.0/1024.0 by default.
|
|
StdArc::Weight *min_sum = NULL, StdArc::Weight *max_sum = NULL);
|
|
|
|
} // end namespace fst
|
|
|
|
#include "fstext/fstext-utils-inl.h"
|
|
|
|
#endif // KALDI_FSTEXT_FSTEXT_UTILS_H_
|