// util/kaldi-table.h
// Copyright 2009-2011 Microsoft Corporation
// 2013 Johns Hopkins University (author: Daniel Povey)
// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
#ifndef KALDI_UTIL_KALDI_TABLE_H_
#define KALDI_UTIL_KALDI_TABLE_H_
#include <string>
#include <vector>
#include <utility>
#include "base/kaldi-common.h"
#include "util/kaldi-holder.h"
namespace kaldi {
// Forward declarations
template<class Holder> class RandomAccessTableReaderImplBase;
template<class Holder> class SequentialTableReaderImplBase;
template<class Holder> class TableWriterImplBase;
/// \addtogroup table_group
/// @{
// This header defines the Table classes (RandomAccessTableReader,
// SequentialTableReader and TableWriter) and explains what the Holder classes,
// which the Table class requires as a template argument, are like. It also
// explains the "rspecifier" and "wspecifier" concepts (these are strings that
// explain how to read/write objects via archives or scp files. A table is
// conceptually a collection of objects of a particular type T indexed by keys
// of type std::string (these Keys additionally have an order within
// each table).
// The Table classes are templated on a type (call it Holder) such that
// Holder::T is a typedef equal to T.
// see kaldi-holder.h for detail on the Holder classes.
typedef std::vector<std::string> KeyList;
// Documentation for "wspecifier"
// "wspecifier" describes how we write a set of objects indexed by keys.
// The basic, unadorned wspecifiers are as follows:
//
// ark:wxfilename
// scp:rxfilename
// ark,scp:filename,wxfilename
// ark,scp:filename,wxfilename
//
//
// We also allow the following modifiers:
// t means text mode.
// b means binary mode.
// f means flush the stream after writing each entry.
// (nf means don't flush, and the default is not to flush).
// p means permissive mode, when writing to an "scp" file only: will ignore
// missing scp entries, i.e. won't write anything for those files but will
// return success status).
//
// So the following are valid wspecifiers:
// ark,b,f:foo
// "ark,b,b:| gzip -c > foo"
// "ark,scp,t,nf:foo.ark,|gzip -c > foo.scp.gz"
// ark,b:-
//
// The meanings of rxfilename and wxfilename are as described in
// kaldi-io.h (they are filenames but include pipes, stdin/stdout
// and so on; filename is a regular filename.
//
// The ark:wxfilename type of wspecifier instructs the class to
// write directly to an archive. For small objects (e.g. lists of ints),
// the text archive format will generally be human readable with one line
// per entry in the archive.
//
// The type "scp:xfilename" refers to an scp file which should
// already exist on disk, and tells us where to write the data for
// each key (usually an actual file); each line of the scp file
// would be:
// key xfilename
//
// The type ark,scp:filename,wxfilename means
// we write both an archive and an scp file that specifies offsets into the
// archive, with lines like:
// key filename:12407
// where the number is the byte offset into the file.
// In this case we restrict the archive-filename to be an actual filename,
// as we can't see a situation where an extended filename would make sense
// for this (we can't fseek() in pipes).
enum WspecifierType {
kNoWspecifier,
kArchiveWspecifier,
kScriptWspecifier,
kBothWspecifier
};
struct WspecifierOptions {
bool binary;
bool flush;
bool permissive; // will ignore absent scp entries.
WspecifierOptions(): binary(true), flush(false), permissive(false) { }
};
// ClassifyWspecifier returns the type of the wspecifier string,
// and (if pointers are non-NULL) outputs the extra information
// about the options, and the script and archive
// filenames.
WspecifierType ClassifyWspecifier(const std::string &wspecifier,
std::string *archive_wxfilename,
std::string *script_wxfilename,
WspecifierOptions *opts);
// ReadScriptFile reads an .scp file in its entirety, and appends it
// (in order as it was in the scp file) in script_out_, which contains
// pairs of (key, xfilename). The .scp
// file format is: on each line, key xfilename
// where xfilename means rxfilename or wxfilename, and may contain internal
// spaces (we trim away any leading or trailing space). The key is space-free.
// ReadScriptFile returns true if the format was valid (empty files
// are valid).
// If 'print_warnings', it will print out warning messages that explain what
// kind of error there was.
bool ReadScriptFile(const std::string &rxfilename,
bool print_warnings,
std::vector<std::pair<std::string, std::string> >
*script_out);
// This version of ReadScriptFile works from an istream.
bool ReadScriptFile(std::istream &is,
bool print_warnings,
std::vector<std::pair<std::string, std::string> >
*script_out);
// Writes, for each entry in script, the first element, then ' ', then the
// second element then '\n'. Checks that the keys (first elements of pairs) are
// valid tokens (nonempty, no whitespace), and the values (second elements of
// pairs) are newline-free and contain no leading or trailing space. Returns
// true on success.
bool WriteScriptFile(const std::string &wxfilename,
const std::vector<std::pair<std::string, std::string> >
&script);
// This version writes to an ostream.
bool WriteScriptFile(std::ostream &os,
const std::vector<std::pair<std::string, std::string> >
&script);
// Documentation for "rspecifier"
// "rspecifier" describes how we read a set of objects indexed by keys.
// The possibilities are:
//
// ark:rxfilename
// scp:rxfilename
//
// We also allow various modifiers:
// o means the program will only ask for each key once, which enables
// the reader to discard already-asked-for values.
// s means the keys are sorted on input (means we don't have to read till
// eof if someone asked for a key that wasn't there).
// cs means that it is called in sorted order (we are generally asserting
// this based on knowledge of how the program works).
// p means "permissive", and causes it to skip over keys whose corresponding
// scp-file entries cannot be read. [and to ignore errors in archives and
// script files, and just consider the "good" entries].
// We allow the negation of the options above, as in no, ns, np,
// but these aren't currently very useful (just equivalent to omitting the
// corresponding option).
// [any of the above options can be prefixed by n to negate them, e.g. no,
// ns, ncs, np; but these aren't currently useful as you could just omit
// the option].
// bg means "background". It currently has no effect for random-access readers,
// but for sequential readers it will cause it to "read ahead" to the next
// value, in a background thread. Recommended when reading larger objects
// such as neural-net training examples, especially when you want to
// maximize GPU usage.
//
// b is ignored [for scripting convenience]
// t is ignored [for scripting convenience]
//
//
// So for instance the following would be a valid rspecifier:
//
// "o, s, p, ark:gunzip -c foo.gz|"
struct RspecifierOptions {
// These options only make a difference for the RandomAccessTableReader class.
bool once; // we assert that the program will only ask for each key once.
bool sorted; // we assert that the keys are sorted.
bool called_sorted; // we assert that the (HasKey(), Value() functions will
// also be called in sorted order. [this implies "once" but not vice versa].
bool permissive; // If "permissive", when reading from scp files it treats
// scp files that can't be read as if the corresponding key were not there.
// For archive files it will suppress errors getting thrown if the archive
// is corrupted and can't be read to the end.
bool background; // For sequential readers, if the background option ("bg")
// is provided, it will read ahead to the next object in a
// background thread.
RspecifierOptions(): once(false), sorted(false),
called_sorted(false), permissive(false),
background(false) { }
};
enum RspecifierType {
kNoRspecifier,
kArchiveRspecifier,
kScriptRspecifier
};
RspecifierType ClassifyRspecifier(const std::string &rspecifier,
std::string *rxfilename,
RspecifierOptions *opts);
/// Allows random access to a collection
/// of objects in an archive or script file; see \ref io_sec_tables.
template<class Holder>
class RandomAccessTableReader {
public:
typedef typename Holder::T T;
RandomAccessTableReader(): impl_(NULL) { }
// This constructor is equivalent to default constructor + "open", but
// throws on error.
explicit RandomAccessTableReader(const std::string &rspecifier);
// Opens the table.
bool Open(const std::string &rspecifier);
// Returns true if table is open.
bool IsOpen() const { return (impl_ != NULL); }
// Close() will close the table [throws if it was not open],
// and returns true on success (false if we were reading an
// archive and we discovered an error in the archive).
bool Close();
// Says if it has this key.
// If you are using the "permissive" (p) read option,
// it will return false for keys whose corresponding entry
// in the scp file cannot be read.
bool HasKey(const std::string &key);
// Value() may throw if you are reading an scp file, you
// do not have the "permissive" (p) option, and an entry
// in the scp file cannot be read. Typically you won't
// want to catch this error.
const T &Value(const std::string &key);
~RandomAccessTableReader();
// Allow copy-constructor only for non-opened readers (needed for inclusion in
// stl vector)
RandomAccessTableReader(const RandomAccessTableReader<Holder>
&other):
impl_(NULL) { KALDI_ASSERT(other.impl_ == NULL); }
private:
// Disallow assignment.
RandomAccessTableReader &operator=(const RandomAccessTableReader<Holder>&);
void CheckImpl() const; // Checks that impl_ is non-NULL; prints an error
// message and dies (with KALDI_ERR) if NULL.
RandomAccessTableReaderImplBase<Holder> *impl_;
};
/// A templated class for reading objects sequentially from an archive or script
/// file; see \ref io_sec_tables.
template<class Holder>
class SequentialTableReader {
public:
typedef typename Holder::T T;
SequentialTableReader(): impl_(NULL) { }
// This constructor equivalent to default constructor + "open", but
// throws on error.
explicit SequentialTableReader(const std::string &rspecifier);
// Opens the table. Returns exit status; but does throw if previously open
// stream was in error state. You can call Close to prevent this; anyway,
// calling Open more than once is not usually needed.
bool Open(const std::string &rspecifier);
// Returns true if we're done. It will also return true if there's some kind
// of error and we can't read any more; in this case, you can detect the
// error by calling Close and checking the return status; otherwise
// the destructor will throw.
inline bool Done();
// Only valid to call Key() if Done() returned false.
inline std::string Key();
// FreeCurrent() is provided as an optimization to save memory, for large
// objects. It instructs the class to deallocate the current value. The
// reference Value() will be invalidated by this.
void FreeCurrent();
// Return reference to the current value. It's only valid to call this if
// Done() returned false. The reference is valid till next call to this
// object. It will throw if you are reading an scp file, did not specify the
// "permissive" (p) option and the file cannot be read. [The permissive
// option makes it behave as if that key does not even exist, if the
// corresponding file cannot be read.] You probably wouldn't want to catch
// this exception; the user can just specify the p option in the rspecifier.
// We make this non-const to enable things like shallow swap on the held
// object in situations where this would avoid making a redundant copy.
T &Value();
// Next goes to the next key. It will not throw; any error will
// result in Done() returning true, and then the destructor will
// throw unless you call Close().
void Next();
// Returns true if table is open for reading (does not imply
// stream is in good state).
bool IsOpen() const;
// Close() will return false (failure) if Done() became true
// because of an error/ condition rather than because we are
// really done [e.g. because of an error or early termination
// in the archive].
// If there is an error and you don't call Close(), the destructor
// will fail.
// Close()
bool Close();
// The destructor may throw. This is the desired behaviour, as it's the way
// we signal the error to the user (to detect it, call Close(). The issue is
// that otherwise the user has no way to tell whether Done() returned true
// because we reached the end of the archive or script, or because there was
// an error that prevented further reading.
~SequentialTableReader();
// Allow copy-constructor only for non-opened readers (needed for inclusion in
// stl vector)
SequentialTableReader(const SequentialTableReader<Holder> &other):
impl_(NULL) { KALDI_ASSERT(other.impl_ == NULL); }
private:
// Disallow assignment.
SequentialTableReader &operator = (const SequentialTableReader<Holder>&);
void CheckImpl() const; // Checks that impl_ is non-NULL; prints an error
// message and dies (with KALDI_ERR) if NULL.
SequentialTableReaderImplBase<Holder> *impl_;
};
/// A templated class for writing objects to an
/// archive or script file; see \ref io_sec_tables.
template<class Holder>
class TableWriter {
public:
typedef typename Holder::T T;
TableWriter(): impl_(NULL) { }
// This constructor equivalent to default constructor
// + "open", but throws on error. See docs for
// wspecifier above.
explicit TableWriter(const std::string &wspecifier);
// Opens the table. See docs for wspecifier above.
// If it returns true, it is open.
bool Open(const std::string &wspecifier);
// Returns true if open for writing.
bool IsOpen() const;
// Write the object. Throws KaldiFatalError on error via the KALDI_ERR macro.
inline void Write(const std::string &key, const T &value) const;
// Flush will flush any archive; it does not return error status
// or throw, any errors will be reported on the next Write or Close.
// Useful if we may be writing to a command in a pipe and want
// to ensure good CPU utilization.
void Flush();
// Close() is not necessary to call, as the destructor
// closes it; it's mainly useful if you want to handle
// error states because the destructor will throw on
// error if you do not call Close().
bool Close();
~TableWriter();
// Allow copy-constructor only for non-opened writers (needed for inclusion in
// stl vector)
TableWriter(const TableWriter &other): impl_(NULL) {
KALDI_ASSERT(other.impl_ == NULL);
}
private:
TableWriter &operator = (const TableWriter&); // Disallow assignment.
void CheckImpl() const; // Checks that impl_ is non-NULL; prints an error
// message and dies (with KALDI_ERR) if NULL.
TableWriterImplBase<Holder> *impl_;
};
/// This class is for when you are reading something in random access, but
/// it may actually be stored per-speaker (or something similar) but the
/// keys you're using are per utterance. So you also provide an "rxfilename"
/// for a file containing lines like
/// utt1 spk1
/// utt2 spk1
/// utt3 spk1
/// and so on. Note: this is optional; if it is an empty string, we just won't
/// do the mapping. Also, "table_rxfilename" may be the empty string (as for
/// a regular table), in which case the table just won't be opened.
/// We provide only the most frequently used of the functions of
/// RandomAccessTableReader.
template<class Holder>
class RandomAccessTableReaderMapped {
public:
typedef typename Holder::T T;
/// Note: "utt2spk_rxfilename" will in the normal case be an rxfilename
/// for an utterance to speaker map, but this code is general; it accepts
/// a generic map.
RandomAccessTableReaderMapped(const std::string &table_rxfilename,
const std::string &utt2spk_rxfilename);
RandomAccessTableReaderMapped() {}
/// Note: when calling Open, utt2spk_rxfilename may be empty.
bool Open(const std::string &table_rxfilename,
const std::string &utt2spk_rxfilename);
bool HasKey(const std::string &key);
const T &Value(const std::string &key);
inline bool IsOpen() const { return reader_.IsOpen(); }
inline bool Close() { return reader_.Close(); }
// The default copy-constructor will do what we want: it will crash for
// already-opened readers, by calling the member-variable copy-constructors.
private:
// Disallow assignment.
RandomAccessTableReaderMapped &operator =
(const RandomAccessTableReaderMapped<Holder>&);
RandomAccessTableReader<Holder> reader_;
RandomAccessTableReader<TokenHolder> token_reader_;
std::string utt2spk_rxfilename_; // Used only in diagnostic messages.
};
/// @} end "addtogroup table_group"
} // end namespace kaldi
#include "util/kaldi-table-inl.h"
#endif // KALDI_UTIL_KALDI_TABLE_H_