mirror of
https://github.com/m5stack/StackFlow.git
synced 2026-05-20 11:32:11 -07:00
Add text normalization for Chinese, Japanese, and English
Implement regex-based text normalization functionality to support trilingual (CJE) content processing
This commit is contained in:
yuyun2000
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,248 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// FST implementation class to attach an arbitrary object with a read/write
|
||||
// method to an FST and its file representation. The FST is given a new type
|
||||
// name.
|
||||
|
||||
#ifndef FST_ADD_ON_H_
|
||||
#define FST_ADD_ON_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include <fst/log.h>
|
||||
|
||||
#include <fst/fst.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Identifies stream data as an add-on FST.
|
||||
static constexpr int32 kAddOnMagicNumber = 446681434;
|
||||
|
||||
// Nothing to save.
|
||||
class NullAddOn {
|
||||
public:
|
||||
NullAddOn() {}
|
||||
|
||||
static NullAddOn *Read(std::istream &strm, const FstReadOptions &opts) {
|
||||
return new NullAddOn();
|
||||
}
|
||||
|
||||
bool Write(std::ostream &ostrm, const FstWriteOptions &opts) const {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
// Create a new add-on from a pair of add-ons.
|
||||
template <class A1, class A2>
|
||||
class AddOnPair {
|
||||
public:
|
||||
// Argument reference count incremented.
|
||||
AddOnPair(std::shared_ptr<A1> a1, std::shared_ptr<A2> a2)
|
||||
: a1_(std::move(a1)), a2_(std::move(a2)) {}
|
||||
|
||||
const A1 *First() const { return a1_.get(); }
|
||||
|
||||
const A2 *Second() const { return a2_.get(); }
|
||||
|
||||
std::shared_ptr<A1> SharedFirst() const { return a1_; }
|
||||
|
||||
std::shared_ptr<A2> SharedSecond() const { return a2_; }
|
||||
|
||||
static AddOnPair<A1, A2> *Read(std::istream &istrm,
|
||||
const FstReadOptions &opts) {
|
||||
A1 *a1 = nullptr;
|
||||
bool have_addon1 = false;
|
||||
ReadType(istrm, &have_addon1);
|
||||
if (have_addon1) a1 = A1::Read(istrm, opts);
|
||||
|
||||
A2 *a2 = nullptr;
|
||||
bool have_addon2 = false;
|
||||
ReadType(istrm, &have_addon2);
|
||||
if (have_addon2) a2 = A2::Read(istrm, opts);
|
||||
|
||||
return new AddOnPair<A1, A2>(std::shared_ptr<A1>(a1),
|
||||
std::shared_ptr<A2>(a2));
|
||||
}
|
||||
|
||||
bool Write(std::ostream &ostrm, const FstWriteOptions &opts) const {
|
||||
bool have_addon1 = a1_ != nullptr;
|
||||
WriteType(ostrm, have_addon1);
|
||||
if (have_addon1) a1_->Write(ostrm, opts);
|
||||
bool have_addon2 = a2_ != nullptr;
|
||||
WriteType(ostrm, have_addon2);
|
||||
if (have_addon2) a2_->Write(ostrm, opts);
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
std::shared_ptr<A1> a1_;
|
||||
std::shared_ptr<A2> a2_;
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
|
||||
// Adds an object of type T to an FST. T must support:
|
||||
//
|
||||
// T* Read(std::istream &);
|
||||
// bool Write(std::ostream &);
|
||||
//
|
||||
// The resulting type is a new FST implementation.
|
||||
template <class FST, class T>
|
||||
class AddOnImpl : public FstImpl<typename FST::Arc> {
|
||||
public:
|
||||
using Arc = typename FST::Arc;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
|
||||
using FstImpl<Arc>::SetType;
|
||||
using FstImpl<Arc>::SetInputSymbols;
|
||||
using FstImpl<Arc>::SetOutputSymbols;
|
||||
using FstImpl<Arc>::SetProperties;
|
||||
using FstImpl<Arc>::WriteHeader;
|
||||
|
||||
// We make a thread-safe copy of the FST by default since an FST
|
||||
// implementation is expected to not share mutable data between objects.
|
||||
AddOnImpl(const FST &fst, const string &type,
|
||||
std::shared_ptr<T> t = std::shared_ptr<T>())
|
||||
: fst_(fst, true), t_(std::move(t)) {
|
||||
SetType(type);
|
||||
SetProperties(fst_.Properties(kFstProperties, false));
|
||||
SetInputSymbols(fst_.InputSymbols());
|
||||
SetOutputSymbols(fst_.OutputSymbols());
|
||||
}
|
||||
|
||||
// Conversion from const Fst<Arc> & to F always copies the underlying
|
||||
// implementation.
|
||||
AddOnImpl(const Fst<Arc> &fst, const string &type,
|
||||
std::shared_ptr<T> t = std::shared_ptr<T>())
|
||||
: fst_(fst), t_(std::move(t)) {
|
||||
SetType(type);
|
||||
SetProperties(fst_.Properties(kFstProperties, false));
|
||||
SetInputSymbols(fst_.InputSymbols());
|
||||
SetOutputSymbols(fst_.OutputSymbols());
|
||||
}
|
||||
|
||||
// We make a thread-safe copy of the FST by default since an FST
|
||||
// implementation is expected to not share mutable data between objects.
|
||||
AddOnImpl(const AddOnImpl<FST, T> &impl)
|
||||
: fst_(impl.fst_, true), t_(impl.t_) {
|
||||
SetType(impl.Type());
|
||||
SetProperties(fst_.Properties(kCopyProperties, false));
|
||||
SetInputSymbols(fst_.InputSymbols());
|
||||
SetOutputSymbols(fst_.OutputSymbols());
|
||||
}
|
||||
|
||||
StateId Start() const { return fst_.Start(); }
|
||||
|
||||
Weight Final(StateId s) const { return fst_.Final(s); }
|
||||
|
||||
size_t NumArcs(StateId s) const { return fst_.NumArcs(s); }
|
||||
|
||||
size_t NumInputEpsilons(StateId s) const { return fst_.NumInputEpsilons(s); }
|
||||
|
||||
size_t NumOutputEpsilons(StateId s) const {
|
||||
return fst_.NumOutputEpsilons(s);
|
||||
}
|
||||
|
||||
size_t NumStates() const { return fst_.NumStates(); }
|
||||
|
||||
static AddOnImpl<FST, T> *Read(std::istream &strm,
|
||||
const FstReadOptions &opts) {
|
||||
FstReadOptions nopts(opts);
|
||||
FstHeader hdr;
|
||||
if (!nopts.header) {
|
||||
hdr.Read(strm, nopts.source);
|
||||
nopts.header = &hdr;
|
||||
}
|
||||
std::unique_ptr<AddOnImpl<FST, T>> impl(
|
||||
new AddOnImpl<FST, T>(nopts.header->FstType()));
|
||||
if (!impl->ReadHeader(strm, nopts, kMinFileVersion, &hdr)) return nullptr;
|
||||
impl.reset();
|
||||
int32 magic_number = 0;
|
||||
ReadType(strm, &magic_number); // Ensures this is an add-on FST.
|
||||
if (magic_number != kAddOnMagicNumber) {
|
||||
LOG(ERROR) << "AddOnImpl::Read: Bad add-on header: " << nopts.source;
|
||||
return nullptr;
|
||||
}
|
||||
FstReadOptions fopts(opts);
|
||||
fopts.header = nullptr; // Contained header was written out.
|
||||
std::unique_ptr<FST> fst(FST::Read(strm, fopts));
|
||||
if (!fst) return nullptr;
|
||||
std::shared_ptr<T> t;
|
||||
bool have_addon = false;
|
||||
ReadType(strm, &have_addon);
|
||||
if (have_addon) { // Reads add-on object if present.
|
||||
t = std::shared_ptr<T>(T::Read(strm, fopts));
|
||||
if (!t) return nullptr;
|
||||
}
|
||||
return new AddOnImpl<FST, T>(*fst, nopts.header->FstType(), t);
|
||||
}
|
||||
|
||||
bool Write(std::ostream &strm, const FstWriteOptions &opts) const {
|
||||
FstHeader hdr;
|
||||
FstWriteOptions nopts(opts);
|
||||
nopts.write_isymbols = false; // Allows contained FST to hold any symbols.
|
||||
nopts.write_osymbols = false;
|
||||
WriteHeader(strm, nopts, kFileVersion, &hdr);
|
||||
WriteType(strm, kAddOnMagicNumber); // Ensures this is an add-on FST.
|
||||
FstWriteOptions fopts(opts);
|
||||
fopts.write_header = true; // Forces writing contained header.
|
||||
if (!fst_.Write(strm, fopts)) return false;
|
||||
bool have_addon = !!t_;
|
||||
WriteType(strm, have_addon);
|
||||
// Writes add-on object if present.
|
||||
if (have_addon) t_->Write(strm, opts);
|
||||
return true;
|
||||
}
|
||||
|
||||
void InitStateIterator(StateIteratorData<Arc> *data) const {
|
||||
fst_.InitStateIterator(data);
|
||||
}
|
||||
|
||||
void InitArcIterator(StateId s, ArcIteratorData<Arc> *data) const {
|
||||
fst_.InitArcIterator(s, data);
|
||||
}
|
||||
|
||||
FST &GetFst() { return fst_; }
|
||||
|
||||
const FST &GetFst() const { return fst_; }
|
||||
|
||||
const T *GetAddOn() const { return t_.get(); }
|
||||
|
||||
std::shared_ptr<T> GetSharedAddOn() const { return t_; }
|
||||
|
||||
void SetAddOn(std::shared_ptr<T> t) { t_ = t; }
|
||||
|
||||
private:
|
||||
explicit AddOnImpl(const string &type) : t_() {
|
||||
SetType(type);
|
||||
SetProperties(kExpanded);
|
||||
}
|
||||
|
||||
// Current file format version.
|
||||
static constexpr int kFileVersion = 1;
|
||||
// Minimum file format version supported.
|
||||
static constexpr int kMinFileVersion = 1;
|
||||
|
||||
FST fst_;
|
||||
std::shared_ptr<T> t_;
|
||||
|
||||
AddOnImpl &operator=(const AddOnImpl &) = delete;
|
||||
};
|
||||
|
||||
template <class FST, class T>
|
||||
constexpr int AddOnImpl<FST, T>::kFileVersion;
|
||||
|
||||
template <class FST, class T>
|
||||
constexpr int AddOnImpl<FST, T>::kMinFileVersion;
|
||||
|
||||
} // namespace internal
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_ADD_ON_H_
|
||||
@@ -0,0 +1,232 @@
|
||||
#ifndef FST_ARC_ARENA_H_
|
||||
#define FST_ARC_ARENA_H_
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <fst/fst.h>
|
||||
#include <fst/memory.h>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace fst {
|
||||
|
||||
// ArcArena is used for fast allocation of contiguous arrays of arcs.
|
||||
//
|
||||
// To create an arc array:
|
||||
// for each state:
|
||||
// for each arc:
|
||||
// arena.PushArc();
|
||||
// // Commits these arcs and returns pointer to them.
|
||||
// Arc *arcs = arena.GetArcs();
|
||||
//
|
||||
// OR
|
||||
//
|
||||
// arena.DropArcs(); // Throws away current arcs, reuse the space.
|
||||
//
|
||||
// The arcs returned are guaranteed to be contiguous and the pointer returned
|
||||
// will never be invalidated until the arena is cleared for reuse.
|
||||
//
|
||||
// The contents of the arena can be released with a call to arena.Clear() after
|
||||
// which the arena will restart with an initial allocation capable of holding at
|
||||
// least all of the arcs requested in the last usage before Clear() making
|
||||
// subsequent uses of the Arena more efficient.
|
||||
//
|
||||
// The max_retained_size option can limit the amount of arc space requested on
|
||||
// Clear() to avoid excess growth from intermittent high usage.
|
||||
template <typename Arc>
|
||||
class ArcArena {
|
||||
public:
|
||||
explicit ArcArena(size_t block_size = 256,
|
||||
size_t max_retained_size = 1e6)
|
||||
: block_size_(block_size),
|
||||
max_retained_size_(max_retained_size) {
|
||||
blocks_.emplace_back(MakeSharedBlock(block_size_));
|
||||
first_block_size_ = block_size_;
|
||||
total_size_ = block_size_;
|
||||
arcs_ = blocks_.back().get();
|
||||
end_ = arcs_ + block_size_;
|
||||
next_ = arcs_;
|
||||
}
|
||||
|
||||
ArcArena(const ArcArena& copy)
|
||||
: arcs_(copy.arcs_), next_(copy.next_), end_(copy.end_),
|
||||
block_size_(copy.block_size_),
|
||||
first_block_size_(copy.first_block_size_),
|
||||
total_size_(copy.total_size_),
|
||||
max_retained_size_(copy.max_retained_size_),
|
||||
blocks_(copy.blocks_) {
|
||||
NewBlock(block_size_);
|
||||
}
|
||||
|
||||
void ReserveArcs(size_t n) {
|
||||
if (next_ + n < end_) return;
|
||||
NewBlock(n);
|
||||
}
|
||||
|
||||
void PushArc(const Arc& arc) {
|
||||
if (next_ == end_) {
|
||||
size_t length = next_ - arcs_;
|
||||
NewBlock(length * 2);
|
||||
}
|
||||
*next_ = arc;
|
||||
++next_;
|
||||
}
|
||||
|
||||
const Arc* GetArcs() {
|
||||
const auto *arcs = arcs_;
|
||||
arcs_ = next_;
|
||||
return arcs;
|
||||
}
|
||||
|
||||
void DropArcs() { next_ = arcs_; }
|
||||
|
||||
size_t Size() { return total_size_; }
|
||||
|
||||
void Clear() {
|
||||
blocks_.resize(1);
|
||||
if (total_size_ > first_block_size_) {
|
||||
first_block_size_ = std::min(max_retained_size_, total_size_);
|
||||
blocks_.back() = MakeSharedBlock(first_block_size_);
|
||||
}
|
||||
total_size_ = first_block_size_;
|
||||
arcs_ = blocks_.back().get();
|
||||
end_ = arcs_ + first_block_size_;
|
||||
next_ = arcs_;
|
||||
}
|
||||
|
||||
private:
|
||||
// Allocates a new block with capacity of at least n or block_size,
|
||||
// copying incomplete arc sequence from old block to new block.
|
||||
void NewBlock(size_t n) {
|
||||
const auto length = next_ - arcs_;
|
||||
const auto new_block_size = std::max(n, block_size_);
|
||||
total_size_ += new_block_size;
|
||||
blocks_.emplace_back(MakeSharedBlock(new_block_size));
|
||||
std::copy(arcs_, next_, blocks_.back().get());
|
||||
arcs_ = blocks_.back().get();
|
||||
next_ = arcs_ + length;
|
||||
end_ = arcs_ + new_block_size;
|
||||
}
|
||||
|
||||
std::shared_ptr<Arc> MakeSharedBlock(size_t size) {
|
||||
return std::shared_ptr<Arc>(new Arc[size], std::default_delete<Arc[]>());
|
||||
}
|
||||
|
||||
Arc *arcs_;
|
||||
Arc *next_;
|
||||
const Arc *end_;
|
||||
size_t block_size_;
|
||||
size_t first_block_size_;
|
||||
size_t total_size_;
|
||||
size_t max_retained_size_;
|
||||
std::list<std::shared_ptr<Arc>> blocks_;
|
||||
};
|
||||
|
||||
// ArcArenaStateStore uses a resusable ArcArena to store arc arrays and does not
|
||||
// require that the Expander call ReserveArcs first.
|
||||
//
|
||||
// TODO(tombagby): Make cache type configurable.
|
||||
// TODO(tombagby): Provide ThreadLocal/Concurrent configuration.
|
||||
template <class A>
|
||||
class ArcArenaStateStore {
|
||||
public:
|
||||
using Arc = A;
|
||||
using Weight = typename Arc::Weight;
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
ArcArenaStateStore() : arena_(64 * 1024) {
|
||||
}
|
||||
|
||||
class State {
|
||||
public:
|
||||
Weight Final() const { return final_; }
|
||||
|
||||
size_t NumInputEpsilons() const { return niepsilons_; }
|
||||
|
||||
size_t NumOutputEpsilons() const { return noepsilons_; }
|
||||
|
||||
size_t NumArcs() const { return narcs_; }
|
||||
|
||||
const Arc &GetArc(size_t n) const { return arcs_[n]; }
|
||||
|
||||
const Arc *Arcs() const { return arcs_; }
|
||||
|
||||
int* MutableRefCount() const { return nullptr; }
|
||||
|
||||
private:
|
||||
State(Weight weight, int32 niepsilons, int32 noepsilons, int32 narcs,
|
||||
const Arc *arcs)
|
||||
: final_(std::move(weight)),
|
||||
niepsilons_(niepsilons),
|
||||
noepsilons_(noepsilons),
|
||||
narcs_(narcs),
|
||||
arcs_(arcs) {}
|
||||
|
||||
Weight final_;
|
||||
size_t niepsilons_;
|
||||
size_t noepsilons_;
|
||||
size_t narcs_;
|
||||
const Arc *arcs_;
|
||||
|
||||
friend class ArcArenaStateStore<Arc>;
|
||||
};
|
||||
|
||||
template <class Expander>
|
||||
State *FindOrExpand(Expander &expander, StateId state_id) { // NOLINT
|
||||
auto it = cache_.insert(std::pair<StateId, State*>(state_id, nullptr));
|
||||
if (!it.second) return it.first->second;
|
||||
// Needs a new state.
|
||||
StateBuilder builder(&arena_);
|
||||
expander.Expand(state_id, &builder);
|
||||
const auto arcs = arena_.GetArcs();
|
||||
size_t narcs = builder.narcs_;
|
||||
size_t niepsilons = 0;
|
||||
size_t noepsilons = 0;
|
||||
for (size_t i = 0; i < narcs; ++i) {
|
||||
if (arcs[i].ilabel == 0) ++niepsilons;
|
||||
if (arcs[i].olabel == 0) ++noepsilons;
|
||||
}
|
||||
states_.emplace_back(
|
||||
State(builder.final_, niepsilons, noepsilons, narcs, arcs));
|
||||
// Places it in the cache.
|
||||
auto state = &states_.back();
|
||||
it.first->second = state;
|
||||
return state;
|
||||
}
|
||||
|
||||
State *Find(StateId state_id) const {
|
||||
auto it = cache_.find(state_id);
|
||||
return (it == cache_.end()) ? nullptr : it->second;
|
||||
}
|
||||
|
||||
private:
|
||||
class StateBuilder {
|
||||
public:
|
||||
explicit StateBuilder(ArcArena<Arc>* arena)
|
||||
: arena_(arena), final_(Weight::Zero()), narcs_(0) {}
|
||||
|
||||
void SetFinal(Weight weight) { final_ = std::move(weight); }
|
||||
|
||||
void ReserveArcs(size_t n) { arena_->ReserveArcs(n); }
|
||||
|
||||
void AddArc(const Arc &arc) {
|
||||
++narcs_;
|
||||
arena_->PushArc(arc);
|
||||
}
|
||||
|
||||
private:
|
||||
friend class ArcArenaStateStore<Arc>;
|
||||
|
||||
ArcArena<Arc> *arena_;
|
||||
Weight final_;
|
||||
size_t narcs_;
|
||||
};
|
||||
|
||||
std::unordered_map<StateId, State *> cache_;
|
||||
std::deque<State> states_;
|
||||
ArcArena<Arc> arena_;
|
||||
};
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_ARC_ARENA_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,317 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Commonly used FST arc types.
|
||||
|
||||
#ifndef FST_ARC_H_
|
||||
#define FST_ARC_H_
|
||||
|
||||
#include <climits>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
|
||||
|
||||
#include <fst/expectation-weight.h>
|
||||
#include <fst/float-weight.h>
|
||||
#include <fst/lexicographic-weight.h>
|
||||
#include <fst/power-weight.h>
|
||||
#include <fst/product-weight.h>
|
||||
#include <fst/signed-log-weight.h>
|
||||
#include <fst/sparse-power-weight.h>
|
||||
#include <fst/string-weight.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
template <class W>
|
||||
struct ArcTpl {
|
||||
public:
|
||||
using Weight = W;
|
||||
using Label = int;
|
||||
using StateId = int;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
ArcTpl() noexcept(std::is_nothrow_default_constructible<Weight>::value) {}
|
||||
|
||||
template <class T>
|
||||
ArcTpl(Label ilabel, Label olabel, T &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<T>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const auto *const type =
|
||||
new string(Weight::Type() == "tropical" ? "standard" : Weight::Type());
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
using StdArc = ArcTpl<TropicalWeight>;
|
||||
using LogArc = ArcTpl<LogWeight>;
|
||||
using Log64Arc = ArcTpl<Log64Weight>;
|
||||
using SignedLogArc = ArcTpl<SignedLogWeight>;
|
||||
using SignedLog64Arc = ArcTpl<SignedLog64Weight>;
|
||||
using MinMaxArc = ArcTpl<MinMaxWeight>;
|
||||
|
||||
// Arc with integer labels and state IDs and string weights.
|
||||
template <StringType S = STRING_LEFT>
|
||||
struct StringArc {
|
||||
public:
|
||||
using Label = int;
|
||||
using Weight = StringWeight<int, S>;
|
||||
using StateId = int;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
StringArc() = default;
|
||||
|
||||
template <class W>
|
||||
StringArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const auto *const type = new string(
|
||||
S == STRING_LEFT ? "left_standard_string"
|
||||
: (S == STRING_RIGHT ? "right_standard_string"
|
||||
: "restricted_standard_string"));
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
// Arc with label and state Id type the same as template arg and with
|
||||
// weights over the Gallic semiring w.r.t the output labels and weights of A.
|
||||
template <class A, GallicType G = GALLIC_LEFT>
|
||||
struct GallicArc {
|
||||
using Arc = A;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = GallicWeight<Label, typename Arc::Weight, G>;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
GallicArc() = default;
|
||||
|
||||
template <class W>
|
||||
GallicArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
explicit GallicArc(const Arc &arc)
|
||||
: ilabel(arc.ilabel), olabel(arc.ilabel), weight(arc.olabel, arc.weight),
|
||||
nextstate(arc.nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const auto *const type = new string(
|
||||
(G == GALLIC_LEFT
|
||||
? "left_gallic_"
|
||||
: (G == GALLIC_RIGHT
|
||||
? "right_gallic_"
|
||||
: (G == GALLIC_RESTRICT
|
||||
? "restricted_gallic_"
|
||||
: (G == GALLIC_MIN ? "min_gallic_" : "gallic_")))) +
|
||||
Arc::Type());
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
// Arc with the reverse of the weight found in its template arg.
|
||||
template <class A>
|
||||
struct ReverseArc {
|
||||
using Arc = A;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::StateId;
|
||||
using AWeight = typename Arc::Weight;
|
||||
using Weight = typename AWeight::ReverseWeight;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
ReverseArc() = default;
|
||||
|
||||
template <class W>
|
||||
ReverseArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const auto *const type = new string("reverse_" + Arc::Type());
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
// Arc with integer labels and state IDs and lexicographic weights.
|
||||
template <class Weight1, class Weight2>
|
||||
struct LexicographicArc {
|
||||
using Label = int;
|
||||
using StateId = int;
|
||||
using Weight = LexicographicWeight<Weight1, Weight2>;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
LexicographicArc() = default;
|
||||
|
||||
template <class W>
|
||||
LexicographicArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const string *const type = new string(Weight::Type());
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
// Arc with integer labels and state IDs and product weights.
|
||||
template <class Weight1, class Weight2>
|
||||
struct ProductArc {
|
||||
using Label = int;
|
||||
using StateId = int;
|
||||
using Weight = ProductWeight<Weight1, Weight2>;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
ProductArc() = default;
|
||||
|
||||
template <class W>
|
||||
ProductArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const auto *const type = new string(Weight::Type());
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
// Arc with label and state ID type the same as first template argument and with
|
||||
// weights over the n-th Cartesian power of the weight type of the template
|
||||
// argument.
|
||||
template <class A, size_t n>
|
||||
struct PowerArc {
|
||||
using Arc = A;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = PowerWeight<typename Arc::Weight, n>;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
PowerArc() = default;
|
||||
|
||||
template <class W>
|
||||
PowerArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const auto *const type =
|
||||
new string(Arc::Type() + "_^" + std::to_string(n));
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
// Arc with label and state ID type the same as first template argument and with
|
||||
// weights over the arbitrary Cartesian power of the weight type.
|
||||
template <class A, class K = int>
|
||||
struct SparsePowerArc {
|
||||
using Arc = A;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::Label;
|
||||
using Weight = SparsePowerWeight<typename Arc::Weight, K>;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
SparsePowerArc() = default;
|
||||
|
||||
template <class W>
|
||||
SparsePowerArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const string *const type = [] {
|
||||
string type = Arc::Type() + "_^n";
|
||||
if (sizeof(K) != sizeof(uint32)) {
|
||||
type += "_" + std::to_string(CHAR_BIT * sizeof(K));
|
||||
}
|
||||
return new string(type);
|
||||
}();
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
// Arc with label and state ID type the same as first template argument and with
|
||||
// expectation weight over the first template argument's weight type and the
|
||||
// second template argument.
|
||||
template <class A, class X2>
|
||||
struct ExpectationArc {
|
||||
using Arc = A;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::StateId;
|
||||
using X1 = typename Arc::Weight;
|
||||
using Weight = ExpectationWeight<X1, X2>;
|
||||
|
||||
Label ilabel;
|
||||
Label olabel;
|
||||
Weight weight;
|
||||
StateId nextstate;
|
||||
|
||||
ExpectationArc() = default;
|
||||
|
||||
template <class W>
|
||||
ExpectationArc(Label ilabel, Label olabel, W &&weight, StateId nextstate)
|
||||
: ilabel(ilabel),
|
||||
olabel(olabel),
|
||||
weight(std::forward<W>(weight)),
|
||||
nextstate(nextstate) {}
|
||||
|
||||
static const string &Type() {
|
||||
static const auto *const type =
|
||||
new string("expectation_" + Arc::Type() + "_" + X2::Type());
|
||||
return *type;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_ARC_H_
|
||||
@@ -0,0 +1,93 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Function objects to restrict which arcs are traversed in an FST.
|
||||
|
||||
#ifndef FST_ARCFILTER_H_
|
||||
#define FST_ARCFILTER_H_
|
||||
|
||||
|
||||
#include <fst/fst.h>
|
||||
#include <fst/util.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
// True for all arcs.
|
||||
template <class Arc>
|
||||
class AnyArcFilter {
|
||||
public:
|
||||
bool operator()(const Arc &arc) const { return true; }
|
||||
};
|
||||
|
||||
// True for (input/output) epsilon arcs.
|
||||
template <class Arc>
|
||||
class EpsilonArcFilter {
|
||||
public:
|
||||
bool operator()(const Arc &arc) const {
|
||||
return arc.ilabel == 0 && arc.olabel == 0;
|
||||
}
|
||||
};
|
||||
|
||||
// True for input epsilon arcs.
|
||||
template <class Arc>
|
||||
class InputEpsilonArcFilter {
|
||||
public:
|
||||
bool operator()(const Arc &arc) const { return arc.ilabel == 0; }
|
||||
};
|
||||
|
||||
// True for output epsilon arcs.
|
||||
template <class Arc>
|
||||
class OutputEpsilonArcFilter {
|
||||
public:
|
||||
bool operator()(const Arc &arc) const { return arc.olabel == 0; }
|
||||
};
|
||||
|
||||
// True if specified label matches (doesn't match) when keep_match is
|
||||
// true (false).
|
||||
template <class Arc>
|
||||
class LabelArcFilter {
|
||||
public:
|
||||
using Label = typename Arc::Label;
|
||||
|
||||
explicit LabelArcFilter(Label label, bool match_input = true,
|
||||
bool keep_match = true)
|
||||
: label_(label), match_input_(match_input), keep_match_(keep_match) {}
|
||||
|
||||
bool operator()(const Arc &arc) const {
|
||||
const bool match = (match_input_ ? arc.ilabel : arc.olabel) == label_;
|
||||
return keep_match_ ? match : !match;
|
||||
}
|
||||
|
||||
private:
|
||||
const Label label_;
|
||||
const bool match_input_;
|
||||
const bool keep_match_;
|
||||
};
|
||||
|
||||
// True if specified labels match (don't match) when keep_match is true (false).
|
||||
template <class Arc>
|
||||
class MultiLabelArcFilter {
|
||||
public:
|
||||
using Label = typename Arc::Label;
|
||||
|
||||
explicit MultiLabelArcFilter(bool match_input = true, bool keep_match = true)
|
||||
: match_input_(match_input), keep_match_(keep_match) {}
|
||||
|
||||
bool operator()(const Arc &arc) const {
|
||||
const Label label = match_input_ ? arc.ilabel : arc.olabel;
|
||||
const bool match = labels_.Find(label) != labels_.End();
|
||||
return keep_match_ ? match : !match;
|
||||
}
|
||||
|
||||
void AddLabel(Label label) { labels_.Insert(label); }
|
||||
|
||||
private:
|
||||
CompactSet<Label, kNoLabel> labels_;
|
||||
const bool match_input_;
|
||||
const bool keep_match_;
|
||||
};
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_ARCFILTER_H_
|
||||
@@ -0,0 +1,211 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Functions and classes to sort arcs in an FST.
|
||||
|
||||
#ifndef FST_ARCSORT_H_
|
||||
#define FST_ARCSORT_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <fst/cache.h>
|
||||
#include <fst/state-map.h>
|
||||
#include <fst/test-properties.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
template <class Arc, class Compare>
|
||||
class ArcSortMapper {
|
||||
public:
|
||||
using FromArc = Arc;
|
||||
using ToArc = Arc;
|
||||
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
|
||||
constexpr ArcSortMapper(const Fst<Arc> &fst, const Compare &comp)
|
||||
: fst_(fst), comp_(comp), i_(0) {}
|
||||
|
||||
// Allows updating Fst argument; pass only if changed.
|
||||
ArcSortMapper(const ArcSortMapper<Arc, Compare> &mapper,
|
||||
const Fst<Arc> *fst = nullptr)
|
||||
: fst_(fst ? *fst : mapper.fst_), comp_(mapper.comp_), i_(0) {}
|
||||
|
||||
StateId Start() { return fst_.Start(); }
|
||||
|
||||
Weight Final(StateId s) const { return fst_.Final(s); }
|
||||
|
||||
void SetState(StateId s) {
|
||||
i_ = 0;
|
||||
arcs_.clear();
|
||||
arcs_.reserve(fst_.NumArcs(s));
|
||||
for (ArcIterator<Fst<Arc>> aiter(fst_, s); !aiter.Done(); aiter.Next()) {
|
||||
arcs_.push_back(aiter.Value());
|
||||
}
|
||||
std::sort(arcs_.begin(), arcs_.end(), comp_);
|
||||
}
|
||||
|
||||
bool Done() const { return i_ >= arcs_.size(); }
|
||||
|
||||
const Arc &Value() const { return arcs_[i_]; }
|
||||
|
||||
void Next() { ++i_; }
|
||||
|
||||
MapSymbolsAction InputSymbolsAction() const { return MAP_COPY_SYMBOLS; }
|
||||
|
||||
MapSymbolsAction OutputSymbolsAction() const { return MAP_COPY_SYMBOLS; }
|
||||
|
||||
uint64 Properties(uint64 props) const { return comp_.Properties(props); }
|
||||
|
||||
private:
|
||||
const Fst<Arc> &fst_;
|
||||
const Compare &comp_;
|
||||
std::vector<Arc> arcs_;
|
||||
ssize_t i_; // current arc position
|
||||
|
||||
ArcSortMapper &operator=(const ArcSortMapper &) = delete;
|
||||
};
|
||||
|
||||
// Sorts the arcs in an FST according to function object 'comp' of type Compare.
|
||||
// This version modifies its input. Comparison function objects ILabelCompare
|
||||
// and OLabelCompare are provided by the library. In general, Compare must meet
|
||||
// the requirements for a comparison function object (e.g., similar to those
|
||||
// used by std::sort). It must also have a member Properties(uint64) that
|
||||
// specifies the known properties of the sorted FST; it takes as argument the
|
||||
// input FST's known properties before the sort.
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// - Time: O(v d log d)
|
||||
// - Space: O(d)
|
||||
//
|
||||
// where v = # of states and d = maximum out-degree.
|
||||
template <class Arc, class Compare>
|
||||
void ArcSort(MutableFst<Arc> *fst, Compare comp) {
|
||||
ArcSortMapper<Arc, Compare> mapper(*fst, comp);
|
||||
StateMap(fst, mapper);
|
||||
}
|
||||
|
||||
using ArcSortFstOptions = CacheOptions;
|
||||
|
||||
// Sorts the arcs in an FST according to function object 'comp' of type Compare.
|
||||
// This version is a delayed FST. Comparsion function objects ILabelCompare and
|
||||
// OLabelCompare are provided by the library. In general, Compare must meet the
|
||||
// requirements for a comparision function object (e.g., similar to those
|
||||
// used by std::sort). It must also have a member Properties(uint64) that
|
||||
// specifies the known properties of the sorted FST; it takes as argument the
|
||||
// input FST's known properties.
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// - Time: O(v d log d)
|
||||
// - Space: O(d)
|
||||
//
|
||||
// where v = # of states visited, d = maximum out-degree of states visited.
|
||||
// Constant time and space to visit an input state is assumed and exclusive of
|
||||
// caching.
|
||||
template <class Arc, class Compare>
|
||||
class ArcSortFst : public StateMapFst<Arc, Arc, ArcSortMapper<Arc, Compare>> {
|
||||
using StateMapFst<Arc, Arc, ArcSortMapper<Arc, Compare>>::GetImpl;
|
||||
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
using Mapper = ArcSortMapper<Arc, Compare>;
|
||||
|
||||
ArcSortFst(const Fst<Arc> &fst, const Compare &comp)
|
||||
: StateMapFst<Arc, Arc, Mapper>(fst,
|
||||
ArcSortMapper<Arc, Compare>(fst, comp)) {}
|
||||
|
||||
ArcSortFst(const Fst<Arc> &fst, const Compare &comp,
|
||||
const ArcSortFstOptions &opts)
|
||||
: StateMapFst<Arc, Arc, Mapper>(fst, Mapper(fst, comp), opts) {}
|
||||
|
||||
// See Fst<>::Copy() for doc.
|
||||
ArcSortFst(const ArcSortFst<Arc, Compare> &fst, bool safe = false)
|
||||
: StateMapFst<Arc, Arc, Mapper>(fst, safe) {}
|
||||
|
||||
// Gets a copy of this ArcSortFst. See Fst<>::Copy() for further doc.
|
||||
ArcSortFst<Arc, Compare> *Copy(bool safe = false) const override {
|
||||
return new ArcSortFst(*this, safe);
|
||||
}
|
||||
|
||||
size_t NumArcs(StateId s) const override {
|
||||
return GetImpl()->GetFst()->NumArcs(s);
|
||||
}
|
||||
|
||||
size_t NumInputEpsilons(StateId s) const override {
|
||||
return GetImpl()->GetFst()->NumInputEpsilons(s);
|
||||
}
|
||||
|
||||
size_t NumOutputEpsilons(StateId s) const override {
|
||||
return GetImpl()->GetFst()->NumOutputEpsilons(s);
|
||||
}
|
||||
};
|
||||
|
||||
// Specialization for ArcSortFst.
|
||||
template <class Arc, class Compare>
|
||||
class StateIterator<ArcSortFst<Arc, Compare>>
|
||||
: public StateIterator<StateMapFst<Arc, Arc, ArcSortMapper<Arc, Compare>>> {
|
||||
public:
|
||||
explicit StateIterator(const ArcSortFst<Arc, Compare> &fst)
|
||||
: StateIterator<StateMapFst<Arc, Arc, ArcSortMapper<Arc, Compare>>>(fst) {
|
||||
}
|
||||
};
|
||||
|
||||
// Specialization for ArcSortFst.
|
||||
template <class Arc, class Compare>
|
||||
class ArcIterator<ArcSortFst<Arc, Compare>>
|
||||
: public ArcIterator<StateMapFst<Arc, Arc, ArcSortMapper<Arc, Compare>>> {
|
||||
public:
|
||||
ArcIterator(const ArcSortFst<Arc, Compare> &fst, typename Arc::StateId s)
|
||||
: ArcIterator<StateMapFst<Arc, Arc, ArcSortMapper<Arc, Compare>>>(fst,
|
||||
s) {}
|
||||
};
|
||||
|
||||
// Compare class for comparing input labels of arcs.
|
||||
template <class Arc>
|
||||
class ILabelCompare {
|
||||
public:
|
||||
constexpr ILabelCompare() {}
|
||||
|
||||
constexpr bool operator()(const Arc &arc1, const Arc &arc2) const {
|
||||
return arc1.ilabel < arc2.ilabel;
|
||||
}
|
||||
|
||||
constexpr uint64 Properties(uint64 props) const {
|
||||
return (props & kArcSortProperties) | kILabelSorted |
|
||||
(props & kAcceptor ? kOLabelSorted : 0);
|
||||
}
|
||||
};
|
||||
|
||||
// Compare class for comparing output labels of arcs.
|
||||
template <class Arc>
|
||||
class OLabelCompare {
|
||||
public:
|
||||
constexpr OLabelCompare() {}
|
||||
|
||||
constexpr bool operator()(const Arc &arc1, const Arc &arc2) const {
|
||||
return arc1.olabel < arc2.olabel;
|
||||
}
|
||||
|
||||
constexpr uint64 Properties(uint64 props) const {
|
||||
return (props & kArcSortProperties) | kOLabelSorted |
|
||||
(props & kAcceptor ? kILabelSorted : 0);
|
||||
}
|
||||
};
|
||||
|
||||
// Useful aliases when using StdArc.
|
||||
|
||||
template <class Compare>
|
||||
using StdArcSortFst = ArcSortFst<StdArc, Compare>;
|
||||
|
||||
using StdILabelCompare = ILabelCompare<StdArc>;
|
||||
|
||||
using StdOLabelCompare = OLabelCompare<StdArc>;
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_ARCSORT_H_
|
||||
@@ -0,0 +1,480 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Classes for representing a bijective mapping between an arbitrary entry
|
||||
// of type T and a signed integral ID.
|
||||
|
||||
#ifndef FST_BI_TABLE_H_
|
||||
#define FST_BI_TABLE_H_
|
||||
|
||||
#include <deque>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include <fst/log.h>
|
||||
#include <fst/memory.h>
|
||||
#include <unordered_set>
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Bitables model bijective mappings between entries of an arbitrary type T and
|
||||
// an signed integral ID of type I. The IDs are allocated starting from 0 in
|
||||
// order.
|
||||
//
|
||||
// template <class I, class T>
|
||||
// class BiTable {
|
||||
// public:
|
||||
//
|
||||
// // Required constructors.
|
||||
// BiTable();
|
||||
//
|
||||
// // Looks up integer ID from entry. If it doesn't exist and insert
|
||||
// / is true, adds it; otherwise, returns -1.
|
||||
// I FindId(const T &entry, bool insert = true);
|
||||
//
|
||||
// // Looks up entry from integer ID.
|
||||
// const T &FindEntry(I) const;
|
||||
//
|
||||
// // Returns number of stored entries.
|
||||
// I Size() const;
|
||||
// };
|
||||
|
||||
// An implementation using a hash map for the entry to ID mapping. H is the
|
||||
// hash function and E is the equality function. If passed to the constructor,
|
||||
// ownership is given to this class.
|
||||
template <class I, class T, class H, class E = std::equal_to<T>>
|
||||
class HashBiTable {
|
||||
public:
|
||||
// Reserves space for table_size elements. If passing H and E to the
|
||||
// constructor, this class owns them.
|
||||
explicit HashBiTable(size_t table_size = 0, H *h = nullptr, E *e = nullptr) :
|
||||
hash_func_(h ? h : new H()), hash_equal_(e ? e : new E()),
|
||||
entry2id_(table_size, *hash_func_, *hash_equal_) {
|
||||
if (table_size) id2entry_.reserve(table_size);
|
||||
}
|
||||
|
||||
HashBiTable(const HashBiTable<I, T, H, E> &table)
|
||||
: hash_func_(new H(*table.hash_func_)),
|
||||
hash_equal_(new E(*table.hash_equal_)),
|
||||
entry2id_(table.entry2id_.begin(), table.entry2id_.end(),
|
||||
table.entry2id_.size(), *hash_func_, *hash_equal_),
|
||||
id2entry_(table.id2entry_) {}
|
||||
|
||||
I FindId(const T &entry, bool insert = true) {
|
||||
if (!insert) {
|
||||
const auto it = entry2id_.find(entry);
|
||||
return it == entry2id_.end() ? -1 : it->second - 1;
|
||||
}
|
||||
I &id_ref = entry2id_[entry];
|
||||
if (id_ref == 0) { // T not found; stores and assigns a new ID.
|
||||
id2entry_.push_back(entry);
|
||||
id_ref = id2entry_.size();
|
||||
}
|
||||
return id_ref - 1; // NB: id_ref = ID + 1.
|
||||
}
|
||||
|
||||
const T &FindEntry(I s) const { return id2entry_[s]; }
|
||||
|
||||
I Size() const { return id2entry_.size(); }
|
||||
|
||||
// TODO(riley): Add fancy clear-to-size, as in CompactHashBiTable.
|
||||
void Clear() {
|
||||
entry2id_.clear();
|
||||
id2entry_.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<H> hash_func_;
|
||||
std::unique_ptr<E> hash_equal_;
|
||||
std::unordered_map<T, I, H, E> entry2id_;
|
||||
std::vector<T> id2entry_;
|
||||
};
|
||||
|
||||
// Enables alternative hash set representations below.
|
||||
enum HSType { HS_STL = 0, HS_DENSE = 1, HS_SPARSE = 2, HS_FLAT = 3 };
|
||||
|
||||
// Default hash set is STL hash_set.
|
||||
template <class K, class H, class E, HSType HS>
|
||||
struct HashSet : public std::unordered_set<K, H, E, PoolAllocator<K>> {
|
||||
explicit HashSet(size_t n = 0, const H &h = H(), const E &e = E())
|
||||
: std::unordered_set<K, H, E, PoolAllocator<K>>(n, h, e) {}
|
||||
|
||||
void rehash(size_t n) {}
|
||||
};
|
||||
|
||||
// An implementation using a hash set for the entry to ID mapping. The hash set
|
||||
// holds keys which are either the ID or kCurrentKey. These keys can be mapped
|
||||
// to entries either by looking up in the entry vector or, if kCurrentKey, in
|
||||
// current_entry_. The hash and key equality functions map to entries first. H
|
||||
// is the hash function and E is the equality function. If passed to the
|
||||
// constructor, ownership is given to this class.
|
||||
// TODO(rybach): remove support for (deprecated and unused) HS_DENSE, HS_SPARSE.
|
||||
template <class I, class T, class H, class E = std::equal_to<T>,
|
||||
HSType HS = HS_FLAT>
|
||||
class CompactHashBiTable {
|
||||
public:
|
||||
friend class HashFunc;
|
||||
friend class HashEqual;
|
||||
|
||||
// Reserves space for table_size elements. If passing H and E to the
|
||||
// constructor, this class owns them.
|
||||
explicit CompactHashBiTable(size_t table_size = 0, H *h = nullptr,
|
||||
E *e = nullptr) :
|
||||
hash_func_(h ? h : new H()), hash_equal_(e ? e : new E()),
|
||||
compact_hash_func_(*this), compact_hash_equal_(*this),
|
||||
keys_(table_size, compact_hash_func_, compact_hash_equal_) {
|
||||
if (table_size) id2entry_.reserve(table_size);
|
||||
}
|
||||
|
||||
CompactHashBiTable(const CompactHashBiTable<I, T, H, E, HS> &table)
|
||||
: hash_func_(new H(*table.hash_func_)),
|
||||
hash_equal_(new E(*table.hash_equal_)),
|
||||
compact_hash_func_(*this), compact_hash_equal_(*this),
|
||||
keys_(table.keys_.size(), compact_hash_func_, compact_hash_equal_),
|
||||
id2entry_(table.id2entry_) {
|
||||
keys_.insert(table.keys_.begin(), table.keys_.end());
|
||||
}
|
||||
|
||||
I FindId(const T &entry, bool insert = true) {
|
||||
current_entry_ = &entry;
|
||||
if (insert) {
|
||||
auto result = keys_.insert(kCurrentKey);
|
||||
if (!result.second) return *result.first; // Already exists.
|
||||
// Overwrites kCurrentKey with a new key value; this is safe because it
|
||||
// doesn't affect hashing or equality testing.
|
||||
I key = id2entry_.size();
|
||||
const_cast<I &>(*result.first) = key;
|
||||
id2entry_.push_back(entry);
|
||||
return key;
|
||||
}
|
||||
const auto it = keys_.find(kCurrentKey);
|
||||
return it == keys_.end() ? -1 : *it;
|
||||
}
|
||||
|
||||
const T &FindEntry(I s) const { return id2entry_[s]; }
|
||||
|
||||
I Size() const { return id2entry_.size(); }
|
||||
|
||||
// Clears content; with argument, erases last n IDs.
|
||||
void Clear(ssize_t n = -1) {
|
||||
if (n < 0 || n >= id2entry_.size()) { // Clears completely.
|
||||
keys_.clear();
|
||||
id2entry_.clear();
|
||||
} else if (n == id2entry_.size() - 1) { // Leaves only key 0.
|
||||
const T entry = FindEntry(0);
|
||||
keys_.clear();
|
||||
id2entry_.clear();
|
||||
FindId(entry, true);
|
||||
} else {
|
||||
while (n-- > 0) {
|
||||
I key = id2entry_.size() - 1;
|
||||
keys_.erase(key);
|
||||
id2entry_.pop_back();
|
||||
}
|
||||
keys_.rehash(0);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
static_assert(std::is_signed<I>::value, "I must be a signed type");
|
||||
// ... otherwise >= kCurrentKey comparisons as used below don't work.
|
||||
// TODO(rybach): (1) remove kEmptyKey, kDeletedKey, (2) don't use >= for key
|
||||
// comparison, (3) allow unsigned key types.
|
||||
static constexpr I kCurrentKey = -1;
|
||||
static constexpr I kEmptyKey = -2;
|
||||
static constexpr I kDeletedKey = -3;
|
||||
|
||||
class HashFunc {
|
||||
public:
|
||||
explicit HashFunc(const CompactHashBiTable &ht) : ht_(&ht) {}
|
||||
|
||||
size_t operator()(I k) const {
|
||||
if (k >= kCurrentKey) {
|
||||
return (*ht_->hash_func_)(ht_->Key2Entry(k));
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const CompactHashBiTable *ht_;
|
||||
};
|
||||
|
||||
class HashEqual {
|
||||
public:
|
||||
explicit HashEqual(const CompactHashBiTable &ht) : ht_(&ht) {}
|
||||
|
||||
bool operator()(I k1, I k2) const {
|
||||
if (k1 == k2) {
|
||||
return true;
|
||||
} else if (k1 >= kCurrentKey && k2 >= kCurrentKey) {
|
||||
return (*ht_->hash_equal_)(ht_->Key2Entry(k1), ht_->Key2Entry(k2));
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const CompactHashBiTable *ht_;
|
||||
};
|
||||
|
||||
using KeyHashSet = HashSet<I, HashFunc, HashEqual, HS>;
|
||||
|
||||
const T &Key2Entry(I k) const {
|
||||
if (k == kCurrentKey) {
|
||||
return *current_entry_;
|
||||
} else {
|
||||
return id2entry_[k];
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<H> hash_func_;
|
||||
std::unique_ptr<E> hash_equal_;
|
||||
HashFunc compact_hash_func_;
|
||||
HashEqual compact_hash_equal_;
|
||||
KeyHashSet keys_;
|
||||
std::vector<T> id2entry_;
|
||||
const T *current_entry_;
|
||||
};
|
||||
|
||||
template <class I, class T, class H, class E, HSType HS>
|
||||
constexpr I CompactHashBiTable<I, T, H, E, HS>::kCurrentKey;
|
||||
|
||||
template <class I, class T, class H, class E, HSType HS>
|
||||
constexpr I CompactHashBiTable<I, T, H, E, HS>::kEmptyKey;
|
||||
|
||||
template <class I, class T, class H, class E, HSType HS>
|
||||
constexpr I CompactHashBiTable<I, T, H, E, HS>::kDeletedKey;
|
||||
|
||||
// An implementation using a vector for the entry to ID mapping. It is passed a
|
||||
// function object FP that should fingerprint entries uniquely to an integer
|
||||
// that can used as a vector index. Normally, VectorBiTable constructs the FP
|
||||
// object. The user can instead pass in this object; in that case, VectorBiTable
|
||||
// takes its ownership.
|
||||
template <class I, class T, class FP>
|
||||
class VectorBiTable {
|
||||
public:
|
||||
// Reserves table_size cells of space. If passing FP argument to the
|
||||
// constructor, this class owns it.
|
||||
explicit VectorBiTable(FP *fp = nullptr, size_t table_size = 0) :
|
||||
fp_(fp ? fp : new FP()) {
|
||||
if (table_size) id2entry_.reserve(table_size);
|
||||
}
|
||||
|
||||
VectorBiTable(const VectorBiTable<I, T, FP> &table)
|
||||
: fp_(new FP(*table.fp_)), fp2id_(table.fp2id_),
|
||||
id2entry_(table.id2entry_) {}
|
||||
|
||||
I FindId(const T &entry, bool insert = true) {
|
||||
ssize_t fp = (*fp_)(entry);
|
||||
if (fp >= fp2id_.size()) fp2id_.resize(fp + 1);
|
||||
I &id_ref = fp2id_[fp];
|
||||
if (id_ref == 0) { // T not found.
|
||||
if (insert) { // Stores and assigns a new ID.
|
||||
id2entry_.push_back(entry);
|
||||
id_ref = id2entry_.size();
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return id_ref - 1; // NB: id_ref = ID + 1.
|
||||
}
|
||||
|
||||
const T &FindEntry(I s) const { return id2entry_[s]; }
|
||||
|
||||
I Size() const { return id2entry_.size(); }
|
||||
|
||||
const FP &Fingerprint() const { return *fp_; }
|
||||
|
||||
private:
|
||||
std::unique_ptr<FP> fp_;
|
||||
std::vector<I> fp2id_;
|
||||
std::vector<T> id2entry_;
|
||||
};
|
||||
|
||||
// An implementation using a vector and a compact hash table. The selecting
|
||||
// functor S returns true for entries to be hashed in the vector. The
|
||||
// fingerprinting functor FP returns a unique fingerprint for each entry to be
|
||||
// hashed in the vector (these need to be suitable for indexing in a vector).
|
||||
// The hash functor H is used when hashing entry into the compact hash table.
|
||||
// If passed to the constructor, ownership is given to this class.
|
||||
template <class I, class T, class S, class FP, class H, HSType HS = HS_DENSE>
|
||||
class VectorHashBiTable {
|
||||
public:
|
||||
friend class HashFunc;
|
||||
friend class HashEqual;
|
||||
|
||||
explicit VectorHashBiTable(S *s, FP *fp, H *h, size_t vector_size = 0,
|
||||
size_t entry_size = 0)
|
||||
: selector_(s), fp_(fp), h_(h), hash_func_(*this), hash_equal_(*this),
|
||||
keys_(0, hash_func_, hash_equal_) {
|
||||
if (vector_size) fp2id_.reserve(vector_size);
|
||||
if (entry_size) id2entry_.reserve(entry_size);
|
||||
}
|
||||
|
||||
VectorHashBiTable(const VectorHashBiTable<I, T, S, FP, H, HS> &table)
|
||||
: selector_(new S(table.s_)), fp_(new FP(*table.fp_)),
|
||||
h_(new H(*table.h_)), id2entry_(table.id2entry_),
|
||||
fp2id_(table.fp2id_), hash_func_(*this), hash_equal_(*this),
|
||||
keys_(table.keys_.size(), hash_func_, hash_equal_) {
|
||||
keys_.insert(table.keys_.begin(), table.keys_.end());
|
||||
}
|
||||
|
||||
I FindId(const T &entry, bool insert = true) {
|
||||
if ((*selector_)(entry)) { // Uses the vector if selector_(entry) == true.
|
||||
uint64 fp = (*fp_)(entry);
|
||||
if (fp2id_.size() <= fp) fp2id_.resize(fp + 1, 0);
|
||||
if (fp2id_[fp] == 0) { // T not found.
|
||||
if (insert) { // Stores and assigns a new ID.
|
||||
id2entry_.push_back(entry);
|
||||
fp2id_[fp] = id2entry_.size();
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return fp2id_[fp] - 1; // NB: assoc_value = ID + 1.
|
||||
} else { // Uses the hash table otherwise.
|
||||
current_entry_ = &entry;
|
||||
const auto it = keys_.find(kCurrentKey);
|
||||
if (it == keys_.end()) {
|
||||
if (insert) {
|
||||
I key = id2entry_.size();
|
||||
id2entry_.push_back(entry);
|
||||
keys_.insert(key);
|
||||
return key;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
return *it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const T &FindEntry(I s) const { return id2entry_[s]; }
|
||||
|
||||
I Size() const { return id2entry_.size(); }
|
||||
|
||||
const S &Selector() const { return *selector_; }
|
||||
|
||||
const FP &Fingerprint() const { return *fp_; }
|
||||
|
||||
const H &Hash() const { return *h_; }
|
||||
|
||||
private:
|
||||
static constexpr I kCurrentKey = -1;
|
||||
static constexpr I kEmptyKey = -2;
|
||||
|
||||
class HashFunc {
|
||||
public:
|
||||
explicit HashFunc(const VectorHashBiTable &ht) : ht_(&ht) {}
|
||||
|
||||
size_t operator()(I k) const {
|
||||
if (k >= kCurrentKey) {
|
||||
return (*(ht_->h_))(ht_->Key2Entry(k));
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const VectorHashBiTable *ht_;
|
||||
};
|
||||
|
||||
class HashEqual {
|
||||
public:
|
||||
explicit HashEqual(const VectorHashBiTable &ht) : ht_(&ht) {}
|
||||
|
||||
bool operator()(I k1, I k2) const {
|
||||
if (k1 >= kCurrentKey && k2 >= kCurrentKey) {
|
||||
return ht_->Key2Entry(k1) == ht_->Key2Entry(k2);
|
||||
} else {
|
||||
return k1 == k2;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const VectorHashBiTable *ht_;
|
||||
};
|
||||
|
||||
using KeyHashSet = HashSet<I, HashFunc, HashEqual, HS>;
|
||||
|
||||
const T &Key2Entry(I k) const {
|
||||
if (k == kCurrentKey) {
|
||||
return *current_entry_;
|
||||
} else {
|
||||
return id2entry_[k];
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<S> selector_; // True if entry hashed into vector.
|
||||
std::unique_ptr<FP> fp_; // Fingerprint used for hashing into vector.
|
||||
std::unique_ptr<H> h_; // Hash funcion used for hashing into hash_set.
|
||||
|
||||
std::vector<T> id2entry_; // Maps state IDs to entry.
|
||||
std::vector<I> fp2id_; // Maps entry fingerprints to IDs.
|
||||
|
||||
// Compact implementation of the hash table mapping entries to state IDs
|
||||
// using the hash function h_.
|
||||
HashFunc hash_func_;
|
||||
HashEqual hash_equal_;
|
||||
KeyHashSet keys_;
|
||||
const T *current_entry_;
|
||||
};
|
||||
|
||||
template <class I, class T, class S, class FP, class H, HSType HS>
|
||||
constexpr I VectorHashBiTable<I, T, S, FP, H, HS>::kCurrentKey;
|
||||
|
||||
template <class I, class T, class S, class FP, class H, HSType HS>
|
||||
constexpr I VectorHashBiTable<I, T, S, FP, H, HS>::kEmptyKey;
|
||||
|
||||
// An implementation using a hash map for the entry to ID mapping. This version
|
||||
// permits erasing of arbitrary states. The entry T must have == defined and
|
||||
// its default constructor must produce a entry that will never be seen. F is
|
||||
// the hash function.
|
||||
template <class I, class T, class F>
|
||||
class ErasableBiTable {
|
||||
public:
|
||||
ErasableBiTable() : first_(0) {}
|
||||
|
||||
I FindId(const T &entry, bool insert = true) {
|
||||
I &id_ref = entry2id_[entry];
|
||||
if (id_ref == 0) { // T not found.
|
||||
if (insert) { // Stores and assigns a new ID.
|
||||
id2entry_.push_back(entry);
|
||||
id_ref = id2entry_.size() + first_;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return id_ref - 1; // NB: id_ref = ID + 1.
|
||||
}
|
||||
|
||||
const T &FindEntry(I s) const { return id2entry_[s - first_]; }
|
||||
|
||||
I Size() const { return id2entry_.size(); }
|
||||
|
||||
void Erase(I s) {
|
||||
auto &ref = id2entry_[s - first_];
|
||||
entry2id_.erase(ref);
|
||||
ref = empty_entry_;
|
||||
while (!id2entry_.empty() && id2entry_.front() == empty_entry_) {
|
||||
id2entry_.pop_front();
|
||||
++first_;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
std::unordered_map<T, I, F> entry2id_;
|
||||
std::deque<T> id2entry_;
|
||||
const T empty_entry_;
|
||||
I first_; // I of first element in the deque.
|
||||
};
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_BI_TABLE_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,134 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Functions and classes to compute the concatenative closure of an FST.
|
||||
|
||||
#ifndef FST_CLOSURE_H_
|
||||
#define FST_CLOSURE_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
#include <fst/mutable-fst.h>
|
||||
#include <fst/rational.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Computes the concatenative closure. This version modifies its
|
||||
// MutableFst input. If an FST transduces string x to y with weight a,
|
||||
// then its closure transduces x to y with weight a, xx to yy with
|
||||
// weight Times(a, a), xxx to yyy with with Times(Times(a, a), a),
|
||||
// etc. If closure_type == CLOSURE_STAR, then the empty string is
|
||||
// transduced to itself with weight Weight::One() as well.
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// Time: O(V)
|
||||
// Space: O(V)
|
||||
//
|
||||
// where V is the number of states.
|
||||
template <class Arc>
|
||||
void Closure(MutableFst<Arc> *fst, ClosureType closure_type) {
|
||||
using Weight = typename Arc::Weight;
|
||||
const auto props = fst->Properties(kFstProperties, false);
|
||||
const auto start = fst->Start();
|
||||
for (StateIterator<MutableFst<Arc>> siter(*fst); !siter.Done();
|
||||
siter.Next()) {
|
||||
const auto s = siter.Value();
|
||||
const auto weight = fst->Final(s);
|
||||
if (weight != Weight::Zero()) fst->AddArc(s, Arc(0, 0, weight, start));
|
||||
}
|
||||
if (closure_type == CLOSURE_STAR) {
|
||||
fst->ReserveStates(fst->NumStates() + 1);
|
||||
const auto nstart = fst->AddState();
|
||||
fst->SetStart(nstart);
|
||||
fst->SetFinal(nstart, Weight::One());
|
||||
if (start != kNoLabel) fst->AddArc(nstart, Arc(0, 0, Weight::One(), start));
|
||||
}
|
||||
fst->SetProperties(ClosureProperties(props, closure_type == CLOSURE_STAR),
|
||||
kFstProperties);
|
||||
}
|
||||
|
||||
// Computes the concatenative closure. This version modifies its
|
||||
// RationalFst input.
|
||||
template <class Arc>
|
||||
void Closure(RationalFst<Arc> *fst, ClosureType closure_type) {
|
||||
fst->GetMutableImpl()->AddClosure(closure_type);
|
||||
}
|
||||
|
||||
struct ClosureFstOptions : RationalFstOptions {
|
||||
ClosureType type;
|
||||
|
||||
ClosureFstOptions(const RationalFstOptions &opts,
|
||||
ClosureType type = CLOSURE_STAR)
|
||||
: RationalFstOptions(opts), type(type) {}
|
||||
|
||||
explicit ClosureFstOptions(ClosureType type = CLOSURE_STAR) : type(type) {}
|
||||
};
|
||||
|
||||
// Computes the concatenative closure. This version is a delayed FST. If an FST
|
||||
// transduces string x to y with weight a, then its closure transduces x to y
|
||||
// with weight a, xx to yy with weight Times(a, a), xxx to yyy with weight
|
||||
// Times(Times(a, a), a), etc. If closure_type == CLOSURE_STAR, then the empty
|
||||
// string is transduced to itself with weight Weight::One() as well.
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// Time: O(v)
|
||||
// Space: O(v)
|
||||
//
|
||||
// where v is the number of states visited. Constant time and space to visit an
|
||||
// input state or arc is assumed and exclusive of caching.
|
||||
template <class A>
|
||||
class ClosureFst : public RationalFst<A> {
|
||||
public:
|
||||
using Arc = A;
|
||||
|
||||
ClosureFst(const Fst<Arc> &fst, ClosureType closure_type) {
|
||||
GetMutableImpl()->InitClosure(fst, closure_type);
|
||||
}
|
||||
|
||||
ClosureFst(const Fst<Arc> &fst, const ClosureFstOptions &opts)
|
||||
: RationalFst<A>(opts) {
|
||||
GetMutableImpl()->InitClosure(fst, opts.type);
|
||||
}
|
||||
|
||||
// See Fst<>::Copy() for doc.
|
||||
ClosureFst(const ClosureFst<Arc> &fst, bool safe = false)
|
||||
: RationalFst<A>(fst, safe) {}
|
||||
|
||||
// Gets a copy of this ClosureFst. See Fst<>::Copy() for further doc.
|
||||
ClosureFst<A> *Copy(bool safe = false) const override {
|
||||
return new ClosureFst<A>(*this, safe);
|
||||
}
|
||||
|
||||
private:
|
||||
using ImplToFst<internal::RationalFstImpl<Arc>>::GetImpl;
|
||||
using ImplToFst<internal::RationalFstImpl<Arc>>::GetMutableImpl;
|
||||
};
|
||||
|
||||
// Specialization for ClosureFst.
|
||||
template <class Arc>
|
||||
class StateIterator<ClosureFst<Arc>> : public StateIterator<RationalFst<Arc>> {
|
||||
public:
|
||||
explicit StateIterator(const ClosureFst<Arc> &fst)
|
||||
: StateIterator<RationalFst<Arc>>(fst) {}
|
||||
};
|
||||
|
||||
// Specialization for ClosureFst.
|
||||
template <class Arc>
|
||||
class ArcIterator<ClosureFst<Arc>> : public ArcIterator<RationalFst<Arc>> {
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
ArcIterator(const ClosureFst<Arc> &fst, StateId s)
|
||||
: ArcIterator<RationalFst<Arc>>(fst, s) {}
|
||||
};
|
||||
|
||||
// Useful alias when using StdArc.
|
||||
using StdClosureFst = ClosureFst<StdArc>;
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_CLOSURE_H_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,130 @@
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
|
||||
#ifndef FST_LIB_COMPAT_H_
|
||||
#define FST_LIB_COMPAT_H_
|
||||
|
||||
#include <climits>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Makes copy constructor and operator= private
|
||||
// Deprecated: now just use =delete.
|
||||
#define DISALLOW_COPY_AND_ASSIGN(type) \
|
||||
type(const type&); \
|
||||
void operator=(const type&)
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
#define OPENFST_DEPRECATED(message) __attribute__((deprecated(message)))
|
||||
#elif defined(_MSC_VER)
|
||||
#define OPENFST_DEPRECATED(message) __declspec(deprecated(message))
|
||||
#else
|
||||
#define OPENFST_DEPRECATED(message)
|
||||
#endif
|
||||
|
||||
#include <fst/config.h>
|
||||
#include <fst/types.h>
|
||||
#include <fst/lock.h>
|
||||
#include <fst/flags.h>
|
||||
#include <fst/log.h>
|
||||
#include <fst/icu.h>
|
||||
|
||||
using std::string;
|
||||
|
||||
void FailedNewHandler();
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#include <intrin.h>
|
||||
const char* basename(const char* path);
|
||||
#define __builtin_popcount __popcnt
|
||||
|
||||
#ifdef _M_X64
|
||||
// Using 64-bit MSVC intrinsics.
|
||||
#define __builtin_popcountll __popcnt64
|
||||
inline unsigned int __builtin_ctzll(std::uint64_t w) {
|
||||
unsigned long v;
|
||||
return _BitScanForward64(&v, std::uint32_t(w)) ? v : 0;
|
||||
}
|
||||
#else
|
||||
// Using 32-bit MSVC intrinsics.
|
||||
inline unsigned int __builtin_popcountll(std::uint64_t w) {
|
||||
return __popcnt(std::uint32_t(w)) + __popcnt(std::uint32_t(w >> 32));
|
||||
}
|
||||
inline unsigned int __builtin_ctzll(std::uint64_t w) {
|
||||
unsigned long v;
|
||||
return (_BitScanForward(&v, std::uint32_t(w)) ? v :
|
||||
_BitScanForward(&v, std::uint32_t(w >> 32)) ? v + 32 : 0);
|
||||
}
|
||||
#endif // _M_X64
|
||||
#endif // _MSC_VER
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Downcasting.
|
||||
template<typename To, typename From>
|
||||
inline To down_cast(From* f) { return static_cast<To>(f); }
|
||||
|
||||
// Bitcasting.
|
||||
template <class Dest, class Source>
|
||||
inline Dest bit_cast(const Source &source) {
|
||||
static_assert(sizeof(Dest) == sizeof(Source),
|
||||
"Bitcasting unsafe for specified types");
|
||||
Dest dest;
|
||||
memcpy(&dest, &source, sizeof(dest));
|
||||
return dest;
|
||||
}
|
||||
|
||||
// Check sums
|
||||
class CheckSummer {
|
||||
public:
|
||||
CheckSummer() : count_(0) {
|
||||
check_sum_.resize(kCheckSumLength, '\0');
|
||||
}
|
||||
|
||||
void Reset() {
|
||||
count_ = 0;
|
||||
for (int i = 0; i < kCheckSumLength; ++i) check_sum_[i] = '\0';
|
||||
}
|
||||
|
||||
void Update(void const *data, int size) {
|
||||
const char *p = reinterpret_cast<const char *>(data);
|
||||
for (int i = 0; i < size; ++i) {
|
||||
check_sum_[(count_++) % kCheckSumLength] ^= p[i];
|
||||
}
|
||||
}
|
||||
|
||||
void Update(string const &data) {
|
||||
for (int i = 0; i < data.size(); ++i) {
|
||||
check_sum_[(count_++) % kCheckSumLength] ^= data[i];
|
||||
}
|
||||
}
|
||||
|
||||
string Digest() { return check_sum_; }
|
||||
|
||||
private:
|
||||
static const int kCheckSumLength = 32;
|
||||
int count_;
|
||||
string check_sum_;
|
||||
|
||||
CheckSummer(const CheckSummer &) = delete;
|
||||
CheckSummer &operator=(const CheckSummer &) = delete;
|
||||
};
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_LIB_COMPAT_H_
|
||||
@@ -0,0 +1,277 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Class to complement an FST.
|
||||
|
||||
#ifndef FST_COMPLEMENT_H_
|
||||
#define FST_COMPLEMENT_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <fst/log.h>
|
||||
|
||||
#include <fst/fst.h>
|
||||
#include <fst/test-properties.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
template <class Arc>
|
||||
class ComplementFst;
|
||||
|
||||
namespace internal {
|
||||
|
||||
// Implementation of delayed ComplementFst. The algorithm used completes the
|
||||
// (deterministic) FSA and then exchanges final and non-final states.
|
||||
// Completion, i.e. ensuring that all labels can be read from every state, is
|
||||
// accomplished by using ρ-labels, which match all labels that are otherwise
|
||||
// not found leaving a state. The first state in the output is reserved to be a
|
||||
// new state that is the destination of all ρ-labels. Each remaining output
|
||||
// state s corresponds to input state s - 1. The first arc in the output at
|
||||
// these states is the ρ-label, the remaining arcs correspond to the input
|
||||
// arcs.
|
||||
template <class A>
|
||||
class ComplementFstImpl : public FstImpl<A> {
|
||||
public:
|
||||
using Arc = A;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
|
||||
using FstImpl<A>::SetType;
|
||||
using FstImpl<A>::SetProperties;
|
||||
using FstImpl<A>::SetInputSymbols;
|
||||
using FstImpl<A>::SetOutputSymbols;
|
||||
|
||||
friend class StateIterator<ComplementFst<Arc>>;
|
||||
friend class ArcIterator<ComplementFst<Arc>>;
|
||||
|
||||
explicit ComplementFstImpl(const Fst<Arc> &fst) : fst_(fst.Copy()) {
|
||||
SetType("complement");
|
||||
uint64 props = fst.Properties(kILabelSorted, false);
|
||||
SetProperties(ComplementProperties(props), kCopyProperties);
|
||||
SetInputSymbols(fst.InputSymbols());
|
||||
SetOutputSymbols(fst.OutputSymbols());
|
||||
}
|
||||
|
||||
ComplementFstImpl(const ComplementFstImpl<Arc> &impl)
|
||||
: fst_(impl.fst_->Copy()) {
|
||||
SetType("complement");
|
||||
SetProperties(impl.Properties(), kCopyProperties);
|
||||
SetInputSymbols(impl.InputSymbols());
|
||||
SetOutputSymbols(impl.OutputSymbols());
|
||||
}
|
||||
|
||||
StateId Start() const {
|
||||
if (Properties(kError)) return kNoStateId;
|
||||
auto start = fst_->Start();
|
||||
return start != kNoStateId ? start + 1 : 0;
|
||||
}
|
||||
|
||||
// Exchange final and non-final states; makes ρ-destination state final.
|
||||
Weight Final(StateId s) const {
|
||||
if (s == 0 || fst_->Final(s - 1) == Weight::Zero()) {
|
||||
return Weight::One();
|
||||
} else {
|
||||
return Weight::Zero();
|
||||
}
|
||||
}
|
||||
|
||||
size_t NumArcs(StateId s) const {
|
||||
return s == 0 ? 1 : fst_->NumArcs(s - 1) + 1;
|
||||
}
|
||||
|
||||
size_t NumInputEpsilons(StateId s) const {
|
||||
return s == 0 ? 0 : fst_->NumInputEpsilons(s - 1);
|
||||
}
|
||||
|
||||
size_t NumOutputEpsilons(StateId s) const {
|
||||
return s == 0 ? 0 : fst_->NumOutputEpsilons(s - 1);
|
||||
}
|
||||
|
||||
uint64 Properties() const override { return Properties(kFstProperties); }
|
||||
|
||||
// Sets error if found, and returns other FST impl properties.
|
||||
uint64 Properties(uint64 mask) const override {
|
||||
if ((mask & kError) && fst_->Properties(kError, false)) {
|
||||
SetProperties(kError, kError);
|
||||
}
|
||||
return FstImpl<Arc>::Properties(mask);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<const Fst<Arc>> fst_;
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
// Complements an automaton. This is a library-internal operation that
|
||||
// introduces a (negative) ρ-label; use Difference/DifferenceFst in user code,
|
||||
// which will not see this label. This version is a delayed FST.
|
||||
//
|
||||
// This class attaches interface to implementation and handles
|
||||
// reference counting, delegating most methods to ImplToFst.
|
||||
template <class A>
|
||||
class ComplementFst : public ImplToFst<internal::ComplementFstImpl<A>> {
|
||||
public:
|
||||
using Arc = A;
|
||||
using Label = typename Arc::Label;
|
||||
using StateId = typename Arc::StateId;
|
||||
using Impl = internal::ComplementFstImpl<Arc>;
|
||||
|
||||
friend class StateIterator<ComplementFst<Arc>>;
|
||||
friend class ArcIterator<ComplementFst<Arc>>;
|
||||
|
||||
explicit ComplementFst(const Fst<Arc> &fst)
|
||||
: ImplToFst<Impl>(std::make_shared<Impl>(fst)) {
|
||||
static constexpr auto props =
|
||||
kUnweighted | kNoEpsilons | kIDeterministic | kAcceptor;
|
||||
if (fst.Properties(props, true) != props) {
|
||||
FSTERROR() << "ComplementFst: Argument not an unweighted "
|
||||
<< "epsilon-free deterministic acceptor";
|
||||
GetImpl()->SetProperties(kError, kError);
|
||||
}
|
||||
}
|
||||
|
||||
// See Fst<>::Copy() for doc.
|
||||
ComplementFst(const ComplementFst<Arc> &fst, bool safe = false)
|
||||
: ImplToFst<Impl>(fst, safe) {}
|
||||
|
||||
// Gets a copy of this FST. See Fst<>::Copy() for further doc.
|
||||
ComplementFst<Arc> *Copy(bool safe = false) const override {
|
||||
return new ComplementFst<Arc>(*this, safe);
|
||||
}
|
||||
|
||||
inline void InitStateIterator(StateIteratorData<Arc> *data) const override;
|
||||
|
||||
inline void InitArcIterator(StateId s,
|
||||
ArcIteratorData<Arc> *data) const override;
|
||||
|
||||
// Label that represents the ρ-transition; we use a negative value private to
|
||||
// the library and which will preserve FST label sort order.
|
||||
static const Label kRhoLabel = -2;
|
||||
|
||||
private:
|
||||
using ImplToFst<Impl>::GetImpl;
|
||||
|
||||
ComplementFst &operator=(const ComplementFst &) = delete;
|
||||
};
|
||||
|
||||
template <class Arc>
|
||||
const typename Arc::Label ComplementFst<Arc>::kRhoLabel;
|
||||
|
||||
// Specialization for ComplementFst.
|
||||
template <class Arc>
|
||||
class StateIterator<ComplementFst<Arc>> : public StateIteratorBase<Arc> {
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
explicit StateIterator(const ComplementFst<Arc> &fst)
|
||||
: siter_(*fst.GetImpl()->fst_), s_(0) {}
|
||||
|
||||
bool Done() const final { return s_ > 0 && siter_.Done(); }
|
||||
|
||||
StateId Value() const final { return s_; }
|
||||
|
||||
void Next() final {
|
||||
if (s_ != 0) siter_.Next();
|
||||
++s_;
|
||||
}
|
||||
|
||||
void Reset() final {
|
||||
siter_.Reset();
|
||||
s_ = 0;
|
||||
}
|
||||
|
||||
private:
|
||||
StateIterator<Fst<Arc>> siter_;
|
||||
StateId s_;
|
||||
};
|
||||
|
||||
// Specialization for ComplementFst.
|
||||
template <class Arc>
|
||||
class ArcIterator<ComplementFst<Arc>> : public ArcIteratorBase<Arc> {
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
|
||||
ArcIterator(const ComplementFst<Arc> &fst, StateId s) : s_(s), pos_(0) {
|
||||
if (s_ != 0) {
|
||||
aiter_.reset(new ArcIterator<Fst<Arc>>(*fst.GetImpl()->fst_, s - 1));
|
||||
}
|
||||
}
|
||||
|
||||
bool Done() const final {
|
||||
if (s_ != 0) {
|
||||
return pos_ > 0 && aiter_->Done();
|
||||
} else {
|
||||
return pos_ > 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Adds the ρ-label to the ρ destination state.
|
||||
const Arc &Value() const final {
|
||||
if (pos_ == 0) {
|
||||
arc_.ilabel = arc_.olabel = ComplementFst<Arc>::kRhoLabel;
|
||||
arc_.weight = Weight::One();
|
||||
arc_.nextstate = 0;
|
||||
} else {
|
||||
arc_ = aiter_->Value();
|
||||
++arc_.nextstate;
|
||||
}
|
||||
return arc_;
|
||||
}
|
||||
|
||||
void Next() final {
|
||||
if (s_ != 0 && pos_ > 0) aiter_->Next();
|
||||
++pos_;
|
||||
}
|
||||
|
||||
size_t Position() const final { return pos_; }
|
||||
|
||||
void Reset() final {
|
||||
if (s_ != 0) aiter_->Reset();
|
||||
pos_ = 0;
|
||||
}
|
||||
|
||||
void Seek(size_t a) final {
|
||||
if (s_ != 0) {
|
||||
if (a == 0) {
|
||||
aiter_->Reset();
|
||||
} else {
|
||||
aiter_->Seek(a - 1);
|
||||
}
|
||||
}
|
||||
pos_ = a;
|
||||
}
|
||||
|
||||
uint32 Flags() const final { return kArcValueFlags; }
|
||||
|
||||
void SetFlags(uint32, uint32) final {}
|
||||
|
||||
private:
|
||||
std::unique_ptr<ArcIterator<Fst<Arc>>> aiter_;
|
||||
StateId s_;
|
||||
size_t pos_;
|
||||
mutable Arc arc_;
|
||||
};
|
||||
|
||||
template <class Arc>
|
||||
inline void ComplementFst<Arc>::InitStateIterator(
|
||||
StateIteratorData<Arc> *data) const {
|
||||
data->base = new StateIterator<ComplementFst<Arc>>(*this);
|
||||
}
|
||||
|
||||
template <class Arc>
|
||||
inline void ComplementFst<Arc>::InitArcIterator(StateId s,
|
||||
ArcIteratorData<Arc> *data) const {
|
||||
data->base = new ArcIterator<ComplementFst<Arc>>(*this, s);
|
||||
}
|
||||
|
||||
// Useful alias when using StdArc.
|
||||
using StdComplementFst = ComplementFst<StdArc>;
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_COMPLEMENT_H_
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,220 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Functions and classes to compute the concatenation of two FSTs.
|
||||
|
||||
#ifndef FST_CONCAT_H_
|
||||
#define FST_CONCAT_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
#include <fst/mutable-fst.h>
|
||||
#include <fst/rational.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Computes the concatenation (product) of two FSTs. If FST1 transduces string
|
||||
// x to y with weight a and FST2 transduces string w to v with weight b, then
|
||||
// their concatenation transduces string xw to yv with weight Times(a, b).
|
||||
//
|
||||
// This version modifies its MutableFst argument (in first position).
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// Time: O(V1 + V2 + E2)
|
||||
// Space: O(V1 + V2 + E2)
|
||||
//
|
||||
// where Vi is the number of states, and Ei is the number of arcs, of the ith
|
||||
// FST.
|
||||
template <class Arc>
|
||||
void Concat(MutableFst<Arc> *fst1, const Fst<Arc> &fst2) {
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
// Checks that the symbol table are compatible.
|
||||
if (!CompatSymbols(fst1->InputSymbols(), fst2.InputSymbols()) ||
|
||||
!CompatSymbols(fst1->OutputSymbols(), fst2.OutputSymbols())) {
|
||||
FSTERROR() << "Concat: Input/output symbol tables of 1st argument "
|
||||
<< "does not match input/output symbol tables of 2nd argument";
|
||||
fst1->SetProperties(kError, kError);
|
||||
return;
|
||||
}
|
||||
const auto props1 = fst1->Properties(kFstProperties, false);
|
||||
const auto props2 = fst2.Properties(kFstProperties, false);
|
||||
const auto start1 = fst1->Start();
|
||||
if (start1 == kNoStateId) {
|
||||
if (props2 & kError) fst1->SetProperties(kError, kError);
|
||||
return;
|
||||
}
|
||||
const auto numstates1 = fst1->NumStates();
|
||||
if (fst2.Properties(kExpanded, false)) {
|
||||
fst1->ReserveStates(numstates1 + CountStates(fst2));
|
||||
}
|
||||
for (StateIterator<Fst<Arc>> siter2(fst2); !siter2.Done(); siter2.Next()) {
|
||||
const auto s1 = fst1->AddState();
|
||||
const auto s2 = siter2.Value();
|
||||
fst1->SetFinal(s1, fst2.Final(s2));
|
||||
fst1->ReserveArcs(s1, fst2.NumArcs(s2));
|
||||
for (ArcIterator<Fst<Arc>> aiter(fst2, s2); !aiter.Done(); aiter.Next()) {
|
||||
auto arc = aiter.Value();
|
||||
arc.nextstate += numstates1;
|
||||
fst1->AddArc(s1, arc);
|
||||
}
|
||||
}
|
||||
const auto start2 = fst2.Start();
|
||||
for (StateId s1 = 0; s1 < numstates1; ++s1) {
|
||||
const auto weight = fst1->Final(s1);
|
||||
if (weight != Weight::Zero()) {
|
||||
fst1->SetFinal(s1, Weight::Zero());
|
||||
if (start2 != kNoStateId) {
|
||||
fst1->AddArc(s1, Arc(0, 0, weight, start2 + numstates1));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (start2 != kNoStateId) {
|
||||
fst1->SetProperties(ConcatProperties(props1, props2), kFstProperties);
|
||||
}
|
||||
}
|
||||
|
||||
// Computes the concatentation of two FSTs. This version modifies its
|
||||
// MutableFst argument (in second position).
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// Time: O(V1 + E1)
|
||||
// Space: O(V1 + E1)
|
||||
//
|
||||
// where Vi is the number of states, and Ei is the number of arcs, of the ith
|
||||
// FST.
|
||||
template <class Arc>
|
||||
void Concat(const Fst<Arc> &fst1, MutableFst<Arc> *fst2) {
|
||||
using Weight = typename Arc::Weight;
|
||||
// Checks that the symbol table are compatible.
|
||||
if (!CompatSymbols(fst1.InputSymbols(), fst2->InputSymbols()) ||
|
||||
!CompatSymbols(fst1.OutputSymbols(), fst2->OutputSymbols())) {
|
||||
FSTERROR() << "Concat: Input/output symbol tables of 1st argument "
|
||||
<< "does not match input/output symbol tables of 2nd argument";
|
||||
fst2->SetProperties(kError, kError);
|
||||
return;
|
||||
}
|
||||
const auto props1 = fst1.Properties(kFstProperties, false);
|
||||
const auto props2 = fst2->Properties(kFstProperties, false);
|
||||
const auto start2 = fst2->Start();
|
||||
if (start2 == kNoStateId) {
|
||||
if (props1 & kError) fst2->SetProperties(kError, kError);
|
||||
return;
|
||||
}
|
||||
const auto numstates2 = fst2->NumStates();
|
||||
if (fst1.Properties(kExpanded, false)) {
|
||||
fst2->ReserveStates(numstates2 + CountStates(fst1));
|
||||
}
|
||||
for (StateIterator<Fst<Arc>> siter(fst1); !siter.Done(); siter.Next()) {
|
||||
const auto s1 = siter.Value();
|
||||
const auto s2 = fst2->AddState();
|
||||
const auto weight = fst1.Final(s1);
|
||||
if (weight != Weight::Zero()) {
|
||||
fst2->ReserveArcs(s2, fst1.NumArcs(s1) + 1);
|
||||
fst2->AddArc(s2, Arc(0, 0, weight, start2));
|
||||
} else {
|
||||
fst2->ReserveArcs(s2, fst1.NumArcs(s1));
|
||||
}
|
||||
for (ArcIterator<Fst<Arc>> aiter(fst1, s1); !aiter.Done(); aiter.Next()) {
|
||||
auto arc = aiter.Value();
|
||||
arc.nextstate += numstates2;
|
||||
fst2->AddArc(s2, arc);
|
||||
}
|
||||
}
|
||||
const auto start1 = fst1.Start();
|
||||
if (start1 != kNoStateId) {
|
||||
fst2->SetStart(start1 + numstates2);
|
||||
fst2->SetProperties(ConcatProperties(props1, props2), kFstProperties);
|
||||
} else {
|
||||
fst2->SetStart(fst2->AddState());
|
||||
}
|
||||
}
|
||||
|
||||
// Computes the concatentation of two FSTs. This version modifies its
|
||||
// RationalFst input (in first position).
|
||||
template <class Arc>
|
||||
void Concat(RationalFst<Arc> *fst1, const Fst<Arc> &fst2) {
|
||||
fst1->GetMutableImpl()->AddConcat(fst2, true);
|
||||
}
|
||||
|
||||
// Computes the concatentation of two FSTs. This version modifies its
|
||||
// RationalFst input (in second position).
|
||||
template <class Arc>
|
||||
void Concat(const Fst<Arc> &fst1, RationalFst<Arc> *fst2) {
|
||||
fst2->GetMutableImpl()->AddConcat(fst1, false);
|
||||
}
|
||||
|
||||
using ConcatFstOptions = RationalFstOptions;
|
||||
|
||||
// Computes the concatenation (product) of two FSTs; this version is a delayed
|
||||
// FST. If FST1 transduces string x to y with weight a and FST2 transduces
|
||||
// string w to v with weight b, then their concatenation transduces string xw
|
||||
// to yv with Times(a, b).
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// Time: O(v1 + e1 + v2 + e2),
|
||||
// Space: O(v1 + v2)
|
||||
//
|
||||
// where vi is the number of states visited, and ei is the number of arcs
|
||||
// visited, of the ith FST. Constant time and space to visit an input state or
|
||||
// arc is assumed and exclusive of caching.
|
||||
template <class A>
|
||||
class ConcatFst : public RationalFst<A> {
|
||||
public:
|
||||
using Arc = A;
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
|
||||
ConcatFst(const Fst<Arc> &fst1, const Fst<Arc> &fst2) {
|
||||
GetMutableImpl()->InitConcat(fst1, fst2);
|
||||
}
|
||||
|
||||
ConcatFst(const Fst<Arc> &fst1, const Fst<Arc> &fst2,
|
||||
const ConcatFstOptions &opts)
|
||||
: RationalFst<Arc>(opts) {
|
||||
GetMutableImpl()->InitConcat(fst1, fst2);
|
||||
}
|
||||
|
||||
// See Fst<>::Copy() for doc.
|
||||
ConcatFst(const ConcatFst<Arc> &fst, bool safe = false)
|
||||
: RationalFst<Arc>(fst, safe) {}
|
||||
|
||||
// Get a copy of this ConcatFst. See Fst<>::Copy() for further doc.
|
||||
ConcatFst<Arc> *Copy(bool safe = false) const override {
|
||||
return new ConcatFst<Arc>(*this, safe);
|
||||
}
|
||||
|
||||
private:
|
||||
using ImplToFst<internal::RationalFstImpl<Arc>>::GetImpl;
|
||||
using ImplToFst<internal::RationalFstImpl<Arc>>::GetMutableImpl;
|
||||
};
|
||||
|
||||
// Specialization for ConcatFst.
|
||||
template <class Arc>
|
||||
class StateIterator<ConcatFst<Arc>> : public StateIterator<RationalFst<Arc>> {
|
||||
public:
|
||||
explicit StateIterator(const ConcatFst<Arc> &fst)
|
||||
: StateIterator<RationalFst<Arc>>(fst) {}
|
||||
};
|
||||
|
||||
// Specialization for ConcatFst.
|
||||
template <class Arc>
|
||||
class ArcIterator<ConcatFst<Arc>> : public ArcIterator<RationalFst<Arc>> {
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
ArcIterator(const ConcatFst<Arc> &fst, StateId s)
|
||||
: ArcIterator<RationalFst<Arc>>(fst, s) {}
|
||||
};
|
||||
|
||||
// Useful alias when using StdArc.
|
||||
using StdConcatFst = ConcatFst<StdArc>;
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_CONCAT_H_
|
||||
@@ -0,0 +1,3 @@
|
||||
// Windows-specific OpenFst config file
|
||||
// No dynamic registration.
|
||||
#define FST_NO_DYNAMIC_LINKING 1
|
||||
@@ -0,0 +1,11 @@
|
||||
// OpenFst config file
|
||||
|
||||
/* Define to 1 if you have the ICU library. */
|
||||
#undef HAVE_ICU
|
||||
|
||||
/* Define to 1 if the system has the type `std::tr1::hash<long long
|
||||
unsigned>'. */
|
||||
#define HAVE_STD__TR1__HASH_LONG_LONG_UNSIGNED_ 1
|
||||
|
||||
/* Define to 1 if the system has the type `__gnu_cxx::slist<int>'. */
|
||||
#define HAVE___GNU_CXX__SLIST_INT_ 1
|
||||
@@ -0,0 +1,323 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Classes and functions to remove unsuccessful paths from an FST.
|
||||
|
||||
#ifndef FST_CONNECT_H_
|
||||
#define FST_CONNECT_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include <fst/dfs-visit.h>
|
||||
#include <fst/mutable-fst.h>
|
||||
#include <fst/union-find.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
// Finds and returns connected components. Use with Visit().
|
||||
template <class Arc>
|
||||
class CcVisitor {
|
||||
public:
|
||||
using Weight = typename Arc::Weight;
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
// cc[i]: connected component number for state i.
|
||||
explicit CcVisitor(std::vector<StateId> *cc)
|
||||
: comps_(new UnionFind<StateId>(0, kNoStateId)), cc_(cc), nstates_(0) {}
|
||||
|
||||
// comps: connected components equiv classes.
|
||||
explicit CcVisitor(UnionFind<StateId> *comps)
|
||||
: comps_(comps), cc_(nullptr), nstates_(0) {}
|
||||
|
||||
~CcVisitor() {
|
||||
if (cc_) delete comps_;
|
||||
}
|
||||
|
||||
void InitVisit(const Fst<Arc> &fst) {}
|
||||
|
||||
bool InitState(StateId s, StateId root) {
|
||||
++nstates_;
|
||||
if (comps_->FindSet(s) == kNoStateId) comps_->MakeSet(s);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WhiteArc(StateId s, const Arc &arc) {
|
||||
comps_->MakeSet(arc.nextstate);
|
||||
comps_->Union(s, arc.nextstate);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GreyArc(StateId s, const Arc &arc) {
|
||||
comps_->Union(s, arc.nextstate);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool BlackArc(StateId s, const Arc &arc) {
|
||||
comps_->Union(s, arc.nextstate);
|
||||
return true;
|
||||
}
|
||||
|
||||
void FinishState(StateId s) {}
|
||||
|
||||
void FinishVisit() {
|
||||
if (cc_) GetCcVector(cc_);
|
||||
}
|
||||
|
||||
// Returns number of components.
|
||||
// cc[i]: connected component number for state i.
|
||||
int GetCcVector(std::vector<StateId> *cc) {
|
||||
cc->clear();
|
||||
cc->resize(nstates_, kNoStateId);
|
||||
StateId ncomp = 0;
|
||||
for (StateId s = 0; s < nstates_; ++s) {
|
||||
const auto rep = comps_->FindSet(s);
|
||||
auto &comp = (*cc)[rep];
|
||||
if (comp == kNoStateId) {
|
||||
comp = ncomp;
|
||||
++ncomp;
|
||||
}
|
||||
(*cc)[s] = comp;
|
||||
}
|
||||
return ncomp;
|
||||
}
|
||||
|
||||
private:
|
||||
UnionFind<StateId> *comps_; // Components.
|
||||
std::vector<StateId> *cc_; // State's cc number.
|
||||
StateId nstates_; // State count.
|
||||
};
|
||||
|
||||
// Finds and returns strongly-connected components, accessible and
|
||||
// coaccessible states and related properties. Uses Tarjan's single
|
||||
// DFS SCC algorithm (see Aho, et al, "Design and Analysis of Computer
|
||||
// Algorithms", 189pp). Use with DfsVisit();
|
||||
template <class Arc>
|
||||
class SccVisitor {
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
|
||||
// scc[i]: strongly-connected component number for state i.
|
||||
// SCC numbers will be in topological order for acyclic input.
|
||||
// access[i]: accessibility of state i.
|
||||
// coaccess[i]: coaccessibility of state i.
|
||||
// Any of above can be NULL.
|
||||
// props: related property bits (cyclicity, initial cyclicity,
|
||||
// accessibility, coaccessibility) set/cleared (o.w. unchanged).
|
||||
SccVisitor(std::vector<StateId> *scc, std::vector<bool> *access,
|
||||
std::vector<bool> *coaccess, uint64 *props)
|
||||
: scc_(scc), access_(access), coaccess_(coaccess), props_(props) {}
|
||||
explicit SccVisitor(uint64 *props)
|
||||
: scc_(nullptr), access_(nullptr), coaccess_(nullptr), props_(props) {}
|
||||
|
||||
void InitVisit(const Fst<Arc> &fst);
|
||||
|
||||
bool InitState(StateId s, StateId root);
|
||||
|
||||
bool TreeArc(StateId s, const Arc &arc) { return true; }
|
||||
|
||||
bool BackArc(StateId s, const Arc &arc) {
|
||||
const auto t = arc.nextstate;
|
||||
if ((*dfnumber_)[t] < (*lowlink_)[s]) (*lowlink_)[s] = (*dfnumber_)[t];
|
||||
if ((*coaccess_)[t]) (*coaccess_)[s] = true;
|
||||
*props_ |= kCyclic;
|
||||
*props_ &= ~kAcyclic;
|
||||
if (t == start_) {
|
||||
*props_ |= kInitialCyclic;
|
||||
*props_ &= ~kInitialAcyclic;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ForwardOrCrossArc(StateId s, const Arc &arc) {
|
||||
const auto t = arc.nextstate;
|
||||
if ((*dfnumber_)[t] < (*dfnumber_)[s] /* cross edge */ && (*onstack_)[t] &&
|
||||
(*dfnumber_)[t] < (*lowlink_)[s]) {
|
||||
(*lowlink_)[s] = (*dfnumber_)[t];
|
||||
}
|
||||
if ((*coaccess_)[t]) (*coaccess_)[s] = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Last argument always ignored, but required by the interface.
|
||||
void FinishState(StateId state, StateId p, const Arc *);
|
||||
|
||||
void FinishVisit() {
|
||||
// Numbers SCCs in topological order when acyclic.
|
||||
if (scc_) {
|
||||
for (size_t s = 0; s < scc_->size(); ++s) {
|
||||
(*scc_)[s] = nscc_ - 1 - (*scc_)[s];
|
||||
}
|
||||
}
|
||||
if (coaccess_internal_) delete coaccess_;
|
||||
dfnumber_.reset();
|
||||
lowlink_.reset();
|
||||
onstack_.reset();
|
||||
scc_stack_.reset();
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<StateId> *scc_; // State's scc number.
|
||||
std::vector<bool> *access_; // State's accessibility.
|
||||
std::vector<bool> *coaccess_; // State's coaccessibility.
|
||||
uint64 *props_;
|
||||
const Fst<Arc> *fst_;
|
||||
StateId start_;
|
||||
StateId nstates_; // State count.
|
||||
StateId nscc_; // SCC count.
|
||||
bool coaccess_internal_;
|
||||
std::unique_ptr<std::vector<StateId>> dfnumber_; // State discovery times.
|
||||
std::unique_ptr<std::vector<StateId>>
|
||||
lowlink_; // lowlink[state] == dfnumber[state] => SCC root
|
||||
std::unique_ptr<std::vector<bool>> onstack_; // Is a state on the SCC stack?
|
||||
std::unique_ptr<std::vector<StateId>>
|
||||
scc_stack_; // SCC stack, with random access.
|
||||
};
|
||||
|
||||
template <class Arc>
|
||||
inline void SccVisitor<Arc>::InitVisit(const Fst<Arc> &fst) {
|
||||
if (scc_) scc_->clear();
|
||||
if (access_) access_->clear();
|
||||
if (coaccess_) {
|
||||
coaccess_->clear();
|
||||
coaccess_internal_ = false;
|
||||
} else {
|
||||
coaccess_ = new std::vector<bool>;
|
||||
coaccess_internal_ = true;
|
||||
}
|
||||
*props_ |= kAcyclic | kInitialAcyclic | kAccessible | kCoAccessible;
|
||||
*props_ &= ~(kCyclic | kInitialCyclic | kNotAccessible | kNotCoAccessible);
|
||||
fst_ = &fst;
|
||||
start_ = fst.Start();
|
||||
nstates_ = 0;
|
||||
nscc_ = 0;
|
||||
dfnumber_.reset(new std::vector<StateId>());
|
||||
lowlink_.reset(new std::vector<StateId>());
|
||||
onstack_.reset(new std::vector<bool>());
|
||||
scc_stack_.reset(new std::vector<StateId>());
|
||||
}
|
||||
|
||||
template <class Arc>
|
||||
inline bool SccVisitor<Arc>::InitState(StateId s, StateId root) {
|
||||
scc_stack_->push_back(s);
|
||||
if (static_cast<StateId>(dfnumber_->size()) <= s) {
|
||||
if (scc_) scc_->resize(s + 1, -1);
|
||||
if (access_) access_->resize(s + 1, false);
|
||||
coaccess_->resize(s + 1, false);
|
||||
dfnumber_->resize(s + 1, -1);
|
||||
lowlink_->resize(s + 1, -1);
|
||||
onstack_->resize(s + 1, false);
|
||||
}
|
||||
(*dfnumber_)[s] = nstates_;
|
||||
(*lowlink_)[s] = nstates_;
|
||||
(*onstack_)[s] = true;
|
||||
if (root == start_) {
|
||||
if (access_) (*access_)[s] = true;
|
||||
} else {
|
||||
if (access_) (*access_)[s] = false;
|
||||
*props_ |= kNotAccessible;
|
||||
*props_ &= ~kAccessible;
|
||||
}
|
||||
++nstates_;
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class Arc>
|
||||
inline void SccVisitor<Arc>::FinishState(StateId s, StateId p, const Arc *) {
|
||||
if (fst_->Final(s) != Weight::Zero()) (*coaccess_)[s] = true;
|
||||
if ((*dfnumber_)[s] == (*lowlink_)[s]) { // Root of new SCC.
|
||||
bool scc_coaccess = false;
|
||||
auto i = scc_stack_->size();
|
||||
StateId t;
|
||||
do {
|
||||
t = (*scc_stack_)[--i];
|
||||
if ((*coaccess_)[t]) scc_coaccess = true;
|
||||
} while (s != t);
|
||||
do {
|
||||
t = scc_stack_->back();
|
||||
if (scc_) (*scc_)[t] = nscc_;
|
||||
if (scc_coaccess) (*coaccess_)[t] = true;
|
||||
(*onstack_)[t] = false;
|
||||
scc_stack_->pop_back();
|
||||
} while (s != t);
|
||||
if (!scc_coaccess) {
|
||||
*props_ |= kNotCoAccessible;
|
||||
*props_ &= ~kCoAccessible;
|
||||
}
|
||||
++nscc_;
|
||||
}
|
||||
if (p != kNoStateId) {
|
||||
if ((*coaccess_)[s]) (*coaccess_)[p] = true;
|
||||
if ((*lowlink_)[s] < (*lowlink_)[p]) (*lowlink_)[p] = (*lowlink_)[s];
|
||||
}
|
||||
}
|
||||
|
||||
// Trims an FST, removing states and arcs that are not on successful paths.
|
||||
// This version modifies its input.
|
||||
//
|
||||
// Complexity:
|
||||
//
|
||||
// Time: O(V + E)
|
||||
// Space: O(V + E)
|
||||
//
|
||||
// where V = # of states and E = # of arcs.
|
||||
template <class Arc>
|
||||
void Connect(MutableFst<Arc> *fst) {
|
||||
using StateId = typename Arc::StateId;
|
||||
std::vector<bool> access;
|
||||
std::vector<bool> coaccess;
|
||||
uint64 props = 0;
|
||||
SccVisitor<Arc> scc_visitor(nullptr, &access, &coaccess, &props);
|
||||
DfsVisit(*fst, &scc_visitor);
|
||||
std::vector<StateId> dstates;
|
||||
dstates.reserve(access.size());
|
||||
for (StateId s = 0; s < access.size(); ++s) {
|
||||
if (!access[s] || !coaccess[s]) dstates.push_back(s);
|
||||
}
|
||||
fst->DeleteStates(dstates);
|
||||
fst->SetProperties(kAccessible | kCoAccessible, kAccessible | kCoAccessible);
|
||||
}
|
||||
|
||||
// Returns an acyclic FST where each SCC in the input FST has been condensed to
|
||||
// a single state with transitions between SCCs retained and within SCCs
|
||||
// dropped. Also populates 'scc' with a mapping from input to output states.
|
||||
template <class Arc>
|
||||
void Condense(const Fst<Arc> &ifst, MutableFst<Arc> *ofst,
|
||||
std::vector<typename Arc::StateId> *scc) {
|
||||
using StateId = typename Arc::StateId;
|
||||
ofst->DeleteStates();
|
||||
uint64 props = 0;
|
||||
SccVisitor<Arc> scc_visitor(scc, nullptr, nullptr, &props);
|
||||
DfsVisit(ifst, &scc_visitor);
|
||||
const auto iter = std::max_element(scc->cbegin(), scc->cend());
|
||||
if (iter == scc->cend()) return;
|
||||
const StateId num_condensed_states = 1 + *iter;
|
||||
ofst->ReserveStates(num_condensed_states);
|
||||
for (StateId c = 0; c < num_condensed_states; ++c) {
|
||||
ofst->AddState();
|
||||
}
|
||||
for (StateId s = 0; s < scc->size(); ++s) {
|
||||
const auto c = (*scc)[s];
|
||||
if (s == ifst.Start()) ofst->SetStart(c);
|
||||
const auto weight = ifst.Final(s);
|
||||
if (weight != Arc::Weight::Zero())
|
||||
ofst->SetFinal(c, Plus(ofst->Final(c), weight));
|
||||
for (ArcIterator<Fst<Arc>> aiter(ifst, s); !aiter.Done(); aiter.Next()) {
|
||||
const auto &arc = aiter.Value();
|
||||
const auto nextc = (*scc)[arc.nextstate];
|
||||
if (nextc != c) {
|
||||
Arc condensed_arc = arc;
|
||||
condensed_arc.nextstate = nextc;
|
||||
ofst->AddArc(c, std::move(condensed_arc));
|
||||
}
|
||||
}
|
||||
}
|
||||
ofst->SetProperties(kAcyclic | kInitialAcyclic, kAcyclic | kInitialAcyclic);
|
||||
}
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_CONNECT_H_
|
||||
@@ -0,0 +1,485 @@
|
||||
// See www.openfst.org for extensive documentation on this weighted
|
||||
// finite-state transducer library.
|
||||
//
|
||||
// Simple concrete immutable FST whose states and arcs are each stored in
|
||||
// single arrays.
|
||||
|
||||
#ifndef FST_CONST_FST_H_
|
||||
#define FST_CONST_FST_H_
|
||||
|
||||
#include <climits>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Google-only...
|
||||
// ...Google-only
|
||||
#include <fst/log.h>
|
||||
|
||||
#include <fst/expanded-fst.h>
|
||||
#include <fst/fst-decl.h>
|
||||
#include <fst/mapped-file.h>
|
||||
#include <fst/test-properties.h>
|
||||
#include <fst/util.h>
|
||||
|
||||
|
||||
namespace fst {
|
||||
|
||||
template <class A, class Unsigned>
|
||||
class ConstFst;
|
||||
|
||||
template <class F, class G>
|
||||
void Cast(const F &, G *);
|
||||
|
||||
namespace internal {
|
||||
|
||||
// States and arcs each implemented by single arrays, templated on the
|
||||
// Arc definition. Unsigned is used to represent indices into the arc array.
|
||||
template <class A, class Unsigned>
|
||||
class ConstFstImpl : public FstImpl<A> {
|
||||
public:
|
||||
using Arc = A;
|
||||
using StateId = typename Arc::StateId;
|
||||
using Weight = typename Arc::Weight;
|
||||
|
||||
using FstImpl<A>::SetInputSymbols;
|
||||
using FstImpl<A>::SetOutputSymbols;
|
||||
using FstImpl<A>::SetType;
|
||||
using FstImpl<A>::SetProperties;
|
||||
using FstImpl<A>::Properties;
|
||||
|
||||
ConstFstImpl()
|
||||
: states_(nullptr),
|
||||
arcs_(nullptr),
|
||||
narcs_(0),
|
||||
nstates_(0),
|
||||
start_(kNoStateId) {
|
||||
string type = "const";
|
||||
if (sizeof(Unsigned) != sizeof(uint32)) {
|
||||
type += std::to_string(CHAR_BIT * sizeof(Unsigned));
|
||||
}
|
||||
SetType(type);
|
||||
SetProperties(kNullProperties | kStaticProperties);
|
||||
}
|
||||
|
||||
explicit ConstFstImpl(const Fst<Arc> &fst);
|
||||
|
||||
StateId Start() const { return start_; }
|
||||
|
||||
Weight Final(StateId s) const { return states_[s].weight; }
|
||||
|
||||
StateId NumStates() const { return nstates_; }
|
||||
|
||||
size_t NumArcs(StateId s) const { return states_[s].narcs; }
|
||||
|
||||
size_t NumInputEpsilons(StateId s) const { return states_[s].niepsilons; }
|
||||
|
||||
size_t NumOutputEpsilons(StateId s) const { return states_[s].noepsilons; }
|
||||
|
||||
static ConstFstImpl<Arc, Unsigned> *Read(std::istream &strm,
|
||||
const FstReadOptions &opts);
|
||||
|
||||
const Arc *Arcs(StateId s) const { return arcs_ + states_[s].pos; }
|
||||
|
||||
// Provide information needed for generic state iterator.
|
||||
void InitStateIterator(StateIteratorData<Arc> *data) const {
|
||||
data->base = nullptr;
|
||||
data->nstates = nstates_;
|
||||
}
|
||||
|
||||
// Provide information needed for the generic arc iterator.
|
||||
void InitArcIterator(StateId s, ArcIteratorData<Arc> *data) const {
|
||||
data->base = nullptr;
|
||||
data->arcs = arcs_ + states_[s].pos;
|
||||
data->narcs = states_[s].narcs;
|
||||
data->ref_count = nullptr;
|
||||
}
|
||||
|
||||
private:
|
||||
// Used to find narcs_ and nstates_ in Write.
|
||||
friend class ConstFst<Arc, Unsigned>;
|
||||
|
||||
// States implemented by array *states_ below, arcs by (single) *arcs_.
|
||||
struct ConstState {
|
||||
Weight weight; // Final weight.
|
||||
Unsigned pos; // Start of state's arcs in *arcs_.
|
||||
Unsigned narcs; // Number of arcs (per state).
|
||||
Unsigned niepsilons; // Number of input epsilons.
|
||||
Unsigned noepsilons; // Number of output epsilons.
|
||||
|
||||
ConstState() : weight(Weight::Zero()) {}
|
||||
};
|
||||
|
||||
// Properties always true of this FST class.
|
||||
static constexpr uint64 kStaticProperties = kExpanded;
|
||||
// Current unaligned file format version. The unaligned version was added and
|
||||
// made the default since the aligned version does not work on pipes.
|
||||
static constexpr int kFileVersion = 2;
|
||||
// Current aligned file format version.
|
||||
static constexpr int kAlignedFileVersion = 1;
|
||||
// Minimum file format version supported.
|
||||
static constexpr int kMinFileVersion = 1;
|
||||
|
||||
std::unique_ptr<MappedFile> states_region_; // Mapped file for states.
|
||||
std::unique_ptr<MappedFile> arcs_region_; // Mapped file for arcs.
|
||||
ConstState *states_; // States representation.
|
||||
Arc *arcs_; // Arcs representation.
|
||||
size_t narcs_; // Number of arcs.
|
||||
StateId nstates_; // Number of states.
|
||||
StateId start_; // Initial state.
|
||||
|
||||
ConstFstImpl(const ConstFstImpl &) = delete;
|
||||
ConstFstImpl &operator=(const ConstFstImpl &) = delete;
|
||||
};
|
||||
|
||||
template <class Arc, class Unsigned>
|
||||
constexpr uint64 ConstFstImpl<Arc, Unsigned>::kStaticProperties;
|
||||
|
||||
template <class Arc, class Unsigned>
|
||||
constexpr int ConstFstImpl<Arc, Unsigned>::kFileVersion;
|
||||
|
||||
template <class Arc, class Unsigned>
|
||||
constexpr int ConstFstImpl<Arc, Unsigned>::kAlignedFileVersion;
|
||||
|
||||
template <class Arc, class Unsigned>
|
||||
constexpr int ConstFstImpl<Arc, Unsigned>::kMinFileVersion;
|
||||
|
||||
template <class Arc, class Unsigned>
|
||||
ConstFstImpl<Arc, Unsigned>::ConstFstImpl(const Fst<Arc> &fst)
|
||||
: narcs_(0), nstates_(0) {
|
||||
string type = "const";
|
||||
if (sizeof(Unsigned) != sizeof(uint32)) {
|
||||
type += std::to_string(CHAR_BIT * sizeof(Unsigned));
|
||||
}
|
||||
SetType(type);
|
||||
SetInputSymbols(fst.InputSymbols());
|
||||
SetOutputSymbols(fst.OutputSymbols());
|
||||
start_ = fst.Start();
|
||||
// Counts states and arcs.
|
||||
for (StateIterator<Fst<Arc>> siter(fst); !siter.Done(); siter.Next()) {
|
||||
++nstates_;
|
||||
narcs_ += fst.NumArcs(siter.Value());
|
||||
}
|
||||
states_region_.reset(MappedFile::Allocate(nstates_ * sizeof(*states_)));
|
||||
arcs_region_.reset(MappedFile::Allocate(narcs_ * sizeof(*arcs_)));
|
||||
states_ = reinterpret_cast<ConstState *>(states_region_->mutable_data());
|
||||
arcs_ = reinterpret_cast<Arc *>(arcs_region_->mutable_data());
|
||||
size_t pos = 0;
|
||||
for (StateId s = 0; s < nstates_; ++s) {
|
||||
states_[s].weight = fst.Final(s);
|
||||
states_[s].pos = pos;
|
||||
states_[s].narcs = 0;
|
||||
states_[s].niepsilons = 0;
|
||||
states_[s].noepsilons = 0;
|
||||
for (ArcIterator<Fst<Arc>> aiter(fst, s); !aiter.Done(); aiter.Next()) {
|
||||
const auto &arc = aiter.Value();
|
||||
++states_[s].narcs;
|
||||
if (arc.ilabel == 0) ++states_[s].niepsilons;
|
||||
if (arc.olabel == 0) ++states_[s].noepsilons;
|
||||
arcs_[pos] = arc;
|
||||
++pos;
|
||||
}
|
||||
}
|
||||
const auto props =
|
||||
fst.Properties(kMutable, false)
|
||||
? fst.Properties(kCopyProperties, true)
|
||||
: CheckProperties(
|
||||
fst, kCopyProperties & ~kWeightedCycles & ~kUnweightedCycles,
|
||||
kCopyProperties);
|
||||
SetProperties(props | kStaticProperties);
|
||||
}
|
||||
|
||||
template <class Arc, class Unsigned>
|
||||
ConstFstImpl<Arc, Unsigned> *ConstFstImpl<Arc, Unsigned>::Read(
|
||||
std::istream &strm, const FstReadOptions &opts) {
|
||||
using ConstState = typename ConstFstImpl<Arc, Unsigned>::ConstState;
|
||||
std::unique_ptr<ConstFstImpl<Arc, Unsigned>> impl(
|
||||
new ConstFstImpl<Arc, Unsigned>());
|
||||
FstHeader hdr;
|
||||
if (!impl->ReadHeader(strm, opts, kMinFileVersion, &hdr)) return nullptr;
|
||||
impl->start_ = hdr.Start();
|
||||
impl->nstates_ = hdr.NumStates();
|
||||
impl->narcs_ = hdr.NumArcs();
|
||||
// Ensures compatibility.
|
||||
if (hdr.Version() == kAlignedFileVersion) {
|
||||
hdr.SetFlags(hdr.GetFlags() | FstHeader::IS_ALIGNED);
|
||||
}
|
||||
if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && !AlignInput(strm)) {
|
||||
LOG(ERROR) << "ConstFst::Read: Alignment failed: " << opts.source;
|
||||
return nullptr;
|
||||
}
|
||||
size_t b = impl->nstates_ * sizeof(ConstState);
|
||||
impl->states_region_.reset(
|
||||
MappedFile::Map(&strm, opts.mode == FstReadOptions::MAP, opts.source, b));
|
||||
if (!strm || !impl->states_region_) {
|
||||
LOG(ERROR) << "ConstFst::Read: Read failed: " << opts.source;
|
||||
return nullptr;
|
||||
}
|
||||
impl->states_ =
|
||||
reinterpret_cast<ConstState *>(impl->states_region_->mutable_data());
|
||||
if ((hdr.GetFlags() & FstHeader::IS_ALIGNED) && !AlignInput(strm)) {
|
||||
LOG(ERROR) << "ConstFst::Read: Alignment failed: " << opts.source;
|
||||
return nullptr;
|
||||
}
|
||||
b = impl->narcs_ * sizeof(Arc);
|
||||
impl->arcs_region_.reset(
|
||||
MappedFile::Map(&strm, opts.mode == FstReadOptions::MAP, opts.source, b));
|
||||
if (!strm || !impl->arcs_region_) {
|
||||
LOG(ERROR) << "ConstFst::Read: Read failed: " << opts.source;
|
||||
return nullptr;
|
||||
}
|
||||
impl->arcs_ = reinterpret_cast<Arc *>(impl->arcs_region_->mutable_data());
|
||||
return impl.release();
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
||||
// Simple concrete immutable FST. This class attaches interface to
|
||||
// implementation and handles reference counting, delegating most methods to
|
||||
// ImplToExpandedFst. The unsigned type U is used to represent indices into the
|
||||
// arc array (default declared in fst-decl.h).
|
||||
template <class A, class Unsigned>
|
||||
class ConstFst : public ImplToExpandedFst<internal::ConstFstImpl<A, Unsigned>> {
|
||||
public:
|
||||
using Arc = A;
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
using Impl = internal::ConstFstImpl<A, Unsigned>;
|
||||
using ConstState = typename Impl::ConstState;
|
||||
|
||||
friend class StateIterator<ConstFst<Arc, Unsigned>>;
|
||||
friend class ArcIterator<ConstFst<Arc, Unsigned>>;
|
||||
|
||||
template <class F, class G>
|
||||
void friend Cast(const F &, G *);
|
||||
|
||||
ConstFst() : ImplToExpandedFst<Impl>(std::make_shared<Impl>()) {}
|
||||
|
||||
explicit ConstFst(const Fst<Arc> &fst)
|
||||
: ImplToExpandedFst<Impl>(std::make_shared<Impl>(fst)) {}
|
||||
|
||||
ConstFst(const ConstFst<A, Unsigned> &fst, bool safe = false)
|
||||
: ImplToExpandedFst<Impl>(fst) {}
|
||||
|
||||
// Gets a copy of this ConstFst. See Fst<>::Copy() for further doc.
|
||||
ConstFst<A, Unsigned> *Copy(bool safe = false) const override {
|
||||
return new ConstFst<A, Unsigned>(*this, safe);
|
||||
}
|
||||
|
||||
// Reads a ConstFst from an input stream, returning nullptr on error.
|
||||
static ConstFst<A, Unsigned> *Read(std::istream &strm,
|
||||
const FstReadOptions &opts) {
|
||||
auto *impl = Impl::Read(strm, opts);
|
||||
return impl ? new ConstFst<A, Unsigned>(std::shared_ptr<Impl>(impl))
|
||||
: nullptr;
|
||||
}
|
||||
|
||||
// Read a ConstFst from a file; return nullptr on error; empty filename reads
|
||||
// from standard input.
|
||||
static ConstFst<A, Unsigned> *Read(const string &filename) {
|
||||
auto *impl = ImplToExpandedFst<Impl>::Read(filename);
|
||||
return impl ? new ConstFst<A, Unsigned>(std::shared_ptr<Impl>(impl))
|
||||
: nullptr;
|
||||
}
|
||||
|
||||
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override {
|
||||
return WriteFst(*this, strm, opts);
|
||||
}
|
||||
|
||||
bool Write(const string &filename) const override {
|
||||
return Fst<Arc>::WriteFile(filename);
|
||||
}
|
||||
|
||||
template <class FST>
|
||||
static bool WriteFst(const FST &fst, std::ostream &strm,
|
||||
const FstWriteOptions &opts);
|
||||
|
||||
void InitStateIterator(StateIteratorData<Arc> *data) const override {
|
||||
GetImpl()->InitStateIterator(data);
|
||||
}
|
||||
|
||||
void InitArcIterator(StateId s, ArcIteratorData<Arc> *data) const override {
|
||||
GetImpl()->InitArcIterator(s, data);
|
||||
}
|
||||
|
||||
private:
|
||||
explicit ConstFst(std::shared_ptr<Impl> impl)
|
||||
: ImplToExpandedFst<Impl>(impl) {}
|
||||
|
||||
using ImplToFst<Impl, ExpandedFst<Arc>>::GetImpl;
|
||||
|
||||
// Uses overloading to extract the type of the argument.
|
||||
static const Impl *GetImplIfConstFst(const ConstFst &const_fst) {
|
||||
return const_fst.GetImpl();
|
||||
}
|
||||
|
||||
// NB: this does not give privileged treatment to subtypes of ConstFst.
|
||||
template <typename FST>
|
||||
static Impl *GetImplIfConstFst(const FST &fst) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ConstFst &operator=(const ConstFst &) = delete;
|
||||
};
|
||||
|
||||
// Writes FST in Const format, potentially with a pass over the machine before
|
||||
// writing to compute number of states and arcs.
|
||||
template <class Arc, class Unsigned>
|
||||
template <class FST>
|
||||
bool ConstFst<Arc, Unsigned>::WriteFst(const FST &fst, std::ostream &strm,
|
||||
const FstWriteOptions &opts) {
|
||||
const auto file_version =
|
||||
opts.align ? internal::ConstFstImpl<Arc, Unsigned>::kAlignedFileVersion
|
||||
: internal::ConstFstImpl<Arc, Unsigned>::kFileVersion;
|
||||
size_t num_arcs = 0; // To silence -Wsometimes-uninitialized warnings.
|
||||
size_t num_states = 0; // Ditto.
|
||||
std::streamoff start_offset = 0;
|
||||
bool update_header = true;
|
||||
if (const auto *impl = GetImplIfConstFst(fst)) {
|
||||
num_arcs = impl->narcs_;
|
||||
num_states = impl->nstates_;
|
||||
update_header = false;
|
||||
} else if (opts.stream_write || (start_offset = strm.tellp()) == -1) {
|
||||
// precompute values needed for header when we cannot seek to rewrite it.
|
||||
num_arcs = 0;
|
||||
num_states = 0;
|
||||
for (StateIterator<FST> siter(fst); !siter.Done(); siter.Next()) {
|
||||
num_arcs += fst.NumArcs(siter.Value());
|
||||
++num_states;
|
||||
}
|
||||
update_header = false;
|
||||
}
|
||||
FstHeader hdr;
|
||||
hdr.SetStart(fst.Start());
|
||||
hdr.SetNumStates(num_states);
|
||||
hdr.SetNumArcs(num_arcs);
|
||||
string type = "const";
|
||||
if (sizeof(Unsigned) != sizeof(uint32)) {
|
||||
type += std::to_string(CHAR_BIT * sizeof(Unsigned));
|
||||
}
|
||||
const auto properties =
|
||||
fst.Properties(kCopyProperties, true) |
|
||||
internal::ConstFstImpl<Arc, Unsigned>::kStaticProperties;
|
||||
internal::FstImpl<Arc>::WriteFstHeader(fst, strm, opts, file_version, type,
|
||||
properties, &hdr);
|
||||
if (opts.align && !AlignOutput(strm)) {
|
||||
LOG(ERROR) << "Could not align file during write after header";
|
||||
return false;
|
||||
}
|
||||
size_t pos = 0;
|
||||
size_t states = 0;
|
||||
typename ConstFst<Arc, Unsigned>::ConstState state;
|
||||
for (StateIterator<FST> siter(fst); !siter.Done(); siter.Next()) {
|
||||
const auto s = siter.Value();
|
||||
state.weight = fst.Final(s);
|
||||
state.pos = pos;
|
||||
state.narcs = fst.NumArcs(s);
|
||||
state.niepsilons = fst.NumInputEpsilons(s);
|
||||
state.noepsilons = fst.NumOutputEpsilons(s);
|
||||
strm.write(reinterpret_cast<const char *>(&state), sizeof(state));
|
||||
pos += state.narcs;
|
||||
++states;
|
||||
}
|
||||
hdr.SetNumStates(states);
|
||||
hdr.SetNumArcs(pos);
|
||||
if (opts.align && !AlignOutput(strm)) {
|
||||
LOG(ERROR) << "Could not align file during write after writing states";
|
||||
}
|
||||
for (StateIterator<FST> siter(fst); !siter.Done(); siter.Next()) {
|
||||
for (ArcIterator<FST> aiter(fst, siter.Value()); !aiter.Done();
|
||||
aiter.Next()) {
|
||||
const auto &arc = aiter.Value();
|
||||
// Google-only...
|
||||
#ifdef MEMORY_SANITIZER
|
||||
// arc may contain padding which has unspecified contents. Tell MSAN to
|
||||
// not complain about it when writing it to a file.
|
||||
ANNOTATE_MEMORY_IS_INITIALIZED(reinterpret_cast<const char *>(&arc),
|
||||
sizeof(arc));
|
||||
#endif
|
||||
// ...Google-only
|
||||
strm.write(reinterpret_cast<const char *>(&arc), sizeof(arc));
|
||||
}
|
||||
}
|
||||
strm.flush();
|
||||
if (!strm) {
|
||||
LOG(ERROR) << "ConstFst::WriteFst: write failed: " << opts.source;
|
||||
return false;
|
||||
}
|
||||
if (update_header) {
|
||||
return internal::FstImpl<Arc>::UpdateFstHeader(
|
||||
fst, strm, opts, file_version, type, properties, &hdr, start_offset);
|
||||
} else {
|
||||
if (hdr.NumStates() != num_states) {
|
||||
LOG(ERROR) << "Inconsistent number of states observed during write";
|
||||
return false;
|
||||
}
|
||||
if (hdr.NumArcs() != num_arcs) {
|
||||
LOG(ERROR) << "Inconsistent number of arcs observed during write";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Specialization for ConstFst; see generic version in fst.h for sample usage
|
||||
// (but use the ConstFst type instead). This version should inline.
|
||||
template <class Arc, class Unsigned>
|
||||
class StateIterator<ConstFst<Arc, Unsigned>> {
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
explicit StateIterator(const ConstFst<Arc, Unsigned> &fst)
|
||||
: nstates_(fst.GetImpl()->NumStates()), s_(0) {}
|
||||
|
||||
bool Done() const { return s_ >= nstates_; }
|
||||
|
||||
StateId Value() const { return s_; }
|
||||
|
||||
void Next() { ++s_; }
|
||||
|
||||
void Reset() { s_ = 0; }
|
||||
|
||||
private:
|
||||
const StateId nstates_;
|
||||
StateId s_;
|
||||
};
|
||||
|
||||
// Specialization for ConstFst; see generic version in fst.h for sample usage
|
||||
// (but use the ConstFst type instead). This version should inline.
|
||||
template <class Arc, class Unsigned>
|
||||
class ArcIterator<ConstFst<Arc, Unsigned>> {
|
||||
public:
|
||||
using StateId = typename Arc::StateId;
|
||||
|
||||
ArcIterator(const ConstFst<Arc, Unsigned> &fst, StateId s)
|
||||
: arcs_(fst.GetImpl()->Arcs(s)),
|
||||
narcs_(fst.GetImpl()->NumArcs(s)),
|
||||
i_(0) {}
|
||||
|
||||
bool Done() const { return i_ >= narcs_; }
|
||||
|
||||
const Arc &Value() const { return arcs_[i_]; }
|
||||
|
||||
void Next() { ++i_; }
|
||||
|
||||
size_t Position() const { return i_; }
|
||||
|
||||
void Reset() { i_ = 0; }
|
||||
|
||||
void Seek(size_t a) { i_ = a; }
|
||||
|
||||
constexpr uint32 Flags() const { return kArcValueFlags; }
|
||||
|
||||
void SetFlags(uint32, uint32) {}
|
||||
|
||||
private:
|
||||
const Arc *arcs_;
|
||||
size_t narcs_;
|
||||
size_t i_;
|
||||
};
|
||||
|
||||
// A useful alias when using StdArc.
|
||||
using StdConstFst = ConstFst<StdArc>;
|
||||
|
||||
} // namespace fst
|
||||
|
||||
#endif // FST_CONST_FST_H_
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user