xtink4/crosspoint-reader
0
0
Fork 0
mirror of https://github.com/crosspoint-reader/crosspoint-reader.git synced 2026-04-29 10:26:52 -07:00
Code Issues Packages Projects Releases Wiki Activity

perf: apply (micro) optimization on SerializedHyphenationPatterns (#689)

Browse Source 
## Summary

This PR applies a micro optimization on `SerializedHyphenationPatterns`,
which allow reading `rootOffset` directly without having to parse then
cache it.

It should not affect storage space since no new bytes are added.

This also gets rid of the linear cache search whenever
`liangBreakIndexes` is called. In theory, the performance should be
improved a bit, although it may be too small to be noticeable in
practice.

## Testing

master branch:

```
english: 99.1023%
french: 100%
german: 97.7289%
russian: 97.2167%
spanish: 99.0236%
```

This PR:

```
english: 99.1023%
french: 100%
german: 97.7289%
russian: 97.2167%
spanish: 99.0236%
```

---

### AI Usage

While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.

Did you use AI tools to help write this code? PARTIALLY - mostly IDE
tab-autocompletions
This commit is contained in:
Xuan-Son Nguyen
2026-02-16 20:27:43 +11:00
committed by GitHub
parent 6c3a615fac
commit 0508bfc1f7
12 changed files with 18115 additions and 15296 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/hyphenation-trie-format.md
+4 -17
View File
@@ -45,22 +45,9 @@ byte arrays, and emits headers under
`SerializedHyphenationPatterns` descriptor so the reader can keep the automaton
in flash.
To refresh the firmware assets after updating the `.bin` files, run:
A convenient script `update_hyphenation.sh` is used to update all languages.
To use it, run:
```
./scripts/generate_hyphenation_trie.py \
--input lib/Epub/Epub/hyphenation/tries/en.bin \
--output lib/Epub/Epub/hyphenation/generated/hyph-en.trie.h
./scripts/generate_hyphenation_trie.py \
--input lib/Epub/Epub/hyphenation/tries/fr.bin \
--output lib/Epub/Epub/hyphenation/generated/hyph-fr.trie.h
./scripts/generate_hyphenation_trie.py \
--input lib/Epub/Epub/hyphenation/tries/de.bin \
--output lib/Epub/Epub/hyphenation/generated/hyph-de.trie.h
./scripts/generate_hyphenation_trie.py \
--input lib/Epub/Epub/hyphenation/tries/ru.bin \
--output lib/Epub/Epub/hyphenation/generated/hyph-ru.trie.h
```sh
./scripts/update_hypenation.sh
```
lib/Epub/Epub/hyphenation/LanguageRegistry.cpp
+2 -2
View File
@@ -14,10 +14,10 @@
namespace {
// English hyphenation patterns (3/3 minimum prefix/suffix length)
LanguageHyphenator englishHyphenator(en_us_patterns, isLatinLetter, toLowerLatin, 3, 3);
LanguageHyphenator englishHyphenator(en_patterns, isLatinLetter, toLowerLatin, 3, 3);
LanguageHyphenator frenchHyphenator(fr_patterns, isLatinLetter, toLowerLatin);
LanguageHyphenator germanHyphenator(de_patterns, isLatinLetter, toLowerLatin);
LanguageHyphenator russianHyphenator(ru_ru_patterns, isCyrillicLetter, toLowerCyrillic);
LanguageHyphenator russianHyphenator(ru_patterns, isCyrillicLetter, toLowerCyrillic);
LanguageHyphenator spanishHyphenator(es_patterns, isLatinLetter, toLowerLatin);
LanguageHyphenator italianHyphenator(it_patterns, isLatinLetter, toLowerLatin);
lib/Epub/Epub/hyphenation/LiangHyphenation.cpp
+5 -55
View File
@@ -53,6 +53,8 @@
namespace {
using EmbeddedAutomaton = SerializedHyphenationPatterns;
struct AugmentedWord {
std::vector<uint8_t> bytes;
std::vector<size_t> charByteOffsets;
@@ -141,59 +143,10 @@ struct AutomatonState {
bool valid() const { return data != nullptr; }
};
// Lightweight descriptor for the entire embedded automaton.
// The blob format is:
// [0..3] - big-endian root offset
// [4....] - node heap containing variable-sized headers + transition data
struct EmbeddedAutomaton {
const uint8_t* data = nullptr;
size_t size = 0;
uint32_t rootOffset = 0;
bool valid() const { return data != nullptr && size >= 4 && rootOffset < size; }
};
// Decode the serialized automaton header and root offset.
EmbeddedAutomaton parseAutomaton(const SerializedHyphenationPatterns& patterns) {
EmbeddedAutomaton automaton;
if (!patterns.data || patterns.size < 4) {
return automaton;
}
automaton.data = patterns.data;
automaton.size = patterns.size;
automaton.rootOffset = (static_cast<uint32_t>(patterns.data[0]) << 24) |
(static_cast<uint32_t>(patterns.data[1]) << 16) |
(static_cast<uint32_t>(patterns.data[2]) << 8) | static_cast<uint32_t>(patterns.data[3]);
if (automaton.rootOffset >= automaton.size) {
automaton.data = nullptr;
automaton.size = 0;
}
return automaton;
}
// Cache parsed automata per blob pointer to avoid reparsing.
const EmbeddedAutomaton& getAutomaton(const SerializedHyphenationPatterns& patterns) {
struct CacheEntry {
const SerializedHyphenationPatterns* key;
EmbeddedAutomaton automaton;
};
static std::vector<CacheEntry> cache;
for (const auto& entry : cache) {
if (entry.key == &patterns) {
return entry.automaton;
}
}
cache.push_back({&patterns, parseAutomaton(patterns)});
return cache.back().automaton;
}
// Interpret the node located at `addr`, returning transition metadata.
AutomatonState decodeState(const EmbeddedAutomaton& automaton, size_t addr) {
AutomatonState state;
if (!automaton.valid() || addr >= automaton.size) {
if (addr >= automaton.size) {
return state;
}
@@ -234,7 +187,7 @@ AutomatonState decodeState(const EmbeddedAutomaton& automaton, size_t addr) {
if (offset + levelsLen > automaton.size) {
return AutomatonState{};
}
levelsPtr = automaton.data + offset;
levelsPtr = automaton.data + offset - 4u;
}
if (pos + childCount > remaining) {
@@ -344,10 +297,7 @@ std::vector<size_t> liangBreakIndexes(const std::vector<CodepointInfo>& cps,
return {};
}
const EmbeddedAutomaton& automaton = getAutomaton(patterns);
if (!automaton.valid()) {
return {};
}
const EmbeddedAutomaton& automaton = patterns;
const AutomatonState root = decodeState(automaton, automaton.rootOffset);
if (!root.valid()) {
lib/Epub/Epub/hyphenation/SerializedHyphenationTrie.h
+1
View File
@@ -5,6 +5,7 @@
// Lightweight descriptor that points at a serialized Liang hyphenation trie stored in flash.
struct SerializedHyphenationPatterns {
size_t rootOffset;
const std::uint8_t* data;
size_t size;
};
lib/Epub/Epub/hyphenation/generated/hyph-de.trie.h
+12893 -10856
View File
File diff suppressed because it is too large Load Diff
lib/Epub/Epub/hyphenation/generated/hyph-en.trie.h
+1689 -1423
View File
File diff suppressed because it is too large Load Diff
lib/Epub/Epub/hyphenation/generated/hyph-es.trie.h
+854 -719
View File
File diff suppressed because it is too large Load Diff
lib/Epub/Epub/hyphenation/generated/hyph-fr.trie.h
+438 -368
View File
File diff suppressed because it is too large Load Diff
lib/Epub/Epub/hyphenation/generated/hyph-it.trie.h
+96 -96
View File
@@ -7,107 +7,107 @@
// Auto-generated by generate_hyphenation_trie.py. Do not edit manually.
alignas(4) constexpr uint8_t it_trie_data[] = {
0x00, 0x00, 0x05, 0xC4, 0x17, 0x0C, 0x33, 0x35, 0x0C, 0x29, 0x22, 0x0D, 0x3E, 0x0B, 0x47, 0x20,
0x0D, 0x16, 0x0B, 0x34, 0x0D, 0x21, 0x0C, 0x3D, 0x1F, 0x0C, 0x2A, 0x17, 0x2A, 0x0B, 0x02, 0x0C,
0x01, 0x02, 0x16, 0x02, 0x0D, 0x0C, 0x0C, 0x0D, 0x03, 0x0C, 0x01, 0x0C, 0x0E, 0x0D, 0x04, 0x02,
0x0B, 0xA0, 0x00, 0x42, 0x21, 0x6E, 0xFD, 0xA0, 0x00, 0x72, 0x21, 0x6E, 0xFD, 0xA1, 0x00, 0x61,
0x6D, 0xFD, 0x21, 0x69, 0xFB, 0x21, 0x74, 0xFD, 0x22, 0x70, 0x6E, 0xEC, 0xFD, 0xA0, 0x00, 0x91,
0x21, 0x6F, 0xFD, 0x21, 0x69, 0xFD, 0xA0, 0x00, 0xA2, 0x21, 0x73, 0xFD, 0x21, 0x70, 0xFD, 0xA0,
0x00, 0xC2, 0x21, 0x6D, 0xFD, 0x21, 0x75, 0xFD, 0x21, 0x63, 0xFD, 0x21, 0x72, 0xFD, 0xA0, 0x00,
0xE1, 0x21, 0x6F, 0xFD, 0x21, 0x72, 0xFD, 0x21, 0x74, 0xFD, 0x21, 0x6E, 0xFD, 0xA3, 0x01, 0x11,
0x61, 0x69, 0x6F, 0xDF, 0xEE, 0xFD, 0xA0, 0x00, 0xF2, 0x21, 0x65, 0xFD, 0x21, 0x6E, 0xFD, 0x21,
0x69, 0xFD, 0x21, 0x63, 0xFD, 0x21, 0x73, 0xFD, 0xA1, 0x01, 0x11, 0x69, 0xFD, 0xA0, 0x01, 0x12,
0x21, 0x75, 0xFD, 0x21, 0x65, 0xFD, 0x21, 0x78, 0xFD, 0xA0, 0x01, 0x32, 0x21, 0x6B, 0xFD, 0x21,
0x6E, 0xFD, 0xA0, 0x00, 0x71, 0x21, 0x65, 0xFD, 0x22, 0x61, 0x65, 0xF7, 0xFD, 0x21, 0x72, 0xFB,
0xA0, 0x01, 0x52, 0x21, 0x61, 0xFD, 0x21, 0x73, 0xFD, 0x21, 0x70, 0xFD, 0x21, 0x69, 0xFD, 0xA0,
0x01, 0x71, 0x21, 0x6F, 0xFD, 0x21, 0x63, 0xFD, 0x21, 0x72, 0xFD, 0x21, 0x61, 0xFD, 0xA0, 0x00,
0x61, 0x21, 0x6F, 0xFD, 0x21, 0x74, 0xFD, 0x41, 0x70, 0xFF, 0x50, 0x21, 0x6F, 0xFC, 0x21, 0x74,
0xFD, 0x22, 0x70, 0x72, 0xF3, 0xFD, 0x21, 0x61, 0xE8, 0x21, 0x72, 0xFD, 0xA0, 0x00, 0xF1, 0x22,
0x6C, 0x72, 0xFD, 0xFD, 0x21, 0x69, 0xE3, 0x21, 0x6C, 0xFD, 0x41, 0x65, 0xFF, 0x43, 0xA0, 0x01,
0x11, 0x25, 0x61, 0x68, 0x6F, 0x72, 0x73, 0xE8, 0xEE, 0xF6, 0xF9, 0xFD, 0xA0, 0x01, 0x82, 0x21,
0x72, 0xFD, 0x21, 0x63, 0xFD, 0x21, 0x73, 0xFD, 0x21, 0x69, 0xFD, 0x21, 0x65, 0xFD, 0xA0, 0x01,
0xA2, 0x21, 0x65, 0xFD, 0x21, 0x72, 0xFD, 0x21, 0x61, 0xFD, 0x41, 0x75, 0xFF, 0x4C, 0x42, 0x6C,
0x72, 0xFF, 0xFC, 0xFF, 0x48, 0x21, 0x62, 0xF9, 0x22, 0x68, 0x75, 0xEF, 0xFD, 0x47, 0x63, 0x64,
0x6C, 0x6E, 0x70, 0x72, 0x74, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF,
0x5C, 0xFF, 0x5C, 0x21, 0x73, 0xEA, 0x21, 0x6E, 0xFD, 0x21, 0x61, 0xFD, 0xA1, 0x01, 0x11, 0x72,
0xFD, 0x41, 0x6E, 0xFF, 0x15, 0x21, 0x67, 0xFC, 0xA0, 0x01, 0xC2, 0x21, 0x74, 0xFD, 0x21, 0x6C,
0xFD, 0x22, 0x61, 0x65, 0xF4, 0xFD, 0x52, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x6C, 0x6E, 0x6F,
0x70, 0x72, 0x73, 0x74, 0x77, 0x68, 0x6A, 0x6B, 0x7A, 0xFE, 0xC2, 0xFE, 0xCD, 0xFE, 0xF7, 0xFF,
0x12, 0xFF, 0x20, 0xFF, 0x37, 0xFF, 0x46, 0xFF, 0x55, 0xFF, 0x6B, 0xFF, 0x8B, 0xFF, 0xA5, 0xFF,
0xC2, 0xFF, 0xE6, 0xFF, 0xFB, 0xFF, 0x88, 0xFF, 0x88, 0xFF, 0x88, 0xFF, 0x88, 0xA0, 0x01, 0xE2,
0xA0, 0x00, 0xD1, 0x24, 0x61, 0x65, 0x6F, 0x75, 0xFD, 0xFD, 0xFD, 0xFD, 0x21, 0x6F, 0xF4, 0x21,
0x61, 0xF1, 0xA0, 0x01, 0xE1, 0x21, 0x2E, 0xFD, 0x24, 0x69, 0x75, 0x79, 0x74, 0xEB, 0xF4, 0xF7,
0xFD, 0x21, 0x75, 0xDF, 0xA0, 0x00, 0x51, 0x22, 0x69, 0x77, 0xFA, 0xFD, 0x21, 0x69, 0xD7, 0xAE,
0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x6D, 0x6E, 0x70, 0x73, 0x74, 0x76, 0x6C, 0x72, 0x2E, 0x27,
0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xF5, 0xF5, 0xE3, 0xE3, 0x22, 0x2E,
0x27, 0xC4, 0xC7, 0xC6, 0x00, 0x51, 0x68, 0x2E, 0x27, 0x62, 0x72, 0x6E, 0xFF, 0xBF, 0xFF, 0xBF,
0xFF, 0xFB, 0xFF, 0xBF, 0xFE, 0xFB, 0xFF, 0xBF, 0xD0, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x6B,
0x6D, 0x6E, 0x71, 0x73, 0x74, 0x7A, 0x68, 0x6C, 0x72, 0x2E, 0x27, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF,
0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF,
0xAA, 0xFF, 0xEB, 0xFF, 0xBC, 0xFF, 0xBC, 0xFF, 0xAA, 0xFF, 0xAA, 0xCE, 0x02, 0x01, 0x62, 0x64,
0x67, 0x6C, 0x6D, 0x6E, 0x70, 0x72, 0x73, 0x74, 0x76, 0x77, 0x2E, 0x27, 0xFF, 0x77, 0xFF, 0x77,
0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x89, 0xFF, 0x77, 0xFF, 0x77,
0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xCA, 0x02, 0x01, 0x62, 0x67, 0x66, 0x6E, 0x6C,
0x72, 0x73, 0x74, 0x2E, 0x27, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x5C, 0xFF,
0x5C, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x4A, 0xA0, 0x02, 0x12, 0xA1, 0x00, 0x51, 0x74,
0xFD, 0xD1, 0x02, 0x01, 0x62, 0x64, 0x66, 0x67, 0x68, 0x6C, 0x6D, 0x6E, 0x70, 0x72, 0x73, 0x74,
0x76, 0x77, 0x7A, 0x2E, 0x27, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0xFB, 0xFF,
0x33, 0xFF, 0x21, 0xFF, 0x33, 0xFF, 0x21, 0xFF, 0x33, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF,
0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0x41, 0x70, 0xFD, 0x4D, 0xCB, 0x02, 0x01, 0x62, 0x64,
0x68, 0x69, 0x6C, 0x6D, 0x6E, 0x72, 0x76, 0x2E, 0x27, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFF,
0xFC, 0xFE, 0xF9, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xC2,
0x02, 0x01, 0x2E, 0x27, 0xFE, 0xC3, 0xFE, 0xC3, 0xCB, 0x02, 0x01, 0x67, 0x66, 0x68, 0x6B, 0x6C,
0x6D, 0x72, 0x73, 0x74, 0x2E, 0x27, 0xFE, 0xBA, 0xFE, 0xBA, 0xFE, 0xCC, 0xFE, 0xBA, 0xFE, 0xCC,
0xFE, 0xBA, 0xFE, 0xCC, 0xFE, 0xBA, 0xFE, 0xBA, 0xFE, 0xBA, 0xFE, 0xBA, 0xA0, 0x02, 0x33, 0x42,
0x2E, 0x27, 0xFE, 0x93, 0xFE, 0x93, 0xD5, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x67, 0x68, 0x6A,
0x6B, 0x6C, 0x6D, 0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x77, 0x7A, 0x2E, 0x27, 0xFE, 0x8C,
0xFE, 0x8C, 0xFE, 0x8C, 0xFF, 0xF6, 0xFE, 0x8C, 0xFE, 0x9E, 0xFE, 0x9E, 0xFE, 0x8C, 0xFE, 0x8C,
0x17, 0x0C, 0x33, 0x35, 0x0C, 0x29, 0x22, 0x0D, 0x3E, 0x0B, 0x47, 0x20, 0x0D, 0x16, 0x0B, 0x34,
0x0D, 0x21, 0x0C, 0x3D, 0x1F, 0x0C, 0x2A, 0x17, 0x2A, 0x0B, 0x02, 0x0C, 0x01, 0x02, 0x16, 0x02,
0x0D, 0x0C, 0x0C, 0x0D, 0x03, 0x0C, 0x01, 0x0C, 0x0E, 0x0D, 0x04, 0x02, 0x0B, 0xA0, 0x00, 0x42,
0x21, 0x6E, 0xFD, 0xA0, 0x00, 0x72, 0x21, 0x6E, 0xFD, 0xA1, 0x00, 0x61, 0x6D, 0xFD, 0x21, 0x69,
0xFB, 0x21, 0x74, 0xFD, 0x22, 0x70, 0x6E, 0xEC, 0xFD, 0xA0, 0x00, 0x91, 0x21, 0x6F, 0xFD, 0x21,
0x69, 0xFD, 0xA0, 0x00, 0xA2, 0x21, 0x73, 0xFD, 0x21, 0x70, 0xFD, 0xA0, 0x00, 0xC2, 0x21, 0x6D,
0xFD, 0x21, 0x75, 0xFD, 0x21, 0x63, 0xFD, 0x21, 0x72, 0xFD, 0xA0, 0x00, 0xE1, 0x21, 0x6F, 0xFD,
0x21, 0x72, 0xFD, 0x21, 0x74, 0xFD, 0x21, 0x6E, 0xFD, 0xA3, 0x01, 0x11, 0x61, 0x69, 0x6F, 0xDF,
0xEE, 0xFD, 0xA0, 0x00, 0xF2, 0x21, 0x65, 0xFD, 0x21, 0x6E, 0xFD, 0x21, 0x69, 0xFD, 0x21, 0x63,
0xFD, 0x21, 0x73, 0xFD, 0xA1, 0x01, 0x11, 0x69, 0xFD, 0xA0, 0x01, 0x12, 0x21, 0x75, 0xFD, 0x21,
0x65, 0xFD, 0x21, 0x78, 0xFD, 0xA0, 0x01, 0x32, 0x21, 0x6B, 0xFD, 0x21, 0x6E, 0xFD, 0xA0, 0x00,
0x71, 0x21, 0x65, 0xFD, 0x22, 0x61, 0x65, 0xF7, 0xFD, 0x21, 0x72, 0xFB, 0xA0, 0x01, 0x52, 0x21,
0x61, 0xFD, 0x21, 0x73, 0xFD, 0x21, 0x70, 0xFD, 0x21, 0x69, 0xFD, 0xA0, 0x01, 0x71, 0x21, 0x6F,
0xFD, 0x21, 0x63, 0xFD, 0x21, 0x72, 0xFD, 0x21, 0x61, 0xFD, 0xA0, 0x00, 0x61, 0x21, 0x6F, 0xFD,
0x21, 0x74, 0xFD, 0x41, 0x70, 0xFF, 0x50, 0x21, 0x6F, 0xFC, 0x21, 0x74, 0xFD, 0x22, 0x70, 0x72,
0xF3, 0xFD, 0x21, 0x61, 0xE8, 0x21, 0x72, 0xFD, 0xA0, 0x00, 0xF1, 0x22, 0x6C, 0x72, 0xFD, 0xFD,
0x21, 0x69, 0xE3, 0x21, 0x6C, 0xFD, 0x41, 0x65, 0xFF, 0x43, 0xA0, 0x01, 0x11, 0x25, 0x61, 0x68,
0x6F, 0x72, 0x73, 0xE8, 0xEE, 0xF6, 0xF9, 0xFD, 0xA0, 0x01, 0x82, 0x21, 0x72, 0xFD, 0x21, 0x63,
0xFD, 0x21, 0x73, 0xFD, 0x21, 0x69, 0xFD, 0x21, 0x65, 0xFD, 0xA0, 0x01, 0xA2, 0x21, 0x65, 0xFD,
0x21, 0x72, 0xFD, 0x21, 0x61, 0xFD, 0x41, 0x75, 0xFF, 0x4C, 0x42, 0x6C, 0x72, 0xFF, 0xFC, 0xFF,
0x48, 0x21, 0x62, 0xF9, 0x22, 0x68, 0x75, 0xEF, 0xFD, 0x47, 0x63, 0x64, 0x6C, 0x6E, 0x70, 0x72,
0x74, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x5C, 0x21,
0x73, 0xEA, 0x21, 0x6E, 0xFD, 0x21, 0x61, 0xFD, 0xA1, 0x01, 0x11, 0x72, 0xFD, 0x41, 0x6E, 0xFF,
0x15, 0x21, 0x67, 0xFC, 0xA0, 0x01, 0xC2, 0x21, 0x74, 0xFD, 0x21, 0x6C, 0xFD, 0x22, 0x61, 0x65,
0xF4, 0xFD, 0x52, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x6C, 0x6E, 0x6F, 0x70, 0x72, 0x73, 0x74,
0x77, 0x68, 0x6A, 0x6B, 0x7A, 0xFE, 0xC2, 0xFE, 0xCD, 0xFE, 0xF7, 0xFF, 0x12, 0xFF, 0x20, 0xFF,
0x37, 0xFF, 0x46, 0xFF, 0x55, 0xFF, 0x6B, 0xFF, 0x8B, 0xFF, 0xA5, 0xFF, 0xC2, 0xFF, 0xE6, 0xFF,
0xFB, 0xFF, 0x88, 0xFF, 0x88, 0xFF, 0x88, 0xFF, 0x88, 0xA0, 0x01, 0xE2, 0xA0, 0x00, 0xD1, 0x24,
0x61, 0x65, 0x6F, 0x75, 0xFD, 0xFD, 0xFD, 0xFD, 0x21, 0x6F, 0xF4, 0x21, 0x61, 0xF1, 0xA0, 0x01,
0xE1, 0x21, 0x2E, 0xFD, 0x24, 0x69, 0x75, 0x79, 0x74, 0xEB, 0xF4, 0xF7, 0xFD, 0x21, 0x75, 0xDF,
0xA0, 0x00, 0x51, 0x22, 0x69, 0x77, 0xFA, 0xFD, 0x21, 0x69, 0xD7, 0xAE, 0x02, 0x01, 0x62, 0x63,
0x64, 0x66, 0x6D, 0x6E, 0x70, 0x73, 0x74, 0x76, 0x6C, 0x72, 0x2E, 0x27, 0xE3, 0xE3, 0xE3, 0xE3,
0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xE3, 0xF5, 0xF5, 0xE3, 0xE3, 0x22, 0x2E, 0x27, 0xC4, 0xC7, 0xC6,
0x00, 0x51, 0x68, 0x2E, 0x27, 0x62, 0x72, 0x6E, 0xFF, 0xBF, 0xFF, 0xBF, 0xFF, 0xFB, 0xFF, 0xBF,
0xFE, 0xFB, 0xFF, 0xBF, 0xD0, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x6B, 0x6D, 0x6E, 0x71, 0x73,
0x74, 0x7A, 0x68, 0x6C, 0x72, 0x2E, 0x27, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF,
0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xAA, 0xFF, 0xEB, 0xFF,
0xBC, 0xFF, 0xBC, 0xFF, 0xAA, 0xFF, 0xAA, 0xCE, 0x02, 0x01, 0x62, 0x64, 0x67, 0x6C, 0x6D, 0x6E,
0x70, 0x72, 0x73, 0x74, 0x76, 0x77, 0x2E, 0x27, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77,
0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x89, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77, 0xFF, 0x77,
0xFF, 0x77, 0xFF, 0x77, 0xCA, 0x02, 0x01, 0x62, 0x67, 0x66, 0x6E, 0x6C, 0x72, 0x73, 0x74, 0x2E,
0x27, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x4A, 0xFF, 0x5C, 0xFF, 0x5C, 0xFF, 0x4A, 0xFF,
0x4A, 0xFF, 0x4A, 0xFF, 0x4A, 0xA0, 0x02, 0x12, 0xA1, 0x00, 0x51, 0x74, 0xFD, 0xD1, 0x02, 0x01,
0x62, 0x64, 0x66, 0x67, 0x68, 0x6C, 0x6D, 0x6E, 0x70, 0x72, 0x73, 0x74, 0x76, 0x77, 0x7A, 0x2E,
0x27, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0xFB, 0xFF, 0x33, 0xFF, 0x21, 0xFF,
0x33, 0xFF, 0x21, 0xFF, 0x33, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF, 0x21, 0xFF,
0x21, 0xFF, 0x21, 0x41, 0x70, 0xFD, 0x4D, 0xCB, 0x02, 0x01, 0x62, 0x64, 0x68, 0x69, 0x6C, 0x6D,
0x6E, 0x72, 0x76, 0x2E, 0x27, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFF, 0xFC, 0xFE, 0xF9, 0xFE,
0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xFE, 0xE7, 0xC2, 0x02, 0x01, 0x2E, 0x27,
0xFE, 0xC3, 0xFE, 0xC3, 0xCB, 0x02, 0x01, 0x67, 0x66, 0x68, 0x6B, 0x6C, 0x6D, 0x72, 0x73, 0x74,
0x2E, 0x27, 0xFE, 0xBA, 0xFE, 0xBA, 0xFE, 0xCC, 0xFE, 0xBA, 0xFE, 0xCC, 0xFE, 0xBA, 0xFE, 0xCC,
0xFE, 0xBA, 0xFE, 0xBA, 0xFE, 0xBA, 0xFE, 0xBA, 0xA0, 0x02, 0x33, 0x42, 0x2E, 0x27, 0xFE, 0x93,
0xFE, 0x93, 0xD5, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x67, 0x68, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E,
0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x77, 0x7A, 0x2E, 0x27, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C,
0xFF, 0xF6, 0xFE, 0x8C, 0xFE, 0x9E, 0xFE, 0x9E, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C,
0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C,
0xFE, 0x8C, 0xFE, 0x8C, 0xFE, 0x8C, 0xFF, 0xF9, 0xCF, 0x02, 0x01, 0x62, 0x63, 0x66, 0x6C, 0x6D,
0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x77, 0x2E, 0x27, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A,
0xFE, 0x8C, 0xFF, 0xF9, 0xCF, 0x02, 0x01, 0x62, 0x63, 0x66, 0x6C, 0x6D, 0x6E, 0x70, 0x71, 0x72,
0x73, 0x74, 0x76, 0x77, 0x2E, 0x27, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A,
0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A,
0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xFE, 0x4A, 0xA0, 0x02, 0x62, 0xA1, 0x01, 0xE1, 0x6E, 0xFD,
0x21, 0x72, 0xF8, 0x21, 0x65, 0xFD, 0xA1, 0x01, 0xE1, 0x66, 0xFD, 0x41, 0x74, 0xFE, 0x07, 0x21,
0x69, 0xFC, 0x21, 0x65, 0xFD, 0xD3, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x67, 0x6B, 0x6C, 0x6D,
0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x7A, 0x68, 0x2E, 0x27, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD,
0xFD, 0xFD, 0xFD, 0xFF, 0xE6, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD,
0xFD, 0xFD, 0xFD, 0xFF, 0xF1, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFF, 0xFD, 0xFD, 0xFD, 0xFD,
0xFD, 0xA0, 0x02, 0x82, 0xA1, 0x01, 0xE1, 0x65, 0xFD, 0x21, 0x63, 0xF8, 0xA1, 0x01, 0xE1, 0x69,
0xFD, 0xCB, 0x02, 0x01, 0x64, 0x68, 0x6C, 0x6E, 0x70, 0x72, 0x73, 0x74, 0x7A, 0x2E, 0x27, 0xFD,
0xB1, 0xFD, 0xC3, 0xFD, 0xC3, 0xFF, 0xF3, 0xFD, 0xB1, 0xFD, 0xC3, 0xFF, 0xFB, 0xFD, 0xB1, 0xFD,
0xB1, 0xFD, 0xB1, 0xFD, 0xB1, 0xC3, 0x02, 0x01, 0x71, 0x2E, 0x27, 0xFD, 0x8D, 0xFD, 0x8D, 0xFD,
0x8D, 0xA0, 0x02, 0x53, 0xA1, 0x01, 0xE1, 0x73, 0xFD, 0xD5, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66,
0x68, 0x67, 0x6B, 0x6C, 0x6D, 0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x78, 0x77, 0x7A, 0x2E,
0x27, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x8B, 0xFD, 0x79, 0xFD, 0x79, 0xFD,
0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFF, 0xFB, 0xFD,
0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0x43, 0x6D, 0x2E, 0x27, 0xFD,
0x37, 0xFD, 0x37, 0xFD, 0x37, 0xA0, 0x02, 0xC2, 0xA1, 0x02, 0x32, 0x6D, 0xFD, 0x41, 0x6E, 0xFE,
0x8F, 0x4B, 0x62, 0x63, 0x64, 0x66, 0x67, 0x6D, 0x6E, 0x70, 0x73, 0x74, 0x76, 0xFD, 0x21, 0xFD,
0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD,
0x21, 0xFD, 0x21, 0xA0, 0x02, 0xE1, 0x22, 0x2E, 0x27, 0xFD, 0xFD, 0xC7, 0x02, 0xA2, 0x68, 0x73,
0x70, 0x74, 0x7A, 0x2E, 0x27, 0xFF, 0xC0, 0xFF, 0xCD, 0xFF, 0xD2, 0xFF, 0xD6, 0xFC, 0xF7, 0xFF,
0xF8, 0xFF, 0xFB, 0xC1, 0x00, 0x51, 0x2E, 0xFC, 0xDF, 0x41, 0x68, 0xFF, 0x18, 0xA1, 0x00, 0x51,
0x63, 0xFC, 0xC1, 0x01, 0xE1, 0x73, 0xFE, 0xB6, 0xC2, 0x00, 0x51, 0x6B, 0x73, 0xFC, 0xCA, 0xFC,
0x06, 0xD2, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x67, 0x68, 0x6C, 0x6D, 0x6E, 0x70, 0x72, 0x73,
0x74, 0x76, 0x77, 0x7A, 0x2E, 0x27, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1,
0xFF, 0xE2, 0xFC, 0xD3, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xD3, 0xFF, 0xEC, 0xFF, 0xF1,
0xFC, 0xC1, 0xFC, 0xC1, 0xFF, 0xF7, 0xFC, 0xC1, 0xFE, 0x2E, 0xC6, 0x02, 0x01, 0x63, 0x6C, 0x72,
0x76, 0x2E, 0x27, 0xFC, 0x88, 0xFC, 0x9A, 0xFC, 0x9A, 0xFC, 0x88, 0xFC, 0x88, 0xFD, 0xF5, 0x41,
0x72, 0xFB, 0xAF, 0xA0, 0x02, 0xF2, 0xC5, 0x02, 0x01, 0x68, 0x61, 0x79, 0x2E, 0x27, 0xFC, 0x7E,
0xFF, 0xF9, 0xFF, 0xFD, 0xFC, 0x6C, 0xFC, 0x6C, 0xCA, 0x02, 0x01, 0x62, 0x63, 0x66, 0x68, 0x6D,
0x70, 0x74, 0x77, 0x2E, 0x27, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC,
0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0x42, 0x6F, 0x69, 0xFC, 0x48, 0xFC, 0x27,
0xCB, 0x02, 0x01, 0x62, 0x64, 0x6C, 0x6E, 0x70, 0x74, 0x73, 0x76, 0x7A, 0x2E, 0x27, 0xFC, 0x32,
0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32,
0xFC, 0x32, 0xFD, 0x9F, 0x5A, 0x2E, 0x27, 0x61, 0x65, 0x6F, 0x62, 0x63, 0x64, 0x66, 0x67, 0x68,
0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xFB,
0xC2, 0xFB, 0xF9, 0xFC, 0x14, 0xFC, 0x23, 0xFC, 0x28, 0xFC, 0x2B, 0xFC, 0x64, 0xFC, 0x97, 0xFC,
0xC4, 0xFC, 0xED, 0xFD, 0x27, 0xFD, 0x4B, 0xFD, 0x54, 0xFD, 0x82, 0xFD, 0xC4, 0xFE, 0x11, 0xFE,
0x5D, 0xFE, 0x81, 0xFE, 0x95, 0xFF, 0x17, 0xFF, 0x4D, 0xFF, 0x86, 0xFF, 0xA2, 0xFF, 0xB4, 0xFF,
0xD5, 0xFF, 0xDC,
0xFE, 0x4A, 0xFE, 0x4A, 0xA0, 0x02, 0x62, 0xA1, 0x01, 0xE1, 0x6E, 0xFD, 0x21, 0x72, 0xF8, 0x21,
0x65, 0xFD, 0xA1, 0x01, 0xE1, 0x66, 0xFD, 0x41, 0x74, 0xFE, 0x07, 0x21, 0x69, 0xFC, 0x21, 0x65,
0xFD, 0xD3, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x67, 0x6B, 0x6C, 0x6D, 0x6E, 0x70, 0x71, 0x72,
0x73, 0x74, 0x76, 0x7A, 0x68, 0x2E, 0x27, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFF,
0xE6, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFF,
0xF1, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xFF, 0xFD, 0xFD, 0xFD, 0xFD, 0xFD, 0xA0, 0x02, 0x82,
0xA1, 0x01, 0xE1, 0x65, 0xFD, 0x21, 0x63, 0xF8, 0xA1, 0x01, 0xE1, 0x69, 0xFD, 0xCB, 0x02, 0x01,
0x64, 0x68, 0x6C, 0x6E, 0x70, 0x72, 0x73, 0x74, 0x7A, 0x2E, 0x27, 0xFD, 0xB1, 0xFD, 0xC3, 0xFD,
0xC3, 0xFF, 0xF3, 0xFD, 0xB1, 0xFD, 0xC3, 0xFF, 0xFB, 0xFD, 0xB1, 0xFD, 0xB1, 0xFD, 0xB1, 0xFD,
0xB1, 0xC3, 0x02, 0x01, 0x71, 0x2E, 0x27, 0xFD, 0x8D, 0xFD, 0x8D, 0xFD, 0x8D, 0xA0, 0x02, 0x53,
0xA1, 0x01, 0xE1, 0x73, 0xFD, 0xD5, 0x02, 0x01, 0x62, 0x63, 0x64, 0x66, 0x68, 0x67, 0x6B, 0x6C,
0x6D, 0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x78, 0x77, 0x7A, 0x2E, 0x27, 0xFD, 0x79, 0xFD,
0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x8B, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD,
0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFF, 0xFB, 0xFD, 0x79, 0xFD, 0x79, 0xFD,
0x79, 0xFD, 0x79, 0xFD, 0x79, 0xFD, 0x79, 0x43, 0x6D, 0x2E, 0x27, 0xFD, 0x37, 0xFD, 0x37, 0xFD,
0x37, 0xA0, 0x02, 0xC2, 0xA1, 0x02, 0x32, 0x6D, 0xFD, 0x41, 0x6E, 0xFE, 0x8F, 0x4B, 0x62, 0x63,
0x64, 0x66, 0x67, 0x6D, 0x6E, 0x70, 0x73, 0x74, 0x76, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD,
0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xFD, 0x21, 0xA0,
0x02, 0xE1, 0x22, 0x2E, 0x27, 0xFD, 0xFD, 0xC7, 0x02, 0xA2, 0x68, 0x73, 0x70, 0x74, 0x7A, 0x2E,
0x27, 0xFF, 0xC0, 0xFF, 0xCD, 0xFF, 0xD2, 0xFF, 0xD6, 0xFC, 0xF7, 0xFF, 0xF8, 0xFF, 0xFB, 0xC1,
0x00, 0x51, 0x2E, 0xFC, 0xDF, 0x41, 0x68, 0xFF, 0x18, 0xA1, 0x00, 0x51, 0x63, 0xFC, 0xC1, 0x01,
0xE1, 0x73, 0xFE, 0xB6, 0xC2, 0x00, 0x51, 0x6B, 0x73, 0xFC, 0xCA, 0xFC, 0x06, 0xD2, 0x02, 0x01,
0x62, 0x63, 0x64, 0x66, 0x67, 0x68, 0x6C, 0x6D, 0x6E, 0x70, 0x72, 0x73, 0x74, 0x76, 0x77, 0x7A,
0x2E, 0x27, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1, 0xFF, 0xE2, 0xFC, 0xD3,
0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xC1, 0xFC, 0xD3, 0xFF, 0xEC, 0xFF, 0xF1, 0xFC, 0xC1, 0xFC, 0xC1,
0xFF, 0xF7, 0xFC, 0xC1, 0xFE, 0x2E, 0xC6, 0x02, 0x01, 0x63, 0x6C, 0x72, 0x76, 0x2E, 0x27, 0xFC,
0x88, 0xFC, 0x9A, 0xFC, 0x9A, 0xFC, 0x88, 0xFC, 0x88, 0xFD, 0xF5, 0x41, 0x72, 0xFB, 0xAF, 0xA0,
0x02, 0xF2, 0xC5, 0x02, 0x01, 0x68, 0x61, 0x79, 0x2E, 0x27, 0xFC, 0x7E, 0xFF, 0xF9, 0xFF, 0xFD,
0xFC, 0x6C, 0xFC, 0x6C, 0xCA, 0x02, 0x01, 0x62, 0x63, 0x66, 0x68, 0x6D, 0x70, 0x74, 0x77, 0x2E,
0x27, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0xFC,
0x5A, 0xFC, 0x5A, 0xFC, 0x5A, 0x42, 0x6F, 0x69, 0xFC, 0x48, 0xFC, 0x27, 0xCB, 0x02, 0x01, 0x62,
0x64, 0x6C, 0x6E, 0x70, 0x74, 0x73, 0x76, 0x7A, 0x2E, 0x27, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32,
0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFC, 0x32, 0xFD, 0x9F,
0x5A, 0x2E, 0x27, 0x61, 0x65, 0x6F, 0x62, 0x63, 0x64, 0x66, 0x67, 0x68, 0x6A, 0x6B, 0x6C, 0x6D,
0x6E, 0x70, 0x71, 0x72, 0x73, 0x74, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xFB, 0xC2, 0xFB, 0xF9, 0xFC,
0x14, 0xFC, 0x23, 0xFC, 0x28, 0xFC, 0x2B, 0xFC, 0x64, 0xFC, 0x97, 0xFC, 0xC4, 0xFC, 0xED, 0xFD,
0x27, 0xFD, 0x4B, 0xFD, 0x54, 0xFD, 0x82, 0xFD, 0xC4, 0xFE, 0x11, 0xFE, 0x5D, 0xFE, 0x81, 0xFE,
0x95, 0xFF, 0x17, 0xFF, 0x4D, 0xFF, 0x86, 0xFF, 0xA2, 0xFF, 0xB4, 0xFF, 0xD5, 0xFF, 0xDC,
};
constexpr SerializedHyphenationPatterns it_patterns = {
0x5C0u,
it_trie_data,
sizeof(it_trie_data),
};
lib/Epub/Epub/hyphenation/generated/hyph-ru.trie.h
+2089 -1759
View File
File diff suppressed because it is too large Load Diff
scripts/generate_hyphenation_trie.py
+20 -1
View File
@@ -33,10 +33,28 @@ def _symbol_from_output(path: pathlib.Path) -> str:
def write_header(path: pathlib.Path, blob: bytes, symbol: str) -> None:
# Emit a constexpr header containing the raw bytes plus a SerializedHyphenationPatterns descriptor.
# The binary format has:
# - 4 bytes: big-endian root address
# - levels tape: from byte 4 to root_addr
# - nodes data: from root_addr onwards
if len(blob) < 4:
raise ValueError(f"Blob too small: {len(blob)} bytes")
# Parse root address (big-endian uint32)
root_addr = (blob[0] << 24) | (blob[1] << 16) | (blob[2] << 8) | blob[3]
if root_addr > len(blob):
raise ValueError(f"Root address {root_addr} exceeds blob size {len(blob)}")
# Remove the 4-byte root address and adjust the offset
bytes_literal = _format_bytes(blob[4:])
root_addr_new = root_addr - 4
path.parent.mkdir(parents=True, exist_ok=True)
data_symbol = f"{symbol}_trie_data"
patterns_symbol = f"{symbol}_patterns"
bytes_literal = _format_bytes(blob)
content = f"""#pragma once
#include <cstddef>
@@ -50,6 +68,7 @@ alignas(4) constexpr uint8_t {data_symbol}[] = {{
}};
constexpr SerializedHyphenationPatterns {patterns_symbol} = {{
{f"0x{root_addr_new:02X}"}u,
{data_symbol},
sizeof({data_symbol}),
}};
scripts/update_hypenation.sh
Executable
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
process() {
local lang="$1"
mkdir -p "build"
wget -O "build/$lang.bin" "https://github.com/typst/hypher/raw/refs/heads/main/tries/$lang.bin"
python scripts/generate_hyphenation_trie.py \
--input "build/$lang.bin" \
--output "lib/Epub/Epub/hyphenation/generated/hyph-${lang}.trie.h"
}
process en
process fr
process de
process es
process ru
process it