mirror of
https://github.com/crosspoint-reader/crosspoint-reader.git
synced 2026-04-29 10:26:52 -07:00
feat: dict based Hyphenation (#305)
## Summary * Adds (optional) Hyphenation for English, French, German, Russian languages ## Additional Context * Included hyphenation dictionaries add approximately 280kb to the flash usage (German alone takes 200kb) * Trie encoded dictionaries are adopted from hypher project (https://github.com/typst/hypher) * Soft hyphens (and other explicit hyphens) take precedence over dict-based hyphenation. Overall, the hyphenation rules are quite aggressive, as I believe it makes more sense on our smaller screen. --------- Co-authored-by: Dave Allie <dave@daveallie.com>
This commit is contained in:
Arthur Tazhitdinov
co-authored by
Dave Allie
Dave Allie
parent
5fef99c641
commit
8824c87490
@@ -4,3 +4,5 @@
|
||||
.vscode
|
||||
lib/EpdFont/fontsrc
|
||||
*.generated.h
|
||||
build
|
||||
**/__pycache__/
|
||||
@@ -0,0 +1,66 @@
|
||||
# Hypher Binary Tries
|
||||
|
||||
CrossPoint embeds the exact binary automata produced by
|
||||
[Typst's `hypher`](https://github.com/typst/hypher).
|
||||
|
||||
## File layout
|
||||
|
||||
Each `.bin` blob is a single self-contained automaton:
|
||||
|
||||
```
|
||||
uint32_t root_addr_be; // big-endian offset of the root node
|
||||
uint8_t levels[]; // shared "levels" tape (dist/score pairs)
|
||||
uint8_t nodes[]; // node records packed back-to-back
|
||||
```
|
||||
|
||||
The size of the `levels` tape is implicit. Individual nodes reference slices
|
||||
inside that tape via 12-bit offsets, so no additional pointers are required.
|
||||
|
||||
### Node encoding
|
||||
|
||||
Every node starts with a single control byte:
|
||||
|
||||
- Bit 7 – set when the node stores scores (`levels`).
|
||||
- Bits 5-6 – stride of the target deltas (1, 2, or 3 bytes, big-endian).
|
||||
- Bits 0-4 – transition count (values ≥ 31 spill into an extra byte).
|
||||
|
||||
If the `levels` flag is set, two more bytes follow. Together they encode a
|
||||
12-bit offset into the global `levels` tape and a 4-bit length. Each byte in the
|
||||
levels tape packs a distance/score pair as `dist * 10 + score`, where `dist`
|
||||
counts how many UTF-8 bytes we advanced since the previous digit.
|
||||
|
||||
After the optional levels header come the transition labels (one byte per edge)
|
||||
followed by the signed target deltas. Targets are stored as relative offsets
|
||||
from the current node address. Deltas up to ±128 fit in a single byte, larger
|
||||
distances grow to 2 or 3 bytes. The runtime walks the transitions with a simple
|
||||
linear scan and materializes the absolute address by adding the decoded delta
|
||||
to the current node’s base.
|
||||
|
||||
## Embedding blobs into the firmware
|
||||
|
||||
The helper script `scripts/generate_hyphenation_trie.py` acts as a thin
|
||||
wrapper: it reads the hypher-generated `.bin` files, formats them as `constexpr`
|
||||
byte arrays, and emits headers under
|
||||
`lib/Epub/Epub/hyphenation/generated/`. Each header defines the raw data plus a
|
||||
`SerializedHyphenationPatterns` descriptor so the reader can keep the automaton
|
||||
in flash.
|
||||
|
||||
To refresh the firmware assets after updating the `.bin` files, run:
|
||||
|
||||
```
|
||||
./scripts/generate_hyphenation_trie.py \
|
||||
--input lib/Epub/Epub/hyphenation/tries/en.bin \
|
||||
--output lib/Epub/Epub/hyphenation/generated/hyph-en.trie.h
|
||||
|
||||
./scripts/generate_hyphenation_trie.py \
|
||||
--input lib/Epub/Epub/hyphenation/tries/fr.bin \
|
||||
--output lib/Epub/Epub/hyphenation/generated/hyph-fr.trie.h
|
||||
|
||||
./scripts/generate_hyphenation_trie.py \
|
||||
--input lib/Epub/Epub/hyphenation/tries/de.bin \
|
||||
--output lib/Epub/Epub/hyphenation/generated/hyph-de.trie.h
|
||||
|
||||
./scripts/generate_hyphenation_trie.py \
|
||||
--input lib/Epub/Epub/hyphenation/tries/ru.bin \
|
||||
--output lib/Epub/Epub/hyphenation/generated/hyph-ru.trie.h
|
||||
```
|
||||
@@ -74,6 +74,7 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) {
|
||||
// Grab data from opfParser into epub
|
||||
bookMetadata.title = opfParser.title;
|
||||
bookMetadata.author = opfParser.author;
|
||||
bookMetadata.language = opfParser.language;
|
||||
bookMetadata.coverItemHref = opfParser.coverItemHref;
|
||||
bookMetadata.textReferenceHref = opfParser.textReferenceHref;
|
||||
|
||||
@@ -348,6 +349,15 @@ const std::string& Epub::getAuthor() const {
|
||||
return bookMetadataCache->coreMetadata.author;
|
||||
}
|
||||
|
||||
const std::string& Epub::getLanguage() const {
|
||||
static std::string blank;
|
||||
if (!bookMetadataCache || !bookMetadataCache->isLoaded()) {
|
||||
return blank;
|
||||
}
|
||||
|
||||
return bookMetadataCache->coreMetadata.language;
|
||||
}
|
||||
|
||||
std::string Epub::getCoverBmpPath(bool cropped) const {
|
||||
const auto coverFileName = "cover" + cropped ? "_crop" : "";
|
||||
return cachePath + "/" + coverFileName + ".bmp";
|
||||
|
||||
@@ -44,6 +44,7 @@ class Epub {
|
||||
const std::string& getPath() const;
|
||||
const std::string& getTitle() const;
|
||||
const std::string& getAuthor() const;
|
||||
const std::string& getLanguage() const;
|
||||
std::string getCoverBmpPath(bool cropped = false) const;
|
||||
bool generateCoverBmp(bool cropped = false) const;
|
||||
std::string getThumbBmpPath() const;
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
#include "FsHelpers.h"
|
||||
|
||||
namespace {
|
||||
constexpr uint8_t BOOK_CACHE_VERSION = 4;
|
||||
constexpr uint8_t BOOK_CACHE_VERSION = 5;
|
||||
constexpr char bookBinFile[] = "/book.bin";
|
||||
constexpr char tmpSpineBinFile[] = "/spine.bin.tmp";
|
||||
constexpr char tmpTocBinFile[] = "/toc.bin.tmp";
|
||||
@@ -87,8 +87,9 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
|
||||
|
||||
constexpr uint32_t headerASize =
|
||||
sizeof(BOOK_CACHE_VERSION) + /* LUT Offset */ sizeof(uint32_t) + sizeof(spineCount) + sizeof(tocCount);
|
||||
const uint32_t metadataSize = metadata.title.size() + metadata.author.size() + metadata.coverItemHref.size() +
|
||||
metadata.textReferenceHref.size() + sizeof(uint32_t) * 4;
|
||||
const uint32_t metadataSize = metadata.title.size() + metadata.author.size() + metadata.language.size() +
|
||||
metadata.coverItemHref.size() + metadata.textReferenceHref.size() +
|
||||
sizeof(uint32_t) * 5;
|
||||
const uint32_t lutSize = sizeof(uint32_t) * spineCount + sizeof(uint32_t) * tocCount;
|
||||
const uint32_t lutOffset = headerASize + metadataSize;
|
||||
|
||||
@@ -100,6 +101,7 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta
|
||||
// Metadata
|
||||
serialization::writeString(bookFile, metadata.title);
|
||||
serialization::writeString(bookFile, metadata.author);
|
||||
serialization::writeString(bookFile, metadata.language);
|
||||
serialization::writeString(bookFile, metadata.coverItemHref);
|
||||
serialization::writeString(bookFile, metadata.textReferenceHref);
|
||||
|
||||
@@ -289,6 +291,7 @@ bool BookMetadataCache::load() {
|
||||
|
||||
serialization::readString(bookFile, coreMetadata.title);
|
||||
serialization::readString(bookFile, coreMetadata.author);
|
||||
serialization::readString(bookFile, coreMetadata.language);
|
||||
serialization::readString(bookFile, coreMetadata.coverItemHref);
|
||||
serialization::readString(bookFile, coreMetadata.textReferenceHref);
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ class BookMetadataCache {
|
||||
struct BookMetadata {
|
||||
std::string title;
|
||||
std::string author;
|
||||
std::string language;
|
||||
std::string coverItemHref;
|
||||
std::string textReferenceHref;
|
||||
};
|
||||
|
||||
+205
-12
@@ -5,11 +5,50 @@
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <vector>
|
||||
|
||||
#include "hyphenation/Hyphenator.h"
|
||||
|
||||
constexpr int MAX_COST = std::numeric_limits<int>::max();
|
||||
|
||||
namespace {
|
||||
|
||||
// Soft hyphen byte pattern used throughout EPUBs (UTF-8 for U+00AD).
|
||||
constexpr char SOFT_HYPHEN_UTF8[] = "\xC2\xAD";
|
||||
constexpr size_t SOFT_HYPHEN_BYTES = 2;
|
||||
|
||||
bool containsSoftHyphen(const std::string& word) { return word.find(SOFT_HYPHEN_UTF8) != std::string::npos; }
|
||||
|
||||
// Removes every soft hyphen in-place so rendered glyphs match measured widths.
|
||||
void stripSoftHyphensInPlace(std::string& word) {
|
||||
size_t pos = 0;
|
||||
while ((pos = word.find(SOFT_HYPHEN_UTF8, pos)) != std::string::npos) {
|
||||
word.erase(pos, SOFT_HYPHEN_BYTES);
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the rendered width for a word while ignoring soft hyphen glyphs and optionally appending a visible hyphen.
|
||||
uint16_t measureWordWidth(const GfxRenderer& renderer, const int fontId, const std::string& word,
|
||||
const EpdFontFamily::Style style, const bool appendHyphen = false) {
|
||||
const bool hasSoftHyphen = containsSoftHyphen(word);
|
||||
if (!hasSoftHyphen && !appendHyphen) {
|
||||
return renderer.getTextWidth(fontId, word.c_str(), style);
|
||||
}
|
||||
|
||||
std::string sanitized = word;
|
||||
if (hasSoftHyphen) {
|
||||
stripSoftHyphensInPlace(sanitized);
|
||||
}
|
||||
if (appendHyphen) {
|
||||
sanitized.push_back('-');
|
||||
}
|
||||
return renderer.getTextWidth(fontId, sanitized.c_str(), style);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void ParsedText::addWord(std::string word, const EpdFontFamily::Style fontStyle) {
|
||||
if (word.empty()) return;
|
||||
|
||||
@@ -25,10 +64,19 @@ void ParsedText::layoutAndExtractLines(const GfxRenderer& renderer, const int fo
|
||||
return;
|
||||
}
|
||||
|
||||
// Apply fixed transforms before any per-line layout work.
|
||||
applyParagraphIndent();
|
||||
|
||||
const int pageWidth = viewportWidth;
|
||||
const int spaceWidth = renderer.getSpaceWidth(fontId);
|
||||
const auto wordWidths = calculateWordWidths(renderer, fontId);
|
||||
const auto lineBreakIndices = computeLineBreaks(pageWidth, spaceWidth, wordWidths);
|
||||
auto wordWidths = calculateWordWidths(renderer, fontId);
|
||||
std::vector<size_t> lineBreakIndices;
|
||||
if (hyphenationEnabled) {
|
||||
// Use greedy layout that can split words mid-loop when a hyphenated prefix fits.
|
||||
lineBreakIndices = computeHyphenatedLineBreaks(renderer, fontId, pageWidth, spaceWidth, wordWidths);
|
||||
} else {
|
||||
lineBreakIndices = computeLineBreaks(renderer, fontId, pageWidth, spaceWidth, wordWidths);
|
||||
}
|
||||
const size_t lineCount = includeLastLine ? lineBreakIndices.size() : lineBreakIndices.size() - 1;
|
||||
|
||||
for (size_t i = 0; i < lineCount; ++i) {
|
||||
@@ -42,17 +90,11 @@ std::vector<uint16_t> ParsedText::calculateWordWidths(const GfxRenderer& rendere
|
||||
std::vector<uint16_t> wordWidths;
|
||||
wordWidths.reserve(totalWordCount);
|
||||
|
||||
// add em-space at the beginning of first word in paragraph to indent
|
||||
if ((style == TextBlock::JUSTIFIED || style == TextBlock::LEFT_ALIGN) && !extraParagraphSpacing) {
|
||||
std::string& first_word = words.front();
|
||||
first_word.insert(0, "\xe2\x80\x83");
|
||||
}
|
||||
|
||||
auto wordsIt = words.begin();
|
||||
auto wordStylesIt = wordStyles.begin();
|
||||
|
||||
while (wordsIt != words.end()) {
|
||||
wordWidths.push_back(renderer.getTextWidth(fontId, wordsIt->c_str(), *wordStylesIt));
|
||||
wordWidths.push_back(measureWordWidth(renderer, fontId, *wordsIt, *wordStylesIt));
|
||||
|
||||
std::advance(wordsIt, 1);
|
||||
std::advance(wordStylesIt, 1);
|
||||
@@ -61,8 +103,21 @@ std::vector<uint16_t> ParsedText::calculateWordWidths(const GfxRenderer& rendere
|
||||
return wordWidths;
|
||||
}
|
||||
|
||||
std::vector<size_t> ParsedText::computeLineBreaks(const int pageWidth, const int spaceWidth,
|
||||
const std::vector<uint16_t>& wordWidths) const {
|
||||
std::vector<size_t> ParsedText::computeLineBreaks(const GfxRenderer& renderer, const int fontId, const int pageWidth,
|
||||
const int spaceWidth, std::vector<uint16_t>& wordWidths) {
|
||||
if (words.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Ensure any word that would overflow even as the first entry on a line is split using fallback hyphenation.
|
||||
for (size_t i = 0; i < wordWidths.size(); ++i) {
|
||||
while (wordWidths[i] > pageWidth) {
|
||||
if (!hyphenateWordAtIndex(i, pageWidth, renderer, fontId, wordWidths, /*allowFallbackBreaks=*/true)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const size_t totalWordCount = words.size();
|
||||
|
||||
// DP table to store the minimum badness (cost) of lines starting at index i
|
||||
@@ -140,6 +195,138 @@ std::vector<size_t> ParsedText::computeLineBreaks(const int pageWidth, const int
|
||||
return lineBreakIndices;
|
||||
}
|
||||
|
||||
void ParsedText::applyParagraphIndent() {
|
||||
if (extraParagraphSpacing || words.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (style == TextBlock::JUSTIFIED || style == TextBlock::LEFT_ALIGN) {
|
||||
words.front().insert(0, "\xe2\x80\x83");
|
||||
}
|
||||
}
|
||||
|
||||
// Builds break indices while opportunistically splitting the word that would overflow the current line.
|
||||
std::vector<size_t> ParsedText::computeHyphenatedLineBreaks(const GfxRenderer& renderer, const int fontId,
|
||||
const int pageWidth, const int spaceWidth,
|
||||
std::vector<uint16_t>& wordWidths) {
|
||||
std::vector<size_t> lineBreakIndices;
|
||||
size_t currentIndex = 0;
|
||||
|
||||
while (currentIndex < wordWidths.size()) {
|
||||
const size_t lineStart = currentIndex;
|
||||
int lineWidth = 0;
|
||||
|
||||
// Consume as many words as possible for current line, splitting when prefixes fit
|
||||
while (currentIndex < wordWidths.size()) {
|
||||
const bool isFirstWord = currentIndex == lineStart;
|
||||
const int spacing = isFirstWord ? 0 : spaceWidth;
|
||||
const int candidateWidth = spacing + wordWidths[currentIndex];
|
||||
|
||||
// Word fits on current line
|
||||
if (lineWidth + candidateWidth <= pageWidth) {
|
||||
lineWidth += candidateWidth;
|
||||
++currentIndex;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Word would overflow — try to split based on hyphenation points
|
||||
const int availableWidth = pageWidth - lineWidth - spacing;
|
||||
const bool allowFallbackBreaks = isFirstWord; // Only for first word on line
|
||||
|
||||
if (availableWidth > 0 &&
|
||||
hyphenateWordAtIndex(currentIndex, availableWidth, renderer, fontId, wordWidths, allowFallbackBreaks)) {
|
||||
// Prefix now fits; append it to this line and move to next line
|
||||
lineWidth += spacing + wordWidths[currentIndex];
|
||||
++currentIndex;
|
||||
break;
|
||||
}
|
||||
|
||||
// Could not split: force at least one word per line to avoid infinite loop
|
||||
if (currentIndex == lineStart) {
|
||||
lineWidth += candidateWidth;
|
||||
++currentIndex;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
lineBreakIndices.push_back(currentIndex);
|
||||
}
|
||||
|
||||
return lineBreakIndices;
|
||||
}
|
||||
|
||||
// Splits words[wordIndex] into prefix (adding a hyphen only when needed) and remainder when a legal breakpoint fits the
|
||||
// available width.
|
||||
bool ParsedText::hyphenateWordAtIndex(const size_t wordIndex, const int availableWidth, const GfxRenderer& renderer,
|
||||
const int fontId, std::vector<uint16_t>& wordWidths,
|
||||
const bool allowFallbackBreaks) {
|
||||
// Guard against invalid indices or zero available width before attempting to split.
|
||||
if (availableWidth <= 0 || wordIndex >= words.size()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get iterators to target word and style.
|
||||
auto wordIt = words.begin();
|
||||
auto styleIt = wordStyles.begin();
|
||||
std::advance(wordIt, wordIndex);
|
||||
std::advance(styleIt, wordIndex);
|
||||
|
||||
const std::string& word = *wordIt;
|
||||
const auto style = *styleIt;
|
||||
|
||||
// Collect candidate breakpoints (byte offsets and hyphen requirements).
|
||||
auto breakInfos = Hyphenator::breakOffsets(word, allowFallbackBreaks);
|
||||
if (breakInfos.empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t chosenOffset = 0;
|
||||
int chosenWidth = -1;
|
||||
bool chosenNeedsHyphen = true;
|
||||
|
||||
// Iterate over each legal breakpoint and retain the widest prefix that still fits.
|
||||
for (const auto& info : breakInfos) {
|
||||
const size_t offset = info.byteOffset;
|
||||
if (offset == 0 || offset >= word.size()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const bool needsHyphen = info.requiresInsertedHyphen;
|
||||
const int prefixWidth = measureWordWidth(renderer, fontId, word.substr(0, offset), style, needsHyphen);
|
||||
if (prefixWidth > availableWidth || prefixWidth <= chosenWidth) {
|
||||
continue; // Skip if too wide or not an improvement
|
||||
}
|
||||
|
||||
chosenWidth = prefixWidth;
|
||||
chosenOffset = offset;
|
||||
chosenNeedsHyphen = needsHyphen;
|
||||
}
|
||||
|
||||
if (chosenWidth < 0) {
|
||||
// No hyphenation point produced a prefix that fits in the remaining space.
|
||||
return false;
|
||||
}
|
||||
|
||||
// Split the word at the selected breakpoint and append a hyphen if required.
|
||||
std::string remainder = word.substr(chosenOffset);
|
||||
wordIt->resize(chosenOffset);
|
||||
if (chosenNeedsHyphen) {
|
||||
wordIt->push_back('-');
|
||||
}
|
||||
|
||||
// Insert the remainder word (with matching style) directly after the prefix.
|
||||
auto insertWordIt = std::next(wordIt);
|
||||
auto insertStyleIt = std::next(styleIt);
|
||||
words.insert(insertWordIt, remainder);
|
||||
wordStyles.insert(insertStyleIt, style);
|
||||
|
||||
// Update cached widths to reflect the new prefix/remainder pairing.
|
||||
wordWidths[wordIndex] = static_cast<uint16_t>(chosenWidth);
|
||||
const uint16_t remainderWidth = measureWordWidth(renderer, fontId, remainder, style);
|
||||
wordWidths.insert(wordWidths.begin() + wordIndex + 1, remainderWidth);
|
||||
return true;
|
||||
}
|
||||
|
||||
void ParsedText::extractLine(const size_t breakIndex, const int pageWidth, const int spaceWidth,
|
||||
const std::vector<uint16_t>& wordWidths, const std::vector<size_t>& lineBreakIndices,
|
||||
const std::function<void(std::shared_ptr<TextBlock>)>& processLine) {
|
||||
@@ -191,5 +378,11 @@ void ParsedText::extractLine(const size_t breakIndex, const int pageWidth, const
|
||||
std::list<EpdFontFamily::Style> lineWordStyles;
|
||||
lineWordStyles.splice(lineWordStyles.begin(), wordStyles, wordStyles.begin(), wordStyleEndIt);
|
||||
|
||||
for (auto& word : lineWords) {
|
||||
if (containsSoftHyphen(word)) {
|
||||
stripSoftHyphensInPlace(word);
|
||||
}
|
||||
}
|
||||
|
||||
processLine(std::make_shared<TextBlock>(std::move(lineWords), std::move(lineXPos), std::move(lineWordStyles), style));
|
||||
}
|
||||
}
|
||||
@@ -17,16 +17,24 @@ class ParsedText {
|
||||
std::list<EpdFontFamily::Style> wordStyles;
|
||||
TextBlock::Style style;
|
||||
bool extraParagraphSpacing;
|
||||
bool hyphenationEnabled;
|
||||
|
||||
std::vector<size_t> computeLineBreaks(int pageWidth, int spaceWidth, const std::vector<uint16_t>& wordWidths) const;
|
||||
void applyParagraphIndent();
|
||||
std::vector<size_t> computeLineBreaks(const GfxRenderer& renderer, int fontId, int pageWidth, int spaceWidth,
|
||||
std::vector<uint16_t>& wordWidths);
|
||||
std::vector<size_t> computeHyphenatedLineBreaks(const GfxRenderer& renderer, int fontId, int pageWidth,
|
||||
int spaceWidth, std::vector<uint16_t>& wordWidths);
|
||||
bool hyphenateWordAtIndex(size_t wordIndex, int availableWidth, const GfxRenderer& renderer, int fontId,
|
||||
std::vector<uint16_t>& wordWidths, bool allowFallbackBreaks);
|
||||
void extractLine(size_t breakIndex, int pageWidth, int spaceWidth, const std::vector<uint16_t>& wordWidths,
|
||||
const std::vector<size_t>& lineBreakIndices,
|
||||
const std::function<void(std::shared_ptr<TextBlock>)>& processLine);
|
||||
std::vector<uint16_t> calculateWordWidths(const GfxRenderer& renderer, int fontId);
|
||||
|
||||
public:
|
||||
explicit ParsedText(const TextBlock::Style style, const bool extraParagraphSpacing)
|
||||
: style(style), extraParagraphSpacing(extraParagraphSpacing) {}
|
||||
explicit ParsedText(const TextBlock::Style style, const bool extraParagraphSpacing,
|
||||
const bool hyphenationEnabled = false)
|
||||
: style(style), extraParagraphSpacing(extraParagraphSpacing), hyphenationEnabled(hyphenationEnabled) {}
|
||||
~ParsedText() = default;
|
||||
|
||||
void addWord(std::string word, EpdFontFamily::Style fontStyle);
|
||||
@@ -37,4 +45,4 @@ class ParsedText {
|
||||
void layoutAndExtractLines(const GfxRenderer& renderer, int fontId, uint16_t viewportWidth,
|
||||
const std::function<void(std::shared_ptr<TextBlock>)>& processLine,
|
||||
bool includeLastLine = true);
|
||||
};
|
||||
};
|
||||
@@ -4,12 +4,14 @@
|
||||
#include <Serialization.h>
|
||||
|
||||
#include "Page.h"
|
||||
#include "hyphenation/Hyphenator.h"
|
||||
#include "parsers/ChapterHtmlSlimParser.h"
|
||||
|
||||
namespace {
|
||||
constexpr uint8_t SECTION_FILE_VERSION = 9;
|
||||
constexpr uint8_t SECTION_FILE_VERSION = 10;
|
||||
constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(uint8_t) +
|
||||
sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t);
|
||||
sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(bool) +
|
||||
sizeof(uint32_t);
|
||||
} // namespace
|
||||
|
||||
uint32_t Section::onPageComplete(std::unique_ptr<Page> page) {
|
||||
@@ -31,14 +33,15 @@ uint32_t Section::onPageComplete(std::unique_ptr<Page> page) {
|
||||
|
||||
void Section::writeSectionFileHeader(const int fontId, const float lineCompression, const bool extraParagraphSpacing,
|
||||
const uint8_t paragraphAlignment, const uint16_t viewportWidth,
|
||||
const uint16_t viewportHeight) {
|
||||
const uint16_t viewportHeight, const bool hyphenationEnabled) {
|
||||
if (!file) {
|
||||
Serial.printf("[%lu] [SCT] File not open for writing header\n", millis());
|
||||
return;
|
||||
}
|
||||
static_assert(HEADER_SIZE == sizeof(SECTION_FILE_VERSION) + sizeof(fontId) + sizeof(lineCompression) +
|
||||
sizeof(extraParagraphSpacing) + sizeof(paragraphAlignment) + sizeof(viewportWidth) +
|
||||
sizeof(viewportHeight) + sizeof(pageCount) + sizeof(uint32_t),
|
||||
sizeof(viewportHeight) + sizeof(pageCount) + sizeof(hyphenationEnabled) +
|
||||
sizeof(uint32_t),
|
||||
"Header size mismatch");
|
||||
serialization::writePod(file, SECTION_FILE_VERSION);
|
||||
serialization::writePod(file, fontId);
|
||||
@@ -47,13 +50,14 @@ void Section::writeSectionFileHeader(const int fontId, const float lineCompressi
|
||||
serialization::writePod(file, paragraphAlignment);
|
||||
serialization::writePod(file, viewportWidth);
|
||||
serialization::writePod(file, viewportHeight);
|
||||
serialization::writePod(file, hyphenationEnabled);
|
||||
serialization::writePod(file, pageCount); // Placeholder for page count (will be initially 0 when written)
|
||||
serialization::writePod(file, static_cast<uint32_t>(0)); // Placeholder for LUT offset
|
||||
}
|
||||
|
||||
bool Section::loadSectionFile(const int fontId, const float lineCompression, const bool extraParagraphSpacing,
|
||||
const uint8_t paragraphAlignment, const uint16_t viewportWidth,
|
||||
const uint16_t viewportHeight) {
|
||||
const uint16_t viewportHeight, const bool hyphenationEnabled) {
|
||||
if (!SdMan.openFileForRead("SCT", filePath, file)) {
|
||||
return false;
|
||||
}
|
||||
@@ -74,16 +78,19 @@ bool Section::loadSectionFile(const int fontId, const float lineCompression, con
|
||||
float fileLineCompression;
|
||||
bool fileExtraParagraphSpacing;
|
||||
uint8_t fileParagraphAlignment;
|
||||
bool fileHyphenationEnabled;
|
||||
serialization::readPod(file, fileFontId);
|
||||
serialization::readPod(file, fileLineCompression);
|
||||
serialization::readPod(file, fileExtraParagraphSpacing);
|
||||
serialization::readPod(file, fileParagraphAlignment);
|
||||
serialization::readPod(file, fileViewportWidth);
|
||||
serialization::readPod(file, fileViewportHeight);
|
||||
serialization::readPod(file, fileHyphenationEnabled);
|
||||
|
||||
if (fontId != fileFontId || lineCompression != fileLineCompression ||
|
||||
extraParagraphSpacing != fileExtraParagraphSpacing || paragraphAlignment != fileParagraphAlignment ||
|
||||
viewportWidth != fileViewportWidth || viewportHeight != fileViewportHeight) {
|
||||
viewportWidth != fileViewportWidth || viewportHeight != fileViewportHeight ||
|
||||
hyphenationEnabled != fileHyphenationEnabled) {
|
||||
file.close();
|
||||
Serial.printf("[%lu] [SCT] Deserialization failed: Parameters do not match\n", millis());
|
||||
clearCache();
|
||||
@@ -115,7 +122,8 @@ bool Section::clearCache() const {
|
||||
|
||||
bool Section::createSectionFile(const int fontId, const float lineCompression, const bool extraParagraphSpacing,
|
||||
const uint8_t paragraphAlignment, const uint16_t viewportWidth,
|
||||
const uint16_t viewportHeight, const std::function<void()>& progressSetupFn,
|
||||
const uint16_t viewportHeight, const bool hyphenationEnabled,
|
||||
const std::function<void()>& progressSetupFn,
|
||||
const std::function<void(int)>& progressFn) {
|
||||
constexpr uint32_t MIN_SIZE_FOR_PROGRESS = 50 * 1024; // 50KB
|
||||
const auto localPath = epub->getSpineItem(spineIndex).href;
|
||||
@@ -172,14 +180,15 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
|
||||
return false;
|
||||
}
|
||||
writeSectionFileHeader(fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth,
|
||||
viewportHeight);
|
||||
viewportHeight, hyphenationEnabled);
|
||||
std::vector<uint32_t> lut = {};
|
||||
|
||||
ChapterHtmlSlimParser visitor(
|
||||
tmpHtmlPath, renderer, fontId, lineCompression, extraParagraphSpacing, paragraphAlignment, viewportWidth,
|
||||
viewportHeight,
|
||||
viewportHeight, hyphenationEnabled,
|
||||
[this, &lut](std::unique_ptr<Page> page) { lut.emplace_back(this->onPageComplete(std::move(page))); },
|
||||
progressFn);
|
||||
Hyphenator::setPreferredLanguage(epub->getLanguage());
|
||||
success = visitor.parseAndBuildPages();
|
||||
|
||||
SdMan.remove(tmpHtmlPath.c_str());
|
||||
|
||||
@@ -15,7 +15,7 @@ class Section {
|
||||
FsFile file;
|
||||
|
||||
void writeSectionFileHeader(int fontId, float lineCompression, bool extraParagraphSpacing, uint8_t paragraphAlignment,
|
||||
uint16_t viewportWidth, uint16_t viewportHeight);
|
||||
uint16_t viewportWidth, uint16_t viewportHeight, bool hyphenationEnabled);
|
||||
uint32_t onPageComplete(std::unique_ptr<Page> page);
|
||||
|
||||
public:
|
||||
@@ -29,10 +29,10 @@ class Section {
|
||||
filePath(epub->getCachePath() + "/sections/" + std::to_string(spineIndex) + ".bin") {}
|
||||
~Section() = default;
|
||||
bool loadSectionFile(int fontId, float lineCompression, bool extraParagraphSpacing, uint8_t paragraphAlignment,
|
||||
uint16_t viewportWidth, uint16_t viewportHeight);
|
||||
uint16_t viewportWidth, uint16_t viewportHeight, bool hyphenationEnabled);
|
||||
bool clearCache() const;
|
||||
bool createSectionFile(int fontId, float lineCompression, bool extraParagraphSpacing, uint8_t paragraphAlignment,
|
||||
uint16_t viewportWidth, uint16_t viewportHeight,
|
||||
uint16_t viewportWidth, uint16_t viewportHeight, bool hyphenationEnabled,
|
||||
const std::function<void()>& progressSetupFn = nullptr,
|
||||
const std::function<void(int)>& progressFn = nullptr);
|
||||
std::unique_ptr<Page> loadPageFromSectionFile();
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
#include "HyphenationCommon.h"
|
||||
|
||||
#include <Utf8.h>
|
||||
|
||||
namespace {
|
||||
|
||||
// Convert Latin uppercase letters (ASCII plus Latin-1 supplement) to lowercase
|
||||
uint32_t toLowerLatinImpl(const uint32_t cp) {
|
||||
if (cp >= 'A' && cp <= 'Z') {
|
||||
return cp - 'A' + 'a';
|
||||
}
|
||||
if ((cp >= 0x00C0 && cp <= 0x00D6) || (cp >= 0x00D8 && cp <= 0x00DE)) {
|
||||
return cp + 0x20;
|
||||
}
|
||||
|
||||
switch (cp) {
|
||||
case 0x0152: // Œ
|
||||
return 0x0153; // œ
|
||||
case 0x0178: // Ÿ
|
||||
return 0x00FF; // ÿ
|
||||
case 0x1E9E: // ẞ
|
||||
return 0x00DF; // ß
|
||||
default:
|
||||
return cp;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert Cyrillic uppercase letters to lowercase
|
||||
// Cyrillic uppercase range 0x0410-0x042F maps to lowercase by adding 0x20
|
||||
// Special case: Cyrillic capital IO (0x0401) maps to lowercase io (0x0451)
|
||||
uint32_t toLowerCyrillicImpl(const uint32_t cp) {
|
||||
if (cp >= 0x0410 && cp <= 0x042F) {
|
||||
return cp + 0x20;
|
||||
}
|
||||
if (cp == 0x0401) {
|
||||
return 0x0451;
|
||||
}
|
||||
return cp;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
uint32_t toLowerLatin(const uint32_t cp) { return toLowerLatinImpl(cp); }
|
||||
|
||||
uint32_t toLowerCyrillic(const uint32_t cp) { return toLowerCyrillicImpl(cp); }
|
||||
|
||||
bool isLatinLetter(const uint32_t cp) {
|
||||
if ((cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (((cp >= 0x00C0 && cp <= 0x00D6) || (cp >= 0x00D8 && cp <= 0x00F6) || (cp >= 0x00F8 && cp <= 0x00FF)) &&
|
||||
cp != 0x00D7 && cp != 0x00F7) {
|
||||
return true;
|
||||
}
|
||||
|
||||
switch (cp) {
|
||||
case 0x0152: // Œ
|
||||
case 0x0153: // œ
|
||||
case 0x0178: // Ÿ
|
||||
case 0x1E9E: // ẞ
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool isCyrillicLetter(const uint32_t cp) { return (cp >= 0x0400 && cp <= 0x052F); }
|
||||
|
||||
bool isAlphabetic(const uint32_t cp) { return isLatinLetter(cp) || isCyrillicLetter(cp); }
|
||||
|
||||
bool isPunctuation(const uint32_t cp) {
|
||||
switch (cp) {
|
||||
case '-':
|
||||
case '.':
|
||||
case ',':
|
||||
case '!':
|
||||
case '?':
|
||||
case ';':
|
||||
case ':':
|
||||
case '"':
|
||||
case '\'':
|
||||
case ')':
|
||||
case '(':
|
||||
case 0x00AB: // «
|
||||
case 0x00BB: // »
|
||||
case 0x2018: // ‘
|
||||
case 0x2019: // ’
|
||||
case 0x201C: // “
|
||||
case 0x201D: // ”
|
||||
case 0x00A0: // no-break space
|
||||
case '{':
|
||||
case '}':
|
||||
case '[':
|
||||
case ']':
|
||||
case '/':
|
||||
case 0x203A: // ›
|
||||
case 0x2026: // …
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool isAsciiDigit(const uint32_t cp) { return cp >= '0' && cp <= '9'; }
|
||||
|
||||
bool isExplicitHyphen(const uint32_t cp) {
|
||||
switch (cp) {
|
||||
case '-':
|
||||
case 0x00AD: // soft hyphen
|
||||
case 0x058A: // Armenian hyphen
|
||||
case 0x2010: // hyphen
|
||||
case 0x2011: // non-breaking hyphen
|
||||
case 0x2012: // figure dash
|
||||
case 0x2013: // en dash
|
||||
case 0x2014: // em dash
|
||||
case 0x2015: // horizontal bar
|
||||
case 0x2043: // hyphen bullet
|
||||
case 0x207B: // superscript minus
|
||||
case 0x208B: // subscript minus
|
||||
case 0x2212: // minus sign
|
||||
case 0x2E17: // double oblique hyphen
|
||||
case 0x2E3A: // two-em dash
|
||||
case 0x2E3B: // three-em dash
|
||||
case 0xFE58: // small em dash
|
||||
case 0xFE63: // small hyphen-minus
|
||||
case 0xFF0D: // fullwidth hyphen-minus
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool isSoftHyphen(const uint32_t cp) { return cp == 0x00AD; }
|
||||
|
||||
void trimSurroundingPunctuationAndFootnote(std::vector<CodepointInfo>& cps) {
|
||||
if (cps.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove trailing footnote references like [12], even if punctuation trails after the closing bracket.
|
||||
if (cps.size() >= 3) {
|
||||
int end = static_cast<int>(cps.size()) - 1;
|
||||
while (end >= 0 && isPunctuation(cps[end].value)) {
|
||||
--end;
|
||||
}
|
||||
int pos = end;
|
||||
if (pos >= 0 && isAsciiDigit(cps[pos].value)) {
|
||||
while (pos >= 0 && isAsciiDigit(cps[pos].value)) {
|
||||
--pos;
|
||||
}
|
||||
if (pos >= 0 && cps[pos].value == '[' && end - pos > 1) {
|
||||
cps.erase(cps.begin() + pos, cps.end());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (!cps.empty() && isPunctuation(cps.front().value)) {
|
||||
cps.erase(cps.begin());
|
||||
}
|
||||
while (!cps.empty() && isPunctuation(cps.back().value)) {
|
||||
cps.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<CodepointInfo> collectCodepoints(const std::string& word) {
|
||||
std::vector<CodepointInfo> cps;
|
||||
cps.reserve(word.size());
|
||||
|
||||
const unsigned char* base = reinterpret_cast<const unsigned char*>(word.c_str());
|
||||
const unsigned char* ptr = base;
|
||||
while (*ptr != 0) {
|
||||
const unsigned char* current = ptr;
|
||||
const uint32_t cp = utf8NextCodepoint(&ptr);
|
||||
cps.push_back({cp, static_cast<size_t>(current - base)});
|
||||
}
|
||||
|
||||
return cps;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct CodepointInfo {
|
||||
uint32_t value;
|
||||
size_t byteOffset;
|
||||
};
|
||||
|
||||
uint32_t toLowerLatin(uint32_t cp);
|
||||
uint32_t toLowerCyrillic(uint32_t cp);
|
||||
|
||||
bool isLatinLetter(uint32_t cp);
|
||||
bool isCyrillicLetter(uint32_t cp);
|
||||
|
||||
bool isAlphabetic(uint32_t cp);
|
||||
bool isPunctuation(uint32_t cp);
|
||||
bool isAsciiDigit(uint32_t cp);
|
||||
bool isExplicitHyphen(uint32_t cp);
|
||||
bool isSoftHyphen(uint32_t cp);
|
||||
void trimSurroundingPunctuationAndFootnote(std::vector<CodepointInfo>& cps);
|
||||
std::vector<CodepointInfo> collectCodepoints(const std::string& word);
|
||||
@@ -0,0 +1,97 @@
|
||||
#include "Hyphenator.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "HyphenationCommon.h"
|
||||
#include "LanguageRegistry.h"
|
||||
|
||||
const LanguageHyphenator* Hyphenator::cachedHyphenator_ = nullptr;
|
||||
|
||||
namespace {
|
||||
|
||||
// Maps a BCP-47 language tag to a language-specific hyphenator.
|
||||
const LanguageHyphenator* hyphenatorForLanguage(const std::string& langTag) {
|
||||
if (langTag.empty()) return nullptr;
|
||||
|
||||
// Extract primary subtag and normalize to lowercase (e.g., "en-US" -> "en").
|
||||
std::string primary;
|
||||
primary.reserve(langTag.size());
|
||||
for (char c : langTag) {
|
||||
if (c == '-' || c == '_') break;
|
||||
if (c >= 'A' && c <= 'Z') c = static_cast<char>(c - 'A' + 'a');
|
||||
primary.push_back(c);
|
||||
}
|
||||
if (primary.empty()) return nullptr;
|
||||
|
||||
return getLanguageHyphenatorForPrimaryTag(primary);
|
||||
}
|
||||
|
||||
// Maps a codepoint index back to its byte offset inside the source word.
|
||||
size_t byteOffsetForIndex(const std::vector<CodepointInfo>& cps, const size_t index) {
|
||||
return (index < cps.size()) ? cps[index].byteOffset : (cps.empty() ? 0 : cps.back().byteOffset);
|
||||
}
|
||||
|
||||
// Builds a vector of break information from explicit hyphen markers in the given codepoints.
|
||||
std::vector<Hyphenator::BreakInfo> buildExplicitBreakInfos(const std::vector<CodepointInfo>& cps) {
|
||||
std::vector<Hyphenator::BreakInfo> breaks;
|
||||
|
||||
// Scan every codepoint looking for explicit/soft hyphen markers that are surrounded by letters.
|
||||
for (size_t i = 1; i + 1 < cps.size(); ++i) {
|
||||
const uint32_t cp = cps[i].value;
|
||||
if (!isExplicitHyphen(cp) || !isAlphabetic(cps[i - 1].value) || !isAlphabetic(cps[i + 1].value)) {
|
||||
continue;
|
||||
}
|
||||
// Offset points to the next codepoint so rendering starts after the hyphen marker.
|
||||
breaks.push_back({cps[i + 1].byteOffset, isSoftHyphen(cp)});
|
||||
}
|
||||
|
||||
return breaks;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::vector<Hyphenator::BreakInfo> Hyphenator::breakOffsets(const std::string& word, const bool includeFallback) {
|
||||
if (word.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Convert to codepoints and normalize word boundaries.
|
||||
auto cps = collectCodepoints(word);
|
||||
trimSurroundingPunctuationAndFootnote(cps);
|
||||
const auto* hyphenator = cachedHyphenator_;
|
||||
|
||||
// Explicit hyphen markers (soft or hard) take precedence over language breaks.
|
||||
auto explicitBreakInfos = buildExplicitBreakInfos(cps);
|
||||
if (!explicitBreakInfos.empty()) {
|
||||
return explicitBreakInfos;
|
||||
}
|
||||
|
||||
// Ask language hyphenator for legal break points.
|
||||
std::vector<size_t> indexes;
|
||||
if (hyphenator) {
|
||||
indexes = hyphenator->breakIndexes(cps);
|
||||
}
|
||||
|
||||
// Only add fallback breaks if needed
|
||||
if (includeFallback && indexes.empty()) {
|
||||
const size_t minPrefix = hyphenator ? hyphenator->minPrefix() : LiangWordConfig::kDefaultMinPrefix;
|
||||
const size_t minSuffix = hyphenator ? hyphenator->minSuffix() : LiangWordConfig::kDefaultMinSuffix;
|
||||
for (size_t idx = minPrefix; idx + minSuffix <= cps.size(); ++idx) {
|
||||
indexes.push_back(idx);
|
||||
}
|
||||
}
|
||||
|
||||
if (indexes.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<Hyphenator::BreakInfo> breaks;
|
||||
breaks.reserve(indexes.size());
|
||||
for (const size_t idx : indexes) {
|
||||
breaks.push_back({byteOffsetForIndex(cps, idx), true});
|
||||
}
|
||||
|
||||
return breaks;
|
||||
}
|
||||
|
||||
void Hyphenator::setPreferredLanguage(const std::string& lang) { cachedHyphenator_ = hyphenatorForLanguage(lang); }
|
||||
@@ -0,0 +1,24 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class LanguageHyphenator;
|
||||
|
||||
class Hyphenator {
|
||||
public:
|
||||
struct BreakInfo {
|
||||
size_t byteOffset;
|
||||
bool requiresInsertedHyphen;
|
||||
};
|
||||
// Returns byte offsets where the word may be hyphenated. When includeFallback is true, all positions obeying the
|
||||
// minimum prefix/suffix constraints are returned even if no language-specific rule matches.
|
||||
static std::vector<BreakInfo> breakOffsets(const std::string& word, bool includeFallback);
|
||||
|
||||
// Provide a publication-level language hint (e.g. "en", "en-US", "ru") used to select hyphenation rules.
|
||||
static void setPreferredLanguage(const std::string& lang);
|
||||
|
||||
private:
|
||||
static const LanguageHyphenator* cachedHyphenator_;
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include "LiangHyphenation.h"
|
||||
|
||||
// Generic Liang-backed hyphenator that stores pattern metadata plus language-specific helpers.
|
||||
class LanguageHyphenator {
|
||||
public:
|
||||
LanguageHyphenator(const SerializedHyphenationPatterns& patterns, bool (*isLetterFn)(uint32_t),
|
||||
uint32_t (*toLowerFn)(uint32_t), size_t minPrefix = LiangWordConfig::kDefaultMinPrefix,
|
||||
size_t minSuffix = LiangWordConfig::kDefaultMinSuffix)
|
||||
: patterns_(patterns), config_(isLetterFn, toLowerFn, minPrefix, minSuffix) {}
|
||||
|
||||
std::vector<size_t> breakIndexes(const std::vector<CodepointInfo>& cps) const {
|
||||
return liangBreakIndexes(cps, patterns_, config_);
|
||||
}
|
||||
|
||||
size_t minPrefix() const { return config_.minPrefix; }
|
||||
size_t minSuffix() const { return config_.minSuffix; }
|
||||
|
||||
protected:
|
||||
const SerializedHyphenationPatterns& patterns_;
|
||||
LiangWordConfig config_;
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
#include "LanguageRegistry.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
|
||||
#include "HyphenationCommon.h"
|
||||
#include "generated/hyph-de.trie.h"
|
||||
#include "generated/hyph-en.trie.h"
|
||||
#include "generated/hyph-fr.trie.h"
|
||||
#include "generated/hyph-ru.trie.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// English hyphenation patterns (3/3 minimum prefix/suffix length)
|
||||
LanguageHyphenator englishHyphenator(en_us_patterns, isLatinLetter, toLowerLatin, 3, 3);
|
||||
LanguageHyphenator frenchHyphenator(fr_patterns, isLatinLetter, toLowerLatin);
|
||||
LanguageHyphenator germanHyphenator(de_patterns, isLatinLetter, toLowerLatin);
|
||||
LanguageHyphenator russianHyphenator(ru_ru_patterns, isCyrillicLetter, toLowerCyrillic);
|
||||
|
||||
using EntryArray = std::array<LanguageEntry, 4>;
|
||||
|
||||
const EntryArray& entries() {
|
||||
static const EntryArray kEntries = {{{"english", "en", &englishHyphenator},
|
||||
{"french", "fr", &frenchHyphenator},
|
||||
{"german", "de", &germanHyphenator},
|
||||
{"russian", "ru", &russianHyphenator}}};
|
||||
return kEntries;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const LanguageHyphenator* getLanguageHyphenatorForPrimaryTag(const std::string& primaryTag) {
|
||||
const auto& allEntries = entries();
|
||||
const auto it = std::find_if(allEntries.begin(), allEntries.end(),
|
||||
[&primaryTag](const LanguageEntry& entry) { return primaryTag == entry.primaryTag; });
|
||||
return (it != allEntries.end()) ? it->hyphenator : nullptr;
|
||||
}
|
||||
|
||||
LanguageEntryView getLanguageEntries() {
|
||||
const auto& allEntries = entries();
|
||||
return LanguageEntryView{allEntries.data(), allEntries.size()};
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
#include "LanguageHyphenator.h"
|
||||
|
||||
struct LanguageEntry {
|
||||
const char* cliName;
|
||||
const char* primaryTag;
|
||||
const LanguageHyphenator* hyphenator;
|
||||
};
|
||||
|
||||
struct LanguageEntryView {
|
||||
const LanguageEntry* data;
|
||||
size_t size;
|
||||
|
||||
const LanguageEntry* begin() const { return data; }
|
||||
const LanguageEntry* end() const { return data + size; }
|
||||
};
|
||||
|
||||
// Returns the Liang-backed hyphenator for a given primary language tag (e.g., "en", "fr").
|
||||
const LanguageHyphenator* getLanguageHyphenatorForPrimaryTag(const std::string& primaryTag);
|
||||
|
||||
// Exposes the list of supported languages primarily for tooling/tests.
|
||||
LanguageEntryView getLanguageEntries();
|
||||
@@ -0,0 +1,405 @@
|
||||
#include "LiangHyphenation.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
/*
|
||||
* Liang hyphenation pipeline overview (Typst-style binary trie variant)
|
||||
* --------------------------------------------------------------------
|
||||
* 1. Input normalization (buildAugmentedWord)
|
||||
* - Accepts a vector of CodepointInfo structs emitted by the EPUB text
|
||||
* parser. Each codepoint is validated with LiangWordConfig::isLetter so
|
||||
* we abort early on digits, punctuation, etc. If the word is valid we
|
||||
* build an "augmented" byte sequence: leading '.', lowercase UTF-8 bytes
|
||||
* for every letter, then a trailing '.'. While doing this we capture the
|
||||
* UTF-8 byte offset for each character and a reverse lookup table that
|
||||
* maps UTF-8 byte indexes back to codepoint indexes. This lets the rest
|
||||
* of the algorithm stay byte-oriented (matching the serialized automaton)
|
||||
* while still emitting hyphen positions in codepoint space.
|
||||
*
|
||||
* 2. Automaton decoding
|
||||
* - SerializedHyphenationPatterns stores a contiguous blob generated from
|
||||
* Typst's binary tries. The first 4 bytes contain the root offset. Each
|
||||
* node packs transitions, variable-stride relative offsets to child
|
||||
* nodes, and an optional pointer into a shared "levels" list. We parse
|
||||
* that layout lazily via decodeState/transition, keeping everything in
|
||||
* flash memory; no heap allocations besides the stack-local AutomatonState
|
||||
* structs. getAutomaton caches parseAutomaton results per blob pointer so
|
||||
* multiple words hitting the same language only pay the cost once.
|
||||
*
|
||||
* 3. Pattern application
|
||||
* - We walk the augmented bytes left-to-right. For each starting byte we
|
||||
* stream transitions through the trie, terminating when a transition
|
||||
* fails. Whenever a node exposes level data we expand the packed
|
||||
* "dist+level" bytes: `dist` is the delta (in UTF-8 bytes) from the
|
||||
* starting cursor and `level` is the Liang priority digit. Using the
|
||||
* byte→codepoint lookup we mark the corresponding index in `scores`.
|
||||
* Scores are only updated if the new level is higher, mirroring Liang's
|
||||
* "max digit wins" rule.
|
||||
*
|
||||
* 4. Output filtering
|
||||
* - collectBreakIndexes converts odd-valued score entries back to codepoint
|
||||
* break positions while enforcing `minPrefix`/`minSuffix` constraints from
|
||||
* LiangWordConfig. The caller (language-specific hyphenators) can then
|
||||
* translate these indexes into renderer glyph offsets, page layout data,
|
||||
* etc.
|
||||
*
|
||||
* Keeping the entire algorithm small and deterministic is critical on the
|
||||
* ESP32-C3: we avoid recursion, dynamic allocations per node, or copying the
|
||||
* trie. All lookups stay within the generated blob, which lives in flash, and
|
||||
* the working buffers (augmented bytes/scores) scale with the word length rather
|
||||
* than the pattern corpus.
|
||||
*/
|
||||
|
||||
namespace {
|
||||
|
||||
struct AugmentedWord {
|
||||
std::vector<uint8_t> bytes;
|
||||
std::vector<size_t> charByteOffsets;
|
||||
std::vector<int32_t> byteToCharIndex;
|
||||
|
||||
bool empty() const { return bytes.empty(); }
|
||||
size_t charCount() const { return charByteOffsets.size(); }
|
||||
};
|
||||
|
||||
// Encode a single Unicode codepoint into UTF-8 and append to the provided buffer.
|
||||
size_t encodeUtf8(uint32_t cp, std::vector<uint8_t>& out) {
|
||||
if (cp <= 0x7Fu) {
|
||||
out.push_back(static_cast<uint8_t>(cp));
|
||||
return 1;
|
||||
}
|
||||
if (cp <= 0x7FFu) {
|
||||
out.push_back(static_cast<uint8_t>(0xC0u | ((cp >> 6) & 0x1Fu)));
|
||||
out.push_back(static_cast<uint8_t>(0x80u | (cp & 0x3Fu)));
|
||||
return 2;
|
||||
}
|
||||
if (cp <= 0xFFFFu) {
|
||||
out.push_back(static_cast<uint8_t>(0xE0u | ((cp >> 12) & 0x0Fu)));
|
||||
out.push_back(static_cast<uint8_t>(0x80u | ((cp >> 6) & 0x3Fu)));
|
||||
out.push_back(static_cast<uint8_t>(0x80u | (cp & 0x3Fu)));
|
||||
return 3;
|
||||
}
|
||||
out.push_back(static_cast<uint8_t>(0xF0u | ((cp >> 18) & 0x07u)));
|
||||
out.push_back(static_cast<uint8_t>(0x80u | ((cp >> 12) & 0x3Fu)));
|
||||
out.push_back(static_cast<uint8_t>(0x80u | ((cp >> 6) & 0x3Fu)));
|
||||
out.push_back(static_cast<uint8_t>(0x80u | (cp & 0x3Fu)));
|
||||
return 4;
|
||||
}
|
||||
|
||||
// Build the dotted, lowercase UTF-8 representation plus lookup tables.
|
||||
AugmentedWord buildAugmentedWord(const std::vector<CodepointInfo>& cps, const LiangWordConfig& config) {
|
||||
AugmentedWord word;
|
||||
if (cps.empty()) {
|
||||
return word;
|
||||
}
|
||||
|
||||
word.bytes.reserve(cps.size() * 2 + 2);
|
||||
word.charByteOffsets.reserve(cps.size() + 2);
|
||||
|
||||
word.charByteOffsets.push_back(0);
|
||||
word.bytes.push_back('.');
|
||||
|
||||
for (const auto& info : cps) {
|
||||
if (!config.isLetter(info.value)) {
|
||||
word.bytes.clear();
|
||||
word.charByteOffsets.clear();
|
||||
word.byteToCharIndex.clear();
|
||||
return word;
|
||||
}
|
||||
word.charByteOffsets.push_back(word.bytes.size());
|
||||
encodeUtf8(config.toLower(info.value), word.bytes);
|
||||
}
|
||||
|
||||
word.charByteOffsets.push_back(word.bytes.size());
|
||||
word.bytes.push_back('.');
|
||||
|
||||
word.byteToCharIndex.assign(word.bytes.size(), -1);
|
||||
for (size_t i = 0; i < word.charByteOffsets.size(); ++i) {
|
||||
const size_t offset = word.charByteOffsets[i];
|
||||
if (offset < word.byteToCharIndex.size()) {
|
||||
word.byteToCharIndex[offset] = static_cast<int32_t>(i);
|
||||
}
|
||||
}
|
||||
return word;
|
||||
}
|
||||
|
||||
// Decoded view of a single trie node pulled straight out of the serialized blob.
|
||||
// - transitions: contiguous list of next-byte values
|
||||
// - targets: packed relative offsets (1/2/3 bytes) for each transition
|
||||
// - levels: optional pointer into the global levels list with packed dist/level pairs
|
||||
struct AutomatonState {
|
||||
const uint8_t* data = nullptr;
|
||||
size_t size = 0;
|
||||
size_t addr = 0;
|
||||
uint8_t stride = 1;
|
||||
size_t childCount = 0;
|
||||
const uint8_t* transitions = nullptr;
|
||||
const uint8_t* targets = nullptr;
|
||||
const uint8_t* levels = nullptr;
|
||||
size_t levelsLen = 0;
|
||||
|
||||
bool valid() const { return data != nullptr; }
|
||||
};
|
||||
|
||||
// Lightweight descriptor for the entire embedded automaton.
|
||||
// The blob format is:
|
||||
// [0..3] - big-endian root offset
|
||||
// [4....] - node heap containing variable-sized headers + transition data
|
||||
struct EmbeddedAutomaton {
|
||||
const uint8_t* data = nullptr;
|
||||
size_t size = 0;
|
||||
uint32_t rootOffset = 0;
|
||||
|
||||
bool valid() const { return data != nullptr && size >= 4 && rootOffset < size; }
|
||||
};
|
||||
|
||||
// Decode the serialized automaton header and root offset.
|
||||
EmbeddedAutomaton parseAutomaton(const SerializedHyphenationPatterns& patterns) {
|
||||
EmbeddedAutomaton automaton;
|
||||
if (!patterns.data || patterns.size < 4) {
|
||||
return automaton;
|
||||
}
|
||||
|
||||
automaton.data = patterns.data;
|
||||
automaton.size = patterns.size;
|
||||
automaton.rootOffset = (static_cast<uint32_t>(patterns.data[0]) << 24) |
|
||||
(static_cast<uint32_t>(patterns.data[1]) << 16) |
|
||||
(static_cast<uint32_t>(patterns.data[2]) << 8) | static_cast<uint32_t>(patterns.data[3]);
|
||||
if (automaton.rootOffset >= automaton.size) {
|
||||
automaton.data = nullptr;
|
||||
automaton.size = 0;
|
||||
}
|
||||
return automaton;
|
||||
}
|
||||
|
||||
// Cache parsed automata per blob pointer to avoid reparsing.
|
||||
const EmbeddedAutomaton& getAutomaton(const SerializedHyphenationPatterns& patterns) {
|
||||
struct CacheEntry {
|
||||
const SerializedHyphenationPatterns* key;
|
||||
EmbeddedAutomaton automaton;
|
||||
};
|
||||
static std::vector<CacheEntry> cache;
|
||||
|
||||
for (const auto& entry : cache) {
|
||||
if (entry.key == &patterns) {
|
||||
return entry.automaton;
|
||||
}
|
||||
}
|
||||
|
||||
cache.push_back({&patterns, parseAutomaton(patterns)});
|
||||
return cache.back().automaton;
|
||||
}
|
||||
|
||||
// Interpret the node located at `addr`, returning transition metadata.
|
||||
AutomatonState decodeState(const EmbeddedAutomaton& automaton, size_t addr) {
|
||||
AutomatonState state;
|
||||
if (!automaton.valid() || addr >= automaton.size) {
|
||||
return state;
|
||||
}
|
||||
|
||||
const uint8_t* base = automaton.data + addr;
|
||||
size_t remaining = automaton.size - addr;
|
||||
size_t pos = 0;
|
||||
|
||||
const uint8_t header = base[pos++];
|
||||
// Header layout (bits):
|
||||
// 7 - hasLevels flag
|
||||
// 6..5 - stride selector (0 -> 1 byte, otherwise 1|2|3)
|
||||
// 4..0 - child count (5 bits), 31 == overflow -> extra byte
|
||||
const bool hasLevels = (header >> 7) != 0;
|
||||
uint8_t stride = static_cast<uint8_t>((header >> 5) & 0x03u);
|
||||
if (stride == 0) {
|
||||
stride = 1;
|
||||
}
|
||||
size_t childCount = static_cast<size_t>(header & 0x1Fu);
|
||||
if (childCount == 31u) {
|
||||
if (pos >= remaining) {
|
||||
return AutomatonState{};
|
||||
}
|
||||
childCount = base[pos++];
|
||||
}
|
||||
|
||||
const uint8_t* levelsPtr = nullptr;
|
||||
size_t levelsLen = 0;
|
||||
if (hasLevels) {
|
||||
if (pos + 1 >= remaining) {
|
||||
return AutomatonState{};
|
||||
}
|
||||
const uint8_t offsetHi = base[pos++];
|
||||
const uint8_t offsetLoLen = base[pos++];
|
||||
// The 12-bit offset (hi<<4 | top nibble) points into the blob-level levels list.
|
||||
// The bottom nibble stores how many packed entries belong to this node.
|
||||
const size_t offset = (static_cast<size_t>(offsetHi) << 4) | (offsetLoLen >> 4);
|
||||
levelsLen = offsetLoLen & 0x0Fu;
|
||||
if (offset + levelsLen > automaton.size) {
|
||||
return AutomatonState{};
|
||||
}
|
||||
levelsPtr = automaton.data + offset;
|
||||
}
|
||||
|
||||
if (pos + childCount > remaining) {
|
||||
return AutomatonState{};
|
||||
}
|
||||
const uint8_t* transitions = base + pos;
|
||||
pos += childCount;
|
||||
|
||||
const size_t targetsBytes = childCount * stride;
|
||||
if (pos + targetsBytes > remaining) {
|
||||
return AutomatonState{};
|
||||
}
|
||||
const uint8_t* targets = base + pos;
|
||||
|
||||
state.data = automaton.data;
|
||||
state.size = automaton.size;
|
||||
state.addr = addr;
|
||||
state.stride = stride;
|
||||
state.childCount = childCount;
|
||||
state.transitions = transitions;
|
||||
state.targets = targets;
|
||||
state.levels = levelsPtr;
|
||||
state.levelsLen = levelsLen;
|
||||
return state;
|
||||
}
|
||||
|
||||
// Convert the packed stride-sized delta back into a signed offset.
|
||||
int32_t decodeDelta(const uint8_t* buf, uint8_t stride) {
|
||||
if (stride == 1) {
|
||||
return static_cast<int8_t>(buf[0]);
|
||||
}
|
||||
if (stride == 2) {
|
||||
return static_cast<int16_t>((static_cast<uint16_t>(buf[0]) << 8) | static_cast<uint16_t>(buf[1]));
|
||||
}
|
||||
const int32_t unsignedVal =
|
||||
(static_cast<int32_t>(buf[0]) << 16) | (static_cast<int32_t>(buf[1]) << 8) | static_cast<int32_t>(buf[2]);
|
||||
return unsignedVal - (1 << 23);
|
||||
}
|
||||
|
||||
// Follow a single byte transition from `state`, decoding the child node on success.
|
||||
bool transition(const EmbeddedAutomaton& automaton, const AutomatonState& state, uint8_t letter, AutomatonState& out) {
|
||||
if (!state.valid()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Children remain sorted by letter in the serialized blob, but the lists are
|
||||
// short enough that a linear scan keeps code size down compared to binary search.
|
||||
for (size_t idx = 0; idx < state.childCount; ++idx) {
|
||||
if (state.transitions[idx] != letter) {
|
||||
continue;
|
||||
}
|
||||
const uint8_t* deltaPtr = state.targets + idx * state.stride;
|
||||
const int32_t delta = decodeDelta(deltaPtr, state.stride);
|
||||
// Deltas are relative to the current node's address, allowing us to keep all
|
||||
// targets within 24 bits while still referencing further nodes in the blob.
|
||||
const int64_t nextAddr = static_cast<int64_t>(state.addr) + delta;
|
||||
if (nextAddr < 0 || static_cast<size_t>(nextAddr) >= automaton.size) {
|
||||
return false;
|
||||
}
|
||||
out = decodeState(automaton, static_cast<size_t>(nextAddr));
|
||||
return out.valid();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Converts odd score positions back into codepoint indexes, honoring min prefix/suffix constraints.
|
||||
// Each break corresponds to scores[breakIndex + 1] because of the leading '.' sentinel.
|
||||
// Convert odd score entries into hyphen positions while honoring prefix/suffix limits.
|
||||
std::vector<size_t> collectBreakIndexes(const std::vector<CodepointInfo>& cps, const std::vector<uint8_t>& scores,
|
||||
const size_t minPrefix, const size_t minSuffix) {
|
||||
std::vector<size_t> indexes;
|
||||
const size_t cpCount = cps.size();
|
||||
if (cpCount < 2) {
|
||||
return indexes;
|
||||
}
|
||||
|
||||
for (size_t breakIndex = 1; breakIndex < cpCount; ++breakIndex) {
|
||||
if (breakIndex < minPrefix) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const size_t suffixCount = cpCount - breakIndex;
|
||||
if (suffixCount < minSuffix) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const size_t scoreIdx = breakIndex + 1;
|
||||
if (scoreIdx >= scores.size()) {
|
||||
break;
|
||||
}
|
||||
if ((scores[scoreIdx] & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
indexes.push_back(breakIndex);
|
||||
}
|
||||
|
||||
return indexes;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Entry point that runs the full Liang pipeline for a single word.
|
||||
std::vector<size_t> liangBreakIndexes(const std::vector<CodepointInfo>& cps,
|
||||
const SerializedHyphenationPatterns& patterns, const LiangWordConfig& config) {
|
||||
const auto augmented = buildAugmentedWord(cps, config);
|
||||
if (augmented.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const EmbeddedAutomaton& automaton = getAutomaton(patterns);
|
||||
if (!automaton.valid()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const AutomatonState root = decodeState(automaton, automaton.rootOffset);
|
||||
if (!root.valid()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Liang scores: one entry per augmented char (leading/trailing dots included).
|
||||
std::vector<uint8_t> scores(augmented.charCount(), 0);
|
||||
|
||||
// Walk every starting character position and stream bytes through the trie.
|
||||
for (size_t charStart = 0; charStart < augmented.charByteOffsets.size(); ++charStart) {
|
||||
const size_t byteStart = augmented.charByteOffsets[charStart];
|
||||
AutomatonState state = root;
|
||||
|
||||
for (size_t cursor = byteStart; cursor < augmented.bytes.size(); ++cursor) {
|
||||
AutomatonState next;
|
||||
if (!transition(automaton, state, augmented.bytes[cursor], next)) {
|
||||
break; // No more matches for this prefix.
|
||||
}
|
||||
state = next;
|
||||
|
||||
if (state.levels && state.levelsLen > 0) {
|
||||
size_t offset = 0;
|
||||
// Each packed byte stores the byte-distance delta and the Liang level digit.
|
||||
for (size_t i = 0; i < state.levelsLen; ++i) {
|
||||
const uint8_t packed = state.levels[i];
|
||||
const size_t dist = static_cast<size_t>(packed / 10);
|
||||
const uint8_t level = static_cast<uint8_t>(packed % 10);
|
||||
|
||||
offset += dist;
|
||||
const size_t splitByte = byteStart + offset;
|
||||
if (splitByte >= augmented.byteToCharIndex.size()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t boundary = augmented.byteToCharIndex[splitByte];
|
||||
if (boundary < 0) {
|
||||
continue; // Mid-codepoint byte, wait for the next one.
|
||||
}
|
||||
if (boundary < 2 || boundary + 2 > static_cast<int32_t>(augmented.charCount())) {
|
||||
continue; // Skip splits that land in the leading/trailing sentinels.
|
||||
}
|
||||
|
||||
const size_t idx = static_cast<size_t>(boundary);
|
||||
if (idx >= scores.size()) {
|
||||
continue;
|
||||
}
|
||||
scores[idx] = std::max(scores[idx], level);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return collectBreakIndexes(cps, scores, config.minPrefix, config.minSuffix);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
#include "HyphenationCommon.h"
|
||||
#include "SerializedHyphenationTrie.h"
|
||||
|
||||
// Encapsulates every language-specific dial the Liang algorithm needs at runtime. The helpers are
|
||||
// intentionally represented as bare function pointers because we invoke them inside tight loops and
|
||||
// want to avoid the overhead of std::function or functors. The minima default to the TeX-recommended
|
||||
// "2/2" split but individual languages (English, for example) can override them.
|
||||
struct LiangWordConfig {
|
||||
static constexpr size_t kDefaultMinPrefix = 2;
|
||||
static constexpr size_t kDefaultMinSuffix = 2;
|
||||
// Predicate used to reject non-alphabetic characters before pattern lookup. Returning false causes
|
||||
// the entire word to be skipped, matching the behavior of classic TeX hyphenation tables.
|
||||
bool (*isLetter)(uint32_t);
|
||||
// Language-specific case folding that matches how the TeX patterns were authored (usually lower-case
|
||||
// ASCII for Latin and lowercase Cyrillic for Russian). Patterns are stored in UTF-8, so this must
|
||||
// operate on Unicode scalars rather than bytes.
|
||||
uint32_t (*toLower)(uint32_t);
|
||||
// Minimum codepoints required on the left/right of any break. These correspond to TeX's
|
||||
// lefthyphenmin and righthyphenmin knobs.
|
||||
size_t minPrefix;
|
||||
size_t minSuffix;
|
||||
|
||||
// Lightweight aggregate constructor so call sites can declare `const LiangWordConfig config(...)`
|
||||
// without verbose member assignment boilerplate.
|
||||
LiangWordConfig(bool (*letterFn)(uint32_t), uint32_t (*lowerFn)(uint32_t), size_t prefix = kDefaultMinPrefix,
|
||||
size_t suffix = kDefaultMinSuffix)
|
||||
: isLetter(letterFn), toLower(lowerFn), minPrefix(prefix), minSuffix(suffix) {}
|
||||
};
|
||||
|
||||
// Shared Liang pattern evaluator used by every language-specific hyphenator.
|
||||
std::vector<size_t> liangBreakIndexes(const std::vector<CodepointInfo>& cps,
|
||||
const SerializedHyphenationPatterns& patterns, const LiangWordConfig& config);
|
||||
@@ -0,0 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
// Lightweight descriptor that points at a serialized Liang hyphenation trie stored in flash.
|
||||
struct SerializedHyphenationPatterns {
|
||||
const std::uint8_t* data;
|
||||
size_t size;
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user