From 9bc5111c77ae1da20b5ab67328cf1a000c018534 Mon Sep 17 00:00:00 2001 From: CSCMe Date: Mon, 13 Apr 2026 01:41:28 +0200 Subject: [PATCH] fix: increase loadable epub size (#1638) ## Summary * **What is the goal of this PR?** Slightly increase the OOM limit when loading epubs * **What changes are included?** Switched vectors for parsing epubs to deques, allowing use of more memory ## Additional Context * Increases loadable epub size from 2000+ chapter/ToC entries loadable to 5000+ chapter/ToC entries * #1574, but without the complicated stuff --- ### AI Usage While CrossPoint doesn't have restrictions on AI tools in contributing, please be transparent about their usage as it helps set the right context for reviewers. Did you use AI tools to help write this code? _**NO**_ ### Testing | Commit | Book | Time | |----------|------|-------| | 83cd96bc2fd90e5f5a8b0ceaed2022a16a017a1c | 1000.epub | ~30 sec | | 83cd96bc2fd90e5f5a8b0ceaed2022a16a017a1c | 5000.epub | crash | | PR | 1000.epub | ~29 sec | | PR | 5000.epub | ~2 min 20 sec | => No actual loading time regressions [tested_epubs.zip](https://github.com/user-attachments/files/26645243/tested_epubs.zip) --- lib/Epub/Epub/BookMetadataCache.cpp | 16 ++++++++-------- lib/Epub/Epub/BookMetadataCache.h | 4 ++-- lib/Epub/Epub/parsers/ContentOpfParser.h | 3 ++- lib/ZipFile/ZipFile.cpp | 2 +- lib/ZipFile/ZipFile.h | 4 ++-- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/Epub/Epub/BookMetadataCache.cpp b/lib/Epub/Epub/BookMetadataCache.cpp index 3cdee0b0e..ffc456cad 100644 --- a/lib/Epub/Epub/BookMetadataCache.cpp +++ b/lib/Epub/Epub/BookMetadataCache.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include "FsHelpers.h" @@ -50,7 +50,7 @@ bool BookMetadataCache::beginTocPass() { if (spineCount >= LARGE_SPINE_THRESHOLD) { spineHrefIndex.clear(); - spineHrefIndex.reserve(spineCount); + spineHrefIndex.resize(spineCount); spineFile.seek(0); for (int i = 0; i < spineCount; i++) { auto entry = readSpineEntry(spineFile); @@ -58,7 +58,7 @@ bool BookMetadataCache::beginTocPass() { idx.hrefHash = fnvHash64(entry.href); idx.hrefLen = static_cast(entry.href.size()); idx.spineIndex = static_cast(i); - spineHrefIndex.push_back(idx); + spineHrefIndex[i] = idx; } std::sort(spineHrefIndex.begin(), spineHrefIndex.end(), [](const SpineHrefIndexEntry& a, const SpineHrefIndexEntry& b) { @@ -153,7 +153,7 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta // Loop through spines from spine file matching up TOC indexes, calculating cumulative size and writing to book.bin // Build spineIndex->tocIndex mapping in one pass (O(n) instead of O(n*m)) - std::vector spineToTocIndex(spineCount, -1); + std::deque spineToTocIndex(spineCount, -1); tocFile.seek(0); for (int j = 0; j < tocCount; j++) { auto tocEntry = readTocEntry(tocFile); @@ -181,14 +181,14 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta // This is O(n*log(m)) instead of O(n*m) while avoiding memory exhaustion. // See: https://github.com/crosspoint-reader/crosspoint-reader/issues/134 - std::vector spineSizes; + std::deque spineSizes; bool useBatchSizes = false; if (spineCount >= LARGE_SPINE_THRESHOLD) { LOG_DBG("BMC", "Using batch size lookup for %d spine items", spineCount); - std::vector targets; - targets.reserve(spineCount); + std::deque targets; + targets.resize(spineCount); spineFile.seek(0); for (int i = 0; i < spineCount; i++) { @@ -199,7 +199,7 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta t.hash = ZipFile::fnvHash64(path.c_str(), path.size()); t.len = static_cast(path.size()); t.index = static_cast(i); - targets.push_back(t); + targets[i] = t; } std::sort(targets.begin(), targets.end(), [](const ZipFile::SizeTarget& a, const ZipFile::SizeTarget& b) { diff --git a/lib/Epub/Epub/BookMetadataCache.h b/lib/Epub/Epub/BookMetadataCache.h index 9439b37fe..7f45090a9 100644 --- a/lib/Epub/Epub/BookMetadataCache.h +++ b/lib/Epub/Epub/BookMetadataCache.h @@ -3,8 +3,8 @@ #include #include +#include #include -#include class BookMetadataCache { public: @@ -61,7 +61,7 @@ class BookMetadataCache { uint16_t hrefLen; // length for collision reduction int16_t spineIndex; }; - std::vector spineHrefIndex; + std::deque spineHrefIndex; bool useSpineHrefIndex = false; static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400; diff --git a/lib/Epub/Epub/parsers/ContentOpfParser.h b/lib/Epub/Epub/parsers/ContentOpfParser.h index 89fb3379b..485b3a857 100644 --- a/lib/Epub/Epub/parsers/ContentOpfParser.h +++ b/lib/Epub/Epub/parsers/ContentOpfParser.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "Epub.h" @@ -37,7 +38,7 @@ class ContentOpfParser final : public Print { uint16_t idLen; // length for collision reduction uint32_t fileOffset; // offset in .items.bin }; - std::vector itemIndex; + std::deque itemIndex; bool useItemIndex = false; static constexpr uint16_t LARGE_SPINE_THRESHOLD = 400; diff --git a/lib/ZipFile/ZipFile.cpp b/lib/ZipFile/ZipFile.cpp index 2bb19147a..bdf5c0e25 100644 --- a/lib/ZipFile/ZipFile.cpp +++ b/lib/ZipFile/ZipFile.cpp @@ -295,7 +295,7 @@ bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) { return true; } -int ZipFile::fillUncompressedSizes(std::vector& targets, std::vector& sizes) { +int ZipFile::fillUncompressedSizes(std::deque& targets, std::deque& sizes) { if (targets.empty()) { return 0; } diff --git a/lib/ZipFile/ZipFile.h b/lib/ZipFile/ZipFile.h index bc97559dd..60c97a4cf 100644 --- a/lib/ZipFile/ZipFile.h +++ b/lib/ZipFile/ZipFile.h @@ -1,9 +1,9 @@ #pragma once #include +#include #include #include -#include class ZipFile { public: @@ -64,7 +64,7 @@ class ZipFile { // Batch lookup: scan ZIP central dir once and fill sizes for matching targets. // targets must be sorted by (hash, len). sizes[target.index] receives uncompressedSize. // Returns number of targets matched. - int fillUncompressedSizes(std::vector& targets, std::vector& sizes); + int fillUncompressedSizes(std::deque& targets, std::deque& sizes); // Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple // These functions will open and close the zip as needed uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false);