Stream inflated EPUB HTMLs down to disk instead of inflating in memory (#4)

* Downgrade miniz for stability

* Stream HTML from ZIP down to disk instead of loading all in mem

README.md

@@ -6,10 +6,7 @@ Built using **PlatformIO** and targeting the **ESP32-C3** microcontroller.
 CrossPoint Reader is a purpose-built firmware designed to be a drop-in, fully open-source replacement for the official 
 Xteink firmware. It aims to match or improve upon the standard EPUB reading experience.
 
-// TODO include some images
-
-I look at the [**diy-esp32-epub-reader** by atomic14](https://github.com/atomic14/diy-esp32-epub-reader) project a lot 
-when making CrossPoint and a handful of lessons and some direct source code comes directly from that repo.
+![](./docs/cover.jpg)
 
 ## Motivation
 
@@ -129,3 +126,6 @@ Contributions are very welcome!
 ---
 
 CrossPoint Reader is **not affiliated with Xteink or any manufacturer of the X4 hardware**.
+
+Huge shoutout to [**diy-esp32-epub-reader** by atomic14](https://github.com/atomic14/diy-esp32-epub-reader), which was a project I took a lot of inspiration from as I
+was making CrossPoint.

lib/EpdFont/EpdFont.cpp

@@ -2,12 +2,9 @@
 
 #include <Utf8.h>
 
-EpdFont::EpdFont(const EpdFontData* data) { this->data = data; }
-
 inline int min(const int a, const int b) { return a < b ? a : b; }
 inline int max(const int a, const int b) { return a < b ? b : a; }
 
-// TODO: Text properties??
 void EpdFont::getTextBounds(const char* string, const int startX, const int startY, int* minX, int* minY, int* maxX,
                             int* maxY) const {
   *minX = startX;

lib/EpdFont/EpdFont.h

@@ -6,7 +6,7 @@ class EpdFont {
 
  public:
   const EpdFontData* data;
-  explicit EpdFont(const EpdFontData* data);
+  explicit EpdFont(const EpdFontData* data) : data(data) {}
   ~EpdFont() = default;
   void getTextDimensions(const char* string, int* w, int* h) const;
   bool hasPrintableChars(const char* string) const;

lib/EpdFont/EpdFontData.h

@@ -6,13 +6,13 @@
 
 /// Font data stored PER GLYPH
 typedef struct {
-  uint8_t width;            ///< Bitmap dimensions in pixels
-  uint8_t height;           ///< Bitmap dimensions in pixels
-  uint8_t advanceX;         ///< Distance to advance cursor (x axis)
-  int16_t left;             ///< X dist from cursor pos to UL corner
-  int16_t top;              ///< Y dist from cursor pos to UL corner
-  uint16_t compressedSize;  ///< Size of the zlib-compressed font data.
-  uint32_t dataOffset;      ///< Pointer into EpdFont->bitmap
+  uint8_t width;        ///< Bitmap dimensions in pixels
+  uint8_t height;       ///< Bitmap dimensions in pixels
+  uint8_t advanceX;     ///< Distance to advance cursor (x axis)
+  int16_t left;         ///< X dist from cursor pos to UL corner
+  int16_t top;          ///< Y dist from cursor pos to UL corner
+  uint16_t dataLength;  ///< Size of the font data.
+  uint32_t dataOffset;  ///< Pointer into EpdFont->bitmap
 } EpdGlyph;
 
 /// Glyph interval structure
@@ -28,7 +28,6 @@ typedef struct {
   const EpdGlyph* glyph;                ///< Glyph array
   const EpdUnicodeInterval* intervals;  ///< Valid unicode intervals for this font
   uint32_t intervalCount;               ///< Number of unicode intervals.
-  bool compressed;                      ///< Does this font use compressed glyph bitmaps?
   uint8_t advanceY;                     ///< Newline distance (y axis)
   int ascender;                         ///< Maximal height of a glyph above the base line
   int descender;                        ///< Maximal height of a glyph below the base line

lib/EpdFont/scripts/fontconvert.py

@@ -7,20 +7,18 @@ import math
 import argparse
 from collections import namedtuple
 
-# From: https://github.com/vroland/epdiy
+# Originally from https://github.com/vroland/epdiy
 
 parser = argparse.ArgumentParser(description="Generate a header file from a font to be used with epdiy.")
 parser.add_argument("name", action="store", help="name of the font.")
 parser.add_argument("size", type=int, help="font size to use.")
 parser.add_argument("fontstack", action="store", nargs='+', help="list of font files, ordered by descending priority.")
-parser.add_argument("--compress", dest="compress", action="store_true", help="compress glyph bitmaps.")
 parser.add_argument("--additional-intervals", dest="additional_intervals", action="append", help="Additional code point intervals to export as min,max. This argument can be repeated.")
 args = parser.parse_args()
 
-GlyphProps = namedtuple("GlyphProps", ["width", "height", "advance_x", "left", "top", "compressed_size", "data_offset", "code_point"])
+GlyphProps = namedtuple("GlyphProps", ["width", "height", "advance_x", "left", "top", "data_length", "data_offset", "code_point"])
 
 font_stack = [freetype.Face(f) for f in args.fontstack]
-compress = args.compress
 size = args.size
 font_name = args.name
 
@@ -148,19 +146,18 @@ for i_start, i_end in unvalidated_intervals:
         intervals.append((start, i_end))
 
 for face in font_stack:
-    # shift by 6 bytes, because sizes are given as 6-bit fractions
-    # the display has about 150 dpi.
     face.set_char_size(size << 6, size << 6, 150, 150)
 
 total_size = 0
-total_packed = 0
 all_glyphs = []
 
 for i_start, i_end in intervals:
     for code_point in range(i_start, i_end + 1):
         face = load_glyph(code_point)
         bitmap = face.glyph.bitmap
-        pixels = []
+
+        # Build out 4-bit greyscale bitmap
+        pixels4g = []
         px = 0
         for i, v in enumerate(bitmap.buffer):
             y = i / bitmap.width
@@ -169,31 +166,45 @@ for i_start, i_end in intervals:
                 px = (v >> 4)
             else:
                 px = px | (v & 0xF0)
-                pixels.append(px);
+                pixels4g.append(px);
                 px = 0
             # eol
             if x == bitmap.width - 1 and bitmap.width % 2 > 0:
-                pixels.append(px)
+                pixels4g.append(px)
                 px = 0
 
-        packed = bytes(pixels);
-        total_packed += len(packed)
-        compressed = packed
-        if compress:
-            compressed = zlib.compress(packed)
+        # Downsample to 1-bit bitmap - treat any non-zero as black
+        pixelsbw = []
+        px = 0
+        pitch = (bitmap.width // 2) + (bitmap.width % 2)
+        for y in range(bitmap.rows):
+            for x in range(bitmap.width):
+                px = px << 1
+                bm = pixels4g[y * pitch + (x // 2)]
+                px += 1 if ((x & 1) == 0 and bm & 0xF > 0) or ((x & 1) == 1 and bm & 0xF0 > 0) else 0
 
+                if (y * bitmap.width + x) % 8 == 7:
+                    pixelsbw.append(px)
+                    px = 0
+        if (bitmap.width * bitmap.rows) % 8 != 0:
+            px = px << (8 - (bitmap.width * bitmap.rows) % 8)
+            pixelsbw.append(px)
+
+
+        # Build output data
+        packed = bytes(pixelsbw)
         glyph = GlyphProps(
             width = bitmap.width,
             height = bitmap.rows,
             advance_x = norm_floor(face.glyph.advance.x),
             left = face.glyph.bitmap_left,
             top = face.glyph.bitmap_top,
-            compressed_size = len(compressed),
+            data_length = len(packed),
             data_offset = total_size,
             code_point = code_point,
         )
-        total_size += len(compressed)
-        all_glyphs.append((glyph, compressed))
+        total_size += len(packed)
+        all_glyphs.append((glyph, packed))
 
 # pipe seems to be a good heuristic for the "real" descender
 face = load_glyph(ord('|'))
@@ -201,11 +212,11 @@ face = load_glyph(ord('|'))
 glyph_data = []
 glyph_props = []
 for index, glyph in enumerate(all_glyphs):
-    props, compressed = glyph
-    glyph_data.extend([b for b in compressed])
+    props, packed = glyph
+    glyph_data.extend([b for b in packed])
     glyph_props.append(props)
 
-print(f"/**\n * generated by fontconvert.py\n * name: {font_name}\n * size: {size}\n * compressed: {compress}\n */")
+print(f"/**\n * generated by fontconvert.py\n * name: {font_name}\n * size: {size}\n */")
 print("#pragma once")
 print("#include \"EpdFontData.h\"\n")
 print(f"static const uint8_t {font_name}Bitmaps[{len(glyph_data)}] = {{")
@@ -230,7 +241,6 @@ print(f"    {font_name}Bitmaps,")
 print(f"    {font_name}Glyphs,")
 print(f"    {font_name}Intervals,")
 print(f"    {len(intervals)},")
-print(f"    {1 if compress else 0},")
 print(f"    {norm_ceil(face.size.height)},")
 print(f"    {norm_ceil(face.size.ascender)},")
 print(f"    {norm_floor(face.size.descender)},")

lib/EpdFontRenderer/EpdFontRenderer.hpp

@@ -2,42 +2,23 @@
 #include <EpdFontFamily.h>
 #include <HardwareSerial.h>
 #include <Utf8.h>
-#include <miniz.h>
 
 inline int min(const int a, const int b) { return a < b ? a : b; }
 inline int max(const int a, const int b) { return a > b ? a : b; }
 
-static tinfl_decompressor decomp;
-
 template <typename Renderable>
 class EpdFontRenderer {
-  Renderable* renderer;
+  Renderable& renderer;
   void renderChar(uint32_t cp, int* x, const int* y, uint16_t color, EpdFontStyle style = REGULAR);
 
  public:
   const EpdFontFamily* fontFamily;
-  explicit EpdFontRenderer(const EpdFontFamily* fontFamily, Renderable* renderer)
+  explicit EpdFontRenderer(const EpdFontFamily* fontFamily, Renderable& renderer)
       : fontFamily(fontFamily), renderer(renderer) {}
   ~EpdFontRenderer() = default;
   void renderString(const char* string, int* x, int* y, uint16_t color, EpdFontStyle style = REGULAR);
 };
 
-inline int uncompress(uint8_t* dest, size_t uncompressedSize, const uint8_t* source, size_t sourceSize) {
-  if (uncompressedSize == 0 || dest == nullptr || sourceSize == 0 || source == nullptr) {
-    return -1;
-  }
-  tinfl_init(&decomp);
-
-  // we know everything will fit into the buffer.
-  const tinfl_status decomp_status =
-      tinfl_decompress(&decomp, source, &sourceSize, dest, dest, &uncompressedSize,
-                       TINFL_FLAG_PARSE_ZLIB_HEADER | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF);
-  if (decomp_status != TINFL_STATUS_DONE) {
-    return decomp_status;
-  }
-  return 0;
-}
-
 template <typename Renderable>
 void EpdFontRenderer<Renderable>::renderString(const char* string, int* x, int* y, const uint16_t color,
                                                const EpdFontStyle style) {
@@ -79,50 +60,24 @@ void EpdFontRenderer<Renderable>::renderChar(const uint32_t cp, int* x, const in
   const uint8_t height = glyph->height;
   const int left = glyph->left;
 
-  const int byteWidth = width / 2 + width % 2;
-  const unsigned long bitmapSize = byteWidth * height;
   const uint8_t* bitmap = nullptr;
-
-  if (fontFamily->getData(style)->compressed) {
-    auto* tmpBitmap = static_cast<uint8_t*>(malloc(bitmapSize));
-    if (tmpBitmap == nullptr && bitmapSize) {
-      Serial.println("Failed to allocate memory for decompression buffer");
-      return;
-    }
-
-    uncompress(tmpBitmap, bitmapSize, &fontFamily->getData(style)->bitmap[offset], glyph->compressedSize);
-    bitmap = tmpBitmap;
-  } else {
-    bitmap = &fontFamily->getData(style)->bitmap[offset];
-  }
+  bitmap = &fontFamily->getData(style)->bitmap[offset];
 
   if (bitmap != nullptr) {
-    for (int localY = 0; localY < height; localY++) {
-      int yy = *y - glyph->top + localY;
-      const int startPos = *x + left;
-      bool byteComplete = startPos % 2;
-      int localX = max(0, -startPos);
-      const int maxX = startPos + width;
+    for (int glyphY = 0; glyphY < height; glyphY++) {
+      int screenY = *y - glyph->top + glyphY;
+      for (int glyphX = 0; glyphX < width; glyphX++) {
+        const int pixelPosition = glyphY * width + glyphX;
+        int screenX = *x + left + glyphX;
 
-      for (int xx = startPos; xx < maxX; xx++) {
-        uint8_t bm = bitmap[localY * byteWidth + localX / 2];
-        if ((localX & 1) == 0) {
-          bm = bm & 0xF;
-        } else {
-          bm = bm >> 4;
-        }
+        const uint8_t byte = bitmap[pixelPosition / 8];
+        const uint8_t bit_index = 7 - (pixelPosition % 8);
 
-        if (bm) {
-          renderer->drawPixel(xx, yy, color);
+        if ((byte >> bit_index) & 1) {
+          renderer.drawPixel(screenX, screenY, color);
         }
-        byteComplete = !byteComplete;
-        localX++;
       }
     }
-
-    if (fontFamily->getData(style)->compressed) {
-      free(const_cast<uint8_t*>(bitmap));
-    }
   }
 
   *x += glyph->advanceX;

lib/EpdRenderer/EpdRenderer.cpp

@@ -8,21 +8,30 @@
 #include "builtinFonts/ubuntu_10.h"
 #include "builtinFonts/ubuntu_bold_10.h"
 
-EpdRenderer::EpdRenderer(XteinkDisplay* display) {
-  const auto bookerlyFontFamily = new EpdFontFamily(new EpdFont(&bookerly), new EpdFont(&bookerly_bold),
-                                                    new EpdFont(&bookerly_italic), new EpdFont(&bookerly_bold_italic));
-  const auto ubuntuFontFamily = new EpdFontFamily(new EpdFont(&ubuntu_10), new EpdFont(&ubuntu_bold_10));
+EpdFont bookerlyFont(&bookerly);
+EpdFont bookerlyBoldFont(&bookerly_bold);
+EpdFont bookerlyItalicFont(&bookerly_italic);
+EpdFont bookerlyBoldItalicFont(&bookerly_bold_italic);
+EpdFontFamily bookerlyFontFamily(&bookerlyFont, &bookerlyBoldFont, &bookerlyItalicFont, &bookerlyBoldItalicFont);
 
-  this->display = display;
-  this->regularFontRenderer = new EpdFontRenderer<XteinkDisplay>(bookerlyFontFamily, display);
-  this->smallFontRenderer = new EpdFontRenderer<XteinkDisplay>(new EpdFontFamily(new EpdFont(&babyblue)), display);
-  this->uiFontRenderer = new EpdFontRenderer<XteinkDisplay>(ubuntuFontFamily, display);
+EpdFont smallFont(&babyblue);
+EpdFontFamily smallFontFamily(&smallFont);
 
-  this->marginTop = 11;
-  this->marginBottom = 30;
-  this->marginLeft = 10;
-  this->marginRight = 10;
-  this->lineCompression = 0.95f;
+EpdFont ubuntu10Font(&ubuntu_10);
+EpdFont ununtuBold10Font(&ubuntu_bold_10);
+EpdFontFamily ubuntuFontFamily(&ubuntu10Font, &ununtuBold10Font);
+
+EpdRenderer::EpdRenderer(XteinkDisplay& display)
+    : display(display), marginTop(11), marginBottom(30), marginLeft(10), marginRight(10), lineCompression(0.95f) {
+  this->regularFontRenderer = new EpdFontRenderer<XteinkDisplay>(&bookerlyFontFamily, display);
+  this->smallFontRenderer = new EpdFontRenderer<XteinkDisplay>(&smallFontFamily, display);
+  this->uiFontRenderer = new EpdFontRenderer<XteinkDisplay>(&ubuntuFontFamily, display);
+}
+
+EpdRenderer::~EpdRenderer() {
+  delete regularFontRenderer;
+  delete smallFontRenderer;
+  delete uiFontRenderer;
 }
 
 int EpdRenderer::getTextWidth(const char* text, const EpdFontStyle style) const {
@@ -107,24 +116,24 @@ void EpdRenderer::drawTextBox(const int x, const int y, const std::string& text,
 }
 
 void EpdRenderer::drawLine(int x1, int y1, int x2, int y2, uint16_t color) const {
-  display->drawLine(x1 + marginLeft, y1 + marginTop, x2 + marginLeft, y2 + marginTop,
-                    color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
+  display.drawLine(x1 + marginLeft, y1 + marginTop, x2 + marginLeft, y2 + marginTop,
+                   color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
 }
 
 void EpdRenderer::drawRect(const int x, const int y, const int width, const int height, const uint16_t color) const {
-  display->drawRect(x + marginLeft, y + marginTop, width, height, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
+  display.drawRect(x + marginLeft, y + marginTop, width, height, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
 }
 
 void EpdRenderer::fillRect(const int x, const int y, const int width, const int height, const uint16_t color) const {
-  display->fillRect(x + marginLeft, y + marginTop, width, height, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
+  display.fillRect(x + marginLeft, y + marginTop, width, height, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
 }
 
 void EpdRenderer::drawCircle(const int x, const int y, const int radius, const uint16_t color) const {
-  display->drawCircle(x + marginLeft, y + marginTop, radius, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
+  display.drawCircle(x + marginLeft, y + marginTop, radius, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
 }
 
 void EpdRenderer::fillCircle(const int x, const int y, const int radius, const uint16_t color) const {
-  display->fillCircle(x + marginLeft, y + marginTop, radius, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
+  display.fillCircle(x + marginLeft, y + marginTop, radius, color > 0 ? GxEPD_BLACK : GxEPD_WHITE);
 }
 
 void EpdRenderer::drawImage(const uint8_t bitmap[], const int x, const int y, const int width, const int height,
@@ -134,23 +143,23 @@ void EpdRenderer::drawImage(const uint8_t bitmap[], const int x, const int y, co
 
 void EpdRenderer::drawImageNoMargin(const uint8_t bitmap[], const int x, const int y, const int width, const int height,
                                     const bool invert, const bool mirrorY) const {
-  display->drawImage(bitmap, x, y, width, height, invert, mirrorY);
+  display.drawImage(bitmap, x, y, width, height, invert, mirrorY);
 }
 
 void EpdRenderer::clearScreen(const bool black) const {
   Serial.println("Clearing screen");
-  display->fillScreen(black ? GxEPD_BLACK : GxEPD_WHITE);
+  display.fillScreen(black ? GxEPD_BLACK : GxEPD_WHITE);
 }
 
-void EpdRenderer::flushDisplay(const bool partialUpdate) const { display->display(partialUpdate); }
+void EpdRenderer::flushDisplay(const bool partialUpdate) const { display.display(partialUpdate); }
 
 void EpdRenderer::flushArea(const int x, const int y, const int width, const int height) const {
-  display->displayWindow(x + marginLeft, y + marginTop, width, height);
+  display.displayWindow(x + marginLeft, y + marginTop, width, height);
 }
 
-int EpdRenderer::getPageWidth() const { return display->width() - marginLeft - marginRight; }
+int EpdRenderer::getPageWidth() const { return display.width() - marginLeft - marginRight; }
 
-int EpdRenderer::getPageHeight() const { return display->height() - marginTop - marginBottom; }
+int EpdRenderer::getPageHeight() const { return display.height() - marginTop - marginBottom; }
 
 int EpdRenderer::getSpaceWidth() const { return regularFontRenderer->fontFamily->getGlyph(' ', REGULAR)->advanceX; }
 

lib/EpdRenderer/EpdRenderer.h

@@ -7,7 +7,7 @@
 #define XteinkDisplay GxEPD2_BW<GxEPD2_426_GDEQ0426T82, GxEPD2_426_GDEQ0426T82::HEIGHT>
 
 class EpdRenderer {
-  XteinkDisplay* display;
+  XteinkDisplay& display;
   EpdFontRenderer<XteinkDisplay>* regularFontRenderer;
   EpdFontRenderer<XteinkDisplay>* smallFontRenderer;
   EpdFontRenderer<XteinkDisplay>* uiFontRenderer;
@@ -18,8 +18,8 @@ class EpdRenderer {
   float lineCompression;
 
  public:
-  explicit EpdRenderer(XteinkDisplay* display);
-  ~EpdRenderer() = default;
+  explicit EpdRenderer(XteinkDisplay& display);
+  ~EpdRenderer();
   int getTextWidth(const char* text, EpdFontStyle style = REGULAR) const;
   int getUiTextWidth(const char* text, EpdFontStyle style = REGULAR) const;
   int getSmallTextWidth(const char* text, EpdFontStyle style = REGULAR) const;

lib/Epub/Epub.cpp

@@ -9,7 +9,7 @@
 bool Epub::findContentOpfFile(const ZipFile& zip, std::string& contentOpfFile) {
   // open up the meta data to find where the content.opf file lives
   size_t s;
-  const auto metaInfo = zip.readTextFileToMemory("META-INF/container.xml", &s);
+  const auto metaInfo = reinterpret_cast<char*>(zip.readFileToMemory("META-INF/container.xml", &s, true));
   if (!metaInfo) {
     Serial.println("Could not find META-INF/container.xml");
     return false;
@@ -57,7 +57,7 @@ bool Epub::findContentOpfFile(const ZipFile& zip, std::string& contentOpfFile) {
 
 bool Epub::parseContentOpf(ZipFile& zip, std::string& content_opf_file) {
   // read in the content.opf file and parse it
-  auto contents = zip.readTextFileToMemory(content_opf_file.c_str());
+  auto contents = reinterpret_cast<char*>(zip.readFileToMemory(content_opf_file.c_str(), nullptr, true));
 
   // parse the contents
   tinyxml2::XMLDocument doc;
@@ -168,7 +168,7 @@ bool Epub::parseTocNcxFile(const ZipFile& zip) {
     return false;
   }
 
-  const auto ncxData = zip.readTextFileToMemory(tocNcxItem.c_str());
+  const auto ncxData = reinterpret_cast<char*>(zip.readFileToMemory(tocNcxItem.c_str(), nullptr, true));
   if (!ncxData) {
     Serial.printf("Could not find %s\n", tocNcxItem.c_str());
     return false;
@@ -308,11 +308,11 @@ std::string normalisePath(const std::string& path) {
   return result;
 }
 
-uint8_t* Epub::getItemContents(const std::string& itemHref, size_t* size) const {
+uint8_t* Epub::readItemContentsToBytes(const std::string& itemHref, size_t* size, bool trailingNullByte) const {
   const ZipFile zip("/sd" + filepath);
   const std::string path = normalisePath(itemHref);
 
-  const auto content = zip.readFileToMemory(path.c_str(), size);
+  const auto content = zip.readFileToMemory(path.c_str(), size, trailingNullByte);
   if (!content) {
     Serial.printf("Failed to read item %s\n", path.c_str());
     return nullptr;
@@ -321,17 +321,11 @@ uint8_t* Epub::getItemContents(const std::string& itemHref, size_t* size) const
   return content;
 }
 
-char* Epub::getTextItemContents(const std::string& itemHref, size_t* size) const {
+bool Epub::readItemContentsToStream(const std::string& itemHref, Print& out, const size_t chunkSize) const {
   const ZipFile zip("/sd" + filepath);
   const std::string path = normalisePath(itemHref);
 
-  const auto content = zip.readTextFileToMemory(path.c_str(), size);
-  if (!content) {
-    Serial.printf("Failed to read item %s\n", path.c_str());
-    return nullptr;
-  }
-
-  return content;
+  return zip.readFileToStream(path.c_str(), out, chunkSize);
 }
 
 int Epub::getSpineItemsCount() const { return spine.size(); }

lib/Epub/Epub.h

@@ -56,8 +56,9 @@ class Epub {
   const std::string& getPath() const;
   const std::string& getTitle() const;
   const std::string& getCoverImageItem() const;
-  uint8_t* getItemContents(const std::string& itemHref, size_t* size = nullptr) const;
-  char* getTextItemContents(const std::string& itemHref, size_t* size = nullptr) const;
+  uint8_t* readItemContentsToBytes(const std::string& itemHref, size_t* size = nullptr,
+                                   bool trailingNullByte = false) const;
+  bool readItemContentsToStream(const std::string& itemHref, Print& out, size_t chunkSize) const;
   std::string& getSpineItem(int spineIndex);
   int getSpineItemsCount() const;
   EpubTocEntry& getTocItem(int tocTndex);

lib/Epub/Epub/EpubHtmlParser.cpp

@@ -1,181 +0,0 @@
-#include "EpubHtmlParser.h"
-
-#include <EpdRenderer.h>
-#include <HardwareSerial.h>
-
-#include "Page.h"
-#include "htmlEntities.h"
-
-const char* HEADER_TAGS[] = {"h1", "h2", "h3", "h4", "h5", "h6"};
-constexpr int NUM_HEADER_TAGS = sizeof(HEADER_TAGS) / sizeof(HEADER_TAGS[0]);
-
-const char* BLOCK_TAGS[] = {"p", "li", "div", "br"};
-constexpr int NUM_BLOCK_TAGS = sizeof(BLOCK_TAGS) / sizeof(BLOCK_TAGS[0]);
-
-const char* BOLD_TAGS[] = {"b"};
-constexpr int NUM_BOLD_TAGS = sizeof(BOLD_TAGS) / sizeof(BOLD_TAGS[0]);
-
-const char* ITALIC_TAGS[] = {"i"};
-constexpr int NUM_ITALIC_TAGS = sizeof(ITALIC_TAGS) / sizeof(ITALIC_TAGS[0]);
-
-const char* IMAGE_TAGS[] = {"img"};
-constexpr int NUM_IMAGE_TAGS = sizeof(IMAGE_TAGS) / sizeof(IMAGE_TAGS[0]);
-
-const char* SKIP_TAGS[] = {"head", "table"};
-constexpr int NUM_SKIP_TAGS = sizeof(SKIP_TAGS) / sizeof(SKIP_TAGS[0]);
-
-// given the start and end of a tag, check to see if it matches a known tag
-bool matches(const char* tag_name, const char* possible_tags[], const int possible_tag_count) {
-  for (int i = 0; i < possible_tag_count; i++) {
-    if (strcmp(tag_name, possible_tags[i]) == 0) {
-      return true;
-    }
-  }
-  return false;
-}
-
-// start a new text block if needed
-void EpubHtmlParser::startNewTextBlock(const BLOCK_STYLE style) {
-  if (currentTextBlock) {
-    // already have a text block running and it is empty - just reuse it
-    if (currentTextBlock->isEmpty()) {
-      currentTextBlock->set_style(style);
-      return;
-    }
-
-    currentTextBlock->finish();
-    makePages();
-    delete currentTextBlock;
-  }
-  currentTextBlock = new TextBlock(style);
-}
-
-bool EpubHtmlParser::VisitEnter(const tinyxml2::XMLElement& element, const tinyxml2::XMLAttribute* firstAttribute) {
-  const char* tag_name = element.Name();
-  if (matches(tag_name, IMAGE_TAGS, NUM_IMAGE_TAGS)) {
-    const char* src = element.Attribute("src");
-    if (src) {
-      // don't leave an empty text block in the list
-      // const BLOCK_STYLE style = currentTextBlock->get_style();
-      if (currentTextBlock->isEmpty()) {
-        delete currentTextBlock;
-        currentTextBlock = nullptr;
-      }
-      // TODO: Fix this
-      // blocks.push_back(new ImageBlock(m_base_path + src));
-      // start a new text block - with the same style as before
-      // startNewTextBlock(style);
-    } else {
-      // ESP_LOGE(TAG, "Could not find src attribute");
-    }
-    return false;
-  }
-
-  if (matches(tag_name, SKIP_TAGS, NUM_SKIP_TAGS)) {
-    return false;
-  }
-
-  // Serial.printf("Text: %s\n", element.GetText());
-
-  if (matches(tag_name, HEADER_TAGS, NUM_HEADER_TAGS)) {
-    insideBoldTag = true;
-    startNewTextBlock(CENTER_ALIGN);
-  } else if (matches(tag_name, BLOCK_TAGS, NUM_BLOCK_TAGS)) {
-    if (strcmp(tag_name, "br") == 0) {
-      startNewTextBlock(currentTextBlock->get_style());
-    } else {
-      startNewTextBlock(JUSTIFIED);
-    }
-  } else if (matches(tag_name, BOLD_TAGS, NUM_BOLD_TAGS)) {
-    insideBoldTag = true;
-  } else if (matches(tag_name, ITALIC_TAGS, NUM_ITALIC_TAGS)) {
-    insideItalicTag = true;
-  }
-  return true;
-}
-/// Visit a text node.
-bool EpubHtmlParser::Visit(const tinyxml2::XMLText& text) {
-  const char* content = text.Value();
-  currentTextBlock->addSpan(replaceHtmlEntities(content), insideBoldTag, insideItalicTag);
-  return true;
-}
-
-bool EpubHtmlParser::VisitExit(const tinyxml2::XMLElement& element) {
-  const char* tag_name = element.Name();
-  if (matches(tag_name, HEADER_TAGS, NUM_HEADER_TAGS)) {
-    insideBoldTag = false;
-  } else if (matches(tag_name, BLOCK_TAGS, NUM_BLOCK_TAGS)) {
-    // nothing to do
-  } else if (matches(tag_name, BOLD_TAGS, NUM_BOLD_TAGS)) {
-    insideBoldTag = false;
-  } else if (matches(tag_name, ITALIC_TAGS, NUM_ITALIC_TAGS)) {
-    insideItalicTag = false;
-  }
-  return true;
-}
-
-bool EpubHtmlParser::parseAndBuildPages() {
-  startNewTextBlock(JUSTIFIED);
-  tinyxml2::XMLDocument doc(false, tinyxml2::COLLAPSE_WHITESPACE);
-
-  const tinyxml2::XMLError result = doc.LoadFile(filepath);
-  if (result != tinyxml2::XML_SUCCESS) {
-    Serial.printf("Failed to load file, Error: %s\n", tinyxml2::XMLDocument::ErrorIDToName(result));
-    return false;
-  }
-
-  doc.Accept(this);
-  if (currentTextBlock) {
-    makePages();
-    completePageFn(currentPage);
-    currentPage = nullptr;
-    delete currentTextBlock;
-    currentTextBlock = nullptr;
-  }
-
-  return true;
-}
-
-void EpubHtmlParser::makePages() {
-  if (!currentTextBlock) {
-    Serial.println("!! No text block to make pages for !!");
-    return;
-  }
-
-  if (!currentPage) {
-    currentPage = new Page();
-  }
-
-  const int lineHeight = renderer.getLineHeight();
-  const int pageHeight = renderer.getPageHeight();
-
-  // Long running task, make sure to let other things happen
-  vTaskDelay(1);
-
-  if (currentTextBlock->getType() == TEXT_BLOCK) {
-    const auto lines = currentTextBlock->splitIntoLines(renderer);
-
-    for (const auto line : lines) {
-      if (currentPage->nextY + lineHeight > pageHeight) {
-        completePageFn(currentPage);
-        currentPage = new Page();
-      }
-
-      currentPage->elements.push_back(new PageLine(line, currentPage->nextY));
-      currentPage->nextY += lineHeight;
-    }
-    // TODO: Fix spacing between paras
-    // add some extra line between blocks
-    currentPage->nextY += lineHeight / 2;
-  }
-  // TODO: Image block support
-  // if (block->getType() == BlockType::IMAGE_BLOCK) {
-  //   ImageBlock *imageBlock = (ImageBlock *)block;
-  //   if (y + imageBlock->height > page_height) {
-  //     pages.push_back(new Page());
-  //     y = 0;
-  //   }
-  //   pages.back()->elements.push_back(new PageImage(imageBlock, y));
-  //   y += imageBlock->height;
-  // }
-}

lib/Epub/Epub/EpubHtmlParser.h

@@ -1,34 +0,0 @@
-#pragma once
-#include <tinyxml2.h>
-
-#include <functional>
-
-#include "blocks/TextBlock.h"
-
-class Page;
-class EpdRenderer;
-
-class EpubHtmlParser final : public tinyxml2::XMLVisitor {
-  const char* filepath;
-  EpdRenderer& renderer;
-  std::function<void(Page*)> completePageFn;
-
-  bool insideBoldTag = false;
-  bool insideItalicTag = false;
-  TextBlock* currentTextBlock = nullptr;
-  Page* currentPage = nullptr;
-
-  void startNewTextBlock(BLOCK_STYLE style);
-  void makePages();
-
-  // xml parser callbacks
-  bool VisitEnter(const tinyxml2::XMLElement& element, const tinyxml2::XMLAttribute* firstAttribute) override;
-  bool Visit(const tinyxml2::XMLText& text) override;
-  bool VisitExit(const tinyxml2::XMLElement& element) override;
-  // xml parser callbacks
- public:
-  explicit EpubHtmlParser(const char* filepath, EpdRenderer& renderer, const std::function<void(Page*)>& completePageFn)
-      : filepath(filepath), renderer(renderer), completePageFn(completePageFn) {}
-  ~EpubHtmlParser() override = default;
-  bool parseAndBuildPages();
-};