From d5f750ba656b4611ef3e47af21fcc47ff5d9e005 Mon Sep 17 00:00:00 2001 From: Dave Allie Date: Sun, 28 Dec 2025 23:54:09 +1100 Subject: [PATCH] Dynamically allocate miniz internal memory only when zip maintained opened --- lib/Epub/Epub.cpp | 17 ++---- lib/Epub/Epub.h | 3 +- lib/Epub/Epub/BookMetadataCache.cpp | 12 ++++- lib/Epub/Epub/parsers/ContentOpfParser.cpp | 1 - lib/ZipFile/ZipFile.cpp | 61 ++++++++++++++++------ lib/ZipFile/ZipFile.h | 22 +++++--- 6 files changed, 76 insertions(+), 40 deletions(-) diff --git a/lib/Epub/Epub.cpp b/lib/Epub/Epub.cpp index b48d7ea..f1dc25e 100644 --- a/lib/Epub/Epub.cpp +++ b/lib/Epub/Epub.cpp @@ -60,9 +60,6 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata) { } ContentOpfParser opfParser(getCachePath(), getBasePath(), contentOpfSize, bookMetadataCache.get()); - Serial.printf("[%lu] [MEM] Free: %d bytes, Total: %d bytes, Min Free: %d bytes\n", millis(), ESP.getFreeHeap(), - ESP.getHeapSize(), ESP.getMinFreeHeap()); - if (!opfParser.setup()) { Serial.printf("[%lu] [EBP] Could not setup content.opf parser\n", millis()); return false; @@ -321,10 +318,9 @@ bool Epub::generateCoverBmp() const { } uint8_t* Epub::readItemContentsToBytes(const std::string& itemHref, size_t* size, const bool trailingNullByte) const { - const ZipFile zip("/sd" + filepath); const std::string path = FsHelpers::normalisePath(itemHref); - const auto content = zip.readFileToMemory(path.c_str(), size, trailingNullByte); + const auto content = ZipFile("/sd" + filepath).readFileToMemory(path.c_str(), size, trailingNullByte); if (!content) { Serial.printf("[%lu] [EBP] Failed to read item %s\n", millis(), path.c_str()); return nullptr; @@ -334,20 +330,13 @@ uint8_t* Epub::readItemContentsToBytes(const std::string& itemHref, size_t* size } bool Epub::readItemContentsToStream(const std::string& itemHref, Print& out, const size_t chunkSize) const { - const ZipFile zip("/sd" + filepath); const std::string path = FsHelpers::normalisePath(itemHref); - - return zip.readFileToStream(path.c_str(), out, chunkSize); + return ZipFile("/sd" + filepath).readFileToStream(path.c_str(), out, chunkSize); } bool Epub::getItemSize(const std::string& itemHref, size_t* size) const { - const ZipFile zip("/sd" + filepath); - return getItemSize(zip, itemHref, size); -} - -bool Epub::getItemSize(const ZipFile& zip, const std::string& itemHref, size_t* size) { const std::string path = FsHelpers::normalisePath(itemHref); - return zip.getInflatedFileSize(path.c_str(), size); + return ZipFile("/sd" + filepath).getInflatedFileSize(path.c_str(), size); } int Epub::getSpineItemsCount() const { diff --git a/lib/Epub/Epub.h b/lib/Epub/Epub.h index acdd32c..c785008 100644 --- a/lib/Epub/Epub.h +++ b/lib/Epub/Epub.h @@ -24,7 +24,6 @@ class Epub { bool findContentOpfFile(std::string* contentOpfFile) const; bool parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata); bool parseTocNcxFile() const; - static bool getItemSize(const ZipFile& zip, const std::string& itemHref, size_t* size); public: explicit Epub(std::string filepath, const std::string& cacheDir) : filepath(std::move(filepath)) { @@ -54,5 +53,5 @@ class Epub { size_t getCumulativeSpineItemSize(int spineIndex) const; size_t getBookSize() const; - uint8_t calculateProgress(const int currentSpineIndex, const float currentSpineRead) const; + uint8_t calculateProgress(int currentSpineIndex, float currentSpineRead) const; }; diff --git a/lib/Epub/Epub/BookMetadataCache.cpp b/lib/Epub/Epub/BookMetadataCache.cpp index 3cef851..9774dee 100644 --- a/lib/Epub/Epub/BookMetadataCache.cpp +++ b/lib/Epub/Epub/BookMetadataCache.cpp @@ -122,7 +122,15 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta // LUTs complete // Loop through spines from spine file matching up TOC indexes, calculating cumulative size and writing to book.bin - const ZipFile zip("/sd" + epubPath); + ZipFile zip("/sd" + epubPath); + // Pre-open zip file to speed up size calculations + if (!zip.open()) { + Serial.printf("[%lu] [BMC] Could not open EPUB zip for size calculations\n", millis()); + bookFile.close(); + spineFile.close(); + tocFile.close(); + return false; + } size_t cumSize = 0; spineFile.seek(0); for (int i = 0; i < spineCount; i++) { @@ -157,6 +165,8 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta // Write out spine data to book.bin writeSpineEntry(bookFile, spineEntry); } + // Close opened zip file + zip.close(); // Loop through toc entries from toc file writing to book.bin tocFile.seek(0); diff --git a/lib/Epub/Epub/parsers/ContentOpfParser.cpp b/lib/Epub/Epub/parsers/ContentOpfParser.cpp index 3cc6401..801f5c2 100644 --- a/lib/Epub/Epub/parsers/ContentOpfParser.cpp +++ b/lib/Epub/Epub/parsers/ContentOpfParser.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include "../BookMetadataCache.h" diff --git a/lib/ZipFile/ZipFile.cpp b/lib/ZipFile/ZipFile.cpp index 83b1184..d6451b9 100644 --- a/lib/ZipFile/ZipFile.cpp +++ b/lib/ZipFile/ZipFile.cpp @@ -27,28 +27,36 @@ bool inflateOneShot(const uint8_t* inputBuf, const size_t deflatedSize, uint8_t* return true; } -ZipFile::ZipFile(std::string filePath) : filePath(std::move(filePath)) { - const bool status = mz_zip_reader_init_file(&zipArchive, this->filePath.c_str(), 0); - - if (!status) { - Serial.printf("[%lu] [ZIP] mz_zip_reader_init_file() failed for %s! Error: %s\n", millis(), this->filePath.c_str(), - mz_zip_get_error_string(zipArchive.m_last_error)); +bool ZipFile::loadFileStat(const char* filename, mz_zip_archive_file_stat* fileStat) { + const bool wasOpen = isOpen(); + if (!wasOpen) { + if (!open()) { + return false; + } } -} -bool ZipFile::loadFileStat(const char* filename, mz_zip_archive_file_stat* fileStat) const { // find the file mz_uint32 fileIndex = 0; - if (!mz_zip_reader_locate_file_v2(&zipArchive, filename, nullptr, 0, &fileIndex)) { + if (!mz_zip_reader_locate_file_v2(zipArchivePtr.get(), filename, nullptr, 0, &fileIndex)) { Serial.printf("[%lu] [ZIP] Could not find file %s\n", millis(), filename); + if (!wasOpen) { + close(); + } return false; } - if (!mz_zip_reader_file_stat(&zipArchive, fileIndex, fileStat)) { + if (!mz_zip_reader_file_stat(zipArchivePtr.get(), fileIndex, fileStat)) { Serial.printf("[%lu] [ZIP] mz_zip_reader_file_stat() failed! Error: %s\n", millis(), - mz_zip_get_error_string(zipArchive.m_last_error)); + mz_zip_get_error_string(zipArchivePtr->m_last_error)); + if (!wasOpen) { + close(); + } return false; } + + if (!wasOpen) { + close(); + } return true; } @@ -83,8 +91,31 @@ long ZipFile::getDataOffset(const mz_zip_archive_file_stat& fileStat) const { return fileOffset + localHeaderSize + filenameLength + extraOffset; } -bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) const { - mz_zip_archive_file_stat fileStat; +bool ZipFile::open() { + zipArchivePtr.reset(new mz_zip_archive); + mz_zip_zero_struct(zipArchivePtr.get()); + const bool status = + mz_zip_reader_init_file(zipArchivePtr.get(), filePath.c_str(), MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY); + + if (!status) { + Serial.printf("[%lu] [ZIP] mz_zip_reader_init_file() failed for %s! Error: %s\n", millis(), filePath.c_str(), + mz_zip_get_error_string(zipArchivePtr->m_last_error)); + zipArchivePtr.reset(); + return false; + } + return true; +} + +bool ZipFile::close() { + if (zipArchivePtr) { + mz_zip_reader_end(zipArchivePtr.get()); + zipArchivePtr.reset(); + } + return true; +} + +bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) { + mz_zip_archive_file_stat fileStat = {}; if (!loadFileStat(filename, &fileStat)) { return false; } @@ -93,7 +124,7 @@ bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) const { return true; } -uint8_t* ZipFile::readFileToMemory(const char* filename, size_t* size, const bool trailingNullByte) const { +uint8_t* ZipFile::readFileToMemory(const char* filename, size_t* size, const bool trailingNullByte) { mz_zip_archive_file_stat fileStat; if (!loadFileStat(filename, &fileStat)) { return nullptr; @@ -173,7 +204,7 @@ uint8_t* ZipFile::readFileToMemory(const char* filename, size_t* size, const boo return data; } -bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t chunkSize) const { +bool ZipFile::readFileToStream(const char* filename, Print& out, const size_t chunkSize) { mz_zip_archive_file_stat fileStat; if (!loadFileStat(filename, &fileStat)) { return false; diff --git a/lib/ZipFile/ZipFile.h b/lib/ZipFile/ZipFile.h index 58e3ab9..a51cfcd 100644 --- a/lib/ZipFile/ZipFile.h +++ b/lib/ZipFile/ZipFile.h @@ -1,20 +1,28 @@ #pragma once #include +#include #include #include "miniz.h" class ZipFile { std::string filePath; - mutable mz_zip_archive zipArchive = {}; - bool loadFileStat(const char* filename, mz_zip_archive_file_stat* fileStat) const; + std::unique_ptr zipArchivePtr; + bool loadFileStat(const char* filename, mz_zip_archive_file_stat* fileStat); long getDataOffset(const mz_zip_archive_file_stat& fileStat) const; public: - explicit ZipFile(std::string filePath); - ~ZipFile() { mz_zip_reader_end(&zipArchive); } - bool getInflatedFileSize(const char* filename, size_t* size) const; - uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false) const; - bool readFileToStream(const char* filename, Print& out, size_t chunkSize) const; + explicit ZipFile(std::string filePath) : filePath(std::move(filePath)) {} + ~ZipFile() = default; + // Zip file can be opened and closed by hand in order to allow for quick calculation of inflated file size + // It is NOT recommended to pre-open it for any kind of inflation due to memory constraints + bool isOpen() const { return zipArchivePtr != nullptr; } + bool open(); + bool close(); + bool getInflatedFileSize(const char* filename, size_t* size); + // Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple + // These functions will open and close the zip as needed + uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false); + bool readFileToStream(const char* filename, Print& out, size_t chunkSize); };