Files
crosspoint-reader-mod/src/activities/reader/TxtReaderActivity.cpp
Adrian Wilkins-Caruana f1e9dc7f30 perf: font-compression improvements (#1056)
## Purpose

This PR includes some preparatory changes that are needed for an
upcoming performant CJK font feature. The changes have no impact on
render time and heap allocation for latin text. **Despite this, I think
these changes stand on their own as a better font
compression/decompression implementation.**

## Summary

- Font decompressor rewrite: Replaced the 4-slot LRU group cache with a
two-tier system — a page buffer (glyphs prewarmed before rendering
begins) and a hot-group fallback (last decompressed group retained for
non-prewarmed
  glyphs). 
- Byte-aligned compressed bitmap format: Glyph bitmaps within compressed
groups are now stored row-padded rather than tightly packed before
DEFLATE compression, improving compression ratios by making identical
pixel rows produce
identical byte patterns. Glyphs are compacted back to packed format on
demand at render time. Reduces flash size by 155 KB.
- Page prewarm system: Added `Page::collectText` and
`Page::getDominantStyle` to extract per-style glyph requirements before
rendering, and `GfxRenderer::prewarmFontCache` to pre-decompress only
the groups needed for the dominant style
   — eliminating mid-render decompression for the common case.
- UTF-8 robustness fixes: `utf8NextCodepoint` now validates continuation
bytes and returns a replacement glyph on malformed input;
`ChapterHtmlSlimParser` correctly preserves incomplete multi-byte
sequences across word-buffer flush
  boundaries rather than splitting them.

---

### AI Usage

While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.

Did you use AI tools to help write this code? _**YES**_ Architecture and
design was done by me, refined a bit by Claude. Code mostly by Claude,
but not entirely.
2026-03-11 21:05:46 +01:00

557 lines
16 KiB
C++

#include "TxtReaderActivity.h"
#include <FontCacheManager.h>
#include <GfxRenderer.h>
#include <HalStorage.h>
#include <I18n.h>
#include <Serialization.h>
#include <Utf8.h>
#include "CrossPointSettings.h"
#include "CrossPointState.h"
#include "MappedInputManager.h"
#include "ReaderUtils.h"
#include "RecentBooksStore.h"
#include "components/UITheme.h"
#include "fontIds.h"
namespace {
constexpr size_t CHUNK_SIZE = 8 * 1024; // 8KB chunk for reading
// Cache file magic and version
constexpr uint32_t CACHE_MAGIC = 0x54585449; // "TXTI"
constexpr uint8_t CACHE_VERSION = 2; // Increment when cache format changes
} // namespace
void TxtReaderActivity::onEnter() {
Activity::onEnter();
if (!txt) {
return;
}
ReaderUtils::applyOrientation(renderer, SETTINGS.orientation);
txt->setupCacheDir();
// Save current txt as last opened file and add to recent books
auto filePath = txt->getPath();
auto fileName = filePath.substr(filePath.rfind('/') + 1);
APP_STATE.openEpubPath = filePath;
APP_STATE.saveToFile();
RECENT_BOOKS.addBook(filePath, fileName, "", "");
// Trigger first update
requestUpdate();
}
void TxtReaderActivity::onExit() {
Activity::onExit();
// Reset orientation back to portrait for the rest of the UI
renderer.setOrientation(GfxRenderer::Orientation::Portrait);
pageOffsets.clear();
currentPageLines.clear();
APP_STATE.readerActivityLoadCount = 0;
APP_STATE.saveToFile();
txt.reset();
}
void TxtReaderActivity::loop() {
// Long press BACK (1s+) goes to file selection
if (mappedInput.isPressed(MappedInputManager::Button::Back) && mappedInput.getHeldTime() >= ReaderUtils::GO_HOME_MS) {
activityManager.goToFileBrowser(txt ? txt->getPath() : "");
return;
}
// Short press BACK goes directly to home
if (mappedInput.wasReleased(MappedInputManager::Button::Back) &&
mappedInput.getHeldTime() < ReaderUtils::GO_HOME_MS) {
onGoHome();
return;
}
auto [prevTriggered, nextTriggered] = ReaderUtils::detectPageTurn(mappedInput);
if (!prevTriggered && !nextTriggered) {
return;
}
if (prevTriggered && currentPage > 0) {
currentPage--;
requestUpdate();
} else if (nextTriggered && currentPage < totalPages - 1) {
currentPage++;
requestUpdate();
}
}
void TxtReaderActivity::initializeReader() {
if (initialized) {
return;
}
// Store current settings for cache validation
cachedFontId = SETTINGS.getReaderFontId();
cachedScreenMargin = SETTINGS.screenMargin;
cachedParagraphAlignment = SETTINGS.paragraphAlignment;
// Calculate viewport dimensions
renderer.getOrientedViewableTRBL(&cachedOrientedMarginTop, &cachedOrientedMarginRight, &cachedOrientedMarginBottom,
&cachedOrientedMarginLeft);
cachedOrientedMarginTop += cachedScreenMargin;
cachedOrientedMarginLeft += cachedScreenMargin;
cachedOrientedMarginRight += cachedScreenMargin;
cachedOrientedMarginBottom +=
std::max(cachedScreenMargin, static_cast<uint8_t>(UITheme::getInstance().getStatusBarHeight()));
viewportWidth = renderer.getScreenWidth() - cachedOrientedMarginLeft - cachedOrientedMarginRight;
const int viewportHeight = renderer.getScreenHeight() - cachedOrientedMarginTop - cachedOrientedMarginBottom;
const int lineHeight = renderer.getLineHeight(cachedFontId);
linesPerPage = viewportHeight / lineHeight;
if (linesPerPage < 1) linesPerPage = 1;
LOG_DBG("TRS", "Viewport: %dx%d, lines per page: %d", viewportWidth, viewportHeight, linesPerPage);
// Try to load cached page index first
if (!loadPageIndexCache()) {
// Cache not found, build page index
buildPageIndex();
// Save to cache for next time
savePageIndexCache();
}
// Load saved progress
loadProgress();
initialized = true;
}
void TxtReaderActivity::buildPageIndex() {
pageOffsets.clear();
pageOffsets.push_back(0); // First page starts at offset 0
size_t offset = 0;
const size_t fileSize = txt->getFileSize();
LOG_DBG("TRS", "Building page index for %zu bytes...", fileSize);
GUI.drawPopup(renderer, tr(STR_INDEXING));
while (offset < fileSize) {
std::vector<std::string> tempLines;
size_t nextOffset = offset;
if (!loadPageAtOffset(offset, tempLines, nextOffset)) {
break;
}
if (nextOffset <= offset) {
// No progress made, avoid infinite loop
break;
}
offset = nextOffset;
if (offset < fileSize) {
pageOffsets.push_back(offset);
}
// Yield to other tasks periodically
if (pageOffsets.size() % 20 == 0) {
vTaskDelay(1);
}
}
totalPages = pageOffsets.size();
LOG_DBG("TRS", "Built page index: %d pages", totalPages);
}
bool TxtReaderActivity::loadPageAtOffset(size_t offset, std::vector<std::string>& outLines, size_t& nextOffset) {
outLines.clear();
const size_t fileSize = txt->getFileSize();
if (offset >= fileSize) {
return false;
}
// Read a chunk from file
size_t chunkSize = std::min(CHUNK_SIZE, fileSize - offset);
auto* buffer = static_cast<uint8_t*>(malloc(chunkSize + 1));
if (!buffer) {
LOG_ERR("TRS", "Failed to allocate %zu bytes", chunkSize);
return false;
}
if (!txt->readContent(buffer, offset, chunkSize)) {
free(buffer);
return false;
}
buffer[chunkSize] = '\0';
// Parse lines from buffer
size_t pos = 0;
while (pos < chunkSize && static_cast<int>(outLines.size()) < linesPerPage) {
// Find end of line
size_t lineEnd = pos;
while (lineEnd < chunkSize && buffer[lineEnd] != '\n') {
lineEnd++;
}
// Check if we have a complete line
bool lineComplete = (lineEnd < chunkSize) || (offset + lineEnd >= fileSize);
if (!lineComplete && static_cast<int>(outLines.size()) > 0) {
// Incomplete line and we already have some lines, stop here
break;
}
// Calculate the actual length of line content in the buffer (excluding newline)
size_t lineContentLen = lineEnd - pos;
// Check for carriage return
bool hasCR = (lineContentLen > 0 && buffer[pos + lineContentLen - 1] == '\r');
size_t displayLen = hasCR ? lineContentLen - 1 : lineContentLen;
// Extract line content for display (without CR/LF)
std::string line(reinterpret_cast<char*>(buffer + pos), displayLen);
// Track position within this source line (in bytes from pos)
size_t lineBytePos = 0;
// Word wrap if needed
while (!line.empty() && static_cast<int>(outLines.size()) < linesPerPage) {
int lineWidth = renderer.getTextWidth(cachedFontId, line.c_str());
if (lineWidth <= viewportWidth) {
outLines.push_back(line);
lineBytePos = displayLen; // Consumed entire display content
line.clear();
break;
}
// Find break point
size_t breakPos = line.length();
while (breakPos > 0 && renderer.getTextWidth(cachedFontId, line.substr(0, breakPos).c_str()) > viewportWidth) {
// Try to break at space
size_t spacePos = line.rfind(' ', breakPos - 1);
if (spacePos != std::string::npos && spacePos > 0) {
breakPos = spacePos;
} else {
// Break at character boundary for UTF-8
breakPos--;
// Make sure we don't break in the middle of a UTF-8 sequence
while (breakPos > 0 && (line[breakPos] & 0xC0) == 0x80) {
breakPos--;
}
}
}
if (breakPos == 0) {
breakPos = 1;
}
outLines.push_back(line.substr(0, breakPos));
// Skip space at break point
size_t skipChars = breakPos;
if (breakPos < line.length() && line[breakPos] == ' ') {
skipChars++;
}
lineBytePos += skipChars;
line = line.substr(skipChars);
}
// Determine how much of the source buffer we consumed
if (line.empty()) {
// Fully consumed this source line, move past the newline
pos = lineEnd + 1;
} else {
// Partially consumed - page is full mid-line
// Move pos to where we stopped in the line (NOT past the line)
pos = pos + lineBytePos;
break;
}
}
// Ensure we make progress even if calculations go wrong
if (pos == 0 && !outLines.empty()) {
// Fallback: at minimum, consume something to avoid infinite loop
pos = 1;
}
nextOffset = offset + pos;
// Make sure we don't go past the file
if (nextOffset > fileSize) {
nextOffset = fileSize;
}
free(buffer);
return !outLines.empty();
}
void TxtReaderActivity::render(RenderLock&&) {
if (!txt) {
return;
}
// Initialize reader if not done
if (!initialized) {
initializeReader();
}
if (pageOffsets.empty()) {
renderer.clearScreen();
renderer.drawCenteredText(UI_12_FONT_ID, 300, tr(STR_EMPTY_FILE), true, EpdFontFamily::BOLD);
renderer.displayBuffer();
return;
}
// Bounds check
if (currentPage < 0) currentPage = 0;
if (currentPage >= totalPages) currentPage = totalPages - 1;
// Load current page content
size_t offset = pageOffsets[currentPage];
size_t nextOffset;
currentPageLines.clear();
loadPageAtOffset(offset, currentPageLines, nextOffset);
renderer.clearScreen();
renderPage();
// Save progress
saveProgress();
}
void TxtReaderActivity::renderPage() {
const int lineHeight = renderer.getLineHeight(cachedFontId);
const int contentWidth = viewportWidth;
// Render text lines with alignment
auto renderLines = [&]() {
int y = cachedOrientedMarginTop;
for (const auto& line : currentPageLines) {
if (!line.empty()) {
int x = cachedOrientedMarginLeft;
// Apply text alignment
switch (cachedParagraphAlignment) {
case CrossPointSettings::LEFT_ALIGN:
default:
// x already set to left margin
break;
case CrossPointSettings::CENTER_ALIGN: {
int textWidth = renderer.getTextWidth(cachedFontId, line.c_str());
x = cachedOrientedMarginLeft + (contentWidth - textWidth) / 2;
break;
}
case CrossPointSettings::RIGHT_ALIGN: {
int textWidth = renderer.getTextWidth(cachedFontId, line.c_str());
x = cachedOrientedMarginLeft + contentWidth - textWidth;
break;
}
case CrossPointSettings::JUSTIFIED:
// For plain text, justified is treated as left-aligned
// (true justification would require word spacing adjustments)
break;
}
renderer.drawText(cachedFontId, x, y, line.c_str());
}
y += lineHeight;
}
};
// Font prewarm: scan pass accumulates text, then prewarm, then real render
auto* fcm = renderer.getFontCacheManager();
auto scope = fcm->createPrewarmScope();
renderLines(); // scan pass — text accumulated, no drawing
scope.endScanAndPrewarm();
// BW rendering
renderLines();
renderStatusBar();
ReaderUtils::displayWithRefreshCycle(renderer, pagesUntilFullRefresh);
if (SETTINGS.textAntiAliasing) {
ReaderUtils::renderAntiAliased(renderer, [&renderLines]() { renderLines(); });
}
// scope destructor clears font cache via FontCacheManager
}
void TxtReaderActivity::renderStatusBar() const {
const float progress = totalPages > 0 ? (currentPage + 1) * 100.0f / totalPages : 0;
std::string title;
if (SETTINGS.statusBarTitle != CrossPointSettings::STATUS_BAR_TITLE::HIDE_TITLE) {
title = txt->getTitle();
}
GUI.drawStatusBar(renderer, progress, currentPage + 1, totalPages, title);
}
void TxtReaderActivity::saveProgress() const {
FsFile f;
if (Storage.openFileForWrite("TRS", txt->getCachePath() + "/progress.bin", f)) {
uint8_t data[4];
data[0] = currentPage & 0xFF;
data[1] = (currentPage >> 8) & 0xFF;
data[2] = 0;
data[3] = 0;
f.write(data, 4);
f.close();
}
}
void TxtReaderActivity::loadProgress() {
FsFile f;
if (Storage.openFileForRead("TRS", txt->getCachePath() + "/progress.bin", f)) {
uint8_t data[4];
if (f.read(data, 4) == 4) {
currentPage = data[0] + (data[1] << 8);
if (currentPage >= totalPages) {
currentPage = totalPages - 1;
}
if (currentPage < 0) {
currentPage = 0;
}
LOG_DBG("TRS", "Loaded progress: page %d/%d", currentPage, totalPages);
}
f.close();
}
}
bool TxtReaderActivity::loadPageIndexCache() {
// Cache file format (using serialization module):
// - uint32_t: magic "TXTI"
// - uint8_t: cache version
// - uint32_t: file size (to validate cache)
// - int32_t: viewport width
// - int32_t: lines per page
// - int32_t: font ID (to invalidate cache on font change)
// - int32_t: screen margin (to invalidate cache on margin change)
// - uint8_t: paragraph alignment (to invalidate cache on alignment change)
// - uint32_t: total pages count
// - N * uint32_t: page offsets
std::string cachePath = txt->getCachePath() + "/index.bin";
FsFile f;
if (!Storage.openFileForRead("TRS", cachePath, f)) {
LOG_DBG("TRS", "No page index cache found");
return false;
}
// Read and validate header using serialization module
uint32_t magic;
serialization::readPod(f, magic);
if (magic != CACHE_MAGIC) {
LOG_DBG("TRS", "Cache magic mismatch, rebuilding");
f.close();
return false;
}
uint8_t version;
serialization::readPod(f, version);
if (version != CACHE_VERSION) {
LOG_DBG("TRS", "Cache version mismatch (%d != %d), rebuilding", version, CACHE_VERSION);
f.close();
return false;
}
uint32_t fileSize;
serialization::readPod(f, fileSize);
if (fileSize != txt->getFileSize()) {
LOG_DBG("TRS", "Cache file size mismatch, rebuilding");
f.close();
return false;
}
int32_t cachedWidth;
serialization::readPod(f, cachedWidth);
if (cachedWidth != viewportWidth) {
LOG_DBG("TRS", "Cache viewport width mismatch, rebuilding");
f.close();
return false;
}
int32_t cachedLines;
serialization::readPod(f, cachedLines);
if (cachedLines != linesPerPage) {
LOG_DBG("TRS", "Cache lines per page mismatch, rebuilding");
f.close();
return false;
}
int32_t fontId;
serialization::readPod(f, fontId);
if (fontId != cachedFontId) {
LOG_DBG("TRS", "Cache font ID mismatch (%d != %d), rebuilding", fontId, cachedFontId);
f.close();
return false;
}
int32_t margin;
serialization::readPod(f, margin);
if (margin != cachedScreenMargin) {
LOG_DBG("TRS", "Cache screen margin mismatch, rebuilding");
f.close();
return false;
}
uint8_t alignment;
serialization::readPod(f, alignment);
if (alignment != cachedParagraphAlignment) {
LOG_DBG("TRS", "Cache paragraph alignment mismatch, rebuilding");
f.close();
return false;
}
uint32_t numPages;
serialization::readPod(f, numPages);
// Read page offsets
pageOffsets.clear();
pageOffsets.reserve(numPages);
for (uint32_t i = 0; i < numPages; i++) {
uint32_t offset;
serialization::readPod(f, offset);
pageOffsets.push_back(offset);
}
f.close();
totalPages = pageOffsets.size();
LOG_DBG("TRS", "Loaded page index cache: %d pages", totalPages);
return true;
}
void TxtReaderActivity::savePageIndexCache() const {
std::string cachePath = txt->getCachePath() + "/index.bin";
FsFile f;
if (!Storage.openFileForWrite("TRS", cachePath, f)) {
LOG_ERR("TRS", "Failed to save page index cache");
return;
}
// Write header using serialization module
serialization::writePod(f, CACHE_MAGIC);
serialization::writePod(f, CACHE_VERSION);
serialization::writePod(f, static_cast<uint32_t>(txt->getFileSize()));
serialization::writePod(f, static_cast<int32_t>(viewportWidth));
serialization::writePod(f, static_cast<int32_t>(linesPerPage));
serialization::writePod(f, static_cast<int32_t>(cachedFontId));
serialization::writePod(f, static_cast<int32_t>(cachedScreenMargin));
serialization::writePod(f, cachedParagraphAlignment);
serialization::writePod(f, static_cast<uint32_t>(pageOffsets.size()));
// Write page offsets
for (size_t offset : pageOffsets) {
serialization::writePod(f, static_cast<uint32_t>(offset));
}
f.close();
LOG_DBG("TRS", "Saved page index cache: %d pages", totalPages);
}