perf: batch ZIP size lookup in buildBookBin (O(n*m) → O(n log n))
Add ZipFile::fillUncompressedSizes() for single-pass ZIP central directory scan with hash-based target matching. Also apply clang-format fixes for CI. Shadow Slave results: - buildBookBin: 506s → 35s - Total indexing: 8.7min → 50s
This commit is contained in:
@@ -4,6 +4,8 @@
|
||||
#include <SDCardManager.h>
|
||||
#include <miniz.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
bool inflateOneShot(const uint8_t* inputBuf, const size_t deflatedSize, uint8_t* outputBuf, const size_t inflatedSize) {
|
||||
// Setup inflator
|
||||
const auto inflator = static_cast<tinfl_decompressor*>(malloc(sizeof(tinfl_decompressor)));
|
||||
@@ -302,6 +304,80 @@ bool ZipFile::getInflatedFileSize(const char* filename, size_t* size) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int ZipFile::fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes) {
|
||||
if (targets.empty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const bool wasOpen = isOpen();
|
||||
if (!wasOpen && !open()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!loadZipDetails()) {
|
||||
if (!wasOpen) {
|
||||
close();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
file.seek(zipDetails.centralDirOffset);
|
||||
|
||||
int matched = 0;
|
||||
uint32_t sig;
|
||||
char itemName[256];
|
||||
|
||||
while (file.available()) {
|
||||
file.read(&sig, 4);
|
||||
if (sig != 0x02014b50) break;
|
||||
|
||||
file.seekCur(6);
|
||||
uint16_t method;
|
||||
file.read(&method, 2);
|
||||
file.seekCur(8);
|
||||
uint32_t compressedSize, uncompressedSize;
|
||||
file.read(&compressedSize, 4);
|
||||
file.read(&uncompressedSize, 4);
|
||||
uint16_t nameLen, m, k;
|
||||
file.read(&nameLen, 2);
|
||||
file.read(&m, 2);
|
||||
file.read(&k, 2);
|
||||
file.seekCur(8);
|
||||
uint32_t localHeaderOffset;
|
||||
file.read(&localHeaderOffset, 4);
|
||||
|
||||
if (nameLen < 256) {
|
||||
file.read(itemName, nameLen);
|
||||
itemName[nameLen] = '\0';
|
||||
|
||||
uint64_t hash = fnvHash64(itemName, nameLen);
|
||||
SizeTarget key = {hash, nameLen, 0};
|
||||
|
||||
auto it = std::lower_bound(targets.begin(), targets.end(), key, [](const SizeTarget& a, const SizeTarget& b) {
|
||||
return a.hash < b.hash || (a.hash == b.hash && a.len < b.len);
|
||||
});
|
||||
|
||||
while (it != targets.end() && it->hash == hash && it->len == nameLen) {
|
||||
if (it->index < sizes.size()) {
|
||||
sizes[it->index] = uncompressedSize;
|
||||
matched++;
|
||||
}
|
||||
++it;
|
||||
}
|
||||
} else {
|
||||
file.seekCur(nameLen);
|
||||
}
|
||||
|
||||
file.seekCur(m + k);
|
||||
}
|
||||
|
||||
if (!wasOpen) {
|
||||
close();
|
||||
}
|
||||
|
||||
return matched;
|
||||
}
|
||||
|
||||
uint8_t* ZipFile::readFileToMemory(const char* filename, size_t* size, const bool trailingNullByte) {
|
||||
const bool wasOpen = isOpen();
|
||||
if (!wasOpen && !open()) {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
class ZipFile {
|
||||
public:
|
||||
@@ -19,6 +20,23 @@ class ZipFile {
|
||||
bool isSet;
|
||||
};
|
||||
|
||||
// Target for batch uncompressed size lookup (sorted by hash, then len)
|
||||
struct SizeTarget {
|
||||
uint64_t hash; // FNV-1a 64-bit hash of normalized path
|
||||
uint16_t len; // Length of path for collision reduction
|
||||
uint16_t index; // Caller's index (e.g. spine index)
|
||||
};
|
||||
|
||||
// FNV-1a 64-bit hash computed from char buffer (no std::string allocation)
|
||||
static uint64_t fnvHash64(const char* s, size_t len) {
|
||||
uint64_t hash = 14695981039346656037ull;
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
hash ^= static_cast<uint8_t>(s[i]);
|
||||
hash *= 1099511628211ull;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
private:
|
||||
const std::string& filePath;
|
||||
FsFile file;
|
||||
@@ -43,6 +61,10 @@ class ZipFile {
|
||||
bool close();
|
||||
bool loadAllFileStatSlims();
|
||||
bool getInflatedFileSize(const char* filename, size_t* size);
|
||||
// Batch lookup: scan ZIP central dir once and fill sizes for matching targets.
|
||||
// targets must be sorted by (hash, len). sizes[target.index] receives uncompressedSize.
|
||||
// Returns number of targets matched.
|
||||
int fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes);
|
||||
// Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple
|
||||
// These functions will open and close the zip as needed
|
||||
uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false);
|
||||
|
||||
Reference in New Issue
Block a user