perf: batch ZIP size lookup in buildBookBin (O(n*m) → O(n log n))

Add ZipFile::fillUncompressedSizes() for single-pass ZIP central directory
scan with hash-based target matching.

Also apply clang-format fixes for CI.

Shadow Slave results:
- buildBookBin: 506s → 35s
- Total indexing: 8.7min → 50s
This commit is contained in:
Daniel
2026-01-21 19:55:24 -08:00
committed by cottongin
parent cf16d33710
commit 16caa66b4a
6 changed files with 171 additions and 28 deletions

View File

@@ -3,6 +3,7 @@
#include <string>
#include <unordered_map>
#include <vector>
class ZipFile {
public:
@@ -19,6 +20,23 @@ class ZipFile {
bool isSet;
};
// Target for batch uncompressed size lookup (sorted by hash, then len)
struct SizeTarget {
uint64_t hash; // FNV-1a 64-bit hash of normalized path
uint16_t len; // Length of path for collision reduction
uint16_t index; // Caller's index (e.g. spine index)
};
// FNV-1a 64-bit hash computed from char buffer (no std::string allocation)
static uint64_t fnvHash64(const char* s, size_t len) {
uint64_t hash = 14695981039346656037ull;
for (size_t i = 0; i < len; i++) {
hash ^= static_cast<uint8_t>(s[i]);
hash *= 1099511628211ull;
}
return hash;
}
private:
const std::string& filePath;
FsFile file;
@@ -43,6 +61,10 @@ class ZipFile {
bool close();
bool loadAllFileStatSlims();
bool getInflatedFileSize(const char* filename, size_t* size);
// Batch lookup: scan ZIP central dir once and fill sizes for matching targets.
// targets must be sorted by (hash, len). sizes[target.index] receives uncompressedSize.
// Returns number of targets matched.
int fillUncompressedSizes(std::vector<SizeTarget>& targets, std::vector<uint32_t>& sizes);
// Due to the memory required to run each of these, it is recommended to not preopen the zip file for multiple
// These functions will open and close the zip as needed
uint8_t* readFileToMemory(const char* filename, size_t* size = nullptr, bool trailingNullByte = false);