feat: Migrate binary settings to json (#920)

## Summary

* This PR introduces a migration from binary file storage to JSON-based
storage for application settings, state, and various credential stores.
This improves readability, maintainability, and allows for easier manual
configuration editing.
* Benefits:
  - Settings files are now JSON and can be easily read/edited manually
  - Easier to inspect application state and settings during development
  - JSON structure is more flexible for future changes
* Drawback: around 15k of additional flash usage
* Compatibility: Seamless migration preserves existing user data

## Additional Context
1. New JSON I/O Infrastructure files:
- JsonSettingsIO: Core JSON serialization/deserialization logic using
ArduinoJson library
- ObfuscationUtils: XOR-based password obfuscation for sensitive data
2. Migrated Components (now use JSON storage with automatic binary
migration):
     - CrossPointSettings (settings.json): Main application settings
- CrossPointState (state.json): Application state (open book, sleep
mode, etc.)
- WifiCredentialStore (wifi.json): WiFi network credentials (Password
Obfuscation: Sensitive data like WiFi passwords, uses XOR encryption
with fixed keys. Note: This is obfuscation, not cryptographic security -
passwords can be recovered with the key)
- KOReaderCredentialStore (koreader.json): KOReader sync credentials
     - RecentBooksStore (recent.json): Recently opened books list
3. Migration Logic
     - Forward Compatibility: New installations use JSON format
- Backward Compatibility: Existing binary files are automatically
migrated to JSON on first load
- Backup Safety: Original binary files are renamed with .bak extension
after successful migration
- Fallback Handling: If JSON parsing fails, system falls back to binary
loading
4. Infrastructure Updates
     - HalStorage: Added rename() method for backup operations

---

### AI Usage

While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.

Did you use AI tools to help write this code? _** YES**_

---------

Co-authored-by: Dave Allie <dave@daveallie.com>

lib/KOReaderSync/KOReaderCredentialStore.cpp

@@ -3,73 +3,81 @@
 #include <HalStorage.h>
 #include <Logging.h>
 #include <MD5Builder.h>
+#include <ObfuscationUtils.h>
 #include <Serialization.h>
 
+#include "../../src/JsonSettingsIO.h"
+
 // Initialize the static instance
 KOReaderCredentialStore KOReaderCredentialStore::instance;
 
 namespace {
-// File format version
+// File format version (for binary migration)
 constexpr uint8_t KOREADER_FILE_VERSION = 1;
 
-// KOReader credentials file path
-constexpr char KOREADER_FILE[] = "/.crosspoint/koreader.bin";
+// File paths
+constexpr char KOREADER_FILE_BIN[] = "/.crosspoint/koreader.bin";
+constexpr char KOREADER_FILE_JSON[] = "/.crosspoint/koreader.json";
+constexpr char KOREADER_FILE_BAK[] = "/.crosspoint/koreader.bin.bak";
 
 // Default sync server URL
 constexpr char DEFAULT_SERVER_URL[] = "https://sync.koreader.rocks:443";
 
-// Obfuscation key - "KOReader" in ASCII
-// This is NOT cryptographic security, just prevents casual file reading
-constexpr uint8_t OBFUSCATION_KEY[] = {0x4B, 0x4F, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72};
-constexpr size_t KEY_LENGTH = sizeof(OBFUSCATION_KEY);
-}  // namespace
+// Legacy obfuscation key - "KOReader" in ASCII (only used for binary migration)
+constexpr uint8_t LEGACY_OBFUSCATION_KEY[] = {0x4B, 0x4F, 0x52, 0x65, 0x61, 0x64, 0x65, 0x72};
+constexpr size_t LEGACY_KEY_LENGTH = sizeof(LEGACY_OBFUSCATION_KEY);
 
-void KOReaderCredentialStore::obfuscate(std::string& data) const {
+void legacyDeobfuscate(std::string& data) {
   for (size_t i = 0; i < data.size(); i++) {
-    data[i] ^= OBFUSCATION_KEY[i % KEY_LENGTH];
+    data[i] ^= LEGACY_OBFUSCATION_KEY[i % LEGACY_KEY_LENGTH];
   }
 }
+}  // namespace
 
 bool KOReaderCredentialStore::saveToFile() const {
-  // Make sure the directory exists
   Storage.mkdir("/.crosspoint");
-
-  FsFile file;
-  if (!Storage.openFileForWrite("KRS", KOREADER_FILE, file)) {
-    return false;
-  }
-
-  // Write header
-  serialization::writePod(file, KOREADER_FILE_VERSION);
-
-  // Write username (plaintext - not particularly sensitive)
-  serialization::writeString(file, username);
-  LOG_DBG("KRS", "Saving username: %s", username.c_str());
-
-  // Write password (obfuscated)
-  std::string obfuscatedPwd = password;
-  obfuscate(obfuscatedPwd);
-  serialization::writeString(file, obfuscatedPwd);
-
-  // Write server URL
-  serialization::writeString(file, serverUrl);
-
-  // Write match method
-  serialization::writePod(file, static_cast<uint8_t>(matchMethod));
-
-  file.close();
-  LOG_DBG("KRS", "Saved KOReader credentials to file");
-  return true;
+  return JsonSettingsIO::saveKOReader(*this, KOREADER_FILE_JSON);
 }
 
 bool KOReaderCredentialStore::loadFromFile() {
+  // Try JSON first
+  if (Storage.exists(KOREADER_FILE_JSON)) {
+    String json = Storage.readFile(KOREADER_FILE_JSON);
+    if (!json.isEmpty()) {
+      bool resave = false;
+      bool result = JsonSettingsIO::loadKOReader(*this, json.c_str(), &resave);
+      if (result && resave) {
+        saveToFile();
+        LOG_DBG("KRS", "Resaved KOReader credentials to update format");
+      }
+      return result;
+    }
+  }
+
+  // Fall back to binary migration
+  if (Storage.exists(KOREADER_FILE_BIN)) {
+    if (loadFromBinaryFile()) {
+      if (saveToFile()) {
+        Storage.rename(KOREADER_FILE_BIN, KOREADER_FILE_BAK);
+        LOG_DBG("KRS", "Migrated koreader.bin to koreader.json");
+        return true;
+      } else {
+        LOG_ERR("KRS", "Failed to save KOReader credentials during migration");
+        return false;
+      }
+    }
+  }
+
+  LOG_DBG("KRS", "No credentials file found");
+  return false;
+}
+
+bool KOReaderCredentialStore::loadFromBinaryFile() {
   FsFile file;
-  if (!Storage.openFileForRead("KRS", KOREADER_FILE, file)) {
-    LOG_DBG("KRS", "No credentials file found");
+  if (!Storage.openFileForRead("KRS", KOREADER_FILE_BIN, file)) {
     return false;
   }
 
-  // Read and verify version
   uint8_t version;
   serialization::readPod(file, version);
   if (version != KOREADER_FILE_VERSION) {
@@ -78,29 +86,25 @@ bool KOReaderCredentialStore::loadFromFile() {
     return false;
   }
 
-  // Read username
   if (file.available()) {
     serialization::readString(file, username);
   } else {
     username.clear();
   }
 
-  // Read and deobfuscate password
   if (file.available()) {
     serialization::readString(file, password);
-    obfuscate(password);  // XOR is symmetric, so same function deobfuscates
+    legacyDeobfuscate(password);
   } else {
     password.clear();
   }
 
-  // Read server URL
   if (file.available()) {
     serialization::readString(file, serverUrl);
   } else {
     serverUrl.clear();
   }
 
-  // Read match method
   if (file.available()) {
     uint8_t method;
     serialization::readPod(file, method);
@@ -110,7 +114,7 @@ bool KOReaderCredentialStore::loadFromFile() {
   }
 
   file.close();
-  LOG_DBG("KRS", "Loaded KOReader credentials for user: %s", username.c_str());
+  LOG_DBG("KRS", "Loaded KOReader credentials from binary for user: %s", username.c_str());
   return true;
 }
 

lib/KOReaderSync/KOReaderCredentialStore.h

@@ -8,10 +8,17 @@ enum class DocumentMatchMethod : uint8_t {
   BINARY = 1,    // Match by partial MD5 of file content (more accurate, but files must be identical)
 };
 
+class KOReaderCredentialStore;
+namespace JsonSettingsIO {
+bool saveKOReader(const KOReaderCredentialStore& store, const char* path);
+bool loadKOReader(KOReaderCredentialStore& store, const char* json, bool* needsResave);
+}  // namespace JsonSettingsIO
+
 /**
  * Singleton class for storing KOReader sync credentials on the SD card.
- * Credentials are stored in /sd/.crosspoint/koreader.bin with basic
- * XOR obfuscation to prevent casual reading (not cryptographically secure).
+ * Passwords are XOR-obfuscated with the device's unique hardware MAC address
+ * and base64-encoded before writing to JSON (not cryptographically secure,
+ * but prevents casual reading and ties credentials to the specific device).
  */
 class KOReaderCredentialStore {
  private:
@@ -24,8 +31,10 @@ class KOReaderCredentialStore {
   // Private constructor for singleton
   KOReaderCredentialStore() = default;
 
-  // XOR obfuscation (symmetric - same for encode/decode)
-  void obfuscate(std::string& data) const;
+  bool loadFromBinaryFile();
+
+  friend bool JsonSettingsIO::saveKOReader(const KOReaderCredentialStore&, const char*);
+  friend bool JsonSettingsIO::loadKOReader(KOReaderCredentialStore&, const char*, bool*);
 
  public:
   // Delete copy constructor and assignment

lib/Serialization/ObfuscationUtils.cpp

@@ -0,0 +1,98 @@
+#include "ObfuscationUtils.h"
+
+#include <Logging.h>
+#include <base64.h>
+#include <esp_mac.h>
+#include <mbedtls/base64.h>
+
+#include <cstring>
+
+namespace obfuscation {
+
+namespace {
+constexpr size_t HW_KEY_LEN = 6;
+
+// Simple lazy init — no thread-safety concern on single-core ESP32-C3.
+const uint8_t* getHwKey() {
+  static uint8_t key[HW_KEY_LEN] = {};
+  static bool initialized = false;
+  if (!initialized) {
+    esp_efuse_mac_get_default(key);
+    initialized = true;
+  }
+  return key;
+}
+}  // namespace
+
+void xorTransform(std::string& data) {
+  const uint8_t* key = getHwKey();
+  for (size_t i = 0; i < data.size(); i++) {
+    data[i] ^= key[i % HW_KEY_LEN];
+  }
+}
+
+void xorTransform(std::string& data, const uint8_t* key, size_t keyLen) {
+  if (keyLen == 0 || key == nullptr) return;
+  for (size_t i = 0; i < data.size(); i++) {
+    data[i] ^= key[i % keyLen];
+  }
+}
+
+String obfuscateToBase64(const std::string& plaintext) {
+  if (plaintext.empty()) return "";
+  std::string temp = plaintext;
+  xorTransform(temp);
+  return base64::encode(reinterpret_cast<const uint8_t*>(temp.data()), temp.size());
+}
+
+std::string deobfuscateFromBase64(const char* encoded, bool* ok) {
+  if (encoded == nullptr || encoded[0] == '\0') {
+    if (ok) *ok = false;
+    return "";
+  }
+  if (ok) *ok = true;
+  size_t encodedLen = strlen(encoded);
+  // First call: get required output buffer size
+  size_t decodedLen = 0;
+  int ret = mbedtls_base64_decode(nullptr, 0, &decodedLen, reinterpret_cast<const unsigned char*>(encoded), encodedLen);
+  if (ret != 0 && ret != MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL) {
+    LOG_ERR("OBF", "Base64 decode size query failed (ret=%d)", ret);
+    if (ok) *ok = false;
+    return "";
+  }
+  std::string result(decodedLen, '\0');
+  ret = mbedtls_base64_decode(reinterpret_cast<unsigned char*>(&result[0]), decodedLen, &decodedLen,
+                              reinterpret_cast<const unsigned char*>(encoded), encodedLen);
+  if (ret != 0) {
+    LOG_ERR("OBF", "Base64 decode failed (ret=%d)", ret);
+    if (ok) *ok = false;
+    return "";
+  }
+  result.resize(decodedLen);
+  xorTransform(result);
+  return result;
+}
+
+void selfTest() {
+  const char* testInputs[] = {"", "hello", "WiFi P@ssw0rd!", "a"};
+  bool allPassed = true;
+  for (const char* input : testInputs) {
+    String encoded = obfuscateToBase64(std::string(input));
+    std::string decoded = deobfuscateFromBase64(encoded.c_str());
+    if (decoded != input) {
+      LOG_ERR("OBF", "FAIL: \"%s\" -> \"%s\" -> \"%s\"", input, encoded.c_str(), decoded.c_str());
+      allPassed = false;
+    }
+  }
+  // Verify obfuscated form differs from plaintext
+  String enc = obfuscateToBase64("test123");
+  if (enc == "test123") {
+    LOG_ERR("OBF", "FAIL: obfuscated output identical to plaintext");
+    allPassed = false;
+  }
+  if (allPassed) {
+    LOG_DBG("OBF", "Obfuscation self-test PASSED");
+  }
+}
+
+}  // namespace obfuscation

lib/Serialization/ObfuscationUtils.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <Arduino.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+/**
+ * Credential obfuscation utilities using the ESP32's unique hardware MAC address.
+ *
+ * XOR-based obfuscation with the 6-byte eFuse MAC as key. Not cryptographically
+ * secure, but prevents casual reading of credentials on the SD card and ties
+ * obfuscated data to the specific device (cannot be decoded on another chip or PC).
+ *
+ */
+namespace obfuscation {
+
+// XOR obfuscate/deobfuscate in-place using hardware MAC key (symmetric operation)
+void xorTransform(std::string& data);
+
+// Legacy overload for binary migration (uses the old per-store hardcoded keys)
+void xorTransform(std::string& data, const uint8_t* key, size_t keyLen);
+
+// Obfuscate a plaintext string: XOR with hardware key, then base64-encode for JSON storage
+String obfuscateToBase64(const std::string& plaintext);
+
+// Decode base64 and de-obfuscate back to plaintext.
+// Returns empty string on invalid base64 input; sets *ok to false if decode fails.
+std::string deobfuscateFromBase64(const char* encoded, bool* ok = nullptr);
+
+// Self-test: verifies round-trip obfuscation with hardware key. Logs PASS/FAIL.
+void selfTest();
+
+}  // namespace obfuscation

lib/hal/HalStorage.cpp

@@ -36,6 +36,8 @@ bool HalStorage::exists(const char* path) { return SDCard.exists(path); }
 
 bool HalStorage::remove(const char* path) { return SDCard.remove(path); }
 
+bool HalStorage::rename(const char* oldPath, const char* newPath) { return SDCard.rename(oldPath, newPath); }
+
 bool HalStorage::rmdir(const char* path) { return SDCard.rmdir(path); }
 
 bool HalStorage::openFileForRead(const char* moduleName, const char* path, FsFile& file) {

lib/hal/HalStorage.h

@@ -28,6 +28,7 @@ class HalStorage {
   bool mkdir(const char* path, const bool pFlag = true);
   bool exists(const char* path);
   bool remove(const char* path);
+  bool rename(const char* oldPath, const char* newPath);
   bool rmdir(const char* path);
 
   bool openFileForRead(const char* moduleName, const char* path, FsFile& file);