Refactors Calibre Wireless Device & Calibre Library (#404)

Our esp32 consistently dropped the last few packets of the TCP transfer
in the old implementation. Only about 1/5 transfers would complete. I've
refactored that entire system into an actual Calibre Device Plugin that
basically uses the exact same system as the web server's file transfer
protocol. I kept them separate so that we don't muddy up the existing
file transfer stuff even if it's basically the same at the end of the
day I didn't want to limit our ability to change it later.

I've also added basic auth to OPDS and renamed that feature to OPDS
Browser to just disassociate it from Calibre.

---------

Co-authored-by: Arthur Tazhitdinov <lisnake@gmail.com>
Co-authored-by: Dave Allie <dave@daveallie.com>

src/network/CrossPointWebServer.cpp

@@ -18,6 +18,8 @@ namespace {
 // Note: Items starting with "." are automatically hidden
 const char* HIDDEN_ITEMS[] = {"System Volume Information", "XTCache"};
 constexpr size_t HIDDEN_ITEMS_COUNT = sizeof(HIDDEN_ITEMS) / sizeof(HIDDEN_ITEMS[0]);
+constexpr uint16_t UDP_PORTS[] = {54982, 48123, 39001, 44044, 59678};
+constexpr uint16_t LOCAL_UDP_PORT = 8134;
 
 // Static pointer for WebSocket callback (WebSocketsServer requires C-style callback)
 CrossPointWebServer* wsInstance = nullptr;
@@ -30,6 +32,9 @@ size_t wsUploadSize = 0;
 size_t wsUploadReceived = 0;
 unsigned long wsUploadStartTime = 0;
 bool wsUploadInProgress = false;
+String wsLastCompleteName;
+size_t wsLastCompleteSize = 0;
+unsigned long wsLastCompleteAt = 0;
 
 // Helper function to clear epub cache after upload
 void clearEpubCacheIfNeeded(const String& filePath) {
@@ -96,6 +101,7 @@ void CrossPointWebServer::begin() {
 
   server->on("/api/status", HTTP_GET, [this] { handleStatus(); });
   server->on("/api/files", HTTP_GET, [this] { handleFileListData(); });
+  server->on("/download", HTTP_GET, [this] { handleDownload(); });
 
   // Upload endpoint with special handling for multipart form data
   server->on("/upload", HTTP_POST, [this] { handleUploadPost(); }, [this] { handleUpload(); });
@@ -119,6 +125,10 @@ void CrossPointWebServer::begin() {
   wsServer->onEvent(wsEventCallback);
   Serial.printf("[%lu] [WEB] WebSocket server started\n", millis());
 
+  udpActive = udp.begin(LOCAL_UDP_PORT);
+  Serial.printf("[%lu] [WEB] Discovery UDP %s on port %d\n", millis(), udpActive ? "enabled" : "failed",
+                LOCAL_UDP_PORT);
+
   running = true;
 
   Serial.printf("[%lu] [WEB] Web server started on port %d\n", millis(), port);
@@ -156,6 +166,11 @@ void CrossPointWebServer::stop() {
     Serial.printf("[%lu] [WEB] WebSocket server stopped\n", millis());
   }
 
+  if (udpActive) {
+    udp.stop();
+    udpActive = false;
+  }
+
   // Brief delay to allow any in-flight handleClient() calls to complete
   delay(20);
 
@@ -174,7 +189,7 @@ void CrossPointWebServer::stop() {
   Serial.printf("[%lu] [WEB] [MEM] Free heap final: %d bytes\n", millis(), ESP.getFreeHeap());
 }
 
-void CrossPointWebServer::handleClient() const {
+void CrossPointWebServer::handleClient() {
   static unsigned long lastDebugPrint = 0;
 
   // Check running flag FIRST before accessing server
@@ -200,6 +215,40 @@ void CrossPointWebServer::handleClient() const {
   if (wsServer) {
     wsServer->loop();
   }
+
+  // Respond to discovery broadcasts
+  if (udpActive) {
+    int packetSize = udp.parsePacket();
+    if (packetSize > 0) {
+      char buffer[16];
+      int len = udp.read(buffer, sizeof(buffer) - 1);
+      if (len > 0) {
+        buffer[len] = '\0';
+        if (strcmp(buffer, "hello") == 0) {
+          String hostname = WiFi.getHostname();
+          if (hostname.isEmpty()) {
+            hostname = "crosspoint";
+          }
+          String message = "crosspoint (on " + hostname + ");" + String(wsPort);
+          udp.beginPacket(udp.remoteIP(), udp.remotePort());
+          udp.write(reinterpret_cast<const uint8_t*>(message.c_str()), message.length());
+          udp.endPacket();
+        }
+      }
+    }
+  }
+}
+
+CrossPointWebServer::WsUploadStatus CrossPointWebServer::getWsUploadStatus() const {
+  WsUploadStatus status;
+  status.inProgress = wsUploadInProgress;
+  status.received = wsUploadReceived;
+  status.total = wsUploadSize;
+  status.filename = wsUploadFileName.c_str();
+  status.lastCompleteName = wsLastCompleteName.c_str();
+  status.lastCompleteSize = wsLastCompleteSize;
+  status.lastCompleteAt = wsLastCompleteAt;
+  return status;
 }
 
 void CrossPointWebServer::handleRoot() const {
@@ -346,6 +395,69 @@ void CrossPointWebServer::handleFileListData() const {
   Serial.printf("[%lu] [WEB] Served file listing page for path: %s\n", millis(), currentPath.c_str());
 }
 
+void CrossPointWebServer::handleDownload() const {
+  if (!server->hasArg("path")) {
+    server->send(400, "text/plain", "Missing path");
+    return;
+  }
+
+  String itemPath = server->arg("path");
+  if (itemPath.isEmpty() || itemPath == "/") {
+    server->send(400, "text/plain", "Invalid path");
+    return;
+  }
+  if (!itemPath.startsWith("/")) {
+    itemPath = "/" + itemPath;
+  }
+
+  const String itemName = itemPath.substring(itemPath.lastIndexOf('/') + 1);
+  if (itemName.startsWith(".")) {
+    server->send(403, "text/plain", "Cannot access system files");
+    return;
+  }
+  for (size_t i = 0; i < HIDDEN_ITEMS_COUNT; i++) {
+    if (itemName.equals(HIDDEN_ITEMS[i])) {
+      server->send(403, "text/plain", "Cannot access protected items");
+      return;
+    }
+  }
+
+  if (!SdMan.exists(itemPath.c_str())) {
+    server->send(404, "text/plain", "Item not found");
+    return;
+  }
+
+  FsFile file = SdMan.open(itemPath.c_str());
+  if (!file) {
+    server->send(500, "text/plain", "Failed to open file");
+    return;
+  }
+  if (file.isDirectory()) {
+    file.close();
+    server->send(400, "text/plain", "Path is a directory");
+    return;
+  }
+
+  String contentType = "application/octet-stream";
+  if (isEpubFile(itemPath)) {
+    contentType = "application/epub+zip";
+  }
+
+  char nameBuf[128] = {0};
+  String filename = "download";
+  if (file.getName(nameBuf, sizeof(nameBuf))) {
+    filename = nameBuf;
+  }
+
+  server->setContentLength(file.size());
+  server->sendHeader("Content-Disposition", "attachment; filename=\"" + filename + "\"");
+  server->send(200, contentType.c_str(), "");
+
+  WiFiClient client = server->client();
+  client.write(file);
+  file.close();
+}
+
 // Static variables for upload handling
 static FsFile uploadFile;
 static String uploadFileName;
@@ -798,6 +910,10 @@ void CrossPointWebServer::onWebSocketEvent(uint8_t num, WStype_t type, uint8_t*
         wsUploadFile.close();
         wsUploadInProgress = false;
 
+        wsLastCompleteName = wsUploadFileName;
+        wsLastCompleteSize = wsUploadSize;
+        wsLastCompleteAt = millis();
+
         unsigned long elapsed = millis() - wsUploadStartTime;
         float kbps = (elapsed > 0) ? (wsUploadSize / 1024.0) / (elapsed / 1000.0) : 0;
 

src/network/CrossPointWebServer.h

@@ -2,7 +2,10 @@
 
 #include <WebServer.h>
 #include <WebSocketsServer.h>
+#include <WiFiUdp.h>
 
+#include <memory>
+#include <string>
 #include <vector>
 
 // Structure to hold file information
@@ -15,6 +18,16 @@ struct FileInfo {
 
 class CrossPointWebServer {
  public:
+  struct WsUploadStatus {
+    bool inProgress = false;
+    size_t received = 0;
+    size_t total = 0;
+    std::string filename;
+    std::string lastCompleteName;
+    size_t lastCompleteSize = 0;
+    unsigned long lastCompleteAt = 0;
+  };
+
   CrossPointWebServer();
   ~CrossPointWebServer();
 
@@ -25,11 +38,13 @@ class CrossPointWebServer {
   void stop();
 
   // Call this periodically to handle client requests
-  void handleClient() const;
+  void handleClient();
 
   // Check if server is running
   bool isRunning() const { return running; }
 
+  WsUploadStatus getWsUploadStatus() const;
+
   // Get the port number
   uint16_t getPort() const { return port; }
 
@@ -40,6 +55,8 @@ class CrossPointWebServer {
   bool apMode = false;  // true when running in AP mode, false for STA mode
   uint16_t port = 80;
   uint16_t wsPort = 81;  // WebSocket port
+  WiFiUDP udp;
+  bool udpActive = false;
 
   // WebSocket upload state
   void onWebSocketEvent(uint8_t num, WStype_t type, uint8_t* payload, size_t length);
@@ -56,6 +73,7 @@ class CrossPointWebServer {
   void handleStatus() const;
   void handleFileList() const;
   void handleFileListData() const;
+  void handleDownload() const;
   void handleUpload() const;
   void handleUploadPost() const;
   void handleCreateFolder() const;

src/network/HttpDownloader.cpp

@@ -5,9 +5,12 @@
 #include <StreamString.h>
 #include <WiFiClient.h>
 #include <WiFiClientSecure.h>
+#include <base64.h>
 
+#include <cstring>
 #include <memory>
 
+#include "CrossPointSettings.h"
 #include "util/UrlUtils.h"
 
 bool HttpDownloader::fetchUrl(const std::string& url, Stream& outContent) {
@@ -28,6 +31,13 @@ bool HttpDownloader::fetchUrl(const std::string& url, Stream& outContent) {
   http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
   http.addHeader("User-Agent", "CrossPoint-ESP32-" CROSSPOINT_VERSION);
 
+  // Add Basic HTTP auth if credentials are configured
+  if (strlen(SETTINGS.opdsUsername) > 0 && strlen(SETTINGS.opdsPassword) > 0) {
+    std::string credentials = std::string(SETTINGS.opdsUsername) + ":" + SETTINGS.opdsPassword;
+    String encoded = base64::encode(credentials.c_str());
+    http.addHeader("Authorization", "Basic " + encoded);
+  }
+
   const int httpCode = http.GET();
   if (httpCode != HTTP_CODE_OK) {
     Serial.printf("[%lu] [HTTP] Fetch failed: %d\n", millis(), httpCode);
@@ -72,6 +82,13 @@ HttpDownloader::DownloadError HttpDownloader::downloadToFile(const std::string&
   http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
   http.addHeader("User-Agent", "CrossPoint-ESP32-" CROSSPOINT_VERSION);
 
+  // Add Basic HTTP auth if credentials are configured
+  if (strlen(SETTINGS.opdsUsername) > 0 && strlen(SETTINGS.opdsPassword) > 0) {
+    std::string credentials = std::string(SETTINGS.opdsUsername) + ":" + SETTINGS.opdsPassword;
+    String encoded = base64::encode(credentials.c_str());
+    http.addHeader("Authorization", "Basic " + encoded);
+  }
+
   const int httpCode = http.GET();
   if (httpCode != HTTP_CODE_OK) {
     Serial.printf("[%lu] [HTTP] Download failed: %d\n", millis(), httpCode);