fix: harden SQLite fallback folders, Daemon Off badge, and receiver upload safety

This commit is contained in:
2026-11-18 09:13:23 -05:00
parent ec8a414185
commit 6685844cc1
5 changed files with 342 additions and 36 deletions

View File

@@ -785,7 +785,7 @@ struct LvglAppView::Impl {
uint32_t badge_border = pal.border;
bool hollow = false;
if (!state.status.ok) {
if (!state.status.ok || state.status.from_fallback) {
badge_text = "Daemon Off";
badge_bg = pal.status_error_bg;
badge_fg = pal.status_error;

View File

@@ -925,6 +925,7 @@ DaemonStatus IpcClient::read_sqlite_fallback() const {
status.from_fallback = true;
// Populate config fields even if daemon is offline
bool config_loaded = false;
std::vector<FolderStatus> configured_folders;
{
std::ifstream cfg_in(resolved_config_path());
@@ -933,6 +934,7 @@ DaemonStatus IpcClient::read_sqlite_fallback() const {
ss << cfg_in.rdbuf();
JsonValue cfg_root;
if (parse_json(ss.str(), cfg_root) && cfg_root.type == JsonValue::Type::Object) {
config_loaded = true;
if (const auto* v = cfg_root.find("target_host")) {
status.target_host = v->as_string();
}
@@ -1083,20 +1085,22 @@ DaemonStatus IpcClient::read_sqlite_fallback() const {
status.folders.push_back(std::move(wf));
}
for (const auto& fid : seen_folder_order) {
if (included[fid]) {
continue;
if (!config_loaded) {
for (const auto& fid : seen_folder_order) {
if (included[fid]) {
continue;
}
const auto& c = by_folder[fid];
FolderStatus fs;
fs.id = fid;
fs.label = fid;
fs.pending = c.pending;
fs.uploading = c.uploading;
fs.synced = c.synced;
fs.failed = c.failed;
fs.pending_bytes = c.pending_bytes;
status.folders.push_back(std::move(fs));
}
const auto& c = by_folder[fid];
FolderStatus fs;
fs.id = fid;
fs.label = fid;
fs.pending = c.pending;
fs.uploading = c.uploading;
fs.synced = c.synced;
fs.failed = c.failed;
fs.pending_bytes = c.pending_bytes;
status.folders.push_back(std::move(fs));
}
// 3. Read newest 50 log entries

View File

@@ -476,6 +476,99 @@ void test_list_tailscale_peers_subprocess() {
assert(peers[0].online == true);
}
void test_sqlite_fallback_omits_removed_folders() {
TempDir tmp("cp0_sqlite_removed");
fs::path db_path = tmp.path / "state.db";
fs::path cfg_path = tmp.path / "config.json";
fs::path f1_dir = tmp.path / "f1";
fs::create_directories(f1_dir);
sqlite3* db = nullptr;
assert(sqlite3_open(db_path.c_str(), &db) == SQLITE_OK);
const char* schema_sql = R"SQL(
CREATE TABLE files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
folder_id TEXT NOT NULL,
abs_path TEXT NOT NULL,
rel_path TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
mtime_ns INTEGER NOT NULL,
sha256 TEXT NOT NULL,
status TEXT NOT NULL,
retry_count INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
discovered_at INTEGER NOT NULL,
synced_at INTEGER
);
CREATE TABLE logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL,
level TEXT NOT NULL,
message TEXT NOT NULL
);
CREATE TABLE meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
INSERT INTO files(folder_id, abs_path, rel_path, size_bytes, mtime_ns, sha256, status, discovered_at)
VALUES
('f1', '/tmp/f1/a.jpg', 'a.jpg', 1000, 1, 'h1', 'SYNCED', 100),
('f2', '/tmp/f2/b.jpg', 'b.jpg', 2000, 2, 'h2', 'SYNCED', 101);
)SQL";
char* errmsg = nullptr;
assert(sqlite3_exec(db, schema_sql, nullptr, nullptr, &errmsg) == SQLITE_OK);
sqlite3_close(db);
// 1. Valid config.json with f1 only: f2 must NOT be resurrected as a ghost folder
{
std::ofstream cfg_out(cfg_path);
cfg_out << R"({
"target_host": "mac.ts.net",
"target_port": 8765,
"theme": "dark",
"watched_folders": [
{"id": "f1", "path": ")" << f1_dir.string() << R"(", "label": "F1", "enabled": true}
]
})";
}
cp0::IpcClient client((tmp.path / "nonexistent.sock").string(), db_path.string(), cfg_path.string());
auto st = client.read_sqlite_fallback();
assert(st.ok == true);
assert(st.from_fallback == true);
assert(st.synced_count == 2);
assert(st.folders.size() == 1);
assert(st.folders[0].id == "f1");
assert(st.folders[0].synced == 1);
// 2. Valid config.json with empty watched_folders []: no folders resurrected
{
std::ofstream cfg_out(cfg_path, std::ios::trunc);
cfg_out << R"({"target_host": "mac.ts.net", "target_port": 8765, "watched_folders": []})";
}
auto empty_cfg_st = client.read_sqlite_fallback();
assert(empty_cfg_st.ok == true);
assert(empty_cfg_st.folders.empty());
// 3. Missing config.json: fallback synthesizes both f1 and f2 from files
cp0::IpcClient missing_cfg((tmp.path / "nonexistent.sock").string(), db_path.string(), (tmp.path / "missing.json").string());
auto missing_st = missing_cfg.read_sqlite_fallback();
assert(missing_st.ok == true);
assert(missing_st.folders.size() == 2);
assert(missing_st.folders[0].id == "f1");
assert(missing_st.folders[1].id == "f2");
// 4. Corrupt config.json: fallback synthesizes both f1 and f2 from files
fs::path corrupt_cfg = tmp.path / "corrupt.json";
{
std::ofstream bad_out(corrupt_cfg);
bad_out << "{not valid json";
}
cp0::IpcClient corrupt_client((tmp.path / "nonexistent.sock").string(), db_path.string(), corrupt_cfg.string());
auto corrupt_st = corrupt_client.read_sqlite_fallback();
assert(corrupt_st.ok == true);
assert(corrupt_st.folders.size() == 2);
}
} // namespace
int main() {
@@ -483,6 +576,7 @@ int main() {
test_tailscale_peer_parsing();
test_ipc_socket_round_trip();
test_sqlite_fallback_and_config();
test_sqlite_fallback_omits_removed_folders();
test_status_parsing_and_env_paths();
test_list_tailscale_peers_subprocess();
std::cout << "All test_ipc_and_theme assertions passed.\n";

View File

@@ -129,6 +129,7 @@ def _collision_candidate(target_path: Path, mtime: float, sha256_hex: str, index
class _MacReceiverHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
timeout = 60.0
def log_message(self, format: str, *args: object) -> None:
logger.debug("%s - %s", self.client_address[0], format % args)
@@ -142,8 +143,9 @@ class _MacReceiverHandler(BaseHTTPRequestHandler):
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(body)
except (BrokenPipeError, ConnectionResetError):
pass
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError):
self.close_connection = True
def _load_and_authorize(self) -> MacConfig | None:
cfg = load_mac_config(self.server.config_path) # type: ignore[attr-defined]
@@ -170,7 +172,7 @@ class _MacReceiverHandler(BaseHTTPRequestHandler):
"status": "ok",
"version": VERSION,
"hostname": socket.gethostname(),
"dest_root": cfg.dest_root,
"dest_root": str(Path(cfg.dest_root).expanduser()),
},
)
@@ -211,7 +213,12 @@ class _MacReceiverHandler(BaseHTTPRequestHandler):
except ValueError:
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid X-CP0-Mtime"})
return
if not math.isfinite(mtime):
if not math.isfinite(mtime) or mtime < 0:
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid X-CP0-Mtime"})
return
try:
datetime.fromtimestamp(mtime, tz=timezone.utc)
except (OverflowError, OSError, ValueError):
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid X-CP0-Mtime"})
return
@@ -223,47 +230,87 @@ class _MacReceiverHandler(BaseHTTPRequestHandler):
self._send_json(HTTPStatus.BAD_REQUEST, {"error": str(exc)})
return
target_path.parent.mkdir(parents=True, exist_ok=True)
try:
target_path.parent.mkdir(parents=True, exist_ok=True)
except (FileExistsError, NotADirectoryError, IsADirectoryError) as exc:
logger.warning("Destination path conflict for %s: %s", target_path, exc)
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "destination path conflict"})
return
except OSError as exc:
logger.error("Failed creating destination directory for %s: %s", target_path, exc)
self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": "write_failed"})
return
part_path = target_path.parent / f".cp0-{uuid.uuid4().hex}.part"
try:
computed_sha = self._stream_to_part(part_path, content_length)
except _PrematureDisconnectError:
part_path.unlink(missing_ok=True)
self.close_connection = True
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "incomplete request body"})
return
except (FileExistsError, NotADirectoryError, IsADirectoryError) as exc:
part_path.unlink(missing_ok=True)
logger.warning("Destination path conflict writing %s: %s", part_path, exc)
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "destination path conflict"})
return
except OSError as exc:
part_path.unlink(missing_ok=True)
logger.error("Failed writing upload part file %s: %s", part_path, exc)
self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": "write_failed"})
return
if computed_sha is None:
return
if computed_sha != expected_sha:
part_path.unlink(missing_ok=True)
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "sha256 mismatch"})
return
server: MacReceiverServer = self.server # type: ignore[assignment]
status, saved_path = server.commit_part(part_path, target_path, expected_sha, mtime, cfg)
try:
status, saved_path = server.commit_part(part_path, target_path, expected_sha, mtime, cfg)
except (FileExistsError, NotADirectoryError, IsADirectoryError) as exc:
part_path.unlink(missing_ok=True)
logger.warning("Destination path conflict committing %s: %s", target_path, exc)
self._send_json(HTTPStatus.BAD_REQUEST, {"error": "destination path conflict"})
return
except OSError as exc:
part_path.unlink(missing_ok=True)
logger.error("Failed committing upload %s: %s", target_path, exc)
self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": "commit_failed"})
return
if status == "already_exists":
self._send_json(HTTPStatus.OK, {"status": "already_exists"})
return
self._send_json(HTTPStatus.OK, {"status": "saved", "saved_path": str(saved_path)})
def _stream_to_part(self, part_path: Path, content_length: int) -> str:
def _stream_to_part(self, part_path: Path, content_length: int) -> str | None:
digest = hashlib.sha256()
remaining = content_length
with open(part_path, "wb") as handle:
while remaining > 0:
chunk = self.rfile.read(min(CHUNK_SIZE, remaining))
if not chunk:
raise _PrematureDisconnectError()
handle.write(chunk)
digest.update(chunk)
remaining -= len(chunk)
handle.flush()
os.fsync(handle.fileno())
try:
with open(part_path, "wb") as handle:
while remaining > 0:
chunk = self.rfile.read(min(CHUNK_SIZE, remaining))
if not chunk:
raise _PrematureDisconnectError()
handle.write(chunk)
digest.update(chunk)
remaining -= len(chunk)
handle.flush()
os.fsync(handle.fileno())
except (TimeoutError, socket.timeout):
part_path.unlink(missing_ok=True)
self.close_connection = True
self._send_json(
HTTPStatus.REQUEST_TIMEOUT,
{"ok": False, "error": "Upload timed out"},
)
return None
return digest.hexdigest()
@@ -293,7 +340,7 @@ class MacReceiverServer(ThreadingHTTPServer):
self._commit_lock = threading.Lock()
self._notify_lock = threading.Lock()
self._pending_count = 0
self._pending_dest = Path(cfg.dest_root)
self._pending_dest = Path(cfg.dest_root).expanduser()
self._notify_timer: threading.Timer | None = None
self._server_thread: threading.Thread | None = None
@@ -375,12 +422,16 @@ class MacReceiverServer(ThreadingHTTPServer):
return ("already_exists", None)
index += 1
try:
os.utime(part_path, (mtime, mtime))
except OSError as exc:
logger.warning("Failed to set mtime on %s: %s", part_path, exc)
os.replace(part_path, final_path)
except BaseException:
part_path.unlink(missing_ok=True)
raise
os.utime(final_path, (mtime, mtime))
logger.info("Saved photo %s", final_path)
if cfg.notify_on_batch:
self._schedule_notification(Path(cfg.dest_root).expanduser())

View File

@@ -1,6 +1,7 @@
import hashlib
import json
import os
import socket
import tempfile
import threading
import time
@@ -16,6 +17,7 @@ from urllib.request import Request, urlopen
from cp0_mac_receiver.config import MacConfig, load_mac_config, save_mac_config
from cp0_mac_receiver.server import (
MacReceiverServer,
_MacReceiverHandler,
_default_osascript_notifier,
is_allowed_client_ip,
)
@@ -248,9 +250,142 @@ class TestMacReceiverServer(unittest.TestCase):
ctx2.exception.close()
camera_dir = self.dest_root / "Camera"
if camera_dir.exists():
self.assertEqual(list(camera_dir.rglob("*.part")), [])
self.assertFalse((camera_dir / "a.jpg").exists())
self.assertTrue(camera_dir.is_dir())
self.assertEqual(list(camera_dir.rglob("*.part")), [])
self.assertFalse((camera_dir / "a.jpg").exists())
def _read_http_response(self, sock: socket.socket) -> tuple[int, dict]:
chunks = []
while True:
chunk = sock.recv(4096)
if not chunk:
break
chunks.append(chunk)
raw = b"".join(chunks)
head, _, body = raw.partition(b"\r\n\r\n")
status_line = head.split(b"\r\n", 1)[0].decode("ascii", errors="replace")
status_code = int(status_line.split()[1])
return status_code, json.loads(body.decode("utf-8"))
def test_rejects_out_of_range_mtime_with_400(self):
payload = b"mtime-test-bytes"
sha = hashlib.sha256(payload).hexdigest()
for bad_mtime in ("1e20", "1e12", "-1", "-1e20", "nan", "inf", "not-a-number"):
req = Request(
f"{self.base_url}/upload",
data=payload,
method="POST",
headers={
"Content-Length": str(len(payload)),
"X-CP0-Folder-Label": "Camera",
"X-CP0-Rel-Path": "bad_mtime.jpg",
"X-CP0-SHA256": sha,
"X-CP0-Mtime": bad_mtime,
},
)
with self.assertRaises(HTTPError) as ctx:
urlopen(req)
self.assertEqual(ctx.exception.code, 400, f"mtime={bad_mtime}")
body = json.loads(ctx.exception.read())
self.assertEqual(body["error"], "invalid X-CP0-Mtime")
ctx.exception.close()
self.assertEqual(list(self.dest_root.rglob("*.part")), [])
def test_filesystem_collision_returns_400_without_leaking_part_files(self):
self.dest_root.mkdir(parents=True, exist_ok=True)
blocking_file = self.dest_root / "BlockedFile"
blocking_file.write_bytes(b"original-file")
for rel_path in ("a.jpg", "sub/a.jpg"):
with self.assertRaises(HTTPError) as ctx:
self._upload("BlockedFile", rel_path, b"photo-bytes")
self.assertEqual(ctx.exception.code, 400)
body = json.loads(ctx.exception.read())
self.assertIn("error", body)
ctx.exception.close()
self.assertEqual(blocking_file.read_bytes(), b"original-file")
blocking_dir = self.dest_root / "Camera" / "existing_dir.jpg"
blocking_dir.mkdir(parents=True, exist_ok=True)
with self.assertRaises(HTTPError) as ctx2:
self._upload("Camera", "existing_dir.jpg", b"photo-bytes")
self.assertEqual(ctx2.exception.code, 400)
body2 = json.loads(ctx2.exception.read())
self.assertIn("error", body2)
ctx2.exception.close()
self.assertTrue(blocking_dir.is_dir())
self.assertEqual(list(self.dest_root.rglob("*.part")), [])
def test_premature_client_disconnect_cleans_part_file(self):
port = self.srv.server_address[1]
payload = b"partial"
sha = hashlib.sha256(b"x" * 1024).hexdigest()
raw_req = (
"POST /upload HTTP/1.1\r\n"
f"Host: 127.0.0.1:{port}\r\n"
"Content-Length: 1024\r\n"
"X-CP0-Folder-Label: Camera\r\n"
"X-CP0-Rel-Path: partial.jpg\r\n"
f"X-CP0-SHA256: {sha}\r\n"
"X-CP0-Mtime: 1700000000\r\n"
"\r\n"
).encode("ascii") + payload
with socket.create_connection(("127.0.0.1", port), timeout=5.0) as sock:
sock.sendall(raw_req)
sock.shutdown(socket.SHUT_WR)
status_code, body = self._read_http_response(sock)
self.assertEqual(status_code, 400)
self.assertEqual(body["error"], "incomplete request body")
camera_dir = self.dest_root / "Camera"
self.assertTrue(camera_dir.is_dir())
self.assertEqual(list(camera_dir.rglob("*.part")), [])
self.assertFalse((camera_dir / "partial.jpg").exists())
def test_upload_timeout_returns_408_and_cleans_part_file(self):
port = self.srv.server_address[1]
sha = hashlib.sha256(b"x" * 1024).hexdigest()
raw_req = (
"POST /upload HTTP/1.1\r\n"
f"Host: 127.0.0.1:{port}\r\n"
"Content-Length: 1024\r\n"
"X-CP0-Folder-Label: Camera\r\n"
"X-CP0-Rel-Path: slow.jpg\r\n"
f"X-CP0-SHA256: {sha}\r\n"
"X-CP0-Mtime: 1700000000\r\n"
"\r\n"
"partial"
).encode("ascii")
with patch.object(_MacReceiverHandler, "timeout", 0.15):
with socket.create_connection(("127.0.0.1", port), timeout=5.0) as sock:
sock.sendall(raw_req)
status_code, body = self._read_http_response(sock)
self.assertEqual(status_code, 408)
self.assertEqual(body, {"ok": False, "error": "Upload timed out"})
camera_dir = self.dest_root / "Camera"
self.assertTrue(camera_dir.is_dir())
self.assertEqual(list(camera_dir.rglob("*.part")), [])
self.assertFalse((camera_dir / "slow.jpg").exists())
def test_utime_oserror_does_not_fail_commit(self):
with (
patch("cp0_mac_receiver.server.os.utime", side_effect=OSError("utime denied")),
self.assertLogs("cp0_mac_receiver.server", level="WARNING"),
):
with self._upload("Camera", "utime_warn.jpg", b"utime-fallback", 1700000000.0) as resp:
self.assertEqual(resp.status, 200)
body = json.loads(resp.read())
self.assertEqual(body["status"], "saved")
target = self.dest_root / "Camera" / "utime_warn.jpg"
self.assertEqual(target.read_bytes(), b"utime-fallback")
self.assertEqual(list(self.dest_root.rglob("*.part")), [])
def test_debounced_batch_notification_fires_once_after_burst(self):
with self._upload("Camera", "img1.jpg", b"photo-1", 1700000001.0) as r1:
@@ -287,6 +422,28 @@ class TestMacReceiverServer(unittest.TestCase):
self.assertEqual(res["status"], "saved")
self.assertTrue((new_dest / "DCIM" / "p1.jpg").is_file())
def test_health_and_server_expand_tilde_in_dest_root(self):
save_mac_config(
MacConfig(
dest_root="~/Pictures/CardputerZero",
listen_host="127.0.0.1",
listen_port=0,
notify_on_batch=True,
tailscale_only=True,
),
self.cfg_path,
)
expected = str(Path("~/Pictures/CardputerZero").expanduser())
with urlopen(f"{self.base_url}/health") as resp:
body = json.loads(resp.read())
self.assertEqual(body["dest_root"], expected)
srv2 = MacReceiverServer(self.cfg_path, notifier=lambda _c, _d: None, debounce_sec=0.05)
try:
self.assertEqual(srv2._pending_dest, Path(expected))
finally:
srv2.server_close()
if __name__ == "__main__":
unittest.main()