From 6685844cc12e070a12575717cc100448247dd7fb Mon Sep 17 00:00:00 2001 From: cottongin Date: Wed, 18 Nov 2026 09:13:23 -0500 Subject: [PATCH] fix: harden SQLite fallback folders, Daemon Off badge, and receiver upload safety --- device-ui/src/app_ui.cpp | 2 +- device-ui/src/ipc_client.cpp | 30 +++-- device-ui/tests/test_ipc_and_theme.cpp | 94 ++++++++++++++ mac-receiver/cp0_mac_receiver/server.py | 89 ++++++++++--- mac-receiver/tests/test_server.py | 163 +++++++++++++++++++++++- 5 files changed, 342 insertions(+), 36 deletions(-) diff --git a/device-ui/src/app_ui.cpp b/device-ui/src/app_ui.cpp index f2c68de..4b92cd9 100644 --- a/device-ui/src/app_ui.cpp +++ b/device-ui/src/app_ui.cpp @@ -785,7 +785,7 @@ struct LvglAppView::Impl { uint32_t badge_border = pal.border; bool hollow = false; - if (!state.status.ok) { + if (!state.status.ok || state.status.from_fallback) { badge_text = "Daemon Off"; badge_bg = pal.status_error_bg; badge_fg = pal.status_error; diff --git a/device-ui/src/ipc_client.cpp b/device-ui/src/ipc_client.cpp index ee7049f..96f061a 100644 --- a/device-ui/src/ipc_client.cpp +++ b/device-ui/src/ipc_client.cpp @@ -925,6 +925,7 @@ DaemonStatus IpcClient::read_sqlite_fallback() const { status.from_fallback = true; // Populate config fields even if daemon is offline + bool config_loaded = false; std::vector configured_folders; { std::ifstream cfg_in(resolved_config_path()); @@ -933,6 +934,7 @@ DaemonStatus IpcClient::read_sqlite_fallback() const { ss << cfg_in.rdbuf(); JsonValue cfg_root; if (parse_json(ss.str(), cfg_root) && cfg_root.type == JsonValue::Type::Object) { + config_loaded = true; if (const auto* v = cfg_root.find("target_host")) { status.target_host = v->as_string(); } @@ -1083,20 +1085,22 @@ DaemonStatus IpcClient::read_sqlite_fallback() const { status.folders.push_back(std::move(wf)); } - for (const auto& fid : seen_folder_order) { - if (included[fid]) { - continue; + if (!config_loaded) { + for (const auto& fid : seen_folder_order) { + if (included[fid]) { + continue; + } + const auto& c = by_folder[fid]; + FolderStatus fs; + fs.id = fid; + fs.label = fid; + fs.pending = c.pending; + fs.uploading = c.uploading; + fs.synced = c.synced; + fs.failed = c.failed; + fs.pending_bytes = c.pending_bytes; + status.folders.push_back(std::move(fs)); } - const auto& c = by_folder[fid]; - FolderStatus fs; - fs.id = fid; - fs.label = fid; - fs.pending = c.pending; - fs.uploading = c.uploading; - fs.synced = c.synced; - fs.failed = c.failed; - fs.pending_bytes = c.pending_bytes; - status.folders.push_back(std::move(fs)); } // 3. Read newest 50 log entries diff --git a/device-ui/tests/test_ipc_and_theme.cpp b/device-ui/tests/test_ipc_and_theme.cpp index 40216be..438156b 100644 --- a/device-ui/tests/test_ipc_and_theme.cpp +++ b/device-ui/tests/test_ipc_and_theme.cpp @@ -476,6 +476,99 @@ void test_list_tailscale_peers_subprocess() { assert(peers[0].online == true); } +void test_sqlite_fallback_omits_removed_folders() { + TempDir tmp("cp0_sqlite_removed"); + fs::path db_path = tmp.path / "state.db"; + fs::path cfg_path = tmp.path / "config.json"; + fs::path f1_dir = tmp.path / "f1"; + fs::create_directories(f1_dir); + + sqlite3* db = nullptr; + assert(sqlite3_open(db_path.c_str(), &db) == SQLITE_OK); + const char* schema_sql = R"SQL( + CREATE TABLE files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + folder_id TEXT NOT NULL, + abs_path TEXT NOT NULL, + rel_path TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + mtime_ns INTEGER NOT NULL, + sha256 TEXT NOT NULL, + status TEXT NOT NULL, + retry_count INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + discovered_at INTEGER NOT NULL, + synced_at INTEGER + ); + CREATE TABLE logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL, + level TEXT NOT NULL, + message TEXT NOT NULL + ); + CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT INTO files(folder_id, abs_path, rel_path, size_bytes, mtime_ns, sha256, status, discovered_at) + VALUES + ('f1', '/tmp/f1/a.jpg', 'a.jpg', 1000, 1, 'h1', 'SYNCED', 100), + ('f2', '/tmp/f2/b.jpg', 'b.jpg', 2000, 2, 'h2', 'SYNCED', 101); + )SQL"; + char* errmsg = nullptr; + assert(sqlite3_exec(db, schema_sql, nullptr, nullptr, &errmsg) == SQLITE_OK); + sqlite3_close(db); + + // 1. Valid config.json with f1 only: f2 must NOT be resurrected as a ghost folder + { + std::ofstream cfg_out(cfg_path); + cfg_out << R"({ + "target_host": "mac.ts.net", + "target_port": 8765, + "theme": "dark", + "watched_folders": [ + {"id": "f1", "path": ")" << f1_dir.string() << R"(", "label": "F1", "enabled": true} + ] + })"; + } + cp0::IpcClient client((tmp.path / "nonexistent.sock").string(), db_path.string(), cfg_path.string()); + auto st = client.read_sqlite_fallback(); + assert(st.ok == true); + assert(st.from_fallback == true); + assert(st.synced_count == 2); + assert(st.folders.size() == 1); + assert(st.folders[0].id == "f1"); + assert(st.folders[0].synced == 1); + + // 2. Valid config.json with empty watched_folders []: no folders resurrected + { + std::ofstream cfg_out(cfg_path, std::ios::trunc); + cfg_out << R"({"target_host": "mac.ts.net", "target_port": 8765, "watched_folders": []})"; + } + auto empty_cfg_st = client.read_sqlite_fallback(); + assert(empty_cfg_st.ok == true); + assert(empty_cfg_st.folders.empty()); + + // 3. Missing config.json: fallback synthesizes both f1 and f2 from files + cp0::IpcClient missing_cfg((tmp.path / "nonexistent.sock").string(), db_path.string(), (tmp.path / "missing.json").string()); + auto missing_st = missing_cfg.read_sqlite_fallback(); + assert(missing_st.ok == true); + assert(missing_st.folders.size() == 2); + assert(missing_st.folders[0].id == "f1"); + assert(missing_st.folders[1].id == "f2"); + + // 4. Corrupt config.json: fallback synthesizes both f1 and f2 from files + fs::path corrupt_cfg = tmp.path / "corrupt.json"; + { + std::ofstream bad_out(corrupt_cfg); + bad_out << "{not valid json"; + } + cp0::IpcClient corrupt_client((tmp.path / "nonexistent.sock").string(), db_path.string(), corrupt_cfg.string()); + auto corrupt_st = corrupt_client.read_sqlite_fallback(); + assert(corrupt_st.ok == true); + assert(corrupt_st.folders.size() == 2); +} + } // namespace int main() { @@ -483,6 +576,7 @@ int main() { test_tailscale_peer_parsing(); test_ipc_socket_round_trip(); test_sqlite_fallback_and_config(); + test_sqlite_fallback_omits_removed_folders(); test_status_parsing_and_env_paths(); test_list_tailscale_peers_subprocess(); std::cout << "All test_ipc_and_theme assertions passed.\n"; diff --git a/mac-receiver/cp0_mac_receiver/server.py b/mac-receiver/cp0_mac_receiver/server.py index 6ac7a88..1d895cf 100644 --- a/mac-receiver/cp0_mac_receiver/server.py +++ b/mac-receiver/cp0_mac_receiver/server.py @@ -129,6 +129,7 @@ def _collision_candidate(target_path: Path, mtime: float, sha256_hex: str, index class _MacReceiverHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" + timeout = 60.0 def log_message(self, format: str, *args: object) -> None: logger.debug("%s - %s", self.client_address[0], format % args) @@ -142,8 +143,9 @@ class _MacReceiverHandler(BaseHTTPRequestHandler): self.send_header("Connection", "close") self.end_headers() self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): - pass + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError): + self.close_connection = True def _load_and_authorize(self) -> MacConfig | None: cfg = load_mac_config(self.server.config_path) # type: ignore[attr-defined] @@ -170,7 +172,7 @@ class _MacReceiverHandler(BaseHTTPRequestHandler): "status": "ok", "version": VERSION, "hostname": socket.gethostname(), - "dest_root": cfg.dest_root, + "dest_root": str(Path(cfg.dest_root).expanduser()), }, ) @@ -211,7 +213,12 @@ class _MacReceiverHandler(BaseHTTPRequestHandler): except ValueError: self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid X-CP0-Mtime"}) return - if not math.isfinite(mtime): + if not math.isfinite(mtime) or mtime < 0: + self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid X-CP0-Mtime"}) + return + try: + datetime.fromtimestamp(mtime, tz=timezone.utc) + except (OverflowError, OSError, ValueError): self._send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid X-CP0-Mtime"}) return @@ -223,47 +230,87 @@ class _MacReceiverHandler(BaseHTTPRequestHandler): self._send_json(HTTPStatus.BAD_REQUEST, {"error": str(exc)}) return - target_path.parent.mkdir(parents=True, exist_ok=True) + try: + target_path.parent.mkdir(parents=True, exist_ok=True) + except (FileExistsError, NotADirectoryError, IsADirectoryError) as exc: + logger.warning("Destination path conflict for %s: %s", target_path, exc) + self._send_json(HTTPStatus.BAD_REQUEST, {"error": "destination path conflict"}) + return + except OSError as exc: + logger.error("Failed creating destination directory for %s: %s", target_path, exc) + self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": "write_failed"}) + return + part_path = target_path.parent / f".cp0-{uuid.uuid4().hex}.part" try: computed_sha = self._stream_to_part(part_path, content_length) except _PrematureDisconnectError: part_path.unlink(missing_ok=True) + self.close_connection = True self._send_json(HTTPStatus.BAD_REQUEST, {"error": "incomplete request body"}) return + except (FileExistsError, NotADirectoryError, IsADirectoryError) as exc: + part_path.unlink(missing_ok=True) + logger.warning("Destination path conflict writing %s: %s", part_path, exc) + self._send_json(HTTPStatus.BAD_REQUEST, {"error": "destination path conflict"}) + return except OSError as exc: part_path.unlink(missing_ok=True) logger.error("Failed writing upload part file %s: %s", part_path, exc) self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": "write_failed"}) return + if computed_sha is None: + return + if computed_sha != expected_sha: part_path.unlink(missing_ok=True) self._send_json(HTTPStatus.BAD_REQUEST, {"error": "sha256 mismatch"}) return server: MacReceiverServer = self.server # type: ignore[assignment] - status, saved_path = server.commit_part(part_path, target_path, expected_sha, mtime, cfg) + try: + status, saved_path = server.commit_part(part_path, target_path, expected_sha, mtime, cfg) + except (FileExistsError, NotADirectoryError, IsADirectoryError) as exc: + part_path.unlink(missing_ok=True) + logger.warning("Destination path conflict committing %s: %s", target_path, exc) + self._send_json(HTTPStatus.BAD_REQUEST, {"error": "destination path conflict"}) + return + except OSError as exc: + part_path.unlink(missing_ok=True) + logger.error("Failed committing upload %s: %s", target_path, exc) + self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, {"error": "commit_failed"}) + return + if status == "already_exists": self._send_json(HTTPStatus.OK, {"status": "already_exists"}) return self._send_json(HTTPStatus.OK, {"status": "saved", "saved_path": str(saved_path)}) - def _stream_to_part(self, part_path: Path, content_length: int) -> str: + def _stream_to_part(self, part_path: Path, content_length: int) -> str | None: digest = hashlib.sha256() remaining = content_length - with open(part_path, "wb") as handle: - while remaining > 0: - chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) - if not chunk: - raise _PrematureDisconnectError() - handle.write(chunk) - digest.update(chunk) - remaining -= len(chunk) - handle.flush() - os.fsync(handle.fileno()) + try: + with open(part_path, "wb") as handle: + while remaining > 0: + chunk = self.rfile.read(min(CHUNK_SIZE, remaining)) + if not chunk: + raise _PrematureDisconnectError() + handle.write(chunk) + digest.update(chunk) + remaining -= len(chunk) + handle.flush() + os.fsync(handle.fileno()) + except (TimeoutError, socket.timeout): + part_path.unlink(missing_ok=True) + self.close_connection = True + self._send_json( + HTTPStatus.REQUEST_TIMEOUT, + {"ok": False, "error": "Upload timed out"}, + ) + return None return digest.hexdigest() @@ -293,7 +340,7 @@ class MacReceiverServer(ThreadingHTTPServer): self._commit_lock = threading.Lock() self._notify_lock = threading.Lock() self._pending_count = 0 - self._pending_dest = Path(cfg.dest_root) + self._pending_dest = Path(cfg.dest_root).expanduser() self._notify_timer: threading.Timer | None = None self._server_thread: threading.Thread | None = None @@ -375,12 +422,16 @@ class MacReceiverServer(ThreadingHTTPServer): return ("already_exists", None) index += 1 + try: + os.utime(part_path, (mtime, mtime)) + except OSError as exc: + logger.warning("Failed to set mtime on %s: %s", part_path, exc) + os.replace(part_path, final_path) except BaseException: part_path.unlink(missing_ok=True) raise - os.utime(final_path, (mtime, mtime)) logger.info("Saved photo %s", final_path) if cfg.notify_on_batch: self._schedule_notification(Path(cfg.dest_root).expanduser()) diff --git a/mac-receiver/tests/test_server.py b/mac-receiver/tests/test_server.py index 7d0a1cc..f28d342 100644 --- a/mac-receiver/tests/test_server.py +++ b/mac-receiver/tests/test_server.py @@ -1,6 +1,7 @@ import hashlib import json import os +import socket import tempfile import threading import time @@ -16,6 +17,7 @@ from urllib.request import Request, urlopen from cp0_mac_receiver.config import MacConfig, load_mac_config, save_mac_config from cp0_mac_receiver.server import ( MacReceiverServer, + _MacReceiverHandler, _default_osascript_notifier, is_allowed_client_ip, ) @@ -248,9 +250,142 @@ class TestMacReceiverServer(unittest.TestCase): ctx2.exception.close() camera_dir = self.dest_root / "Camera" - if camera_dir.exists(): - self.assertEqual(list(camera_dir.rglob("*.part")), []) - self.assertFalse((camera_dir / "a.jpg").exists()) + self.assertTrue(camera_dir.is_dir()) + self.assertEqual(list(camera_dir.rglob("*.part")), []) + self.assertFalse((camera_dir / "a.jpg").exists()) + + def _read_http_response(self, sock: socket.socket) -> tuple[int, dict]: + chunks = [] + while True: + chunk = sock.recv(4096) + if not chunk: + break + chunks.append(chunk) + raw = b"".join(chunks) + head, _, body = raw.partition(b"\r\n\r\n") + status_line = head.split(b"\r\n", 1)[0].decode("ascii", errors="replace") + status_code = int(status_line.split()[1]) + return status_code, json.loads(body.decode("utf-8")) + + def test_rejects_out_of_range_mtime_with_400(self): + payload = b"mtime-test-bytes" + sha = hashlib.sha256(payload).hexdigest() + for bad_mtime in ("1e20", "1e12", "-1", "-1e20", "nan", "inf", "not-a-number"): + req = Request( + f"{self.base_url}/upload", + data=payload, + method="POST", + headers={ + "Content-Length": str(len(payload)), + "X-CP0-Folder-Label": "Camera", + "X-CP0-Rel-Path": "bad_mtime.jpg", + "X-CP0-SHA256": sha, + "X-CP0-Mtime": bad_mtime, + }, + ) + with self.assertRaises(HTTPError) as ctx: + urlopen(req) + self.assertEqual(ctx.exception.code, 400, f"mtime={bad_mtime}") + body = json.loads(ctx.exception.read()) + self.assertEqual(body["error"], "invalid X-CP0-Mtime") + ctx.exception.close() + + self.assertEqual(list(self.dest_root.rglob("*.part")), []) + + def test_filesystem_collision_returns_400_without_leaking_part_files(self): + self.dest_root.mkdir(parents=True, exist_ok=True) + blocking_file = self.dest_root / "BlockedFile" + blocking_file.write_bytes(b"original-file") + + for rel_path in ("a.jpg", "sub/a.jpg"): + with self.assertRaises(HTTPError) as ctx: + self._upload("BlockedFile", rel_path, b"photo-bytes") + self.assertEqual(ctx.exception.code, 400) + body = json.loads(ctx.exception.read()) + self.assertIn("error", body) + ctx.exception.close() + + self.assertEqual(blocking_file.read_bytes(), b"original-file") + + blocking_dir = self.dest_root / "Camera" / "existing_dir.jpg" + blocking_dir.mkdir(parents=True, exist_ok=True) + with self.assertRaises(HTTPError) as ctx2: + self._upload("Camera", "existing_dir.jpg", b"photo-bytes") + self.assertEqual(ctx2.exception.code, 400) + body2 = json.loads(ctx2.exception.read()) + self.assertIn("error", body2) + ctx2.exception.close() + + self.assertTrue(blocking_dir.is_dir()) + self.assertEqual(list(self.dest_root.rglob("*.part")), []) + + def test_premature_client_disconnect_cleans_part_file(self): + port = self.srv.server_address[1] + payload = b"partial" + sha = hashlib.sha256(b"x" * 1024).hexdigest() + raw_req = ( + "POST /upload HTTP/1.1\r\n" + f"Host: 127.0.0.1:{port}\r\n" + "Content-Length: 1024\r\n" + "X-CP0-Folder-Label: Camera\r\n" + "X-CP0-Rel-Path: partial.jpg\r\n" + f"X-CP0-SHA256: {sha}\r\n" + "X-CP0-Mtime: 1700000000\r\n" + "\r\n" + ).encode("ascii") + payload + + with socket.create_connection(("127.0.0.1", port), timeout=5.0) as sock: + sock.sendall(raw_req) + sock.shutdown(socket.SHUT_WR) + status_code, body = self._read_http_response(sock) + + self.assertEqual(status_code, 400) + self.assertEqual(body["error"], "incomplete request body") + camera_dir = self.dest_root / "Camera" + self.assertTrue(camera_dir.is_dir()) + self.assertEqual(list(camera_dir.rglob("*.part")), []) + self.assertFalse((camera_dir / "partial.jpg").exists()) + + def test_upload_timeout_returns_408_and_cleans_part_file(self): + port = self.srv.server_address[1] + sha = hashlib.sha256(b"x" * 1024).hexdigest() + raw_req = ( + "POST /upload HTTP/1.1\r\n" + f"Host: 127.0.0.1:{port}\r\n" + "Content-Length: 1024\r\n" + "X-CP0-Folder-Label: Camera\r\n" + "X-CP0-Rel-Path: slow.jpg\r\n" + f"X-CP0-SHA256: {sha}\r\n" + "X-CP0-Mtime: 1700000000\r\n" + "\r\n" + "partial" + ).encode("ascii") + + with patch.object(_MacReceiverHandler, "timeout", 0.15): + with socket.create_connection(("127.0.0.1", port), timeout=5.0) as sock: + sock.sendall(raw_req) + status_code, body = self._read_http_response(sock) + + self.assertEqual(status_code, 408) + self.assertEqual(body, {"ok": False, "error": "Upload timed out"}) + camera_dir = self.dest_root / "Camera" + self.assertTrue(camera_dir.is_dir()) + self.assertEqual(list(camera_dir.rglob("*.part")), []) + self.assertFalse((camera_dir / "slow.jpg").exists()) + + def test_utime_oserror_does_not_fail_commit(self): + with ( + patch("cp0_mac_receiver.server.os.utime", side_effect=OSError("utime denied")), + self.assertLogs("cp0_mac_receiver.server", level="WARNING"), + ): + with self._upload("Camera", "utime_warn.jpg", b"utime-fallback", 1700000000.0) as resp: + self.assertEqual(resp.status, 200) + body = json.loads(resp.read()) + + self.assertEqual(body["status"], "saved") + target = self.dest_root / "Camera" / "utime_warn.jpg" + self.assertEqual(target.read_bytes(), b"utime-fallback") + self.assertEqual(list(self.dest_root.rglob("*.part")), []) def test_debounced_batch_notification_fires_once_after_burst(self): with self._upload("Camera", "img1.jpg", b"photo-1", 1700000001.0) as r1: @@ -287,6 +422,28 @@ class TestMacReceiverServer(unittest.TestCase): self.assertEqual(res["status"], "saved") self.assertTrue((new_dest / "DCIM" / "p1.jpg").is_file()) + def test_health_and_server_expand_tilde_in_dest_root(self): + save_mac_config( + MacConfig( + dest_root="~/Pictures/CardputerZero", + listen_host="127.0.0.1", + listen_port=0, + notify_on_batch=True, + tailscale_only=True, + ), + self.cfg_path, + ) + expected = str(Path("~/Pictures/CardputerZero").expanduser()) + with urlopen(f"{self.base_url}/health") as resp: + body = json.loads(resp.read()) + self.assertEqual(body["dest_root"], expected) + + srv2 = MacReceiverServer(self.cfg_path, notifier=lambda _c, _d: None, debounce_sec=0.05) + try: + self.assertEqual(srv2._pending_dest, Path(expected)) + finally: + srv2.server_close() + if __name__ == "__main__": unittest.main()