From c6a672f25beb228f044a489d1d859bcab4b8b8d4 Mon Sep 17 00:00:00 2001 From: cottongin Date: Wed, 18 Nov 2026 08:08:53 -0500 Subject: [PATCH] fix(device-daemon): recover interrupted uploads, fix wake race, and filter ignored subdirs --- device-daemon/cp0_syncd/daemon.py | 39 +++- device-daemon/cp0_syncd/store.py | 13 ++ device-daemon/cp0_syncd/uploader.py | 37 ++-- device-daemon/cp0_syncd/watcher.py | 64 ++++-- .../tests/test_daemon_integration.py | 203 ++++++++++++++++++ 5 files changed, 315 insertions(+), 41 deletions(-) diff --git a/device-daemon/cp0_syncd/daemon.py b/device-daemon/cp0_syncd/daemon.py index e01eaa6..ad7efdd 100644 --- a/device-daemon/cp0_syncd/daemon.py +++ b/device-daemon/cp0_syncd/daemon.py @@ -23,7 +23,7 @@ from .config import ( from .ipc import IpcServer, default_sock_path from .store import SyncStore, compute_sha256, default_db_path, is_media_file from .uploader import Uploader -from .watcher import InotifyWatcher, scan_folder +from .watcher import InotifyWatcher, is_ignored_name, scan_folder logger = logging.getLogger(__name__) @@ -37,6 +37,10 @@ def _safe_int(value: str | None, default: int = 0) -> int: return default +def _has_ignored_dir_part(parts: tuple[str, ...], ignore_prefixes: list[str]) -> bool: + return any(is_ignored_name(part, ignore_prefixes) for part in parts) + + class SyncDaemon: """On-device CardputerZero photo sync daemon.""" @@ -55,6 +59,7 @@ class SyncDaemon: self._config_lock = threading.RLock() self.config: DaemonConfig = load_daemon_config(self.config_path) self.store = SyncStore(self.db_path) + self._recover_interrupted_uploads() self.wake_event = threading.Event() self.uploader = Uploader( @@ -67,10 +72,18 @@ class SyncDaemon: on_file_ready=self._on_file_ready, on_dir_created=self._on_dir_created, poll_interval=watcher_poll_interval, + ignore_prefixes=self.config.ignore_prefixes, + get_ignore_prefixes=lambda: self.get_config().ignore_prefixes, ) self.ipc_server = IpcServer(self.sock_path, self.handle_ipc) self._started = False + def _recover_interrupted_uploads(self) -> int: + recovered = self.store.recover_interrupted_uploads() + if recovered > 0: + self.store.log("WARN", f"Recovered {recovered} interrupted upload(s) to PENDING") + return recovered + def get_config(self) -> DaemonConfig: """Return the current DaemonConfig safely across threads.""" with self._config_lock: @@ -91,17 +104,25 @@ class SyncDaemon: active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None) if active is None: return + + root = Path(os.path.expanduser(active.path)) + try: + rel_obj = file_path.relative_to(root) + except ValueError: + return + + if _has_ignored_dir_part(rel_obj.parts[:-1], cfg.ignore_prefixes): + return if not is_media_file(file_path, cfg, store=self.store): return - root = Path(os.path.expanduser(active.path)) try: st = file_path.stat() if st.st_size <= 0: return sha256 = compute_sha256(file_path) - rel_path = file_path.relative_to(root).as_posix() - except (OSError, ValueError): + rel_path = rel_obj.as_posix() + except OSError: return outcome = self.store.enqueue_file( @@ -115,11 +136,18 @@ class SyncDaemon: if outcome == "enqueued": self.uploader.notify_new_file() - def _on_dir_created(self, folder: WatchedFolder, _dir_path: Path) -> None: + def _on_dir_created(self, folder: WatchedFolder, dir_path: Path) -> None: cfg = self.get_config() active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None) if active is None: return + root = Path(os.path.expanduser(active.path)) + try: + rel_parts = dir_path.relative_to(root).parts + except ValueError: + return + if rel_parts and _has_ignored_dir_part(rel_parts, cfg.ignore_prefixes): + return added = scan_folder(active, cfg, self.store) if added > 0: self.uploader.notify_new_file() @@ -130,6 +158,7 @@ class SyncDaemon: return self._started = True self.store.log("INFO", "cp0-syncd started") + self._recover_interrupted_uploads() self.watcher.sync_folders(self.get_config().watched_folders) self.watcher.start() self.scan_all() diff --git a/device-daemon/cp0_syncd/store.py b/device-daemon/cp0_syncd/store.py index ded661b..086987d 100644 --- a/device-daemon/cp0_syncd/store.py +++ b/device-daemon/cp0_syncd/store.py @@ -211,6 +211,19 @@ class SyncStore: ) self._conn.commit() + def recover_interrupted_uploads(self) -> int: + """Reset stale UPLOADING rows back to PENDING after an unclean restart.""" + with self._lock: + cur = self._conn.execute( + """ + UPDATE files + SET status = 'PENDING' + WHERE status = 'UPLOADING' + """ + ) + self._conn.commit() + return cur.rowcount + def mark_synced(self, file_id: int) -> None: """Mark a file as SYNCED and clear any prior error.""" now_ms = _now_ms() diff --git a/device-daemon/cp0_syncd/uploader.py b/device-daemon/cp0_syncd/uploader.py index 8a0a324..46ba332 100644 --- a/device-daemon/cp0_syncd/uploader.py +++ b/device-daemon/cp0_syncd/uploader.py @@ -105,6 +105,8 @@ class Uploader: def wake(self, reset_backoff: bool = True, probe: bool = False) -> None: """Interrupt backoff sleep immediately and optionally trigger a /health probe.""" + if reset_backoff: + self._store.set_meta("next_retry_ts", "0") with self._state_lock: if reset_backoff: self._backoff_index = 0 @@ -112,27 +114,26 @@ class Uploader: self._next_retry_deadline = 0.0 if probe: self._probe_requested = True - if reset_backoff: - self._store.set_meta("next_retry_ts", "0") - self._wake_event.set() + self._wake_event.set() def start(self) -> None: """Start the background upload loop thread.""" if self._thread is not None and self._thread.is_alive(): return - self._stop_event.clear() cfg = self._get_config() - if cfg.target_host.strip(): - with self._state_lock: + with self._state_lock: + self._stop_event.clear() + if cfg.target_host.strip(): self._probe_requested = True - self._wake_event.set() + self._wake_event.set() self._thread = threading.Thread(target=self._run_loop, name="cp0-uploader", daemon=True) self._thread.start() def stop(self) -> None: """Stop the background upload loop cleanly.""" - self._stop_event.set() - self._wake_event.set() + with self._state_lock: + self._stop_event.set() + self._wake_event.set() if self._thread is not None and self._thread is not threading.current_thread(): self._thread.join(timeout=5.0) self._thread = None @@ -142,7 +143,7 @@ class Uploader: with self._state_lock: probe = self._probe_requested self._probe_requested = False - self._wake_event.clear() + self._wake_event.clear() has_pending = self._store.next_pending() is not None if has_pending or probe: @@ -150,17 +151,15 @@ class Uploader: else: result = "idle" - if self._stop_event.is_set(): - break - if self._wake_event.is_set(): + with self._state_lock: + if self._stop_event.is_set(): + break + woken = self._wake_event.is_set() or self._probe_requested + wait_sec = max(1, self._current_delay) if result in ("offline", "paused") else 60.0 + if woken: continue - if result in ("offline", "paused"): - with self._state_lock: - wait_sec = max(1, self._current_delay) - self._wake_event.wait(timeout=wait_sec) - else: - self._wake_event.wait(timeout=60.0) + self._wake_event.wait(timeout=wait_sec) def _reset_backoff(self) -> None: with self._state_lock: diff --git a/device-daemon/cp0_syncd/watcher.py b/device-daemon/cp0_syncd/watcher.py index 441845e..946089a 100644 --- a/device-daemon/cp0_syncd/watcher.py +++ b/device-daemon/cp0_syncd/watcher.py @@ -13,9 +13,9 @@ import sys import threading from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence -from .config import DaemonConfig, WatchedFolder +from .config import DEFAULT_IGNORE_PREFIXES, DaemonConfig, WatchedFolder from .store import SyncStore, compute_sha256, is_media_file logger = logging.getLogger(__name__) @@ -67,6 +67,11 @@ def parse_inotify_events(data: bytes) -> list[InotifyEvent]: return events +def is_ignored_name(name: str, ignore_prefixes: Sequence[str]) -> bool: + """Return True if a file or directory basename matches any configured ignore prefix.""" + return bool(name) and any(name.startswith(prefix) for prefix in ignore_prefixes if prefix) + + def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> int: """Recursively scan an enabled watched folder and enqueue new media files.""" if not folder.enabled: @@ -77,13 +82,10 @@ def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> i return 0 enqueued = 0 - ignore_prefixes = [prefix for prefix in cfg.ignore_prefixes if prefix] for dirpath, dirnames, filenames in os.walk(root, followlinks=False): dirnames[:] = sorted( - name - for name in dirnames - if not any(name.startswith(prefix) for prefix in ignore_prefixes) + name for name in dirnames if not is_ignored_name(name, cfg.ignore_prefixes) ) current_dir = Path(dirpath) for filename in sorted(filenames): @@ -159,13 +161,16 @@ def _nearest_existing_ancestor(target: Path) -> Path: candidate = parent -def _iter_subdirs(root: Path) -> list[Path]: - """Yield root and all non-symlink subdirectories beneath root.""" +def _iter_subdirs( + root: Path, + ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES, +) -> list[Path]: + """Yield root and all non-symlink, non-ignored subdirectories beneath root.""" dirs: list[Path] = [] if not root.is_dir(): return dirs for dirpath, dirnames, _ in os.walk(root, followlinks=False): - dirnames[:] = sorted(d for d in dirnames if not d.startswith(".")) + dirnames[:] = sorted(d for d in dirnames if not is_ignored_name(d, ignore_prefixes)) dirs.append(Path(dirpath)) return dirs @@ -178,16 +183,25 @@ class PollingWatcher: on_file_ready: Callable[[WatchedFolder, Path], None], on_dir_created: Callable[[WatchedFolder, Path], None] | None = None, poll_interval: float = 0.5, + ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES, + get_ignore_prefixes: Callable[[], Sequence[str]] | None = None, ) -> None: self._on_file_ready = on_file_ready self._on_dir_created = on_dir_created self._poll_interval = max(0.02, float(poll_interval)) + self._ignore_prefixes = tuple(ignore_prefixes) + self._get_ignore_prefixes = get_ignore_prefixes self._lock = threading.RLock() self._stop_event = threading.Event() self._thread: threading.Thread | None = None self._folders: dict[str, WatchedFolder] = {} self._snapshots: dict[str, dict[str, Any]] = {} + def _current_ignore_prefixes(self) -> Sequence[str]: + if self._get_ignore_prefixes is not None: + return self._get_ignore_prefixes() + return self._ignore_prefixes + def sync_folders(self, folders: list[WatchedFolder]) -> None: with self._lock: target_ids = {wf.id for wf in folders if wf.enabled} @@ -219,15 +233,18 @@ class PollingWatcher: root = Path(os.path.expanduser(folder.path)) if not root.is_dir(): return {"present": False, "dirs": set(), "files": {}} - dirs, files = self._scan_tree(root) + dirs, files = self._scan_tree(root, self._current_ignore_prefixes()) return {"present": True, "dirs": dirs, "files": files} @staticmethod - def _scan_tree(root: Path) -> tuple[set[Path], dict[Path, tuple[int, int]]]: + def _scan_tree( + root: Path, + ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES, + ) -> tuple[set[Path], dict[Path, tuple[int, int]]]: dirs: set[Path] = set() files: dict[Path, tuple[int, int]] = {} for dirpath, dirnames, filenames in os.walk(root, followlinks=False): - dirnames[:] = sorted(d for d in dirnames if not d.startswith(".")) + dirnames[:] = sorted(d for d in dirnames if not is_ignored_name(d, ignore_prefixes)) cur = Path(dirpath) dirs.add(cur) for name in filenames: @@ -242,6 +259,7 @@ class PollingWatcher: def poll_once(self) -> None: events_dirs: list[tuple[WatchedFolder, Path]] = [] events_files: list[tuple[WatchedFolder, Path]] = [] + ignore_prefixes = self._current_ignore_prefixes() with self._lock: items = list(self._folders.items()) @@ -260,7 +278,7 @@ class PollingWatcher: snap["files"] = {} continue - dirs, files = self._scan_tree(root) + dirs, files = self._scan_tree(root, ignore_prefixes) prev_dirs: set[Path] = snap["dirs"] prev_files: dict[Path, tuple[int, int]] = snap["files"] @@ -315,10 +333,14 @@ class InotifyWatcher: on_dir_created: Callable[[WatchedFolder, Path], None] | None = None, poll_interval: float = 0.5, libc: Any | None = None, + ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES, + get_ignore_prefixes: Callable[[], Sequence[str]] | None = None, ) -> None: self._on_file_ready = on_file_ready self._on_dir_created = on_dir_created self._poll_interval = max(0.05, float(poll_interval)) + self._ignore_prefixes = tuple(ignore_prefixes) + self._get_ignore_prefixes = get_ignore_prefixes self._lock = threading.RLock() self._stop_event = threading.Event() self._thread: threading.Thread | None = None @@ -353,8 +375,15 @@ class InotifyWatcher: on_file_ready=on_file_ready, on_dir_created=on_dir_created, poll_interval=self._poll_interval, + ignore_prefixes=self._ignore_prefixes, + get_ignore_prefixes=self._get_ignore_prefixes, ) + def _current_ignore_prefixes(self) -> Sequence[str]: + if self._get_ignore_prefixes is not None: + return self._get_ignore_prefixes() + return self._ignore_prefixes + @property def is_using_polling(self) -> bool: """Return True if running with the PollingWatcher fallback.""" @@ -438,7 +467,7 @@ class InotifyWatcher: def _attach_tree_locked(self, folder: WatchedFolder, root: Path) -> None: wds = self._folder_wds.setdefault(folder.id, set()) - for subdir in _iter_subdirs(root): + for subdir in _iter_subdirs(root, self._current_ignore_prefixes()): wd = self._add_watch_locked(subdir, _DIR_WATCH_MASK) if wd is not None: self._active_by_wd[wd] = (folder, subdir) @@ -584,9 +613,10 @@ class InotifyWatcher: active = self._active_by_wd.get(wd) if active is not None: folder, _ = active - if created_dir.is_dir(): - self._attach_tree_locked(folder, created_dir) - dir_callbacks.append((folder, created_dir)) + if not is_ignored_name(created_dir.name, self._current_ignore_prefixes()): + if created_dir.is_dir(): + self._attach_tree_locked(folder, created_dir) + dir_callbacks.append((folder, created_dir)) anc = self._ancestors_by_wd.get(wd) if anc is not None: diff --git a/device-daemon/tests/test_daemon_integration.py b/device-daemon/tests/test_daemon_integration.py index d88e15b..e885bf2 100644 --- a/device-daemon/tests/test_daemon_integration.py +++ b/device-daemon/tests/test_daemon_integration.py @@ -711,6 +711,209 @@ class TestWatchersAndLinuxInotify(unittest.TestCase): finally: db.close() + def test_recovers_interrupted_uploading_rows_on_startup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + cam_dir = Path(tmp) / "Camera" + dest_root = Path(tmp) / "MacDest" + cam_dir.mkdir() + photo = cam_dir / "crashed_mid_upload.jpg" + photo.write_bytes(b"recovered-photo-payload") + + mac_cfg = Path(tmp) / "mac.json" + save_mac_config( + MacConfig( + dest_root=str(dest_root), + listen_host="127.0.0.1", + listen_port=0, + notify_on_batch=False, + tailscale_only=True, + ), + mac_cfg, + ) + mac_srv = MacReceiverServer(mac_cfg) + mac_srv.start() + try: + cfg_path = Path(tmp) / "daemon.json" + db_path = Path(tmp) / "state.db" + sock_path = Path(tmp) / "cp0.sock" + save_daemon_config( + DaemonConfig( + target_host="127.0.0.1", + target_port=mac_srv.server_port, + watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")], + ), + cfg_path, + ) + + # Simulate crash leaving row stuck in UPLOADING in state.db + pre_store = SyncStore(db_path) + try: + pre_store.enqueue_file( + "cam", + str(photo), + "crashed_mid_upload.jpg", + len(b"recovered-photo-payload"), + photo.stat().st_mtime_ns, + compute_sha256(photo), + ) + row = pre_store.next_pending() + assert row is not None + pre_store.mark_uploading(row["id"]) + self.assertIsNone(pre_store.next_pending()) + finally: + pre_store.close() + + daemon = SyncDaemon(cfg_path, db_path, sock_path, watcher_poll_interval=0.05) + daemon.start() + try: + self.assertTrue(_wait_until(lambda: (dest_root / "Camera" / "crashed_mid_upload.jpg").exists())) + status = _send_ipc(sock_path, {"cmd": "status"}) + self.assertEqual(status["synced_count"], 1) + self.assertEqual(status["pending_count"], 0) + finally: + daemon.stop() + finally: + mac_srv.shutdown() + mac_srv.server_close() + + def test_concurrent_wake_probe_not_lost_by_run_loop(self) -> None: + class _SlowClearEvent(threading.Event): + def __init__(self) -> None: + super().__init__() + self.clear_entered = threading.Event() + self.delay_clear = True + + def clear(self) -> None: + self.clear_entered.set() + if self.delay_clear: + time.sleep(0.05) + super().clear() + + with tempfile.TemporaryDirectory() as tmp: + dest_root = Path(tmp) / "MacDest" + mac_cfg = Path(tmp) / "mac.json" + save_mac_config( + MacConfig( + dest_root=str(dest_root), + listen_host="127.0.0.1", + listen_port=0, + notify_on_batch=False, + tailscale_only=True, + ), + mac_cfg, + ) + mac_srv = MacReceiverServer(mac_cfg) + mac_srv.start() + try: + cfg_holder = [DaemonConfig(target_host="", target_port=mac_srv.server_port, watched_folders=[])] + store = SyncStore(Path(tmp) / "state.db") + wake_event = _SlowClearEvent() + from cp0_syncd.uploader import Uploader + + uploader = Uploader(store=store, get_config=lambda: cfg_holder[0], wake_event=wake_event) + uploader.start() + try: + self.assertTrue(wake_event.clear_entered.wait(timeout=2.0)) + # Configure target and fire wake(probe=True) while _run_loop is inside clear(). + cfg_holder[0] = DaemonConfig( + target_host="127.0.0.1", + target_port=mac_srv.server_port, + watched_folders=[], + ) + wake_event.delay_clear = False + uploader.wake(reset_backoff=True, probe=True) + self.assertTrue( + _wait_until( + lambda: store.get_meta("mac_online") == "true" + and store.get_meta("mac_dest_root") == str(dest_root), + timeout=1.5, + ) + ) + finally: + uploader.stop() + store.close() + finally: + mac_srv.shutdown() + mac_srv.server_close() + + def test_ignored_subdirectories_skipped_by_watchers_and_on_file_ready(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + cam_dir = root / "Camera" + (cam_dir / ".hidden").mkdir(parents=True) + (cam_dir / "~tmp").mkdir() + (cam_dir / "2026").mkdir() + wf = WatchedFolder(id="cam", path=str(cam_dir), label="Camera") + + # 1. PollingWatcher ignores files inside .hidden/ and ~tmp/ + polled_files: list[Path] = [] + poller = PollingWatcher( + on_file_ready=lambda _wf, p: polled_files.append(p), + ) + poller.sync_folders([wf]) + (cam_dir / ".hidden" / "a.jpg").write_bytes(b"hidden") + (cam_dir / "~tmp" / "b.jpg").write_bytes(b"tilde") + (cam_dir / "2026" / "ok.jpg").write_bytes(b"real") + poller.poll_once() + self.assertEqual(polled_files, [cam_dir / "2026" / "ok.jpg"]) + + # 2. InotifyWatcher ignores .hidden/ and ~tmp/ on initial attach and dynamic IN_CREATE | IN_ISDIR + fake_libc = _FakeLibc() + inotify_dirs: list[Path] = [] + watcher = InotifyWatcher( + on_file_ready=lambda _wf, _p: None, + on_dir_created=lambda _wf, p: inotify_dirs.append(p), + libc=fake_libc, + ) + watcher.sync_folders([wf]) + watched_paths = {p for p, _ in fake_libc.watches.values()} + self.assertIn(str(cam_dir), watched_paths) + self.assertIn(str(cam_dir / "2026"), watched_paths) + self.assertNotIn(str(cam_dir / ".hidden"), watched_paths) + self.assertNotIn(str(cam_dir / "~tmp"), watched_paths) + + cam_wd = next(wd for wd, (p, _) in fake_libc.watches.items() if p == str(cam_dir)) + (cam_dir / "~newtmp").mkdir() + (cam_dir / ".newdot").mkdir() + watcher.process_inotify_bytes( + _pack_inotify_event(cam_wd, IN_CREATE | IN_ISDIR, 0, b"~newtmp") + + _pack_inotify_event(cam_wd, IN_CREATE | IN_ISDIR, 0, b".newdot") + ) + watched_after = {p for p, _ in fake_libc.watches.values()} + self.assertNotIn(str(cam_dir / "~newtmp"), watched_after) + self.assertNotIn(str(cam_dir / ".newdot"), watched_after) + self.assertEqual(inotify_dirs, []) + + # 3. SyncDaemon._on_file_ready skips files whose relative parent parts start with ignore_prefixes + cfg_path = root / "daemon.json" + db_path = root / "state.db" + sock_path = root / "cp0.sock" + save_daemon_config( + DaemonConfig( + watched_folders=[wf], + ignore_prefixes=[".", "~", "SKIP_"], + ), + cfg_path, + ) + daemon = SyncDaemon(cfg_path, db_path, sock_path) + try: + (cam_dir / "SKIP_dir").mkdir() + skip_file = cam_dir / "SKIP_dir" / "ignored.jpg" + tilde_file = cam_dir / "~tmp" / "b.jpg" + dot_file = cam_dir / ".hidden" / "a.jpg" + skip_file.write_bytes(b"skip-me") + daemon._on_file_ready(wf, dot_file) + daemon._on_file_ready(wf, tilde_file) + daemon._on_file_ready(wf, skip_file) + self.assertIsNone(daemon.store.next_pending()) + + daemon._on_file_ready(wf, cam_dir / "2026" / "ok.jpg") + pending = daemon.store.next_pending() + assert pending is not None + self.assertEqual(pending["rel_path"], "2026/ok.jpg") + finally: + daemon.store.close() + class TestDaemonBinaryEntrypoint(unittest.TestCase): def test_cp0_syncd_binary_runs_via_symlink_and_stops_on_sigterm(self) -> None: