fix(device-daemon): recover interrupted uploads, fix wake race, and filter ignored subdirs

This commit is contained in:
2026-11-18 08:08:53 -05:00
parent da4d963919
commit c6a672f25b
5 changed files with 315 additions and 41 deletions

View File

@@ -23,7 +23,7 @@ from .config import (
from .ipc import IpcServer, default_sock_path
from .store import SyncStore, compute_sha256, default_db_path, is_media_file
from .uploader import Uploader
from .watcher import InotifyWatcher, scan_folder
from .watcher import InotifyWatcher, is_ignored_name, scan_folder
logger = logging.getLogger(__name__)
@@ -37,6 +37,10 @@ def _safe_int(value: str | None, default: int = 0) -> int:
return default
def _has_ignored_dir_part(parts: tuple[str, ...], ignore_prefixes: list[str]) -> bool:
return any(is_ignored_name(part, ignore_prefixes) for part in parts)
class SyncDaemon:
"""On-device CardputerZero photo sync daemon."""
@@ -55,6 +59,7 @@ class SyncDaemon:
self._config_lock = threading.RLock()
self.config: DaemonConfig = load_daemon_config(self.config_path)
self.store = SyncStore(self.db_path)
self._recover_interrupted_uploads()
self.wake_event = threading.Event()
self.uploader = Uploader(
@@ -67,10 +72,18 @@ class SyncDaemon:
on_file_ready=self._on_file_ready,
on_dir_created=self._on_dir_created,
poll_interval=watcher_poll_interval,
ignore_prefixes=self.config.ignore_prefixes,
get_ignore_prefixes=lambda: self.get_config().ignore_prefixes,
)
self.ipc_server = IpcServer(self.sock_path, self.handle_ipc)
self._started = False
def _recover_interrupted_uploads(self) -> int:
recovered = self.store.recover_interrupted_uploads()
if recovered > 0:
self.store.log("WARN", f"Recovered {recovered} interrupted upload(s) to PENDING")
return recovered
def get_config(self) -> DaemonConfig:
"""Return the current DaemonConfig safely across threads."""
with self._config_lock:
@@ -91,17 +104,25 @@ class SyncDaemon:
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
if active is None:
return
root = Path(os.path.expanduser(active.path))
try:
rel_obj = file_path.relative_to(root)
except ValueError:
return
if _has_ignored_dir_part(rel_obj.parts[:-1], cfg.ignore_prefixes):
return
if not is_media_file(file_path, cfg, store=self.store):
return
root = Path(os.path.expanduser(active.path))
try:
st = file_path.stat()
if st.st_size <= 0:
return
sha256 = compute_sha256(file_path)
rel_path = file_path.relative_to(root).as_posix()
except (OSError, ValueError):
rel_path = rel_obj.as_posix()
except OSError:
return
outcome = self.store.enqueue_file(
@@ -115,11 +136,18 @@ class SyncDaemon:
if outcome == "enqueued":
self.uploader.notify_new_file()
def _on_dir_created(self, folder: WatchedFolder, _dir_path: Path) -> None:
def _on_dir_created(self, folder: WatchedFolder, dir_path: Path) -> None:
cfg = self.get_config()
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
if active is None:
return
root = Path(os.path.expanduser(active.path))
try:
rel_parts = dir_path.relative_to(root).parts
except ValueError:
return
if rel_parts and _has_ignored_dir_part(rel_parts, cfg.ignore_prefixes):
return
added = scan_folder(active, cfg, self.store)
if added > 0:
self.uploader.notify_new_file()
@@ -130,6 +158,7 @@ class SyncDaemon:
return
self._started = True
self.store.log("INFO", "cp0-syncd started")
self._recover_interrupted_uploads()
self.watcher.sync_folders(self.get_config().watched_folders)
self.watcher.start()
self.scan_all()

View File

@@ -211,6 +211,19 @@ class SyncStore:
)
self._conn.commit()
def recover_interrupted_uploads(self) -> int:
"""Reset stale UPLOADING rows back to PENDING after an unclean restart."""
with self._lock:
cur = self._conn.execute(
"""
UPDATE files
SET status = 'PENDING'
WHERE status = 'UPLOADING'
"""
)
self._conn.commit()
return cur.rowcount
def mark_synced(self, file_id: int) -> None:
"""Mark a file as SYNCED and clear any prior error."""
now_ms = _now_ms()

View File

@@ -105,6 +105,8 @@ class Uploader:
def wake(self, reset_backoff: bool = True, probe: bool = False) -> None:
"""Interrupt backoff sleep immediately and optionally trigger a /health probe."""
if reset_backoff:
self._store.set_meta("next_retry_ts", "0")
with self._state_lock:
if reset_backoff:
self._backoff_index = 0
@@ -112,27 +114,26 @@ class Uploader:
self._next_retry_deadline = 0.0
if probe:
self._probe_requested = True
if reset_backoff:
self._store.set_meta("next_retry_ts", "0")
self._wake_event.set()
self._wake_event.set()
def start(self) -> None:
"""Start the background upload loop thread."""
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
cfg = self._get_config()
if cfg.target_host.strip():
with self._state_lock:
with self._state_lock:
self._stop_event.clear()
if cfg.target_host.strip():
self._probe_requested = True
self._wake_event.set()
self._wake_event.set()
self._thread = threading.Thread(target=self._run_loop, name="cp0-uploader", daemon=True)
self._thread.start()
def stop(self) -> None:
"""Stop the background upload loop cleanly."""
self._stop_event.set()
self._wake_event.set()
with self._state_lock:
self._stop_event.set()
self._wake_event.set()
if self._thread is not None and self._thread is not threading.current_thread():
self._thread.join(timeout=5.0)
self._thread = None
@@ -142,7 +143,7 @@ class Uploader:
with self._state_lock:
probe = self._probe_requested
self._probe_requested = False
self._wake_event.clear()
self._wake_event.clear()
has_pending = self._store.next_pending() is not None
if has_pending or probe:
@@ -150,17 +151,15 @@ class Uploader:
else:
result = "idle"
if self._stop_event.is_set():
break
if self._wake_event.is_set():
with self._state_lock:
if self._stop_event.is_set():
break
woken = self._wake_event.is_set() or self._probe_requested
wait_sec = max(1, self._current_delay) if result in ("offline", "paused") else 60.0
if woken:
continue
if result in ("offline", "paused"):
with self._state_lock:
wait_sec = max(1, self._current_delay)
self._wake_event.wait(timeout=wait_sec)
else:
self._wake_event.wait(timeout=60.0)
self._wake_event.wait(timeout=wait_sec)
def _reset_backoff(self) -> None:
with self._state_lock:

View File

@@ -13,9 +13,9 @@ import sys
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from typing import Any, Callable, Sequence
from .config import DaemonConfig, WatchedFolder
from .config import DEFAULT_IGNORE_PREFIXES, DaemonConfig, WatchedFolder
from .store import SyncStore, compute_sha256, is_media_file
logger = logging.getLogger(__name__)
@@ -67,6 +67,11 @@ def parse_inotify_events(data: bytes) -> list[InotifyEvent]:
return events
def is_ignored_name(name: str, ignore_prefixes: Sequence[str]) -> bool:
"""Return True if a file or directory basename matches any configured ignore prefix."""
return bool(name) and any(name.startswith(prefix) for prefix in ignore_prefixes if prefix)
def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> int:
"""Recursively scan an enabled watched folder and enqueue new media files."""
if not folder.enabled:
@@ -77,13 +82,10 @@ def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> i
return 0
enqueued = 0
ignore_prefixes = [prefix for prefix in cfg.ignore_prefixes if prefix]
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
dirnames[:] = sorted(
name
for name in dirnames
if not any(name.startswith(prefix) for prefix in ignore_prefixes)
name for name in dirnames if not is_ignored_name(name, cfg.ignore_prefixes)
)
current_dir = Path(dirpath)
for filename in sorted(filenames):
@@ -159,13 +161,16 @@ def _nearest_existing_ancestor(target: Path) -> Path:
candidate = parent
def _iter_subdirs(root: Path) -> list[Path]:
"""Yield root and all non-symlink subdirectories beneath root."""
def _iter_subdirs(
root: Path,
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
) -> list[Path]:
"""Yield root and all non-symlink, non-ignored subdirectories beneath root."""
dirs: list[Path] = []
if not root.is_dir():
return dirs
for dirpath, dirnames, _ in os.walk(root, followlinks=False):
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
dirnames[:] = sorted(d for d in dirnames if not is_ignored_name(d, ignore_prefixes))
dirs.append(Path(dirpath))
return dirs
@@ -178,16 +183,25 @@ class PollingWatcher:
on_file_ready: Callable[[WatchedFolder, Path], None],
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
poll_interval: float = 0.5,
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
get_ignore_prefixes: Callable[[], Sequence[str]] | None = None,
) -> None:
self._on_file_ready = on_file_ready
self._on_dir_created = on_dir_created
self._poll_interval = max(0.02, float(poll_interval))
self._ignore_prefixes = tuple(ignore_prefixes)
self._get_ignore_prefixes = get_ignore_prefixes
self._lock = threading.RLock()
self._stop_event = threading.Event()
self._thread: threading.Thread | None = None
self._folders: dict[str, WatchedFolder] = {}
self._snapshots: dict[str, dict[str, Any]] = {}
def _current_ignore_prefixes(self) -> Sequence[str]:
if self._get_ignore_prefixes is not None:
return self._get_ignore_prefixes()
return self._ignore_prefixes
def sync_folders(self, folders: list[WatchedFolder]) -> None:
with self._lock:
target_ids = {wf.id for wf in folders if wf.enabled}
@@ -219,15 +233,18 @@ class PollingWatcher:
root = Path(os.path.expanduser(folder.path))
if not root.is_dir():
return {"present": False, "dirs": set(), "files": {}}
dirs, files = self._scan_tree(root)
dirs, files = self._scan_tree(root, self._current_ignore_prefixes())
return {"present": True, "dirs": dirs, "files": files}
@staticmethod
def _scan_tree(root: Path) -> tuple[set[Path], dict[Path, tuple[int, int]]]:
def _scan_tree(
root: Path,
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
) -> tuple[set[Path], dict[Path, tuple[int, int]]]:
dirs: set[Path] = set()
files: dict[Path, tuple[int, int]] = {}
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
dirnames[:] = sorted(d for d in dirnames if not is_ignored_name(d, ignore_prefixes))
cur = Path(dirpath)
dirs.add(cur)
for name in filenames:
@@ -242,6 +259,7 @@ class PollingWatcher:
def poll_once(self) -> None:
events_dirs: list[tuple[WatchedFolder, Path]] = []
events_files: list[tuple[WatchedFolder, Path]] = []
ignore_prefixes = self._current_ignore_prefixes()
with self._lock:
items = list(self._folders.items())
@@ -260,7 +278,7 @@ class PollingWatcher:
snap["files"] = {}
continue
dirs, files = self._scan_tree(root)
dirs, files = self._scan_tree(root, ignore_prefixes)
prev_dirs: set[Path] = snap["dirs"]
prev_files: dict[Path, tuple[int, int]] = snap["files"]
@@ -315,10 +333,14 @@ class InotifyWatcher:
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
poll_interval: float = 0.5,
libc: Any | None = None,
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
get_ignore_prefixes: Callable[[], Sequence[str]] | None = None,
) -> None:
self._on_file_ready = on_file_ready
self._on_dir_created = on_dir_created
self._poll_interval = max(0.05, float(poll_interval))
self._ignore_prefixes = tuple(ignore_prefixes)
self._get_ignore_prefixes = get_ignore_prefixes
self._lock = threading.RLock()
self._stop_event = threading.Event()
self._thread: threading.Thread | None = None
@@ -353,8 +375,15 @@ class InotifyWatcher:
on_file_ready=on_file_ready,
on_dir_created=on_dir_created,
poll_interval=self._poll_interval,
ignore_prefixes=self._ignore_prefixes,
get_ignore_prefixes=self._get_ignore_prefixes,
)
def _current_ignore_prefixes(self) -> Sequence[str]:
if self._get_ignore_prefixes is not None:
return self._get_ignore_prefixes()
return self._ignore_prefixes
@property
def is_using_polling(self) -> bool:
"""Return True if running with the PollingWatcher fallback."""
@@ -438,7 +467,7 @@ class InotifyWatcher:
def _attach_tree_locked(self, folder: WatchedFolder, root: Path) -> None:
wds = self._folder_wds.setdefault(folder.id, set())
for subdir in _iter_subdirs(root):
for subdir in _iter_subdirs(root, self._current_ignore_prefixes()):
wd = self._add_watch_locked(subdir, _DIR_WATCH_MASK)
if wd is not None:
self._active_by_wd[wd] = (folder, subdir)
@@ -584,9 +613,10 @@ class InotifyWatcher:
active = self._active_by_wd.get(wd)
if active is not None:
folder, _ = active
if created_dir.is_dir():
self._attach_tree_locked(folder, created_dir)
dir_callbacks.append((folder, created_dir))
if not is_ignored_name(created_dir.name, self._current_ignore_prefixes()):
if created_dir.is_dir():
self._attach_tree_locked(folder, created_dir)
dir_callbacks.append((folder, created_dir))
anc = self._ancestors_by_wd.get(wd)
if anc is not None:

View File

@@ -711,6 +711,209 @@ class TestWatchersAndLinuxInotify(unittest.TestCase):
finally:
db.close()
def test_recovers_interrupted_uploading_rows_on_startup(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cam_dir = Path(tmp) / "Camera"
dest_root = Path(tmp) / "MacDest"
cam_dir.mkdir()
photo = cam_dir / "crashed_mid_upload.jpg"
photo.write_bytes(b"recovered-photo-payload")
mac_cfg = Path(tmp) / "mac.json"
save_mac_config(
MacConfig(
dest_root=str(dest_root),
listen_host="127.0.0.1",
listen_port=0,
notify_on_batch=False,
tailscale_only=True,
),
mac_cfg,
)
mac_srv = MacReceiverServer(mac_cfg)
mac_srv.start()
try:
cfg_path = Path(tmp) / "daemon.json"
db_path = Path(tmp) / "state.db"
sock_path = Path(tmp) / "cp0.sock"
save_daemon_config(
DaemonConfig(
target_host="127.0.0.1",
target_port=mac_srv.server_port,
watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")],
),
cfg_path,
)
# Simulate crash leaving row stuck in UPLOADING in state.db
pre_store = SyncStore(db_path)
try:
pre_store.enqueue_file(
"cam",
str(photo),
"crashed_mid_upload.jpg",
len(b"recovered-photo-payload"),
photo.stat().st_mtime_ns,
compute_sha256(photo),
)
row = pre_store.next_pending()
assert row is not None
pre_store.mark_uploading(row["id"])
self.assertIsNone(pre_store.next_pending())
finally:
pre_store.close()
daemon = SyncDaemon(cfg_path, db_path, sock_path, watcher_poll_interval=0.05)
daemon.start()
try:
self.assertTrue(_wait_until(lambda: (dest_root / "Camera" / "crashed_mid_upload.jpg").exists()))
status = _send_ipc(sock_path, {"cmd": "status"})
self.assertEqual(status["synced_count"], 1)
self.assertEqual(status["pending_count"], 0)
finally:
daemon.stop()
finally:
mac_srv.shutdown()
mac_srv.server_close()
def test_concurrent_wake_probe_not_lost_by_run_loop(self) -> None:
class _SlowClearEvent(threading.Event):
def __init__(self) -> None:
super().__init__()
self.clear_entered = threading.Event()
self.delay_clear = True
def clear(self) -> None:
self.clear_entered.set()
if self.delay_clear:
time.sleep(0.05)
super().clear()
with tempfile.TemporaryDirectory() as tmp:
dest_root = Path(tmp) / "MacDest"
mac_cfg = Path(tmp) / "mac.json"
save_mac_config(
MacConfig(
dest_root=str(dest_root),
listen_host="127.0.0.1",
listen_port=0,
notify_on_batch=False,
tailscale_only=True,
),
mac_cfg,
)
mac_srv = MacReceiverServer(mac_cfg)
mac_srv.start()
try:
cfg_holder = [DaemonConfig(target_host="", target_port=mac_srv.server_port, watched_folders=[])]
store = SyncStore(Path(tmp) / "state.db")
wake_event = _SlowClearEvent()
from cp0_syncd.uploader import Uploader
uploader = Uploader(store=store, get_config=lambda: cfg_holder[0], wake_event=wake_event)
uploader.start()
try:
self.assertTrue(wake_event.clear_entered.wait(timeout=2.0))
# Configure target and fire wake(probe=True) while _run_loop is inside clear().
cfg_holder[0] = DaemonConfig(
target_host="127.0.0.1",
target_port=mac_srv.server_port,
watched_folders=[],
)
wake_event.delay_clear = False
uploader.wake(reset_backoff=True, probe=True)
self.assertTrue(
_wait_until(
lambda: store.get_meta("mac_online") == "true"
and store.get_meta("mac_dest_root") == str(dest_root),
timeout=1.5,
)
)
finally:
uploader.stop()
store.close()
finally:
mac_srv.shutdown()
mac_srv.server_close()
def test_ignored_subdirectories_skipped_by_watchers_and_on_file_ready(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
cam_dir = root / "Camera"
(cam_dir / ".hidden").mkdir(parents=True)
(cam_dir / "~tmp").mkdir()
(cam_dir / "2026").mkdir()
wf = WatchedFolder(id="cam", path=str(cam_dir), label="Camera")
# 1. PollingWatcher ignores files inside .hidden/ and ~tmp/
polled_files: list[Path] = []
poller = PollingWatcher(
on_file_ready=lambda _wf, p: polled_files.append(p),
)
poller.sync_folders([wf])
(cam_dir / ".hidden" / "a.jpg").write_bytes(b"hidden")
(cam_dir / "~tmp" / "b.jpg").write_bytes(b"tilde")
(cam_dir / "2026" / "ok.jpg").write_bytes(b"real")
poller.poll_once()
self.assertEqual(polled_files, [cam_dir / "2026" / "ok.jpg"])
# 2. InotifyWatcher ignores .hidden/ and ~tmp/ on initial attach and dynamic IN_CREATE | IN_ISDIR
fake_libc = _FakeLibc()
inotify_dirs: list[Path] = []
watcher = InotifyWatcher(
on_file_ready=lambda _wf, _p: None,
on_dir_created=lambda _wf, p: inotify_dirs.append(p),
libc=fake_libc,
)
watcher.sync_folders([wf])
watched_paths = {p for p, _ in fake_libc.watches.values()}
self.assertIn(str(cam_dir), watched_paths)
self.assertIn(str(cam_dir / "2026"), watched_paths)
self.assertNotIn(str(cam_dir / ".hidden"), watched_paths)
self.assertNotIn(str(cam_dir / "~tmp"), watched_paths)
cam_wd = next(wd for wd, (p, _) in fake_libc.watches.items() if p == str(cam_dir))
(cam_dir / "~newtmp").mkdir()
(cam_dir / ".newdot").mkdir()
watcher.process_inotify_bytes(
_pack_inotify_event(cam_wd, IN_CREATE | IN_ISDIR, 0, b"~newtmp")
+ _pack_inotify_event(cam_wd, IN_CREATE | IN_ISDIR, 0, b".newdot")
)
watched_after = {p for p, _ in fake_libc.watches.values()}
self.assertNotIn(str(cam_dir / "~newtmp"), watched_after)
self.assertNotIn(str(cam_dir / ".newdot"), watched_after)
self.assertEqual(inotify_dirs, [])
# 3. SyncDaemon._on_file_ready skips files whose relative parent parts start with ignore_prefixes
cfg_path = root / "daemon.json"
db_path = root / "state.db"
sock_path = root / "cp0.sock"
save_daemon_config(
DaemonConfig(
watched_folders=[wf],
ignore_prefixes=[".", "~", "SKIP_"],
),
cfg_path,
)
daemon = SyncDaemon(cfg_path, db_path, sock_path)
try:
(cam_dir / "SKIP_dir").mkdir()
skip_file = cam_dir / "SKIP_dir" / "ignored.jpg"
tilde_file = cam_dir / "~tmp" / "b.jpg"
dot_file = cam_dir / ".hidden" / "a.jpg"
skip_file.write_bytes(b"skip-me")
daemon._on_file_ready(wf, dot_file)
daemon._on_file_ready(wf, tilde_file)
daemon._on_file_ready(wf, skip_file)
self.assertIsNone(daemon.store.next_pending())
daemon._on_file_ready(wf, cam_dir / "2026" / "ok.jpg")
pending = daemon.store.next_pending()
assert pending is not None
self.assertEqual(pending["rel_path"], "2026/ok.jpg")
finally:
daemon.store.close()
class TestDaemonBinaryEntrypoint(unittest.TestCase):
def test_cp0_syncd_binary_runs_via_symlink_and_stops_on_sigterm(self) -> None: