feat(device-daemon): add inotify watcher, adaptive wake uploader, and IPC daemon

This commit is contained in:
2026-11-18 07:39:38 -05:00
parent 69224f3929
commit da4d963919
7 changed files with 2306 additions and 0 deletions

57
device-daemon/bin/cp0-syncd Executable file
View File

@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Service entrypoint for the CardputerZero background photo sync daemon."""
from __future__ import annotations
import argparse
import logging
import os
import signal
import sys
import threading
from pathlib import Path
def _ensure_package_on_path() -> None:
script_root = str(Path(os.path.realpath(__file__)).resolve().parents[1])
if script_root not in sys.path:
sys.path.insert(0, script_root)
installed_lib = "/usr/lib/cp0-sync"
if os.path.isdir(installed_lib) and installed_lib not in sys.path:
sys.path.insert(0, installed_lib)
_ensure_package_on_path()
from cp0_syncd.daemon import SyncDaemon # noqa: E402
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Run the CardputerZero background photo sync daemon.")
parser.add_argument("--config", type=Path, default=None, help="Path to config.json.")
parser.add_argument("--db", type=Path, default=None, help="Path to SQLite state.db.")
parser.add_argument("--sock", type=Path, default=None, help="Path to Unix IPC socket.")
parser.add_argument("--verbose", action="store_true", help="Enable debug logging.")
args = parser.parse_args(argv)
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
stop_event = threading.Event()
def _handle_signal(_signum: int, _frame: object) -> None:
stop_event.set()
signal.signal(signal.SIGINT, _handle_signal)
signal.signal(signal.SIGTERM, _handle_signal)
daemon = SyncDaemon(config_path=args.config, db_path=args.db, sock_path=args.sock)
daemon.run_forever(stop_event=stop_event)
logging.getLogger(__name__).info("cp0-syncd stopped cleanly")
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -9,23 +9,34 @@ from .config import (
remove_watched_folder,
save_daemon_config,
)
from .daemon import SyncDaemon
from .ipc import IpcServer, default_sock_path
from .store import (
SyncStore,
compute_sha256,
default_db_path,
is_media_file,
)
from .uploader import Uploader
from .watcher import InotifyWatcher, PollingWatcher, scan_folder
__all__ = [
"DaemonConfig",
"InotifyWatcher",
"IpcServer",
"PollingWatcher",
"SyncDaemon",
"SyncStore",
"Uploader",
"WatchedFolder",
"add_watched_folder",
"compute_sha256",
"default_config_path",
"default_db_path",
"default_sock_path",
"is_media_file",
"load_daemon_config",
"remove_watched_folder",
"save_daemon_config",
"scan_folder",
]

View File

@@ -0,0 +1,307 @@
"""Orchestrator daemon wiring watcher, SQLite store, uploader, and IPC server."""
from __future__ import annotations
import logging
import os
import threading
from dataclasses import replace
from pathlib import Path
from typing import Any
from .config import (
DEFAULT_TARGET_PORT,
VALID_THEMES,
DaemonConfig,
WatchedFolder,
add_watched_folder,
default_config_path,
load_daemon_config,
remove_watched_folder,
save_daemon_config,
)
from .ipc import IpcServer, default_sock_path
from .store import SyncStore, compute_sha256, default_db_path, is_media_file
from .uploader import Uploader
from .watcher import InotifyWatcher, scan_folder
logger = logging.getLogger(__name__)
def _safe_int(value: str | None, default: int = 0) -> int:
if value is None:
return default
try:
return int(value)
except ValueError:
return default
class SyncDaemon:
"""On-device CardputerZero photo sync daemon."""
def __init__(
self,
config_path: Path | None = None,
db_path: Path | None = None,
sock_path: Path | None = None,
probe_timeout: float = 3.0,
watcher_poll_interval: float = 0.5,
) -> None:
self.config_path = Path(config_path).expanduser() if config_path is not None else default_config_path()
self.db_path = Path(db_path).expanduser() if db_path is not None else default_db_path()
self.sock_path = Path(sock_path).expanduser() if sock_path is not None else default_sock_path()
self._config_lock = threading.RLock()
self.config: DaemonConfig = load_daemon_config(self.config_path)
self.store = SyncStore(self.db_path)
self.wake_event = threading.Event()
self.uploader = Uploader(
store=self.store,
get_config=self.get_config,
wake_event=self.wake_event,
probe_timeout=probe_timeout,
)
self.watcher = InotifyWatcher(
on_file_ready=self._on_file_ready,
on_dir_created=self._on_dir_created,
poll_interval=watcher_poll_interval,
)
self.ipc_server = IpcServer(self.sock_path, self.handle_ipc)
self._started = False
def get_config(self) -> DaemonConfig:
"""Return the current DaemonConfig safely across threads."""
with self._config_lock:
return self.config
def scan_all(self) -> int:
"""Recursively scan all enabled watched folders and enqueue unseen media files."""
cfg = self.get_config()
total = 0
for folder in cfg.watched_folders:
total += scan_folder(folder, cfg, self.store)
if total > 0:
self.uploader.notify_new_file()
return total
def _on_file_ready(self, folder: WatchedFolder, file_path: Path) -> None:
cfg = self.get_config()
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
if active is None:
return
if not is_media_file(file_path, cfg, store=self.store):
return
root = Path(os.path.expanduser(active.path))
try:
st = file_path.stat()
if st.st_size <= 0:
return
sha256 = compute_sha256(file_path)
rel_path = file_path.relative_to(root).as_posix()
except (OSError, ValueError):
return
outcome = self.store.enqueue_file(
active.id,
str(file_path),
rel_path,
st.st_size,
st.st_mtime_ns,
sha256,
)
if outcome == "enqueued":
self.uploader.notify_new_file()
def _on_dir_created(self, folder: WatchedFolder, _dir_path: Path) -> None:
cfg = self.get_config()
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
if active is None:
return
added = scan_folder(active, cfg, self.store)
if added > 0:
self.uploader.notify_new_file()
def start(self) -> None:
"""Start watcher, initial scan, IPC server, and uploader loop."""
if self._started:
return
self._started = True
self.store.log("INFO", "cp0-syncd started")
self.watcher.sync_folders(self.get_config().watched_folders)
self.watcher.start()
self.scan_all()
self.ipc_server.start()
self.uploader.start()
def stop(self) -> None:
"""Stop IPC server, watcher, uploader, and close the SQLite store."""
self.ipc_server.stop()
self.watcher.stop()
self.uploader.stop()
self.store.close()
self._started = False
def run_forever(self, stop_event: threading.Event | None = None) -> None:
"""Run the daemon until stop_event is set."""
waiter = stop_event if stop_event is not None else threading.Event()
self.start()
try:
waiter.wait()
finally:
self.stop()
def handle_ipc(self, req: dict[str, Any]) -> dict[str, Any]:
"""Dispatch a parsed IPC JSON request dict and return a JSON-serializable response."""
cmd = req.get("cmd")
if not isinstance(cmd, str) or not cmd:
return {"ok": False, "error": "Missing 'cmd' field"}
if cmd == "status":
return self._cmd_status()
if cmd == "sync_now":
return self._cmd_sync_now()
if cmd == "add_folder":
return self._cmd_add_folder(req)
if cmd == "remove_folder":
return self._cmd_remove_folder(req)
if cmd == "set_target":
return self._cmd_set_target(req)
if cmd == "set_theme":
return self._cmd_set_theme(req)
if cmd == "retry_failed":
return self._cmd_retry_failed()
return {"ok": False, "error": f"Unknown command: {cmd}"}
def _cmd_status(self) -> dict[str, Any]:
cfg = self.get_config()
snap = self.store.get_status_snapshot(cfg.watched_folders)
meta: dict[str, str] = snap["meta"]
counts: dict[str, int] = snap["counts"]
return {
"ok": True,
"target_host": cfg.target_host,
"target_port": cfg.target_port,
"theme": cfg.theme,
"mac_online": meta.get("mac_online") == "true",
"mac_dest_root": meta.get("mac_dest_root", ""),
"syncing": self.uploader.syncing,
"current_file": self.uploader.current_file,
"pending_count": counts["pending"],
"pending_bytes": counts["pending_bytes"],
"synced_count": counts["synced"],
"failed_count": counts["failed"],
"next_retry_sec": self.uploader.next_retry_sec(),
"last_sync_ts": _safe_int(meta.get("last_sync_ts"), 0),
"last_synced_file": meta.get("last_synced_file", ""),
"last_error": meta.get("last_error", ""),
"folders": [
{
"id": str(item["id"]),
"path": str(item["path"]),
"label": str(item["label"]),
"enabled": bool(item["enabled"]),
"exists": bool(item["exists"]),
"pending": int(item["pending"]),
"synced": int(item["synced"]),
}
for item in snap["folders"]
],
"logs": snap["logs"],
}
def _cmd_sync_now(self) -> dict[str, Any]:
added = self.scan_all()
self.uploader.wake(reset_backoff=True, probe=True)
return {"ok": True, "enqueued": added}
def _cmd_add_folder(self, req: dict[str, Any]) -> dict[str, Any]:
path = req.get("path")
label = req.get("label")
if label is not None and not isinstance(label, str):
return {"ok": False, "error": "Folder label must be a string"}
with self._config_lock:
try:
new_cfg, folder = add_watched_folder(self.config, path, label=label) # type: ignore[arg-type]
except ValueError as exc:
return {"ok": False, "error": str(exc)}
save_daemon_config(new_cfg, self.config_path)
self.config = new_cfg
self.watcher.add_folder(folder)
added = scan_folder(folder, new_cfg, self.store)
self.store.log("INFO", f"Added watched folder: {folder.path} ({folder.label})")
self.uploader.wake(reset_backoff=True, probe=False)
return {
"ok": True,
"folder": {
"id": folder.id,
"path": folder.path,
"label": folder.label,
"enabled": folder.enabled,
},
"enqueued": added,
}
def _cmd_remove_folder(self, req: dict[str, Any]) -> dict[str, Any]:
folder_id = req.get("folder_id")
if not isinstance(folder_id, str) or not folder_id.strip():
return {"ok": False, "error": "Missing or invalid 'folder_id'"}
with self._config_lock:
if not any(wf.id == folder_id for wf in self.config.watched_folders):
return {"ok": False, "error": f"Unknown folder_id: {folder_id}"}
new_cfg = remove_watched_folder(self.config, folder_id)
save_daemon_config(new_cfg, self.config_path)
self.config = new_cfg
self.watcher.remove_folder(folder_id)
purged = self.store.purge_pending_for_folder(folder_id)
self.store.log("INFO", f"Removed watched folder {folder_id} (purged {purged} pending)")
return {"ok": True, "purged": purged}
def _cmd_set_target(self, req: dict[str, Any]) -> dict[str, Any]:
raw_host = req.get("host")
if not isinstance(raw_host, str):
return {"ok": False, "error": "Target host must be a string"}
host = raw_host.strip()
if not host or any(ch.isspace() for ch in host) or "/" in host:
return {"ok": False, "error": "Invalid target host"}
port = req.get("port", DEFAULT_TARGET_PORT)
if not isinstance(port, int) or isinstance(port, bool) or not (1 <= port <= 65535):
return {"ok": False, "error": "Target port must be an integer between 1 and 65535"}
with self._config_lock:
new_cfg = replace(self.config, target_host=host, target_port=port)
save_daemon_config(new_cfg, self.config_path)
self.config = new_cfg
self.store.set_meta("mac_online", "false")
self.store.set_meta("mac_dest_root", "")
self.store.set_meta("last_error", "")
self.store.log("INFO", f"Updated target to {host}:{port}")
self.uploader.wake(reset_backoff=True, probe=True)
return {"ok": True, "target_host": host, "target_port": port}
def _cmd_set_theme(self, req: dict[str, Any]) -> dict[str, Any]:
theme = req.get("theme")
if not isinstance(theme, str) or theme not in VALID_THEMES:
return {"ok": False, "error": "Theme must be 'dark' or 'light'"}
with self._config_lock:
new_cfg = replace(self.config, theme=theme)
save_daemon_config(new_cfg, self.config_path)
self.config = new_cfg
return {"ok": True, "theme": theme}
def _cmd_retry_failed(self) -> dict[str, Any]:
retried = self.store.retry_all_failed()
if retried > 0:
self.store.log("INFO", f"Queued {retried} failed files for retry")
self.uploader.wake(reset_backoff=True, probe=True)
return {"ok": True, "retried": retried}

View File

@@ -0,0 +1,137 @@
"""Newline-delimited JSON Unix domain socket IPC server for cp0-syncd."""
from __future__ import annotations
import json
import logging
import os
import socket
import threading
from pathlib import Path
from typing import Any, Callable
logger = logging.getLogger(__name__)
ENV_SOCK_PATH = "CP0_SYNC_SOCK"
def default_sock_path() -> Path:
"""Return the default Unix socket path, honoring CP0_SYNC_SOCK then XDG_RUNTIME_DIR."""
override = os.environ.get(ENV_SOCK_PATH, "").strip()
if override:
return Path(override).expanduser()
runtime_dir = os.environ.get("XDG_RUNTIME_DIR", "").strip() or "/tmp"
return Path(runtime_dir) / "cp0-sync.sock"
class IpcServer:
"""AF_UNIX stream server speaking newline-delimited JSON commands."""
def __init__(
self,
sock_path: Path,
handler: Callable[[dict[str, Any]], dict[str, Any]],
) -> None:
self.sock_path = Path(sock_path).expanduser()
self._handler = handler
self._stop_event = threading.Event()
self._sock: socket.socket | None = None
self._accept_thread: threading.Thread | None = None
def start(self) -> None:
"""Unlink any stale socket, bind AF_UNIX socket, and start accept loop."""
if self._accept_thread is not None and self._accept_thread.is_alive():
return
self.sock_path.parent.mkdir(parents=True, exist_ok=True)
try:
os.unlink(self.sock_path)
except FileNotFoundError:
pass
srv_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
srv_sock.bind(str(self.sock_path))
os.chmod(self.sock_path, 0o600)
srv_sock.listen(16)
srv_sock.settimeout(0.2)
self._sock = srv_sock
self._stop_event.clear()
self._accept_thread = threading.Thread(
target=self._accept_loop,
name="cp0-ipc-server",
daemon=True,
)
self._accept_thread.start()
def stop(self) -> None:
"""Stop the IPC accept loop, close socket, and unlink socket path."""
self._stop_event.set()
if self._sock is not None:
try:
self._sock.close()
except OSError:
pass
self._sock = None
if self._accept_thread is not None and self._accept_thread is not threading.current_thread():
self._accept_thread.join(timeout=2.0)
self._accept_thread = None
try:
os.unlink(self.sock_path)
except FileNotFoundError:
pass
def _accept_loop(self) -> None:
while not self._stop_event.is_set() and self._sock is not None:
try:
conn, _ = self._sock.accept()
except socket.timeout:
continue
except OSError:
if self._stop_event.is_set():
break
continue
client_thread = threading.Thread(
target=self._serve_client,
args=(conn,),
name="cp0-ipc-client",
daemon=True,
)
client_thread.start()
def _serve_client(self, conn: socket.socket) -> None:
conn.settimeout(2.0)
try:
with conn:
with conn.makefile("r", encoding="utf-8", errors="replace") as reader:
for raw_line in reader:
line = raw_line.strip()
if not line:
continue
response = self._dispatch_line(line)
payload = (json.dumps(response, separators=(",", ":")) + "\n").encode("utf-8")
conn.sendall(payload)
except (OSError, TimeoutError):
pass
def _dispatch_line(self, line: str) -> dict[str, Any]:
try:
req = json.loads(line)
except json.JSONDecodeError as exc:
return {"ok": False, "error": f"Invalid JSON: {exc}"}
if not isinstance(req, dict):
return {"ok": False, "error": "Request must be a JSON object"}
try:
resp = self._handler(req)
except Exception as exc:
logger.exception("IPC handler raised exception")
return {"ok": False, "error": str(exc)}
if not isinstance(resp, dict):
return {"ok": False, "error": "Internal handler returned invalid response"}
return resp

View File

@@ -0,0 +1,374 @@
"""Adaptive wake-probe HTTP uploader for CardputerZero photos."""
from __future__ import annotations
import hashlib
import http.client
import json
import logging
import math
import os
import threading
import time
from http import HTTPStatus
from pathlib import Path
from typing import Callable
from urllib.parse import quote
from .config import DaemonConfig, WatchedFolder
from .store import SyncStore
logger = logging.getLogger(__name__)
BACKOFF_SCHEDULE = (5, 10, 20, 40, 60)
CHUNK_SIZE = 65536
PERMANENT_FAIL_STATUSES = frozenset({400, 413, 422})
def _extract_error_message(body: bytes) -> str:
if not body:
return ""
try:
parsed = json.loads(body.decode("utf-8"))
if isinstance(parsed, dict) and "error" in parsed:
return str(parsed["error"])
except (UnicodeDecodeError, json.JSONDecodeError):
pass
return body.decode("utf-8", errors="replace").strip()[:200]
def _hash_open_file(handle: object) -> str:
digest = hashlib.sha256()
handle.seek(0) # type: ignore[attr-defined]
while True:
chunk = handle.read(CHUNK_SIZE) # type: ignore[attr-defined]
if not chunk:
break
digest.update(chunk)
handle.seek(0) # type: ignore[attr-defined]
return digest.hexdigest()
class Uploader:
"""Sequential FIFO photo uploader with /health wake-probe and adaptive backoff."""
def __init__(
self,
store: SyncStore,
get_config: Callable[[], DaemonConfig],
wake_event: threading.Event,
probe_timeout: float = 3.0,
upload_timeout: float = 30.0,
) -> None:
self._store = store
self._get_config = get_config
self._wake_event = wake_event
self._probe_timeout = float(probe_timeout)
self._upload_timeout = float(upload_timeout)
self._state_lock = threading.Lock()
self._process_lock = threading.Lock()
self._stop_event = threading.Event()
self._thread: threading.Thread | None = None
self._syncing = False
self._current_file = ""
self._backoff_index = 0
self._current_delay = 0
self._next_retry_deadline = 0.0
self._probe_requested = False
self._logged_no_target = False
@property
def syncing(self) -> bool:
"""Return True while actively uploading or probing."""
with self._state_lock:
return self._syncing
@property
def current_file(self) -> str:
"""Return the relative path of the file currently being uploaded, or ''."""
with self._state_lock:
return self._current_file
def next_retry_sec(self) -> int:
"""Return whole seconds remaining until the next backoff retry, or 0."""
with self._state_lock:
if self._next_retry_deadline <= 0.0:
return 0
remaining = self._next_retry_deadline - time.monotonic()
return max(0, int(math.ceil(remaining))) if remaining > 0 else 0
def notify_new_file(self) -> None:
"""Reset retry backoff and wake the uploader when a new photo arrives."""
self.wake(reset_backoff=True, probe=False)
def wake(self, reset_backoff: bool = True, probe: bool = False) -> None:
"""Interrupt backoff sleep immediately and optionally trigger a /health probe."""
with self._state_lock:
if reset_backoff:
self._backoff_index = 0
self._current_delay = 0
self._next_retry_deadline = 0.0
if probe:
self._probe_requested = True
if reset_backoff:
self._store.set_meta("next_retry_ts", "0")
self._wake_event.set()
def start(self) -> None:
"""Start the background upload loop thread."""
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
cfg = self._get_config()
if cfg.target_host.strip():
with self._state_lock:
self._probe_requested = True
self._wake_event.set()
self._thread = threading.Thread(target=self._run_loop, name="cp0-uploader", daemon=True)
self._thread.start()
def stop(self) -> None:
"""Stop the background upload loop cleanly."""
self._stop_event.set()
self._wake_event.set()
if self._thread is not None and self._thread is not threading.current_thread():
self._thread.join(timeout=5.0)
self._thread = None
def _run_loop(self) -> None:
while not self._stop_event.is_set():
with self._state_lock:
probe = self._probe_requested
self._probe_requested = False
self._wake_event.clear()
has_pending = self._store.next_pending() is not None
if has_pending or probe:
result = self.process_once(probe_when_empty=probe)
else:
result = "idle"
if self._stop_event.is_set():
break
if self._wake_event.is_set():
continue
if result in ("offline", "paused"):
with self._state_lock:
wait_sec = max(1, self._current_delay)
self._wake_event.wait(timeout=wait_sec)
else:
self._wake_event.wait(timeout=60.0)
def _reset_backoff(self) -> None:
with self._state_lock:
self._backoff_index = 0
self._current_delay = 0
self._next_retry_deadline = 0.0
self._store.set_meta("next_retry_ts", "0")
def _record_backoff(self) -> int:
with self._state_lock:
delay = BACKOFF_SCHEDULE[min(self._backoff_index, len(BACKOFF_SCHEDULE) - 1)]
if self._backoff_index < len(BACKOFF_SCHEDULE) - 1:
self._backoff_index += 1
self._current_delay = delay
self._next_retry_deadline = time.monotonic() + delay
next_ms = int(time.time() * 1000) + delay * 1000
self._store.set_meta("next_retry_ts", str(next_ms))
return delay
def _set_syncing(self, syncing: bool, current_file: str = "") -> None:
with self._state_lock:
self._syncing = syncing
self._current_file = current_file
def _probe_health(self, host: str, port: int) -> tuple[bool, str]:
conn: http.client.HTTPConnection | None = None
try:
conn = http.client.HTTPConnection(host, port, timeout=self._probe_timeout)
conn.request("GET", "/health", headers={"Connection": "close"})
resp = conn.getresponse()
body = resp.read()
except (OSError, http.client.HTTPException, TimeoutError) as exc:
return (False, str(exc) or exc.__class__.__name__)
finally:
if conn is not None:
conn.close()
if resp.status != HTTPStatus.OK:
detail = _extract_error_message(body)
return (False, f"HTTP {resp.status}: {detail}" if detail else f"HTTP {resp.status}")
try:
data = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
return (False, f"Invalid /health JSON: {exc}")
if not isinstance(data, dict) or data.get("status") != "ok":
return (False, "Unexpected /health payload")
dest_root = data.get("dest_root", "")
return (True, str(dest_root) if isinstance(dest_root, str) else "")
def process_once(self, probe_when_empty: bool = False) -> str:
"""Probe Mac /health if needed and upload queued PENDING files sequentially."""
with self._process_lock:
cfg = self._get_config()
host = cfg.target_host.strip()
if not host:
self._set_syncing(False, "")
self._reset_backoff()
self._store.set_meta("mac_online", "false")
if not self._logged_no_target:
self._store.log("INFO", "No target host configured")
self._logged_no_target = True
return "no_target"
self._logged_no_target = False
first_row = self._store.next_pending()
if first_row is None and not probe_when_empty:
self._reset_backoff()
return "idle"
ok, detail = self._probe_health(host, cfg.target_port)
self._store.set_meta("last_probe_ts", str(int(time.time() * 1000)))
if not ok:
self._set_syncing(False, "")
self._store.set_meta("mac_online", "false")
self._store.set_meta("last_error", detail)
self._record_backoff()
return "offline"
self._reset_backoff()
self._store.set_meta("mac_online", "true")
self._store.set_meta("mac_dest_root", detail)
if first_row is None:
self._store.set_meta("last_error", "")
return "online"
try:
while not self._stop_event.is_set():
cfg = self._get_config()
host = cfg.target_host.strip()
if not host:
return "no_target"
folders_by_id = {wf.id: wf for wf in cfg.watched_folders}
row = self._store.next_pending()
if row is None:
return "synced"
folder = folders_by_id.get(str(row["folder_id"]))
if folder is None:
self._store.purge_pending_for_folder(str(row["folder_id"]))
continue
outcome = self._upload_row(row, folder, host, cfg.target_port)
if outcome in ("paused", "offline"):
return outcome
return "stopped"
finally:
self._set_syncing(False, "")
def _upload_row(
self,
row: dict[str, object],
folder: WatchedFolder,
host: str,
port: int,
) -> str:
file_id = int(row["id"]) # type: ignore[arg-type]
rel_path = str(row["rel_path"])
abs_path = Path(str(row["abs_path"]))
try:
handle = open(abs_path, "rb")
except FileNotFoundError:
self._store.delete_file(file_id)
self._store.log("WARN", f"Removed missing queued file: {rel_path}")
return "missing"
except OSError as exc:
err = f"Read error for {rel_path}: {exc}"
self._store.mark_retry(file_id, err)
self._store.set_meta("last_error", err)
self._store.log("WARN", err)
self._record_backoff()
return "paused"
with handle:
st = os.fstat(handle.fileno())
if st.st_size <= 0:
self._store.delete_file(file_id)
self._store.log("WARN", f"Removed 0-byte queued file: {rel_path}")
return "missing"
if st.st_size != int(row["size_bytes"]) or st.st_mtime_ns != int(row["mtime_ns"]): # type: ignore[arg-type]
sha256 = _hash_open_file(handle)
else:
sha256 = str(row["sha256"])
size_bytes = st.st_size
mtime_sec = st.st_mtime_ns / 1_000_000_000
self._set_syncing(True, rel_path)
self._store.mark_uploading(file_id)
conn: http.client.HTTPConnection | None = None
try:
conn = http.client.HTTPConnection(host, port, timeout=self._upload_timeout)
conn.putrequest("POST", "/upload")
conn.putheader("Content-Length", str(size_bytes))
conn.putheader("X-CP0-Folder-Label", quote(folder.label, safe=""))
conn.putheader("X-CP0-Rel-Path", quote(rel_path, safe="/"))
conn.putheader("X-CP0-SHA256", sha256)
conn.putheader("X-CP0-Mtime", f"{mtime_sec:.6f}")
conn.putheader("Connection", "close")
conn.endheaders()
sent = 0
while sent < size_bytes:
chunk = handle.read(min(CHUNK_SIZE, size_bytes - sent))
if not chunk:
raise OSError("File truncated while streaming upload")
conn.send(chunk)
sent += len(chunk)
resp = conn.getresponse()
body = resp.read()
except (OSError, http.client.HTTPException, TimeoutError) as exc:
err = f"Upload network error ({rel_path}): {exc}"
self._store.mark_retry(file_id, err)
self._store.set_meta("mac_online", "false")
self._store.set_meta("last_error", err)
self._store.log("WARN", err)
self._record_backoff()
return "offline"
finally:
if conn is not None:
conn.close()
if 200 <= resp.status < 300:
self._store.mark_synced(file_id)
self._store.set_meta("last_sync_ts", str(int(time.time() * 1000)))
self._store.set_meta("last_synced_file", rel_path)
self._store.set_meta("last_error", "")
self._reset_backoff()
self._store.log("INFO", f"Synced {folder.label}/{rel_path}")
return "synced"
detail = _extract_error_message(body)
err = f"HTTP {resp.status}: {detail}" if detail else f"HTTP {resp.status}"
if resp.status in PERMANENT_FAIL_STATUSES:
self._store.mark_failed(file_id, err)
self._store.set_meta("last_error", err)
self._store.log("ERROR", f"Rejected {rel_path}: {err}")
return "failed"
self._store.mark_retry(file_id, err)
self._store.set_meta("last_error", err)
self._store.log("ERROR", f"Paused on {rel_path}: {err}")
self._record_backoff()
return "paused"

View File

@@ -0,0 +1,660 @@
"""Recursive startup scanner, Linux ctypes inotify watcher, and polling fallback."""
from __future__ import annotations
import ctypes
import ctypes.util
import errno
import logging
import os
import select
import struct
import sys
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from .config import DaemonConfig, WatchedFolder
from .store import SyncStore, compute_sha256, is_media_file
logger = logging.getLogger(__name__)
# Linux <sys/inotify.h> event masks
IN_CLOSE_WRITE = 0x00000008
IN_MOVED_TO = 0x00000080
IN_CREATE = 0x00000100
IN_DELETE = 0x00000200
IN_DELETE_SELF = 0x00000400
IN_MOVE_SELF = 0x00000800
IN_IGNORED = 0x00008000
IN_ISDIR = 0x40000000
_DIR_WATCH_MASK = IN_CLOSE_WRITE | IN_MOVED_TO | IN_CREATE | IN_DELETE_SELF | IN_MOVE_SELF
_ANCESTOR_WATCH_MASK = IN_CREATE | IN_MOVED_TO | IN_DELETE_SELF | IN_MOVE_SELF
_EVENT_HEADER = struct.Struct("=iIII")
_READ_BUFFER_SIZE = 65536
@dataclass(frozen=True)
class InotifyEvent:
"""Decoded Linux struct inotify_event record."""
wd: int
mask: int
cookie: int
name: str
def parse_inotify_events(data: bytes) -> list[InotifyEvent]:
"""Decode zero or more packed Linux struct inotify_event entries from raw bytes."""
events: list[InotifyEvent] = []
offset = 0
total = len(data)
header_size = _EVENT_HEADER.size
while offset + header_size <= total:
wd, mask, cookie, name_len = _EVENT_HEADER.unpack_from(data, offset)
offset += header_size
if name_len > 0:
end = min(offset + name_len, total)
raw_name = data[offset:end].split(b"\x00", 1)[0]
name = os.fsdecode(raw_name)
offset += name_len
else:
name = ""
events.append(InotifyEvent(wd=wd, mask=mask, cookie=cookie, name=name))
return events
def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> int:
"""Recursively scan an enabled watched folder and enqueue new media files."""
if not folder.enabled:
return 0
root = Path(os.path.expanduser(folder.path))
if not root.is_dir():
return 0
enqueued = 0
ignore_prefixes = [prefix for prefix in cfg.ignore_prefixes if prefix]
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
dirnames[:] = sorted(
name
for name in dirnames
if not any(name.startswith(prefix) for prefix in ignore_prefixes)
)
current_dir = Path(dirpath)
for filename in sorted(filenames):
full_path = current_dir / filename
if not is_media_file(full_path, cfg, store=store):
continue
try:
st = full_path.stat()
if st.st_size <= 0:
continue
sha256 = compute_sha256(full_path)
rel_path = full_path.relative_to(root).as_posix()
except (OSError, ValueError):
continue
outcome = store.enqueue_file(
folder.id,
str(full_path),
rel_path,
st.st_size,
st.st_mtime_ns,
sha256,
)
if outcome == "enqueued":
enqueued += 1
return enqueued
def _load_linux_libc() -> Any | None:
"""Load Linux libc.so.6 with inotify symbols, or return None on unsupported OS."""
if not sys.platform.startswith("linux"):
return None
candidates = ["libc.so.6"]
found = ctypes.util.find_library("c")
if found and found not in candidates:
candidates.append(found)
for candidate in candidates:
try:
libc = ctypes.CDLL(candidate, use_errno=True)
if (
hasattr(libc, "inotify_init1")
and hasattr(libc, "inotify_add_watch")
and hasattr(libc, "inotify_rm_watch")
):
libc.inotify_init1.argtypes = [ctypes.c_int]
libc.inotify_init1.restype = ctypes.c_int
libc.inotify_add_watch.argtypes = [
ctypes.c_int,
ctypes.c_char_p,
ctypes.c_uint32,
]
libc.inotify_add_watch.restype = ctypes.c_int
libc.inotify_rm_watch.argtypes = [ctypes.c_int, ctypes.c_int]
libc.inotify_rm_watch.restype = ctypes.c_int
return libc
except OSError:
continue
return None
def _nearest_existing_ancestor(target: Path) -> Path:
"""Return target or its nearest existing ancestor directory."""
candidate = target
while True:
if candidate.is_dir():
return candidate
parent = candidate.parent
if parent == candidate:
return Path("/")
candidate = parent
def _iter_subdirs(root: Path) -> list[Path]:
"""Yield root and all non-symlink subdirectories beneath root."""
dirs: list[Path] = []
if not root.is_dir():
return dirs
for dirpath, dirnames, _ in os.walk(root, followlinks=False):
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
dirs.append(Path(dirpath))
return dirs
class PollingWatcher:
"""Portable mtime/size polling watcher fallback for non-Linux hosts."""
def __init__(
self,
on_file_ready: Callable[[WatchedFolder, Path], None],
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
poll_interval: float = 0.5,
) -> None:
self._on_file_ready = on_file_ready
self._on_dir_created = on_dir_created
self._poll_interval = max(0.02, float(poll_interval))
self._lock = threading.RLock()
self._stop_event = threading.Event()
self._thread: threading.Thread | None = None
self._folders: dict[str, WatchedFolder] = {}
self._snapshots: dict[str, dict[str, Any]] = {}
def sync_folders(self, folders: list[WatchedFolder]) -> None:
with self._lock:
target_ids = {wf.id for wf in folders if wf.enabled}
for existing_id in list(self._folders.keys()):
if existing_id not in target_ids:
self._folders.pop(existing_id, None)
self._snapshots.pop(existing_id, None)
for wf in folders:
if wf.enabled:
if wf.id not in self._folders or self._folders[wf.id].path != wf.path:
self._folders[wf.id] = wf
self._snapshots[wf.id] = self._build_baseline(wf)
else:
self._folders[wf.id] = wf
def add_folder(self, folder: WatchedFolder) -> None:
with self._lock:
if not folder.enabled:
return
self._folders[folder.id] = folder
self._snapshots[folder.id] = self._build_baseline(folder)
def remove_folder(self, folder_id: str) -> None:
with self._lock:
self._folders.pop(folder_id, None)
self._snapshots.pop(folder_id, None)
def _build_baseline(self, folder: WatchedFolder) -> dict[str, Any]:
root = Path(os.path.expanduser(folder.path))
if not root.is_dir():
return {"present": False, "dirs": set(), "files": {}}
dirs, files = self._scan_tree(root)
return {"present": True, "dirs": dirs, "files": files}
@staticmethod
def _scan_tree(root: Path) -> tuple[set[Path], dict[Path, tuple[int, int]]]:
dirs: set[Path] = set()
files: dict[Path, tuple[int, int]] = {}
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
cur = Path(dirpath)
dirs.add(cur)
for name in filenames:
fpath = cur / name
try:
st = fpath.stat()
except OSError:
continue
files[fpath] = (st.st_size, st.st_mtime_ns)
return dirs, files
def poll_once(self) -> None:
events_dirs: list[tuple[WatchedFolder, Path]] = []
events_files: list[tuple[WatchedFolder, Path]] = []
with self._lock:
items = list(self._folders.items())
for folder_id, folder in items:
root = Path(os.path.expanduser(folder.path))
is_present = root.is_dir()
with self._lock:
snap = self._snapshots.get(folder_id)
if snap is None:
continue
was_present = bool(snap["present"])
if not is_present:
snap["present"] = False
snap["dirs"] = set()
snap["files"] = {}
continue
dirs, files = self._scan_tree(root)
prev_dirs: set[Path] = snap["dirs"]
prev_files: dict[Path, tuple[int, int]] = snap["files"]
if not was_present:
events_dirs.append((folder, root))
for new_dir in sorted(dirs - prev_dirs):
if new_dir != root:
events_dirs.append((folder, new_dir))
for fpath, sig in sorted(files.items()):
if prev_files.get(fpath) != sig:
events_files.append((folder, fpath))
snap["present"] = True
snap["dirs"] = dirs
snap["files"] = files
if self._on_dir_created is not None:
for folder, dir_path in events_dirs:
self._on_dir_created(folder, dir_path)
for folder, file_path in events_files:
self._on_file_ready(folder, file_path)
def start(self) -> None:
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
self._thread = threading.Thread(target=self._run_loop, name="cp0-polling-watcher", daemon=True)
self._thread.start()
def stop(self) -> None:
self._stop_event.set()
if self._thread is not None and self._thread is not threading.current_thread():
self._thread.join(timeout=2.0)
self._thread = None
def _run_loop(self) -> None:
while not self._stop_event.is_set():
try:
self.poll_once()
except Exception:
logger.exception("PollingWatcher tick failed")
self._stop_event.wait(timeout=self._poll_interval)
class InotifyWatcher:
"""Linux ctypes inotify watcher with ancestor tracking and PollingWatcher fallback."""
def __init__(
self,
on_file_ready: Callable[[WatchedFolder, Path], None],
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
poll_interval: float = 0.5,
libc: Any | None = None,
) -> None:
self._on_file_ready = on_file_ready
self._on_dir_created = on_dir_created
self._poll_interval = max(0.05, float(poll_interval))
self._lock = threading.RLock()
self._stop_event = threading.Event()
self._thread: threading.Thread | None = None
self._owns_real_fd = libc is None
self._libc = libc if libc is not None else _load_linux_libc()
self._polling: PollingWatcher | None = None
self._fd: int = -1
self._folders: dict[str, WatchedFolder] = {}
self._wd_paths: dict[int, Path] = {}
self._path_wds: dict[Path, int] = {}
self._active_by_wd: dict[int, tuple[WatchedFolder, Path]] = {}
self._ancestors_by_wd: dict[int, dict[str, tuple[WatchedFolder, Path]]] = {}
self._folder_wds: dict[str, set[int]] = {}
self._unmounted: dict[str, tuple[WatchedFolder, Path]] = {}
if self._libc is not None:
fd = int(self._libc.inotify_init1(0))
if fd < 0:
err = ctypes.get_errno()
logger.warning("inotify_init1 failed (errno=%s); falling back to polling", err)
self._libc = None
else:
self._fd = fd
if self._owns_real_fd:
os.set_inheritable(self._fd, False)
os.set_blocking(self._fd, False)
if self._libc is None:
self._polling = PollingWatcher(
on_file_ready=on_file_ready,
on_dir_created=on_dir_created,
poll_interval=self._poll_interval,
)
@property
def is_using_polling(self) -> bool:
"""Return True if running with the PollingWatcher fallback."""
return self._polling is not None
def sync_folders(self, folders: list[WatchedFolder]) -> None:
"""Synchronize active watches with the given WatchedFolder list."""
if self._polling is not None:
self._polling.sync_folders(folders)
return
with self._lock:
enabled_map = {wf.id: wf for wf in folders if wf.enabled}
for existing_id in list(self._folders.keys()):
if existing_id not in enabled_map:
self._detach_folder_locked(existing_id)
for wf in folders:
if not wf.enabled:
continue
prev = self._folders.get(wf.id)
if prev is None or prev.path != wf.path:
self._detach_folder_locked(wf.id)
self._attach_folder_locked(wf)
else:
self._folders[wf.id] = wf
def add_folder(self, folder: WatchedFolder) -> None:
"""Attach watches for a single folder (or its nearest existing ancestor)."""
if self._polling is not None:
self._polling.add_folder(folder)
return
with self._lock:
if not folder.enabled:
return
self._detach_folder_locked(folder.id)
self._attach_folder_locked(folder)
def remove_folder(self, folder_id: str) -> None:
"""Detach all watches associated with folder_id."""
if self._polling is not None:
self._polling.remove_folder(folder_id)
return
with self._lock:
self._detach_folder_locked(folder_id)
def _add_watch_locked(self, dir_path: Path, mask: int) -> int | None:
assert self._libc is not None
existing = self._path_wds.get(dir_path)
if existing is not None:
return existing
wd = int(self._libc.inotify_add_watch(self._fd, os.fsencode(str(dir_path)), mask))
if wd < 0:
err = ctypes.get_errno()
logger.warning("inotify_add_watch failed for %s (errno=%s)", dir_path, err)
return None
self._wd_paths[wd] = dir_path
self._path_wds[dir_path] = wd
return wd
def _maybe_rm_watch_locked(self, wd: int) -> None:
if wd in self._active_by_wd:
return
if self._ancestors_by_wd.get(wd):
return
self._ancestors_by_wd.pop(wd, None)
dir_path = self._wd_paths.pop(wd, None)
if dir_path is not None and self._path_wds.get(dir_path) == wd:
self._path_wds.pop(dir_path, None)
if self._libc is not None and self._fd >= 0:
self._libc.inotify_rm_watch(self._fd, wd)
def _attach_folder_locked(self, folder: WatchedFolder) -> None:
self._folders[folder.id] = folder
root = Path(os.path.expanduser(folder.path))
if root.is_dir():
self._unmounted.pop(folder.id, None)
self._attach_tree_locked(folder, root)
else:
self._watch_ancestor_locked(folder, root)
def _attach_tree_locked(self, folder: WatchedFolder, root: Path) -> None:
wds = self._folder_wds.setdefault(folder.id, set())
for subdir in _iter_subdirs(root):
wd = self._add_watch_locked(subdir, _DIR_WATCH_MASK)
if wd is not None:
self._active_by_wd[wd] = (folder, subdir)
wds.add(wd)
def _watch_ancestor_locked(self, folder: WatchedFolder, target_root: Path) -> None:
ancestor = _nearest_existing_ancestor(target_root)
self._unmounted[folder.id] = (folder, target_root)
wd = self._add_watch_locked(ancestor, _ANCESTOR_WATCH_MASK)
if wd is not None:
self._ancestors_by_wd.setdefault(wd, {})[folder.id] = (folder, target_root)
self._folder_wds.setdefault(folder.id, set()).add(wd)
def _detach_folder_locked(self, folder_id: str) -> None:
self._folders.pop(folder_id, None)
self._unmounted.pop(folder_id, None)
wds = self._folder_wds.pop(folder_id, set())
for wd in list(wds):
active = self._active_by_wd.get(wd)
if active is not None and active[0].id == folder_id:
self._active_by_wd.pop(wd, None)
anc = self._ancestors_by_wd.get(wd)
if anc is not None:
anc.pop(folder_id, None)
self._maybe_rm_watch_locked(wd)
def _recheck_unmounted_mounts(self) -> None:
mounted_callbacks: list[tuple[WatchedFolder, Path]] = []
with self._lock:
for folder_id, (folder, target_root) in list(self._unmounted.items()):
if target_root.is_dir():
self._detach_folder_locked(folder_id)
self._attach_folder_locked(folder)
mounted_callbacks.append((folder, target_root))
continue
# If an intermediate mountpoint appeared, tighten ancestor watch closer to target_root.
best_ancestor = _nearest_existing_ancestor(target_root)
watched_ancestors = {
self._wd_paths[wd]
for wd in self._folder_wds.get(folder_id, set())
if wd in self._wd_paths
}
if best_ancestor not in watched_ancestors:
self._detach_folder_locked(folder_id)
self._attach_folder_locked(folder)
if self._on_dir_created is not None:
for folder, target_root in mounted_callbacks:
self._on_dir_created(folder, target_root)
def process_inotify_bytes(self, data: bytes) -> None:
"""Parse raw inotify buffer bytes and dispatch callbacks."""
file_callbacks: list[tuple[WatchedFolder, Path]] = []
dir_callbacks: list[tuple[WatchedFolder, Path]] = []
events = parse_inotify_events(data)
with self._lock:
for ev in events:
watched_dir = self._wd_paths.get(ev.wd)
if ev.mask & IN_IGNORED:
self._cleanup_ignored_wd_locked(ev.wd)
continue
if watched_dir is None:
continue
if ev.mask & (IN_DELETE_SELF | IN_MOVE_SELF):
self._handle_self_removed_locked(ev.wd, watched_dir)
continue
if ev.mask & IN_ISDIR:
if ev.name and (ev.mask & (IN_CREATE | IN_MOVED_TO)):
created_dir = watched_dir / ev.name
self._handle_dir_created_locked(
ev.wd, created_dir, dir_callbacks
)
continue
# File event: react ONLY to IN_CLOSE_WRITE or IN_MOVED_TO (never IN_CREATE).
if ev.name and (ev.mask & (IN_CLOSE_WRITE | IN_MOVED_TO)):
active = self._active_by_wd.get(ev.wd)
if active is not None:
folder, base_dir = active
file_callbacks.append((folder, base_dir / ev.name))
if self._on_dir_created is not None:
for folder, dir_path in dir_callbacks:
self._on_dir_created(folder, dir_path)
for folder, file_path in file_callbacks:
self._on_file_ready(folder, file_path)
def _cleanup_ignored_wd_locked(self, wd: int) -> None:
dir_path = self._wd_paths.pop(wd, None)
if dir_path is not None and self._path_wds.get(dir_path) == wd:
self._path_wds.pop(dir_path, None)
active = self._active_by_wd.pop(wd, None)
if active is not None:
folder, _ = active
self._folder_wds.get(folder.id, set()).discard(wd)
root = Path(os.path.expanduser(folder.path))
if dir_path == root and folder.id in self._folders:
self._detach_folder_locked(folder.id)
self._attach_folder_locked(folder)
anc = self._ancestors_by_wd.pop(wd, None)
if anc is not None:
affected = list(anc.keys())
for fid in affected:
self._folder_wds.get(fid, set()).discard(wd)
folder = self._folders.get(fid)
if folder is not None:
self._detach_folder_locked(fid)
self._attach_folder_locked(folder)
def _handle_self_removed_locked(self, wd: int, watched_dir: Path) -> None:
active = self._active_by_wd.get(wd)
if active is not None:
folder, _ = active
root = Path(os.path.expanduser(folder.path))
if watched_dir == root:
self._detach_folder_locked(folder.id)
self._attach_folder_locked(folder)
return
self._active_by_wd.pop(wd, None)
self._folder_wds.get(folder.id, set()).discard(wd)
anc = self._ancestors_by_wd.get(wd)
if anc is not None:
affected = list(anc.values())
for folder, target_root in affected:
self._detach_folder_locked(folder.id)
self._attach_folder_locked(folder)
return
self._maybe_rm_watch_locked(wd)
def _handle_dir_created_locked(
self,
wd: int,
created_dir: Path,
dir_callbacks: list[tuple[WatchedFolder, Path]],
) -> None:
active = self._active_by_wd.get(wd)
if active is not None:
folder, _ = active
if created_dir.is_dir():
self._attach_tree_locked(folder, created_dir)
dir_callbacks.append((folder, created_dir))
anc = self._ancestors_by_wd.get(wd)
if anc is not None:
for folder_id, (folder, target_root) in list(anc.items()):
if created_dir == target_root or created_dir in target_root.parents:
self._detach_folder_locked(folder_id)
self._attach_folder_locked(folder)
if target_root.is_dir():
dir_callbacks.append((folder, target_root))
def start(self) -> None:
"""Start background filesystem watching thread."""
if self._polling is not None:
self._polling.start()
return
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
self._thread = threading.Thread(target=self._run_loop, name="cp0-inotify-watcher", daemon=True)
self._thread.start()
def stop(self) -> None:
"""Stop background watcher thread and close watch resources."""
if self._polling is not None:
self._polling.stop()
return
self._stop_event.set()
if self._thread is not None and self._thread is not threading.current_thread():
self._thread.join(timeout=2.0)
self._thread = None
with self._lock:
for folder_id in list(self._folders.keys()):
self._detach_folder_locked(folder_id)
if self._owns_real_fd and self._fd >= 0:
try:
os.close(self._fd)
except OSError:
pass
self._fd = -1
def _run_loop(self) -> None:
while not self._stop_event.is_set():
fd = self._fd
if fd < 0:
break
try:
readable, _, _ = select.select([fd], [], [], self._poll_interval)
except (OSError, ValueError):
if self._stop_event.is_set():
break
continue
if readable:
while not self._stop_event.is_set():
try:
chunk = os.read(fd, _READ_BUFFER_SIZE)
except BlockingIOError:
break
except OSError as exc:
if exc.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
break
if not self._stop_event.is_set():
logger.warning("inotify read failed: %s", exc)
break
if not chunk:
break
self.process_inotify_bytes(chunk)
if not self._stop_event.is_set():
self._recheck_unmounted_mounts()

View File

@@ -0,0 +1,760 @@
"""Integration and unit tests for cp0_syncd watcher, uploader, IPC, and daemon."""
from __future__ import annotations
import json
import os
import signal
import socket
import struct
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from unittest import mock
from cp0_syncd.config import DaemonConfig, WatchedFolder, load_daemon_config, save_daemon_config
from cp0_syncd.daemon import SyncDaemon
from cp0_syncd.ipc import IpcServer, default_sock_path
from cp0_syncd.store import SyncStore, compute_sha256
from cp0_syncd.watcher import (
IN_CLOSE_WRITE,
IN_CREATE,
IN_ISDIR,
IN_MOVED_TO,
InotifyWatcher,
PollingWatcher,
parse_inotify_events,
scan_folder,
)
REPO_ROOT = Path(__file__).resolve().parents[2]
MAC_RECEIVER_ROOT = REPO_ROOT / "mac-receiver"
if str(MAC_RECEIVER_ROOT) not in sys.path:
sys.path.insert(0, str(MAC_RECEIVER_ROOT))
from cp0_mac_receiver.config import MacConfig, save_mac_config # noqa: E402
from cp0_mac_receiver.server import MacReceiverServer # noqa: E402
def _unused_tcp_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
probe.bind(("127.0.0.1", 0))
return int(probe.getsockname()[1])
def _send_ipc(sock_path: Path, payload: dict[str, object]) -> dict[str, object]:
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
client.settimeout(3.0)
client.connect(str(sock_path))
client.sendall((json.dumps(payload) + "\n").encode("utf-8"))
with client.makefile("r", encoding="utf-8") as reader:
line = reader.readline()
return json.loads(line)
def _wait_until(predicate: object, timeout: float = 3.0, interval: float = 0.02) -> bool:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate(): # type: ignore[operator]
return True
time.sleep(interval)
return bool(predicate()) # type: ignore[operator]
def _pack_inotify_event(wd: int, mask: int, cookie: int, name: bytes) -> bytes:
if name:
padded_len = ((len(name) + 1 + 15) // 16) * 16
padded = name + b"\x00" * (padded_len - len(name))
else:
padded_len = 0
padded = b""
return struct.pack("=iIII", wd, mask, cookie, padded_len) + padded
class _ProgrammableUploadServer(ThreadingHTTPServer):
allow_reuse_address = True
daemon_threads = True
def __init__(self, statuses: list[int], health_status: int = 200) -> None:
self.statuses = list(statuses)
self.health_status = health_status
self.upload_requests: list[dict[str, str]] = []
self.health_delay = 0.0
super().__init__(("127.0.0.1", 0), _ProgrammableHandler)
class _ProgrammableHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, format: str, *args: object) -> None:
pass
def _reply_json(self, code: int, payload: dict[str, object]) -> None:
body = json.dumps(payload).encode("utf-8")
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(body)
def do_GET(self) -> None:
srv: _ProgrammableUploadServer = self.server # type: ignore[assignment]
if srv.health_delay > 0:
time.sleep(srv.health_delay)
if srv.health_status != HTTPStatus.OK:
self._reply_json(srv.health_status, {"error": "health_rejected"})
return
self._reply_json(
HTTPStatus.OK,
{
"status": "ok",
"version": "1.0.0",
"hostname": "mock-mac",
"dest_root": "/tmp/MockDest",
},
)
def do_POST(self) -> None:
srv: _ProgrammableUploadServer = self.server # type: ignore[assignment]
length = int(self.headers.get("Content-Length", "0"))
remaining = length
while remaining > 0:
chunk = self.rfile.read(min(65536, remaining))
if not chunk:
break
remaining -= len(chunk)
srv.upload_requests.append(
{
"label": self.headers.get("X-CP0-Folder-Label", ""),
"rel_path": self.headers.get("X-CP0-Rel-Path", ""),
"sha256": self.headers.get("X-CP0-SHA256", ""),
}
)
code = srv.statuses.pop(0) if srv.statuses else int(HTTPStatus.OK)
if code == HTTPStatus.OK:
self._reply_json(code, {"status": "saved"})
else:
self._reply_json(code, {"error": f"status_{code}"})
class TestIpcAndEnvDefaults(unittest.TestCase):
def test_stale_socket_unlinked_and_invalid_json_handled(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
sock_path = Path(tmp) / "cp0-sync.sock"
sock_path.write_text("stale", encoding="utf-8")
srv = IpcServer(sock_path, lambda req: {"ok": True, "cmd": req["cmd"]})
srv.start()
try:
self.assertEqual(_send_ipc(sock_path, {"cmd": "status"}), {"ok": True, "cmd": "status"})
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
client.settimeout(2.0)
client.connect(str(sock_path))
client.sendall(b"{not-json\n")
resp = json.loads(client.makefile("r", encoding="utf-8").readline())
self.assertFalse(resp["ok"])
self.assertIn("error", resp)
finally:
srv.stop()
self.assertFalse(sock_path.exists())
def test_default_sock_path_honors_env_overrides(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
custom_sock = str(Path(tmp) / "custom.sock")
with mock.patch.dict(os.environ, {"CP0_SYNC_SOCK": custom_sock}, clear=False):
self.assertEqual(default_sock_path(), Path(custom_sock))
env_no_sock = {k: v for k, v in os.environ.items() if k != "CP0_SYNC_SOCK"}
env_no_sock["XDG_RUNTIME_DIR"] = tmp
with mock.patch.dict(os.environ, env_no_sock, clear=True):
self.assertEqual(default_sock_path(), Path(tmp) / "cp0-sync.sock")
class TestUploaderAndEndToEndSync(unittest.TestCase):
def test_sleep_wake_end_to_end_and_missing_file_cleanup(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cam_dir = Path(tmp) / "Camera"
dest_root = Path(tmp) / "MacDest"
cam_dir.mkdir()
p1 = cam_dir / "IMG_1.jpg"
p2 = cam_dir / "IMG_2.jpg"
p1.write_bytes(b"real-photo-1")
p2.write_bytes(b"real-photo-2")
port = _unused_tcp_port()
cfg_path = Path(tmp) / "daemon.json"
db_path = Path(tmp) / "state.db"
sock_path = Path(tmp) / "cp0.sock"
save_daemon_config(
DaemonConfig(
target_host="127.0.0.1",
target_port=port,
watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")],
),
cfg_path,
)
daemon = SyncDaemon(cfg_path, db_path, sock_path)
try:
self.assertEqual(daemon.scan_all(), 2)
self.assertEqual(daemon.uploader.process_once(), "offline")
self.assertEqual(daemon.store.get_meta("mac_online"), "false")
# Delete IMG_2.jpg while Mac is sleeping, then wake Mac receiver.
p2.unlink()
mac_cfg = Path(tmp) / "mac.json"
save_mac_config(
MacConfig(
dest_root=str(dest_root),
listen_host="127.0.0.1",
listen_port=port,
notify_on_batch=False,
tailscale_only=True,
),
mac_cfg,
)
srv = MacReceiverServer(mac_cfg)
srv.start()
try:
self.assertEqual(daemon.uploader.process_once(), "synced")
self.assertEqual(daemon.store.get_meta("mac_online"), "true")
self.assertEqual(daemon.store.get_meta("mac_dest_root"), str(dest_root))
self.assertEqual((dest_root / "Camera" / "IMG_1.jpg").read_bytes(), b"real-photo-1")
self.assertFalse((dest_root / "Camera" / "IMG_2.jpg").exists())
self.assertIsNone(daemon.store.next_pending())
logs = daemon.store.list_logs(limit=20)
self.assertTrue(any(entry["level"] == "WARN" and "IMG_2.jpg" in entry["message"] for entry in logs))
finally:
srv.shutdown()
srv.server_close()
finally:
daemon.store.close()
def test_utf8_and_spaces_in_headers_end_to_end(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
watch_dir = Path(tmp) / "Trip Photos"
nested_dir = watch_dir / "2026 Vacay"
dest_root = Path(tmp) / "MacDest"
nested_dir.mkdir(parents=True)
photo = nested_dir / "café 1.jpg"
photo.write_bytes(b"utf8-photo-payload")
mac_cfg = Path(tmp) / "mac.json"
save_mac_config(
MacConfig(
dest_root=str(dest_root),
listen_host="127.0.0.1",
listen_port=0,
notify_on_batch=False,
tailscale_only=True,
),
mac_cfg,
)
srv = MacReceiverServer(mac_cfg)
srv.start()
try:
cfg_path = Path(tmp) / "daemon.json"
db_path = Path(tmp) / "state.db"
sock_path = Path(tmp) / "cp0.sock"
save_daemon_config(
DaemonConfig(
target_host="127.0.0.1",
target_port=srv.server_port,
watched_folders=[WatchedFolder(id="trip", path=str(watch_dir), label="Trip Photos")],
),
cfg_path,
)
daemon = SyncDaemon(cfg_path, db_path, sock_path)
try:
self.assertEqual(daemon.scan_all(), 1)
self.assertEqual(daemon.uploader.process_once(), "synced")
saved = dest_root / "Trip Photos" / "2026 Vacay" / "café 1.jpg"
self.assertTrue(saved.exists())
self.assertEqual(saved.read_bytes(), b"utf8-photo-payload")
finally:
daemon.store.close()
finally:
srv.shutdown()
srv.server_close()
def test_403_and_500_pause_queue_while_400_413_422_fail_only_that_file(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cam_dir = Path(tmp) / "Camera"
cam_dir.mkdir()
f1 = cam_dir / "a.jpg"
f2 = cam_dir / "b.jpg"
f3 = cam_dir / "c.jpg"
f1.write_bytes(b"photo-a")
f2.write_bytes(b"photo-b")
f3.write_bytes(b"photo-c")
srv = _ProgrammableUploadServer([403, 500, 413, 400, 422, 200])
thread = threading.Thread(target=srv.serve_forever, daemon=True)
thread.start()
try:
cfg_path = Path(tmp) / "daemon.json"
db_path = Path(tmp) / "state.db"
sock_path = Path(tmp) / "cp0.sock"
save_daemon_config(
DaemonConfig(
target_host="127.0.0.1",
target_port=srv.server_port,
watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")],
),
cfg_path,
)
daemon = SyncDaemon(cfg_path, db_path, sock_path)
try:
self.assertEqual(daemon.scan_all(), 3)
# 403 pauses queue immediately on a.jpg; b.jpg and c.jpg are not attempted.
self.assertEqual(daemon.uploader.process_once(), "paused")
self.assertEqual(len(srv.upload_requests), 1)
snap = daemon.store.get_status_snapshot(daemon.get_config().watched_folders)
self.assertEqual(snap["counts"]["pending"], 3)
self.assertEqual(snap["counts"]["failed"], 0)
# 500 also pauses queue immediately on a.jpg.
self.assertEqual(daemon.uploader.process_once(), "paused")
self.assertEqual(len(srv.upload_requests), 2)
snap = daemon.store.get_status_snapshot(daemon.get_config().watched_folders)
self.assertEqual(snap["counts"]["pending"], 3)
self.assertEqual(snap["counts"]["failed"], 0)
# Now 413, 400, 422? No, next in srv.statuses is 413, 400, 422!
# Let's add a 4th file d.jpg to get the 200 OK after a(413), b(400), c(422).
f4 = cam_dir / "d.jpg"
f4.write_bytes(b"photo-d")
self.assertEqual(daemon.scan_all(), 1)
self.assertEqual(daemon.uploader.process_once(), "synced")
snap = daemon.store.get_status_snapshot(daemon.get_config().watched_folders)
self.assertEqual(snap["counts"]["failed"], 3)
self.assertEqual(snap["counts"]["synced"], 1)
self.assertEqual(snap["counts"]["pending"], 0)
finally:
daemon.store.close()
finally:
srv.shutdown()
srv.server_close()
def test_backoff_schedule_reset_and_probe_timeout(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cam_dir = Path(tmp) / "Camera"
cam_dir.mkdir()
(cam_dir / "a.jpg").write_bytes(b"photo-a")
dead_port = _unused_tcp_port()
cfg_path = Path(tmp) / "daemon.json"
db_path = Path(tmp) / "state.db"
sock_path = Path(tmp) / "cp0.sock"
save_daemon_config(
DaemonConfig(
target_host="127.0.0.1",
target_port=dead_port,
watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")],
),
cfg_path,
)
daemon = SyncDaemon(cfg_path, db_path, sock_path, probe_timeout=0.15)
try:
daemon.scan_all()
expected_delays = [5, 10, 20, 40, 60, 60]
for want_sec in expected_delays:
self.assertEqual(daemon.uploader.process_once(), "offline")
sec = daemon.uploader.next_retry_sec()
self.assertGreaterEqual(sec, want_sec - 1)
self.assertLessEqual(sec, want_sec)
# New photo resets backoff back to 5s on next failure.
daemon.uploader.notify_new_file()
self.assertEqual(daemon.uploader.next_retry_sec(), 0)
self.assertEqual(daemon.uploader.process_once(), "offline")
self.assertLessEqual(daemon.uploader.next_retry_sec(), 5)
finally:
daemon.store.close()
# Verify probe_timeout aborts a hanging /health response quickly.
slow_srv = _ProgrammableUploadServer([], health_status=200)
slow_srv.health_delay = 0.6
t = threading.Thread(target=slow_srv.serve_forever, daemon=True)
t.start()
try:
save_daemon_config(
DaemonConfig(
target_host="127.0.0.1",
target_port=slow_srv.server_port,
watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")],
),
cfg_path,
)
daemon2 = SyncDaemon(cfg_path, Path(tmp) / "state2.db", Path(tmp) / "s2.sock", probe_timeout=0.15)
try:
daemon2.scan_all()
t0 = time.monotonic()
self.assertEqual(daemon2.uploader.process_once(), "offline")
elapsed = time.monotonic() - t0
self.assertLess(elapsed, 0.5)
finally:
daemon2.store.close()
finally:
slow_srv.shutdown()
slow_srv.server_close()
class TestSyncDaemonIpcLifecycle(unittest.TestCase):
def test_all_ipc_commands_end_to_end(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cam_dir = Path(tmp) / "Camera"
extra_dir = Path(tmp) / "SDCard"
dest_root = Path(tmp) / "MacDest"
cam_dir.mkdir()
extra_dir.mkdir()
(extra_dir / "sd_1.jpg").write_bytes(b"sd-photo-1")
cfg_path = Path(tmp) / "daemon.json"
db_path = Path(tmp) / "state.db"
sock_path = Path(tmp) / "cp0.sock"
save_daemon_config(
DaemonConfig(
target_host="",
target_port=8765,
theme="dark",
watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")],
),
cfg_path,
)
mac_cfg = Path(tmp) / "mac.json"
save_mac_config(
MacConfig(
dest_root=str(dest_root),
listen_host="127.0.0.1",
listen_port=0,
notify_on_batch=False,
tailscale_only=True,
),
mac_cfg,
)
mac_srv = MacReceiverServer(mac_cfg)
mac_srv.start()
daemon = SyncDaemon(cfg_path, db_path, sock_path, watcher_poll_interval=0.05)
daemon.start()
try:
status = _send_ipc(sock_path, {"cmd": "status"})
self.assertEqual(
set(status.keys()),
{
"ok",
"target_host",
"target_port",
"theme",
"mac_online",
"mac_dest_root",
"syncing",
"current_file",
"pending_count",
"pending_bytes",
"synced_count",
"failed_count",
"next_retry_sec",
"last_sync_ts",
"last_synced_file",
"last_error",
"folders",
"logs",
},
)
self.assertTrue(status["ok"])
self.assertEqual(status["theme"], "dark")
self.assertFalse(status["mac_online"])
# set_theme validation + persistence
bad_theme = _send_ipc(sock_path, {"cmd": "set_theme", "theme": "neon"})
self.assertFalse(bad_theme["ok"])
good_theme = _send_ipc(sock_path, {"cmd": "set_theme", "theme": "light"})
self.assertTrue(good_theme["ok"])
self.assertEqual(load_daemon_config(cfg_path).theme, "light")
# add_folder validation + rescan
overlap = _send_ipc(sock_path, {"cmd": "add_folder", "path": str(cam_dir)})
self.assertFalse(overlap["ok"])
added = _send_ipc(sock_path, {"cmd": "add_folder", "path": str(extra_dir), "label": "SD"})
self.assertTrue(added["ok"])
sd_folder_id = added["folder"]["id"] # type: ignore[index]
self.assertEqual(added["enqueued"], 1)
# set_target validation + immediate probe + upload
bad_port = _send_ipc(sock_path, {"cmd": "set_target", "host": "127.0.0.1", "port": 99999})
self.assertFalse(bad_port["ok"])
bad_host = _send_ipc(sock_path, {"cmd": "set_target", "host": "", "port": 8765})
self.assertFalse(bad_host["ok"])
set_tgt = _send_ipc(
sock_path,
{"cmd": "set_target", "host": "127.0.0.1", "port": mac_srv.server_port},
)
self.assertTrue(set_tgt["ok"])
self.assertTrue(_wait_until(lambda: (dest_root / "SD" / "sd_1.jpg").exists()))
status_after = _send_ipc(sock_path, {"cmd": "status"})
self.assertTrue(status_after["mac_online"])
self.assertEqual(status_after["mac_dest_root"], str(dest_root))
self.assertEqual(status_after["synced_count"], 1)
self.assertEqual(status_after["pending_count"], 0)
self.assertEqual(status_after["last_synced_file"], "sd_1.jpg")
self.assertGreater(status_after["last_sync_ts"], 0)
self.assertEqual(
set(status_after["folders"][0].keys()), # type: ignore[index]
{"id", "path", "label", "enabled", "exists", "pending", "synced"},
)
# retry_failed flips FAILED -> PENDING and uploads via wake
failed_file = cam_dir / "retried.jpg"
failed_file.write_bytes(b"retried-bytes")
sha = compute_sha256(failed_file)
daemon.store.enqueue_file("cam", str(failed_file), "retried.jpg", 13, 1, sha)
row = daemon.store.next_pending()
assert row is not None
daemon.store.mark_failed(row["id"], "413")
retry_resp = _send_ipc(sock_path, {"cmd": "retry_failed"})
self.assertTrue(retry_resp["ok"])
self.assertEqual(retry_resp["retried"], 1)
self.assertTrue(_wait_until(lambda: (dest_root / "Camera" / "retried.jpg").exists()))
# sync_now picks up newly written photo
(cam_dir / "manual.jpg").write_bytes(b"manual-bytes")
sync_resp = _send_ipc(sock_path, {"cmd": "sync_now"})
self.assertTrue(sync_resp["ok"])
self.assertTrue(_wait_until(lambda: (dest_root / "Camera" / "manual.jpg").exists()))
# remove_folder removes config entry and purges pending rows without deleting disk files
mac_srv.shutdown()
mac_srv.server_close()
queued_file = extra_dir / "stay_on_disk.jpg"
queued_file.write_bytes(b"local-only")
daemon.store.enqueue_file(
sd_folder_id,
str(queued_file),
"stay_on_disk.jpg",
10,
1,
compute_sha256(queued_file),
)
rm_resp = _send_ipc(sock_path, {"cmd": "remove_folder", "folder_id": sd_folder_id})
self.assertTrue(rm_resp["ok"])
self.assertEqual(rm_resp["purged"], 1)
self.assertTrue(queued_file.exists())
self.assertEqual([wf.id for wf in load_daemon_config(cfg_path).watched_folders], ["cam"])
# Unknown command returns ok=false
unknown = _send_ipc(sock_path, {"cmd": "bogus"})
self.assertFalse(unknown["ok"])
finally:
daemon.stop()
mac_srv.shutdown()
mac_srv.server_close()
class _FakeLibc:
def __init__(self) -> None:
self.next_wd = 1
self.watches: dict[int, tuple[str, int]] = {}
self.removed_wds: list[int] = []
self.init_calls = 0
def inotify_init1(self, flags: int) -> int:
self.init_calls += 1
return 99
def inotify_add_watch(self, fd: int, path_bytes: bytes, mask: int) -> int:
wd = self.next_wd
self.next_wd += 1
self.watches[wd] = (os.fsdecode(path_bytes), mask)
return wd
def inotify_rm_watch(self, fd: int, wd: int) -> int:
self.removed_wds.append(wd)
self.watches.pop(wd, None)
return 0
class TestWatchersAndLinuxInotify(unittest.TestCase):
def test_parse_linux_inotify_event_buffer_with_utf8(self) -> None:
buf = (
_pack_inotify_event(1, IN_CREATE, 0, b"partial.jpg")
+ _pack_inotify_event(1, IN_CLOSE_WRITE, 0, "café.jpg".encode("utf-8"))
+ _pack_inotify_event(2, IN_CREATE | IN_ISDIR, 0, b"2026-11")
+ _pack_inotify_event(3, IN_MOVED_TO, 42, b"moved.png")
)
events = parse_inotify_events(buf)
self.assertEqual(len(events), 4)
self.assertEqual((events[0].wd, events[0].mask, events[0].name), (1, IN_CREATE, "partial.jpg"))
self.assertEqual((events[1].wd, events[1].mask, events[1].name), (1, IN_CLOSE_WRITE, "café.jpg"))
self.assertEqual((events[2].wd, events[2].mask, events[2].name), (2, IN_CREATE | IN_ISDIR, "2026-11"))
self.assertEqual((events[3].wd, events[3].mask, events[3].cookie, events[3].name), (3, IN_MOVED_TO, 42, "moved.png"))
def test_linux_inotify_dispatch_subdir_watch_and_ancestor_mount(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
cam_dir = root / "Camera"
cam_dir.mkdir()
unmounted = root / "media" / "sd" / "DCIM"
(root / "media").mkdir()
fake_libc = _FakeLibc()
ready_files: list[tuple[str, Path]] = []
created_dirs: list[tuple[str, Path]] = []
watcher = InotifyWatcher(
on_file_ready=lambda wf, p: ready_files.append((wf.id, p)),
on_dir_created=lambda wf, p: created_dirs.append((wf.id, p)),
libc=fake_libc,
)
self.assertFalse(watcher.is_using_polling)
cam_wf = WatchedFolder(id="cam", path=str(cam_dir), label="Camera")
sd_wf = WatchedFolder(id="sd", path=str(unmounted), label="SD")
watcher.sync_folders([cam_wf, sd_wf])
# Locate assigned watch descriptors
wd_by_path = {path: wd for wd, (path, _) in fake_libc.watches.items()}
self.assertIn(str(cam_dir), wd_by_path)
self.assertIn(str(root / "media"), wd_by_path)
cam_wd = wd_by_path[str(cam_dir)]
media_wd = wd_by_path[str(root / "media")]
# 1. File IN_CREATE must be ignored; IN_CLOSE_WRITE and IN_MOVED_TO must fire.
(cam_dir / "shot.jpg").write_bytes(b"img")
(cam_dir / "moved.jpg").write_bytes(b"img2")
watcher.process_inotify_bytes(
_pack_inotify_event(cam_wd, IN_CREATE, 0, b"shot.jpg")
+ _pack_inotify_event(cam_wd, IN_CLOSE_WRITE, 0, b"shot.jpg")
+ _pack_inotify_event(cam_wd, IN_MOVED_TO, 7, b"moved.jpg")
)
self.assertEqual(
ready_files,
[("cam", cam_dir / "shot.jpg"), ("cam", cam_dir / "moved.jpg")],
)
# 2. Subdir IN_CREATE | IN_ISDIR attaches watch dynamically and triggers on_dir_created.
subdir = cam_dir / "2026"
subdir.mkdir()
watcher.process_inotify_bytes(_pack_inotify_event(cam_wd, IN_CREATE | IN_ISDIR, 0, b"2026"))
self.assertTrue(any(path == str(subdir) for path, _ in fake_libc.watches.values()))
self.assertIn(("cam", subdir), created_dirs)
# 3. Ancestor mount creation resolves step-by-step down to unmounted SD path.
(root / "media" / "sd").mkdir()
watcher.process_inotify_bytes(_pack_inotify_event(media_wd, IN_CREATE | IN_ISDIR, 0, b"sd"))
sd_wd = next(wd for wd, (p, _) in fake_libc.watches.items() if p == str(root / "media" / "sd"))
unmounted.mkdir()
watcher.process_inotify_bytes(_pack_inotify_event(sd_wd, IN_CREATE | IN_ISDIR, 0, b"DCIM"))
self.assertTrue(any(path == str(unmounted) for path, _ in fake_libc.watches.values()))
self.assertIn(("sd", unmounted), created_dirs)
def test_polling_watcher_detects_new_mount_and_new_file(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
missing_mount = Path(tmp) / "sdcard" / "DCIM"
wf = WatchedFolder(id="sd", path=str(missing_mount), label="SD")
ready: list[Path] = []
mounted: list[Path] = []
watcher = PollingWatcher(
on_file_ready=lambda _wf, p: ready.append(p),
on_dir_created=lambda _wf, p: mounted.append(p),
poll_interval=0.05,
)
watcher.sync_folders([wf])
watcher.start()
try:
missing_mount.mkdir(parents=True)
self.assertTrue(_wait_until(lambda: missing_mount in mounted))
photo = missing_mount / "IMG_9.jpg"
photo.write_bytes(b"sd-card-photo")
self.assertTrue(_wait_until(lambda: photo in ready))
finally:
watcher.stop()
def test_scan_folder_skips_missing_and_hidden_and_dedups(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cfg = DaemonConfig()
db = SyncStore(Path(tmp) / "state.db")
try:
missing = WatchedFolder(id="miss", path=str(Path(tmp) / "missing"), label="Missing")
self.assertEqual(scan_folder(missing, cfg, db), 0)
cam = Path(tmp) / "Camera"
(cam / ".hidden").mkdir(parents=True)
(cam / "2026").mkdir()
(cam / ".hidden" / "ignore.jpg").write_bytes(b"ignored")
(cam / "2026" / "keep.jpg").write_bytes(b"kept")
(cam / "2026" / "empty.jpg").write_bytes(b"")
wf = WatchedFolder(id="cam", path=str(cam), label="Camera")
with self.assertLogs("cp0_syncd.store", level="WARNING") as warn_logs:
self.assertEqual(scan_folder(wf, cfg, db), 1)
self.assertEqual(scan_folder(wf, cfg, db), 0)
self.assertTrue(any("0-byte" in msg for msg in warn_logs.output))
finally:
db.close()
class TestDaemonBinaryEntrypoint(unittest.TestCase):
def test_cp0_syncd_binary_runs_via_symlink_and_stops_on_sigterm(self) -> None:
bin_path = REPO_ROOT / "device-daemon" / "bin" / "cp0-syncd"
self.assertTrue(bin_path.exists())
self.assertTrue(os.access(bin_path, os.X_OK))
with tempfile.TemporaryDirectory() as tmp:
link_path = Path(tmp) / "cp0-syncd-link"
link_path.symlink_to(bin_path)
cfg_path = Path(tmp) / "config.json"
db_path = Path(tmp) / "state.db"
sock_path = Path(tmp) / "cp0.sock"
env = os.environ.copy()
env.pop("PYTHONPATH", None)
env["CP0_SYNC_CONFIG"] = str(cfg_path)
env["CP0_SYNC_DB"] = str(db_path)
env["CP0_SYNC_SOCK"] = str(sock_path)
proc = subprocess.Popen(
[str(link_path)],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
try:
self.assertTrue(_wait_until(lambda: sock_path.exists(), timeout=5.0))
resp = _send_ipc(sock_path, {"cmd": "status"})
self.assertTrue(resp["ok"])
proc.send_signal(signal.SIGTERM)
rc = proc.wait(timeout=5.0)
self.assertEqual(rc, 0)
self.assertFalse(sock_path.exists())
finally:
if proc.poll() is None:
proc.kill()
proc.wait(timeout=2.0)
if proc.stdout is not None:
proc.stdout.close()
if proc.stderr is not None:
proc.stderr.close()
if __name__ == "__main__":
unittest.main()