feat(device-daemon): add daemon config, overlap guard, and SQLite WAL queue store

This commit is contained in:
2026-11-18 07:29:58 -05:00
parent 23ed06f203
commit 69224f3929
4 changed files with 1039 additions and 0 deletions

View File

@@ -0,0 +1,31 @@
"""CardputerZero on-device background photo sync daemon package."""
from .config import (
DaemonConfig,
WatchedFolder,
add_watched_folder,
default_config_path,
load_daemon_config,
remove_watched_folder,
save_daemon_config,
)
from .store import (
SyncStore,
compute_sha256,
default_db_path,
is_media_file,
)
__all__ = [
"DaemonConfig",
"SyncStore",
"WatchedFolder",
"add_watched_folder",
"compute_sha256",
"default_config_path",
"default_db_path",
"is_media_file",
"load_daemon_config",
"remove_watched_folder",
"save_daemon_config",
]

View File

@@ -0,0 +1,266 @@
"""Configuration persistence and watched-folder management for cp0-syncd."""
from __future__ import annotations
import json
import logging
import os
import re
import uuid
from dataclasses import asdict, dataclass, field, replace
from pathlib import Path
logger = logging.getLogger(__name__)
ENV_CONFIG_PATH = "CP0_SYNC_CONFIG"
DEFAULT_TARGET_PORT = 8765
DEFAULT_THEME = "dark"
VALID_THEMES = ("dark", "light")
DEFAULT_EXTENSIONS = [
".jpg",
".jpeg",
".png",
".webp",
".gif",
".bmp",
".dng",
".raw",
".mp4",
".mov",
]
DEFAULT_IGNORE_PREFIXES = [".", "~"]
DEFAULT_IGNORE_SUFFIXES = [".part", ".tmp", ".crdownload"]
@dataclass
class WatchedFolder:
"""A single local directory watched by cp0-syncd."""
id: str
path: str
label: str
enabled: bool = True
@dataclass
class DaemonConfig:
"""Runtime configuration for the on-device sync daemon and companion UI."""
target_host: str = ""
target_port: int = DEFAULT_TARGET_PORT
theme: str = DEFAULT_THEME
watched_folders: list[WatchedFolder] = field(default_factory=list)
extensions: list[str] = field(default_factory=lambda: list(DEFAULT_EXTENSIONS))
ignore_prefixes: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE_PREFIXES))
ignore_suffixes: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE_SUFFIXES))
def default_config_path() -> Path:
"""Return the default daemon config path, honoring CP0_SYNC_CONFIG if set."""
override = os.environ.get(ENV_CONFIG_PATH, "").strip()
if override:
return Path(override).expanduser()
return Path.home() / ".config" / "cp0-sync" / "config.json"
def default_watched_folders(home_dir: Path | None = None) -> list[WatchedFolder]:
"""Return the first-run default watched folder list (<home>/Pictures/DCIM/Camera)."""
base_home = Path(home_dir).expanduser() if home_dir is not None else Path.home()
camera_path = base_home / "Pictures" / "DCIM" / "Camera"
return [
WatchedFolder(
id="camera",
path=str(camera_path),
label="Camera",
enabled=True,
)
]
def _resolve_config_path(path: Path | None) -> Path:
return Path(path).expanduser() if path is not None else default_config_path()
def _valid_port(value: object) -> bool:
return isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535
def _valid_str_list(value: object) -> list[str] | None:
if not isinstance(value, list):
return None
cleaned = [item for item in value if isinstance(item, str) and item]
return cleaned if len(cleaned) == len(value) else None
def _parse_watched_folders(raw_folders: object, home_dir: Path | None) -> list[WatchedFolder]:
if not isinstance(raw_folders, list):
return default_watched_folders(home_dir)
parsed: list[WatchedFolder] = []
for item in raw_folders:
if not isinstance(item, dict):
continue
folder_id = item.get("id")
folder_path = item.get("path")
label = item.get("label")
enabled = item.get("enabled", True)
if (
isinstance(folder_id, str)
and folder_id.strip()
and isinstance(folder_path, str)
and folder_path.strip()
and isinstance(label, str)
and label.strip()
and isinstance(enabled, bool)
):
parsed.append(
WatchedFolder(
id=folder_id.strip(),
path=folder_path.strip(),
label=label.strip(),
enabled=enabled,
)
)
return parsed
def _default_daemon_config(home_dir: Path | None = None) -> DaemonConfig:
return DaemonConfig(watched_folders=default_watched_folders(home_dir))
def load_daemon_config(path: Path | None = None, home_dir: Path | None = None) -> DaemonConfig:
"""Load DaemonConfig from JSON, creating first-run defaults or safely falling back on errors."""
cfg_path = _resolve_config_path(path)
defaults = _default_daemon_config(home_dir)
if not cfg_path.exists():
try:
save_daemon_config(defaults, cfg_path)
except OSError as exc:
logger.warning("Could not persist first-run config %s (%s); continuing in memory", cfg_path, exc)
return defaults
try:
raw = json.loads(cfg_path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
logger.warning("Failed to read config %s (%s); using defaults", cfg_path, exc)
return defaults
if not isinstance(raw, dict):
logger.warning("Config %s is not a JSON object; using defaults", cfg_path)
return defaults
target_host = raw.get("target_host")
target_port = raw.get("target_port")
theme = raw.get("theme")
extensions = _valid_str_list(raw.get("extensions"))
ignore_prefixes = _valid_str_list(raw.get("ignore_prefixes"))
ignore_suffixes = _valid_str_list(raw.get("ignore_suffixes"))
watched_folders = (
_parse_watched_folders(raw.get("watched_folders"), home_dir)
if "watched_folders" in raw
else defaults.watched_folders
)
return DaemonConfig(
target_host=target_host.strip() if isinstance(target_host, str) else defaults.target_host,
target_port=target_port if _valid_port(target_port) else defaults.target_port,
theme=theme if isinstance(theme, str) and theme in VALID_THEMES else defaults.theme,
watched_folders=watched_folders,
extensions=extensions if extensions is not None else defaults.extensions,
ignore_prefixes=ignore_prefixes if ignore_prefixes is not None else defaults.ignore_prefixes,
ignore_suffixes=ignore_suffixes if ignore_suffixes is not None else defaults.ignore_suffixes,
)
def save_daemon_config(cfg: DaemonConfig, path: Path | None = None) -> None:
"""Atomically persist DaemonConfig to disk using a temp file, fsync, and os.replace."""
cfg_path = _resolve_config_path(path)
cfg_path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = cfg_path.parent / f".{cfg_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
payload = json.dumps(asdict(cfg), indent=2) + "\n"
try:
with open(tmp_path, "w", encoding="utf-8") as handle:
handle.write(payload)
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp_path, cfg_path)
except BaseException:
try:
tmp_path.unlink(missing_ok=True)
except OSError:
pass
raise
def _canonical_path(raw_path: str) -> Path:
expanded = os.path.expanduser(raw_path)
return Path(os.path.realpath(expanded))
def _paths_overlap(a: Path, b: Path) -> bool:
return a == b or a in b.parents or b in a.parents
def _disambiguate_label(base_label: str, existing_labels: set[str]) -> str:
if base_label.lower() not in existing_labels:
return base_label
suffix = 2
while True:
candidate = f"{base_label}-{suffix}"
if candidate.lower() not in existing_labels:
return candidate
suffix += 1
def _make_folder_id(label: str, existing_ids: set[str]) -> str:
slug = re.sub(r"[^a-z0-9]+", "-", label.lower()).strip("-") or "folder"
while True:
candidate = f"{slug}-{uuid.uuid4().hex[:8]}"
if candidate not in existing_ids:
return candidate
def add_watched_folder(
cfg: DaemonConfig,
folder_path: str,
label: str | None = None,
) -> tuple[DaemonConfig, WatchedFolder]:
"""Validate and add a watched folder, rejecting missing/overlapping paths and disambiguating labels."""
if not isinstance(folder_path, str) or not folder_path.strip():
raise ValueError("Folder path must be a non-empty string")
candidate = _canonical_path(folder_path.strip())
if not candidate.exists() or not candidate.is_dir():
raise ValueError(f"Watched directory does not exist: {folder_path}")
for existing in cfg.watched_folders:
existing_real = _canonical_path(existing.path)
if _paths_overlap(candidate, existing_real):
raise ValueError(
f"Folder {candidate} overlaps already-watched path {existing_real}"
)
base_label = label.strip() if isinstance(label, str) and label.strip() else (candidate.name or "Folder")
existing_labels = {wf.label.lower() for wf in cfg.watched_folders}
unique_label = _disambiguate_label(base_label, existing_labels)
existing_ids = {wf.id for wf in cfg.watched_folders}
folder_id = _make_folder_id(unique_label, existing_ids)
new_folder = WatchedFolder(
id=folder_id,
path=str(candidate),
label=unique_label,
enabled=True,
)
updated_cfg = replace(cfg, watched_folders=[*cfg.watched_folders, new_folder])
return updated_cfg, new_folder
def remove_watched_folder(cfg: DaemonConfig, folder_id: str) -> DaemonConfig:
"""Return a copy of DaemonConfig without the specified watched folder ID."""
return replace(
cfg,
watched_folders=[wf for wf in cfg.watched_folders if wf.id != folder_id],
)

View File

@@ -0,0 +1,439 @@
"""SQLite WAL queue store and media file filtering for cp0-syncd."""
from __future__ import annotations
import hashlib
import logging
import os
import sqlite3
import stat
import threading
import time
from pathlib import Path
from typing import Any
from .config import DaemonConfig, WatchedFolder
logger = logging.getLogger(__name__)
ENV_DB_PATH = "CP0_SYNC_DB"
MAX_LOG_ROWS = 500
CHUNK_SIZE = 65536
def default_db_path() -> Path:
"""Return the default SQLite state database path, honoring CP0_SYNC_DB if set."""
override = os.environ.get(ENV_DB_PATH, "").strip()
if override:
return Path(override).expanduser()
return Path.home() / ".local" / "share" / "cp0-sync" / "state.db"
def compute_sha256(path: Path, chunk_size: int = CHUNK_SIZE) -> str:
"""Compute the lowercase hex SHA-256 digest of a file incrementally."""
digest = hashlib.sha256()
with open(path, "rb") as handle:
while True:
chunk = handle.read(chunk_size)
if not chunk:
break
digest.update(chunk)
return digest.hexdigest()
def is_media_file(
path: Path,
cfg: DaemonConfig,
store: SyncStore | None = None,
) -> bool:
"""Return True if path is a non-empty regular media file permitted by DaemonConfig."""
name = path.name
if not name:
return False
if any(name.startswith(prefix) for prefix in cfg.ignore_prefixes if prefix):
return False
name_lower = name.lower()
if any(name_lower.endswith(suffix.lower()) for suffix in cfg.ignore_suffixes if suffix):
return False
allowed_exts = {
ext.lower() if ext.startswith(".") else f".{ext.lower()}"
for ext in cfg.extensions
if ext
}
if path.suffix.lower() not in allowed_exts:
return False
try:
st = path.stat()
except OSError:
return False
if not stat.S_ISREG(st.st_mode):
return False
if st.st_size == 0:
logger.warning("Skipping 0-byte media file: %s", path)
if store is not None:
store.log("WARN", f"Skipped 0-byte file: {name}")
return False
return True
def _now_ms() -> int:
return int(time.time() * 1000)
class SyncStore:
"""Thread-safe SQLite WAL queue, metadata, and ring-buffered log store."""
def __init__(self, db_path: Path | None = None) -> None:
resolved = Path(db_path).expanduser() if db_path is not None else default_db_path()
resolved.parent.mkdir(parents=True, exist_ok=True)
self.db_path = resolved
self._lock = threading.RLock()
self._conn = sqlite3.connect(str(resolved), timeout=5.0, check_same_thread=False)
self._conn.row_factory = sqlite3.Row
self._closed = False
self._init_schema()
def _init_schema(self) -> None:
with self._lock:
self._conn.execute("PRAGMA journal_mode=WAL;")
self._conn.execute("PRAGMA busy_timeout=5000;")
self._conn.execute("PRAGMA synchronous=NORMAL;")
self._conn.executescript(
"""
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
folder_id TEXT NOT NULL,
abs_path TEXT NOT NULL,
rel_path TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
mtime_ns INTEGER NOT NULL,
sha256 TEXT NOT NULL,
status TEXT NOT NULL,
retry_count INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
discovered_at INTEGER NOT NULL,
synced_at INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_files_dedup
ON files(folder_id, rel_path, sha256);
CREATE INDEX IF NOT EXISTS idx_files_status_discovered
ON files(status, discovered_at, id);
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL,
level TEXT NOT NULL,
message TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"""
)
self._conn.commit()
def enqueue_file(
self,
folder_id: str,
abs_path: str,
rel_path: str,
size_bytes: int,
mtime_ns: int,
sha256: str,
) -> str:
"""Enqueue a file if not already synced/queued; skip 0-byte payloads with a WARN log."""
if size_bytes <= 0:
self.log("WARN", f"Skipped 0-byte file: {rel_path}")
return "skipped_empty"
now_ms = _now_ms()
with self._lock:
row = self._conn.execute(
"""
SELECT id, status
FROM files
WHERE folder_id = ? AND rel_path = ? AND sha256 = ?
""",
(folder_id, rel_path, sha256),
).fetchone()
if row is not None:
status = str(row["status"])
if status == "SYNCED":
return "already_synced"
if status == "FAILED":
return "already_failed"
return "already_queued"
self._conn.execute(
"""
INSERT INTO files (
folder_id, abs_path, rel_path, size_bytes, mtime_ns,
sha256, status, retry_count, last_error, discovered_at, synced_at
)
VALUES (?, ?, ?, ?, ?, ?, 'PENDING', 0, NULL, ?, NULL)
""",
(folder_id, abs_path, rel_path, size_bytes, mtime_ns, sha256, now_ms),
)
self._conn.commit()
return "enqueued"
def next_pending(self) -> dict[str, Any] | None:
"""Return the oldest PENDING file row as a dict, or None if the queue is empty."""
with self._lock:
row = self._conn.execute(
"""
SELECT id, folder_id, abs_path, rel_path, size_bytes, mtime_ns,
sha256, status, retry_count, last_error, discovered_at, synced_at
FROM files
WHERE status = 'PENDING'
ORDER BY discovered_at ASC, id ASC
LIMIT 1
"""
).fetchone()
return dict(row) if row is not None else None
def mark_uploading(self, file_id: int) -> None:
"""Transition a queued file to UPLOADING."""
with self._lock:
self._conn.execute(
"UPDATE files SET status = 'UPLOADING' WHERE id = ?",
(file_id,),
)
self._conn.commit()
def mark_synced(self, file_id: int) -> None:
"""Mark a file as SYNCED and clear any prior error."""
now_ms = _now_ms()
with self._lock:
self._conn.execute(
"""
UPDATE files
SET status = 'SYNCED', synced_at = ?, last_error = NULL
WHERE id = ?
""",
(now_ms, file_id),
)
self._conn.commit()
def mark_retry(self, file_id: int, err: str) -> None:
"""Return an item to PENDING while recording the retry count and error."""
with self._lock:
self._conn.execute(
"""
UPDATE files
SET status = 'PENDING', retry_count = retry_count + 1, last_error = ?
WHERE id = ?
""",
(err, file_id),
)
self._conn.commit()
def mark_failed(self, file_id: int, err: str) -> None:
"""Mark a file as FAILED after a permanent per-file rejection."""
with self._lock:
self._conn.execute(
"""
UPDATE files
SET status = 'FAILED', retry_count = retry_count + 1, last_error = ?
WHERE id = ?
""",
(err, file_id),
)
self._conn.commit()
def delete_file(self, file_id: int) -> None:
"""Delete a file record (e.g., when the source file vanished before upload)."""
with self._lock:
self._conn.execute("DELETE FROM files WHERE id = ?", (file_id,))
self._conn.commit()
def purge_pending_for_folder(self, folder_id: str) -> int:
"""Delete un-synced PENDING/UPLOADING queue entries for a removed folder."""
with self._lock:
cur = self._conn.execute(
"""
DELETE FROM files
WHERE folder_id = ? AND status IN ('PENDING', 'UPLOADING')
""",
(folder_id,),
)
self._conn.commit()
return cur.rowcount
def retry_all_failed(self) -> int:
"""Flip all FAILED rows back to PENDING and return the count updated."""
with self._lock:
cur = self._conn.execute(
"""
UPDATE files
SET status = 'PENDING', last_error = NULL
WHERE status = 'FAILED'
"""
)
self._conn.commit()
return cur.rowcount
def log(self, level: str, message: str) -> None:
"""Append a log row and prune the logs table to the newest 500 rows."""
now_ms = _now_ms()
normalized_level = level.strip().upper() or "INFO"
with self._lock:
self._conn.execute(
"INSERT INTO logs (ts, level, message) VALUES (?, ?, ?)",
(now_ms, normalized_level, message),
)
self._conn.execute(
"""
DELETE FROM logs
WHERE id NOT IN (
SELECT id FROM logs ORDER BY id DESC LIMIT ?
)
""",
(MAX_LOG_ROWS,),
)
self._conn.commit()
def log_count(self) -> int:
"""Return the number of rows currently stored in the logs table."""
with self._lock:
row = self._conn.execute("SELECT COUNT(*) FROM logs").fetchone()
return int(row[0]) if row is not None else 0
def list_logs(self, limit: int = 100) -> list[dict[str, Any]]:
"""Return up to `limit` newest log rows ordered newest-first."""
safe_limit = max(0, int(limit))
with self._lock:
rows = self._conn.execute(
"""
SELECT id, ts, level, message
FROM logs
ORDER BY id DESC
LIMIT ?
""",
(safe_limit,),
).fetchall()
return [dict(row) for row in rows]
def set_meta(self, key: str, value: str) -> None:
"""Upsert a metadata key-value pair."""
with self._lock:
self._conn.execute(
"""
INSERT INTO meta (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
""",
(key, str(value)),
)
self._conn.commit()
def get_meta(self, key: str) -> str | None:
"""Fetch a metadata value by key, or None if unset."""
with self._lock:
row = self._conn.execute(
"SELECT value FROM meta WHERE key = ?",
(key,),
).fetchone()
return str(row["value"]) if row is not None else None
def get_status_snapshot(self, watched_folders: list[WatchedFolder]) -> dict[str, Any]:
"""Return queue totals, per-folder counts, metadata, and recent logs for status IPC/UI."""
with self._lock:
folder_stats: dict[str, dict[str, int]] = {}
rows = self._conn.execute(
"""
SELECT folder_id, status, COUNT(*) AS cnt, COALESCE(SUM(size_bytes), 0) AS total_bytes
FROM files
GROUP BY folder_id, status
"""
).fetchall()
totals = {
"pending": 0,
"uploading": 0,
"synced": 0,
"failed": 0,
"pending_bytes": 0,
}
for row in rows:
fid = str(row["folder_id"])
status = str(row["status"])
cnt = int(row["cnt"])
total_bytes = int(row["total_bytes"])
bucket = folder_stats.setdefault(
fid,
{
"pending": 0,
"uploading": 0,
"synced": 0,
"failed": 0,
"pending_bytes": 0,
},
)
if status == "PENDING":
bucket["pending"] += cnt
bucket["pending_bytes"] += total_bytes
totals["pending"] += cnt
totals["pending_bytes"] += total_bytes
elif status == "UPLOADING":
bucket["pending"] += cnt
bucket["uploading"] += cnt
bucket["pending_bytes"] += total_bytes
totals["pending"] += cnt
totals["uploading"] += cnt
totals["pending_bytes"] += total_bytes
elif status == "SYNCED":
bucket["synced"] += cnt
totals["synced"] += cnt
elif status == "FAILED":
bucket["failed"] += cnt
totals["failed"] += cnt
meta_rows = self._conn.execute("SELECT key, value FROM meta").fetchall()
meta = {str(r["key"]): str(r["value"]) for r in meta_rows}
recent_logs = self.list_logs(limit=50)
folders_out = []
for wf in watched_folders:
stats = folder_stats.get(
wf.id,
{"pending": 0, "uploading": 0, "synced": 0, "failed": 0, "pending_bytes": 0},
)
folders_out.append(
{
"id": wf.id,
"path": wf.path,
"label": wf.label,
"enabled": wf.enabled,
"exists": os.path.isdir(os.path.expanduser(wf.path)),
"pending": stats["pending"],
"uploading": stats["uploading"],
"synced": stats["synced"],
"failed": stats["failed"],
"pending_bytes": stats["pending_bytes"],
}
)
return {
"counts": totals,
"folders": folders_out,
"meta": meta,
"logs": recent_logs,
}
def close(self) -> None:
"""Close the SQLite connection."""
with self._lock:
if not self._closed:
self._conn.close()
self._closed = True

View File

@@ -0,0 +1,303 @@
"""Unit tests for device-daemon configuration and SQLite WAL queue store."""
from __future__ import annotations
import json
import os
import sqlite3
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from cp0_syncd.config import (
DaemonConfig,
WatchedFolder,
add_watched_folder,
default_config_path,
load_daemon_config,
remove_watched_folder,
save_daemon_config,
)
from cp0_syncd.store import (
SyncStore,
compute_sha256,
default_db_path,
is_media_file,
)
class TestDaemonConfig(unittest.TestCase):
def test_first_run_creates_default_camera_folder_and_theme(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
home = Path(tmp) / "home"
cfg_path = Path(tmp) / "config.json"
cfg = load_daemon_config(cfg_path, home_dir=home)
self.assertEqual(cfg.theme, "dark")
self.assertEqual(cfg.target_port, 8765)
self.assertEqual(cfg.target_host, "")
self.assertEqual(len(cfg.watched_folders), 1)
self.assertEqual(
cfg.watched_folders[0].path,
str(home / "Pictures" / "DCIM" / "Camera"),
)
self.assertEqual(cfg.watched_folders[0].label, "Camera")
self.assertTrue(cfg.watched_folders[0].enabled)
self.assertFalse(hasattr(cfg, "auth_token"))
def test_corrupt_config_falls_back_to_defaults(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cfg_path = Path(tmp) / "config.json"
cfg_path.write_text("{corrupt", encoding="utf-8")
with self.assertLogs("cp0_syncd.config", level="WARNING"):
cfg = load_daemon_config(cfg_path, home_dir=Path(tmp))
self.assertEqual(cfg.theme, "dark")
self.assertEqual(cfg.target_port, 8765)
self.assertEqual(len(cfg.watched_folders), 1)
self.assertEqual(cfg_path.read_text(encoding="utf-8"), "{corrupt")
def test_save_and_load_roundtrip_preserves_empty_folders_and_theme(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cfg_path = Path(tmp) / "nested" / "config.json"
original = DaemonConfig(
target_host="my-mac.tailnet.ts.net",
target_port=9000,
theme="light",
watched_folders=[],
)
save_daemon_config(original, cfg_path)
raw = json.loads(cfg_path.read_text(encoding="utf-8"))
self.assertNotIn("auth_token", raw)
loaded = load_daemon_config(cfg_path, home_dir=Path(tmp))
self.assertEqual(loaded.target_host, "my-mac.tailnet.ts.net")
self.assertEqual(loaded.target_port, 9000)
self.assertEqual(loaded.theme, "light")
self.assertEqual(loaded.watched_folders, [])
def test_env_overrides_default_paths(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cfg_override = str(Path(tmp) / "custom-config.json")
db_override = str(Path(tmp) / "custom-state.db")
with mock.patch.dict(
os.environ,
{"CP0_SYNC_CONFIG": cfg_override, "CP0_SYNC_DB": db_override},
clear=False,
):
self.assertEqual(default_config_path(), Path(cfg_override))
self.assertEqual(default_db_path(), Path(db_override))
def test_overlap_rejected_and_duplicate_labels_disambiguated(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
d1 = Path(tmp) / "a" / "Camera"
d2 = Path(tmp) / "b" / "Camera"
d3 = Path(tmp) / "c" / "Camera"
child = d1 / "sub"
parent = Path(tmp) / "a"
d1.mkdir(parents=True)
d2.mkdir(parents=True)
d3.mkdir(parents=True)
child.mkdir()
cfg = DaemonConfig(watched_folders=[])
cfg, wf1 = add_watched_folder(cfg, str(d1))
cfg, wf2 = add_watched_folder(cfg, str(d2))
cfg, wf3 = add_watched_folder(cfg, str(d3))
self.assertEqual(wf1.label, "Camera")
self.assertEqual(wf2.label, "Camera-2")
self.assertEqual(wf3.label, "Camera-3")
with self.assertRaises(ValueError):
add_watched_folder(cfg, str(child))
with self.assertRaises(ValueError):
add_watched_folder(cfg, str(parent))
with self.assertRaises(ValueError):
add_watched_folder(cfg, str(d1))
symlink_d1 = Path(tmp) / "cam-link"
symlink_d1.symlink_to(d1)
with self.assertRaises(ValueError):
add_watched_folder(cfg, str(symlink_d1))
def test_add_nonexistent_or_file_raises_and_remove_folder(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
missing = Path(tmp) / "missing"
not_dir = Path(tmp) / "file.txt"
not_dir.write_text("hi", encoding="utf-8")
valid = Path(tmp) / "Screenshots"
valid.mkdir()
cfg = DaemonConfig(watched_folders=[])
with self.assertRaises(ValueError):
add_watched_folder(cfg, str(missing))
with self.assertRaises(ValueError):
add_watched_folder(cfg, str(not_dir))
cfg, wf = add_watched_folder(cfg, str(valid), label="Shots")
self.assertEqual(wf.label, "Shots")
self.assertEqual(len(cfg.watched_folders), 1)
cfg = remove_watched_folder(cfg, wf.id)
self.assertEqual(cfg.watched_folders, [])
class TestSyncStore(unittest.TestCase):
def test_dedup_zero_byte_skip_and_log_cap(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db = SyncStore(Path(tmp) / "state.db")
try:
img = Path(tmp) / "IMG_0001.jpg"
img.write_bytes(b"photo-bytes")
zero = Path(tmp) / "empty.jpg"
zero.write_bytes(b"")
cfg = DaemonConfig()
self.assertTrue(is_media_file(img, cfg))
with self.assertLogs("cp0_syncd.store", level="WARNING"):
self.assertFalse(is_media_file(zero, cfg))
sha = compute_sha256(img)
self.assertEqual(
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
"enqueued",
)
self.assertEqual(
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
"already_queued",
)
row = db.next_pending()
self.assertIsNotNone(row)
assert row is not None
db.mark_synced(row["id"])
self.assertEqual(
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
"already_synced",
)
# Different content at same path should enqueue cleanly
img.write_bytes(b"updated-photo-bytes")
sha2 = compute_sha256(img)
self.assertEqual(
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 19, 2, sha2),
"enqueued",
)
for i in range(550):
db.log("INFO", f"msg {i}")
self.assertEqual(db.log_count(), 500)
recent = db.list_logs(limit=10)
self.assertEqual(len(recent), 10)
self.assertEqual(recent[0]["message"], "msg 549")
finally:
db.close()
def test_wal_mode_and_enqueue_zero_byte_logs_warn(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "state.db"
db = SyncStore(db_path)
try:
conn = sqlite3.connect(str(db_path))
mode = conn.execute("PRAGMA journal_mode;").fetchone()[0]
conn.close()
self.assertEqual(mode.lower(), "wal")
status = db.enqueue_file("cam", "/tmp/zero.jpg", "zero.jpg", 0, 1, "0" * 64)
self.assertEqual(status, "skipped_empty")
logs = db.list_logs(limit=5)
self.assertEqual(len(logs), 1)
self.assertEqual(logs[0]["level"], "WARN")
finally:
db.close()
def test_media_filtering_rules(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cfg = DaemonConfig()
for name, expected in [
("IMG_1.JPEG", True),
("clip.mp4", True),
(".hidden.jpg", False),
("~temp.png", False),
("photo.jpg.part", False),
("photo.jpg.tmp", False),
("photo.jpg.crdownload", False),
("notes.txt", False),
]:
p = Path(tmp) / name
p.write_bytes(b"abc")
self.assertEqual(is_media_file(p, cfg), expected, msg=name)
subdir = Path(tmp) / "sub.jpg"
subdir.mkdir()
self.assertFalse(is_media_file(subdir, cfg))
self.assertFalse(is_media_file(Path(tmp) / "nonexistent.jpg", cfg))
def test_queue_lifecycle_meta_and_status_snapshot(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
cam_dir = Path(tmp) / "Camera"
cam_dir.mkdir()
missing_dir = Path(tmp) / "SDCard"
db = SyncStore(Path(tmp) / "state.db")
try:
db.enqueue_file("cam", str(cam_dir / "a.jpg"), "a.jpg", 100, 10, "a" * 64)
db.enqueue_file("cam", str(cam_dir / "b.jpg"), "b.jpg", 200, 20, "b" * 64)
db.enqueue_file("sd", str(missing_dir / "c.jpg"), "c.jpg", 300, 30, "c" * 64)
first = db.next_pending()
assert first is not None
self.assertEqual(first["rel_path"], "a.jpg")
db.mark_uploading(first["id"])
db.mark_retry(first["id"], "timeout")
# Re-fetch after retry: still PENDING with retry_count == 1
again = db.next_pending()
assert again is not None
self.assertEqual(again["id"], first["id"])
self.assertEqual(again["retry_count"], 1)
self.assertEqual(again["last_error"], "timeout")
db.mark_synced(again["id"])
second = db.next_pending()
assert second is not None
self.assertEqual(second["rel_path"], "b.jpg")
db.mark_failed(second["id"], "413 too large")
# Meta roundtrip
self.assertIsNone(db.get_meta("mac_online"))
db.set_meta("mac_online", "true")
db.set_meta("mac_online", "false")
self.assertEqual(db.get_meta("mac_online"), "false")
folders = [
WatchedFolder(id="cam", path=str(cam_dir), label="Camera"),
WatchedFolder(id="sd", path=str(missing_dir), label="SDCard"),
]
snap = db.get_status_snapshot(folders)
self.assertEqual(snap["counts"]["synced"], 1)
self.assertEqual(snap["counts"]["failed"], 1)
self.assertEqual(snap["counts"]["pending"], 1)
self.assertEqual(snap["counts"]["pending_bytes"], 300)
self.assertEqual(snap["meta"]["mac_online"], "false")
self.assertEqual(len(snap["folders"]), 2)
self.assertTrue(snap["folders"][0]["exists"])
self.assertFalse(snap["folders"][1]["exists"])
self.assertEqual(snap["folders"][0]["synced"], 1)
self.assertEqual(snap["folders"][0]["failed"], 1)
self.assertEqual(snap["folders"][1]["pending"], 1)
# retry_all_failed flips FAILED -> PENDING
self.assertEqual(db.retry_all_failed(), 1)
# purge_pending_for_folder deletes pending for sd without touching cam
self.assertEqual(db.purge_pending_for_folder("sd"), 1)
# delete_file removes remaining pending b.jpg
rem = db.next_pending()
assert rem is not None
self.assertEqual(rem["rel_path"], "b.jpg")
db.delete_file(rem["id"])
self.assertIsNone(db.next_pending())
finally:
db.close()
if __name__ == "__main__":
unittest.main()