mirror of
https://github.com/cottongin/cardputerzero-sync.git
synced 2026-11-20 14:44:47 -06:00
feat(device-daemon): add daemon config, overlap guard, and SQLite WAL queue store
This commit is contained in:
31
device-daemon/cp0_syncd/__init__.py
Normal file
31
device-daemon/cp0_syncd/__init__.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""CardputerZero on-device background photo sync daemon package."""
|
||||
|
||||
from .config import (
|
||||
DaemonConfig,
|
||||
WatchedFolder,
|
||||
add_watched_folder,
|
||||
default_config_path,
|
||||
load_daemon_config,
|
||||
remove_watched_folder,
|
||||
save_daemon_config,
|
||||
)
|
||||
from .store import (
|
||||
SyncStore,
|
||||
compute_sha256,
|
||||
default_db_path,
|
||||
is_media_file,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DaemonConfig",
|
||||
"SyncStore",
|
||||
"WatchedFolder",
|
||||
"add_watched_folder",
|
||||
"compute_sha256",
|
||||
"default_config_path",
|
||||
"default_db_path",
|
||||
"is_media_file",
|
||||
"load_daemon_config",
|
||||
"remove_watched_folder",
|
||||
"save_daemon_config",
|
||||
]
|
||||
266
device-daemon/cp0_syncd/config.py
Normal file
266
device-daemon/cp0_syncd/config.py
Normal file
@@ -0,0 +1,266 @@
|
||||
"""Configuration persistence and watched-folder management for cp0-syncd."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from dataclasses import asdict, dataclass, field, replace
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ENV_CONFIG_PATH = "CP0_SYNC_CONFIG"
|
||||
DEFAULT_TARGET_PORT = 8765
|
||||
DEFAULT_THEME = "dark"
|
||||
VALID_THEMES = ("dark", "light")
|
||||
DEFAULT_EXTENSIONS = [
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".png",
|
||||
".webp",
|
||||
".gif",
|
||||
".bmp",
|
||||
".dng",
|
||||
".raw",
|
||||
".mp4",
|
||||
".mov",
|
||||
]
|
||||
DEFAULT_IGNORE_PREFIXES = [".", "~"]
|
||||
DEFAULT_IGNORE_SUFFIXES = [".part", ".tmp", ".crdownload"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class WatchedFolder:
|
||||
"""A single local directory watched by cp0-syncd."""
|
||||
|
||||
id: str
|
||||
path: str
|
||||
label: str
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class DaemonConfig:
|
||||
"""Runtime configuration for the on-device sync daemon and companion UI."""
|
||||
|
||||
target_host: str = ""
|
||||
target_port: int = DEFAULT_TARGET_PORT
|
||||
theme: str = DEFAULT_THEME
|
||||
watched_folders: list[WatchedFolder] = field(default_factory=list)
|
||||
extensions: list[str] = field(default_factory=lambda: list(DEFAULT_EXTENSIONS))
|
||||
ignore_prefixes: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE_PREFIXES))
|
||||
ignore_suffixes: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE_SUFFIXES))
|
||||
|
||||
|
||||
def default_config_path() -> Path:
|
||||
"""Return the default daemon config path, honoring CP0_SYNC_CONFIG if set."""
|
||||
override = os.environ.get(ENV_CONFIG_PATH, "").strip()
|
||||
if override:
|
||||
return Path(override).expanduser()
|
||||
return Path.home() / ".config" / "cp0-sync" / "config.json"
|
||||
|
||||
|
||||
def default_watched_folders(home_dir: Path | None = None) -> list[WatchedFolder]:
|
||||
"""Return the first-run default watched folder list (<home>/Pictures/DCIM/Camera)."""
|
||||
base_home = Path(home_dir).expanduser() if home_dir is not None else Path.home()
|
||||
camera_path = base_home / "Pictures" / "DCIM" / "Camera"
|
||||
return [
|
||||
WatchedFolder(
|
||||
id="camera",
|
||||
path=str(camera_path),
|
||||
label="Camera",
|
||||
enabled=True,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _resolve_config_path(path: Path | None) -> Path:
|
||||
return Path(path).expanduser() if path is not None else default_config_path()
|
||||
|
||||
|
||||
def _valid_port(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535
|
||||
|
||||
|
||||
def _valid_str_list(value: object) -> list[str] | None:
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
cleaned = [item for item in value if isinstance(item, str) and item]
|
||||
return cleaned if len(cleaned) == len(value) else None
|
||||
|
||||
|
||||
def _parse_watched_folders(raw_folders: object, home_dir: Path | None) -> list[WatchedFolder]:
|
||||
if not isinstance(raw_folders, list):
|
||||
return default_watched_folders(home_dir)
|
||||
|
||||
parsed: list[WatchedFolder] = []
|
||||
for item in raw_folders:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
folder_id = item.get("id")
|
||||
folder_path = item.get("path")
|
||||
label = item.get("label")
|
||||
enabled = item.get("enabled", True)
|
||||
if (
|
||||
isinstance(folder_id, str)
|
||||
and folder_id.strip()
|
||||
and isinstance(folder_path, str)
|
||||
and folder_path.strip()
|
||||
and isinstance(label, str)
|
||||
and label.strip()
|
||||
and isinstance(enabled, bool)
|
||||
):
|
||||
parsed.append(
|
||||
WatchedFolder(
|
||||
id=folder_id.strip(),
|
||||
path=folder_path.strip(),
|
||||
label=label.strip(),
|
||||
enabled=enabled,
|
||||
)
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
def _default_daemon_config(home_dir: Path | None = None) -> DaemonConfig:
|
||||
return DaemonConfig(watched_folders=default_watched_folders(home_dir))
|
||||
|
||||
|
||||
def load_daemon_config(path: Path | None = None, home_dir: Path | None = None) -> DaemonConfig:
|
||||
"""Load DaemonConfig from JSON, creating first-run defaults or safely falling back on errors."""
|
||||
cfg_path = _resolve_config_path(path)
|
||||
defaults = _default_daemon_config(home_dir)
|
||||
|
||||
if not cfg_path.exists():
|
||||
try:
|
||||
save_daemon_config(defaults, cfg_path)
|
||||
except OSError as exc:
|
||||
logger.warning("Could not persist first-run config %s (%s); continuing in memory", cfg_path, exc)
|
||||
return defaults
|
||||
|
||||
try:
|
||||
raw = json.loads(cfg_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Failed to read config %s (%s); using defaults", cfg_path, exc)
|
||||
return defaults
|
||||
|
||||
if not isinstance(raw, dict):
|
||||
logger.warning("Config %s is not a JSON object; using defaults", cfg_path)
|
||||
return defaults
|
||||
|
||||
target_host = raw.get("target_host")
|
||||
target_port = raw.get("target_port")
|
||||
theme = raw.get("theme")
|
||||
extensions = _valid_str_list(raw.get("extensions"))
|
||||
ignore_prefixes = _valid_str_list(raw.get("ignore_prefixes"))
|
||||
ignore_suffixes = _valid_str_list(raw.get("ignore_suffixes"))
|
||||
watched_folders = (
|
||||
_parse_watched_folders(raw.get("watched_folders"), home_dir)
|
||||
if "watched_folders" in raw
|
||||
else defaults.watched_folders
|
||||
)
|
||||
|
||||
return DaemonConfig(
|
||||
target_host=target_host.strip() if isinstance(target_host, str) else defaults.target_host,
|
||||
target_port=target_port if _valid_port(target_port) else defaults.target_port,
|
||||
theme=theme if isinstance(theme, str) and theme in VALID_THEMES else defaults.theme,
|
||||
watched_folders=watched_folders,
|
||||
extensions=extensions if extensions is not None else defaults.extensions,
|
||||
ignore_prefixes=ignore_prefixes if ignore_prefixes is not None else defaults.ignore_prefixes,
|
||||
ignore_suffixes=ignore_suffixes if ignore_suffixes is not None else defaults.ignore_suffixes,
|
||||
)
|
||||
|
||||
|
||||
def save_daemon_config(cfg: DaemonConfig, path: Path | None = None) -> None:
|
||||
"""Atomically persist DaemonConfig to disk using a temp file, fsync, and os.replace."""
|
||||
cfg_path = _resolve_config_path(path)
|
||||
cfg_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = cfg_path.parent / f".{cfg_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
|
||||
payload = json.dumps(asdict(cfg), indent=2) + "\n"
|
||||
try:
|
||||
with open(tmp_path, "w", encoding="utf-8") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp_path, cfg_path)
|
||||
except BaseException:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _canonical_path(raw_path: str) -> Path:
|
||||
expanded = os.path.expanduser(raw_path)
|
||||
return Path(os.path.realpath(expanded))
|
||||
|
||||
|
||||
def _paths_overlap(a: Path, b: Path) -> bool:
|
||||
return a == b or a in b.parents or b in a.parents
|
||||
|
||||
|
||||
def _disambiguate_label(base_label: str, existing_labels: set[str]) -> str:
|
||||
if base_label.lower() not in existing_labels:
|
||||
return base_label
|
||||
suffix = 2
|
||||
while True:
|
||||
candidate = f"{base_label}-{suffix}"
|
||||
if candidate.lower() not in existing_labels:
|
||||
return candidate
|
||||
suffix += 1
|
||||
|
||||
|
||||
def _make_folder_id(label: str, existing_ids: set[str]) -> str:
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", label.lower()).strip("-") or "folder"
|
||||
while True:
|
||||
candidate = f"{slug}-{uuid.uuid4().hex[:8]}"
|
||||
if candidate not in existing_ids:
|
||||
return candidate
|
||||
|
||||
|
||||
def add_watched_folder(
|
||||
cfg: DaemonConfig,
|
||||
folder_path: str,
|
||||
label: str | None = None,
|
||||
) -> tuple[DaemonConfig, WatchedFolder]:
|
||||
"""Validate and add a watched folder, rejecting missing/overlapping paths and disambiguating labels."""
|
||||
if not isinstance(folder_path, str) or not folder_path.strip():
|
||||
raise ValueError("Folder path must be a non-empty string")
|
||||
|
||||
candidate = _canonical_path(folder_path.strip())
|
||||
if not candidate.exists() or not candidate.is_dir():
|
||||
raise ValueError(f"Watched directory does not exist: {folder_path}")
|
||||
|
||||
for existing in cfg.watched_folders:
|
||||
existing_real = _canonical_path(existing.path)
|
||||
if _paths_overlap(candidate, existing_real):
|
||||
raise ValueError(
|
||||
f"Folder {candidate} overlaps already-watched path {existing_real}"
|
||||
)
|
||||
|
||||
base_label = label.strip() if isinstance(label, str) and label.strip() else (candidate.name or "Folder")
|
||||
existing_labels = {wf.label.lower() for wf in cfg.watched_folders}
|
||||
unique_label = _disambiguate_label(base_label, existing_labels)
|
||||
|
||||
existing_ids = {wf.id for wf in cfg.watched_folders}
|
||||
folder_id = _make_folder_id(unique_label, existing_ids)
|
||||
|
||||
new_folder = WatchedFolder(
|
||||
id=folder_id,
|
||||
path=str(candidate),
|
||||
label=unique_label,
|
||||
enabled=True,
|
||||
)
|
||||
updated_cfg = replace(cfg, watched_folders=[*cfg.watched_folders, new_folder])
|
||||
return updated_cfg, new_folder
|
||||
|
||||
|
||||
def remove_watched_folder(cfg: DaemonConfig, folder_id: str) -> DaemonConfig:
|
||||
"""Return a copy of DaemonConfig without the specified watched folder ID."""
|
||||
return replace(
|
||||
cfg,
|
||||
watched_folders=[wf for wf in cfg.watched_folders if wf.id != folder_id],
|
||||
)
|
||||
439
device-daemon/cp0_syncd/store.py
Normal file
439
device-daemon/cp0_syncd/store.py
Normal file
@@ -0,0 +1,439 @@
|
||||
"""SQLite WAL queue store and media file filtering for cp0-syncd."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import stat
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import DaemonConfig, WatchedFolder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ENV_DB_PATH = "CP0_SYNC_DB"
|
||||
MAX_LOG_ROWS = 500
|
||||
CHUNK_SIZE = 65536
|
||||
|
||||
|
||||
def default_db_path() -> Path:
|
||||
"""Return the default SQLite state database path, honoring CP0_SYNC_DB if set."""
|
||||
override = os.environ.get(ENV_DB_PATH, "").strip()
|
||||
if override:
|
||||
return Path(override).expanduser()
|
||||
return Path.home() / ".local" / "share" / "cp0-sync" / "state.db"
|
||||
|
||||
|
||||
def compute_sha256(path: Path, chunk_size: int = CHUNK_SIZE) -> str:
|
||||
"""Compute the lowercase hex SHA-256 digest of a file incrementally."""
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def is_media_file(
|
||||
path: Path,
|
||||
cfg: DaemonConfig,
|
||||
store: SyncStore | None = None,
|
||||
) -> bool:
|
||||
"""Return True if path is a non-empty regular media file permitted by DaemonConfig."""
|
||||
name = path.name
|
||||
if not name:
|
||||
return False
|
||||
|
||||
if any(name.startswith(prefix) for prefix in cfg.ignore_prefixes if prefix):
|
||||
return False
|
||||
|
||||
name_lower = name.lower()
|
||||
if any(name_lower.endswith(suffix.lower()) for suffix in cfg.ignore_suffixes if suffix):
|
||||
return False
|
||||
|
||||
allowed_exts = {
|
||||
ext.lower() if ext.startswith(".") else f".{ext.lower()}"
|
||||
for ext in cfg.extensions
|
||||
if ext
|
||||
}
|
||||
if path.suffix.lower() not in allowed_exts:
|
||||
return False
|
||||
|
||||
try:
|
||||
st = path.stat()
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
if not stat.S_ISREG(st.st_mode):
|
||||
return False
|
||||
|
||||
if st.st_size == 0:
|
||||
logger.warning("Skipping 0-byte media file: %s", path)
|
||||
if store is not None:
|
||||
store.log("WARN", f"Skipped 0-byte file: {name}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _now_ms() -> int:
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
class SyncStore:
|
||||
"""Thread-safe SQLite WAL queue, metadata, and ring-buffered log store."""
|
||||
|
||||
def __init__(self, db_path: Path | None = None) -> None:
|
||||
resolved = Path(db_path).expanduser() if db_path is not None else default_db_path()
|
||||
resolved.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.db_path = resolved
|
||||
self._lock = threading.RLock()
|
||||
self._conn = sqlite3.connect(str(resolved), timeout=5.0, check_same_thread=False)
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
self._closed = False
|
||||
self._init_schema()
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._lock:
|
||||
self._conn.execute("PRAGMA journal_mode=WAL;")
|
||||
self._conn.execute("PRAGMA busy_timeout=5000;")
|
||||
self._conn.execute("PRAGMA synchronous=NORMAL;")
|
||||
self._conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
folder_id TEXT NOT NULL,
|
||||
abs_path TEXT NOT NULL,
|
||||
rel_path TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
mtime_ns INTEGER NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
discovered_at INTEGER NOT NULL,
|
||||
synced_at INTEGER
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_files_dedup
|
||||
ON files(folder_id, rel_path, sha256);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_status_discovered
|
||||
ON files(status, discovered_at, id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts INTEGER NOT NULL,
|
||||
level TEXT NOT NULL,
|
||||
message TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def enqueue_file(
|
||||
self,
|
||||
folder_id: str,
|
||||
abs_path: str,
|
||||
rel_path: str,
|
||||
size_bytes: int,
|
||||
mtime_ns: int,
|
||||
sha256: str,
|
||||
) -> str:
|
||||
"""Enqueue a file if not already synced/queued; skip 0-byte payloads with a WARN log."""
|
||||
if size_bytes <= 0:
|
||||
self.log("WARN", f"Skipped 0-byte file: {rel_path}")
|
||||
return "skipped_empty"
|
||||
|
||||
now_ms = _now_ms()
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"""
|
||||
SELECT id, status
|
||||
FROM files
|
||||
WHERE folder_id = ? AND rel_path = ? AND sha256 = ?
|
||||
""",
|
||||
(folder_id, rel_path, sha256),
|
||||
).fetchone()
|
||||
|
||||
if row is not None:
|
||||
status = str(row["status"])
|
||||
if status == "SYNCED":
|
||||
return "already_synced"
|
||||
if status == "FAILED":
|
||||
return "already_failed"
|
||||
return "already_queued"
|
||||
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT INTO files (
|
||||
folder_id, abs_path, rel_path, size_bytes, mtime_ns,
|
||||
sha256, status, retry_count, last_error, discovered_at, synced_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'PENDING', 0, NULL, ?, NULL)
|
||||
""",
|
||||
(folder_id, abs_path, rel_path, size_bytes, mtime_ns, sha256, now_ms),
|
||||
)
|
||||
self._conn.commit()
|
||||
return "enqueued"
|
||||
|
||||
def next_pending(self) -> dict[str, Any] | None:
|
||||
"""Return the oldest PENDING file row as a dict, or None if the queue is empty."""
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"""
|
||||
SELECT id, folder_id, abs_path, rel_path, size_bytes, mtime_ns,
|
||||
sha256, status, retry_count, last_error, discovered_at, synced_at
|
||||
FROM files
|
||||
WHERE status = 'PENDING'
|
||||
ORDER BY discovered_at ASC, id ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
def mark_uploading(self, file_id: int) -> None:
|
||||
"""Transition a queued file to UPLOADING."""
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"UPDATE files SET status = 'UPLOADING' WHERE id = ?",
|
||||
(file_id,),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def mark_synced(self, file_id: int) -> None:
|
||||
"""Mark a file as SYNCED and clear any prior error."""
|
||||
now_ms = _now_ms()
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE files
|
||||
SET status = 'SYNCED', synced_at = ?, last_error = NULL
|
||||
WHERE id = ?
|
||||
""",
|
||||
(now_ms, file_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def mark_retry(self, file_id: int, err: str) -> None:
|
||||
"""Return an item to PENDING while recording the retry count and error."""
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE files
|
||||
SET status = 'PENDING', retry_count = retry_count + 1, last_error = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(err, file_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def mark_failed(self, file_id: int, err: str) -> None:
|
||||
"""Mark a file as FAILED after a permanent per-file rejection."""
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE files
|
||||
SET status = 'FAILED', retry_count = retry_count + 1, last_error = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(err, file_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def delete_file(self, file_id: int) -> None:
|
||||
"""Delete a file record (e.g., when the source file vanished before upload)."""
|
||||
with self._lock:
|
||||
self._conn.execute("DELETE FROM files WHERE id = ?", (file_id,))
|
||||
self._conn.commit()
|
||||
|
||||
def purge_pending_for_folder(self, folder_id: str) -> int:
|
||||
"""Delete un-synced PENDING/UPLOADING queue entries for a removed folder."""
|
||||
with self._lock:
|
||||
cur = self._conn.execute(
|
||||
"""
|
||||
DELETE FROM files
|
||||
WHERE folder_id = ? AND status IN ('PENDING', 'UPLOADING')
|
||||
""",
|
||||
(folder_id,),
|
||||
)
|
||||
self._conn.commit()
|
||||
return cur.rowcount
|
||||
|
||||
def retry_all_failed(self) -> int:
|
||||
"""Flip all FAILED rows back to PENDING and return the count updated."""
|
||||
with self._lock:
|
||||
cur = self._conn.execute(
|
||||
"""
|
||||
UPDATE files
|
||||
SET status = 'PENDING', last_error = NULL
|
||||
WHERE status = 'FAILED'
|
||||
"""
|
||||
)
|
||||
self._conn.commit()
|
||||
return cur.rowcount
|
||||
|
||||
def log(self, level: str, message: str) -> None:
|
||||
"""Append a log row and prune the logs table to the newest 500 rows."""
|
||||
now_ms = _now_ms()
|
||||
normalized_level = level.strip().upper() or "INFO"
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"INSERT INTO logs (ts, level, message) VALUES (?, ?, ?)",
|
||||
(now_ms, normalized_level, message),
|
||||
)
|
||||
self._conn.execute(
|
||||
"""
|
||||
DELETE FROM logs
|
||||
WHERE id NOT IN (
|
||||
SELECT id FROM logs ORDER BY id DESC LIMIT ?
|
||||
)
|
||||
""",
|
||||
(MAX_LOG_ROWS,),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def log_count(self) -> int:
|
||||
"""Return the number of rows currently stored in the logs table."""
|
||||
with self._lock:
|
||||
row = self._conn.execute("SELECT COUNT(*) FROM logs").fetchone()
|
||||
return int(row[0]) if row is not None else 0
|
||||
|
||||
def list_logs(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||
"""Return up to `limit` newest log rows ordered newest-first."""
|
||||
safe_limit = max(0, int(limit))
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT id, ts, level, message
|
||||
FROM logs
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(safe_limit,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def set_meta(self, key: str, value: str) -> None:
|
||||
"""Upsert a metadata key-value pair."""
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"""
|
||||
INSERT INTO meta (key, value) VALUES (?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||
""",
|
||||
(key, str(value)),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def get_meta(self, key: str) -> str | None:
|
||||
"""Fetch a metadata value by key, or None if unset."""
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT value FROM meta WHERE key = ?",
|
||||
(key,),
|
||||
).fetchone()
|
||||
return str(row["value"]) if row is not None else None
|
||||
|
||||
def get_status_snapshot(self, watched_folders: list[WatchedFolder]) -> dict[str, Any]:
|
||||
"""Return queue totals, per-folder counts, metadata, and recent logs for status IPC/UI."""
|
||||
with self._lock:
|
||||
folder_stats: dict[str, dict[str, int]] = {}
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT folder_id, status, COUNT(*) AS cnt, COALESCE(SUM(size_bytes), 0) AS total_bytes
|
||||
FROM files
|
||||
GROUP BY folder_id, status
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
totals = {
|
||||
"pending": 0,
|
||||
"uploading": 0,
|
||||
"synced": 0,
|
||||
"failed": 0,
|
||||
"pending_bytes": 0,
|
||||
}
|
||||
|
||||
for row in rows:
|
||||
fid = str(row["folder_id"])
|
||||
status = str(row["status"])
|
||||
cnt = int(row["cnt"])
|
||||
total_bytes = int(row["total_bytes"])
|
||||
bucket = folder_stats.setdefault(
|
||||
fid,
|
||||
{
|
||||
"pending": 0,
|
||||
"uploading": 0,
|
||||
"synced": 0,
|
||||
"failed": 0,
|
||||
"pending_bytes": 0,
|
||||
},
|
||||
)
|
||||
if status == "PENDING":
|
||||
bucket["pending"] += cnt
|
||||
bucket["pending_bytes"] += total_bytes
|
||||
totals["pending"] += cnt
|
||||
totals["pending_bytes"] += total_bytes
|
||||
elif status == "UPLOADING":
|
||||
bucket["pending"] += cnt
|
||||
bucket["uploading"] += cnt
|
||||
bucket["pending_bytes"] += total_bytes
|
||||
totals["pending"] += cnt
|
||||
totals["uploading"] += cnt
|
||||
totals["pending_bytes"] += total_bytes
|
||||
elif status == "SYNCED":
|
||||
bucket["synced"] += cnt
|
||||
totals["synced"] += cnt
|
||||
elif status == "FAILED":
|
||||
bucket["failed"] += cnt
|
||||
totals["failed"] += cnt
|
||||
|
||||
meta_rows = self._conn.execute("SELECT key, value FROM meta").fetchall()
|
||||
meta = {str(r["key"]): str(r["value"]) for r in meta_rows}
|
||||
recent_logs = self.list_logs(limit=50)
|
||||
|
||||
folders_out = []
|
||||
for wf in watched_folders:
|
||||
stats = folder_stats.get(
|
||||
wf.id,
|
||||
{"pending": 0, "uploading": 0, "synced": 0, "failed": 0, "pending_bytes": 0},
|
||||
)
|
||||
folders_out.append(
|
||||
{
|
||||
"id": wf.id,
|
||||
"path": wf.path,
|
||||
"label": wf.label,
|
||||
"enabled": wf.enabled,
|
||||
"exists": os.path.isdir(os.path.expanduser(wf.path)),
|
||||
"pending": stats["pending"],
|
||||
"uploading": stats["uploading"],
|
||||
"synced": stats["synced"],
|
||||
"failed": stats["failed"],
|
||||
"pending_bytes": stats["pending_bytes"],
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"counts": totals,
|
||||
"folders": folders_out,
|
||||
"meta": meta,
|
||||
"logs": recent_logs,
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the SQLite connection."""
|
||||
with self._lock:
|
||||
if not self._closed:
|
||||
self._conn.close()
|
||||
self._closed = True
|
||||
303
device-daemon/tests/test_store_and_config.py
Normal file
303
device-daemon/tests/test_store_and_config.py
Normal file
@@ -0,0 +1,303 @@
|
||||
"""Unit tests for device-daemon configuration and SQLite WAL queue store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from cp0_syncd.config import (
|
||||
DaemonConfig,
|
||||
WatchedFolder,
|
||||
add_watched_folder,
|
||||
default_config_path,
|
||||
load_daemon_config,
|
||||
remove_watched_folder,
|
||||
save_daemon_config,
|
||||
)
|
||||
from cp0_syncd.store import (
|
||||
SyncStore,
|
||||
compute_sha256,
|
||||
default_db_path,
|
||||
is_media_file,
|
||||
)
|
||||
|
||||
|
||||
class TestDaemonConfig(unittest.TestCase):
|
||||
def test_first_run_creates_default_camera_folder_and_theme(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
home = Path(tmp) / "home"
|
||||
cfg_path = Path(tmp) / "config.json"
|
||||
cfg = load_daemon_config(cfg_path, home_dir=home)
|
||||
self.assertEqual(cfg.theme, "dark")
|
||||
self.assertEqual(cfg.target_port, 8765)
|
||||
self.assertEqual(cfg.target_host, "")
|
||||
self.assertEqual(len(cfg.watched_folders), 1)
|
||||
self.assertEqual(
|
||||
cfg.watched_folders[0].path,
|
||||
str(home / "Pictures" / "DCIM" / "Camera"),
|
||||
)
|
||||
self.assertEqual(cfg.watched_folders[0].label, "Camera")
|
||||
self.assertTrue(cfg.watched_folders[0].enabled)
|
||||
self.assertFalse(hasattr(cfg, "auth_token"))
|
||||
|
||||
def test_corrupt_config_falls_back_to_defaults(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg_path = Path(tmp) / "config.json"
|
||||
cfg_path.write_text("{corrupt", encoding="utf-8")
|
||||
with self.assertLogs("cp0_syncd.config", level="WARNING"):
|
||||
cfg = load_daemon_config(cfg_path, home_dir=Path(tmp))
|
||||
self.assertEqual(cfg.theme, "dark")
|
||||
self.assertEqual(cfg.target_port, 8765)
|
||||
self.assertEqual(len(cfg.watched_folders), 1)
|
||||
self.assertEqual(cfg_path.read_text(encoding="utf-8"), "{corrupt")
|
||||
|
||||
def test_save_and_load_roundtrip_preserves_empty_folders_and_theme(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg_path = Path(tmp) / "nested" / "config.json"
|
||||
original = DaemonConfig(
|
||||
target_host="my-mac.tailnet.ts.net",
|
||||
target_port=9000,
|
||||
theme="light",
|
||||
watched_folders=[],
|
||||
)
|
||||
save_daemon_config(original, cfg_path)
|
||||
raw = json.loads(cfg_path.read_text(encoding="utf-8"))
|
||||
self.assertNotIn("auth_token", raw)
|
||||
|
||||
loaded = load_daemon_config(cfg_path, home_dir=Path(tmp))
|
||||
self.assertEqual(loaded.target_host, "my-mac.tailnet.ts.net")
|
||||
self.assertEqual(loaded.target_port, 9000)
|
||||
self.assertEqual(loaded.theme, "light")
|
||||
self.assertEqual(loaded.watched_folders, [])
|
||||
|
||||
def test_env_overrides_default_paths(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg_override = str(Path(tmp) / "custom-config.json")
|
||||
db_override = str(Path(tmp) / "custom-state.db")
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{"CP0_SYNC_CONFIG": cfg_override, "CP0_SYNC_DB": db_override},
|
||||
clear=False,
|
||||
):
|
||||
self.assertEqual(default_config_path(), Path(cfg_override))
|
||||
self.assertEqual(default_db_path(), Path(db_override))
|
||||
|
||||
def test_overlap_rejected_and_duplicate_labels_disambiguated(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
d1 = Path(tmp) / "a" / "Camera"
|
||||
d2 = Path(tmp) / "b" / "Camera"
|
||||
d3 = Path(tmp) / "c" / "Camera"
|
||||
child = d1 / "sub"
|
||||
parent = Path(tmp) / "a"
|
||||
d1.mkdir(parents=True)
|
||||
d2.mkdir(parents=True)
|
||||
d3.mkdir(parents=True)
|
||||
child.mkdir()
|
||||
|
||||
cfg = DaemonConfig(watched_folders=[])
|
||||
cfg, wf1 = add_watched_folder(cfg, str(d1))
|
||||
cfg, wf2 = add_watched_folder(cfg, str(d2))
|
||||
cfg, wf3 = add_watched_folder(cfg, str(d3))
|
||||
self.assertEqual(wf1.label, "Camera")
|
||||
self.assertEqual(wf2.label, "Camera-2")
|
||||
self.assertEqual(wf3.label, "Camera-3")
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
add_watched_folder(cfg, str(child))
|
||||
with self.assertRaises(ValueError):
|
||||
add_watched_folder(cfg, str(parent))
|
||||
with self.assertRaises(ValueError):
|
||||
add_watched_folder(cfg, str(d1))
|
||||
|
||||
symlink_d1 = Path(tmp) / "cam-link"
|
||||
symlink_d1.symlink_to(d1)
|
||||
with self.assertRaises(ValueError):
|
||||
add_watched_folder(cfg, str(symlink_d1))
|
||||
|
||||
def test_add_nonexistent_or_file_raises_and_remove_folder(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
missing = Path(tmp) / "missing"
|
||||
not_dir = Path(tmp) / "file.txt"
|
||||
not_dir.write_text("hi", encoding="utf-8")
|
||||
valid = Path(tmp) / "Screenshots"
|
||||
valid.mkdir()
|
||||
|
||||
cfg = DaemonConfig(watched_folders=[])
|
||||
with self.assertRaises(ValueError):
|
||||
add_watched_folder(cfg, str(missing))
|
||||
with self.assertRaises(ValueError):
|
||||
add_watched_folder(cfg, str(not_dir))
|
||||
|
||||
cfg, wf = add_watched_folder(cfg, str(valid), label="Shots")
|
||||
self.assertEqual(wf.label, "Shots")
|
||||
self.assertEqual(len(cfg.watched_folders), 1)
|
||||
|
||||
cfg = remove_watched_folder(cfg, wf.id)
|
||||
self.assertEqual(cfg.watched_folders, [])
|
||||
|
||||
|
||||
class TestSyncStore(unittest.TestCase):
|
||||
def test_dedup_zero_byte_skip_and_log_cap(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db = SyncStore(Path(tmp) / "state.db")
|
||||
try:
|
||||
img = Path(tmp) / "IMG_0001.jpg"
|
||||
img.write_bytes(b"photo-bytes")
|
||||
zero = Path(tmp) / "empty.jpg"
|
||||
zero.write_bytes(b"")
|
||||
cfg = DaemonConfig()
|
||||
|
||||
self.assertTrue(is_media_file(img, cfg))
|
||||
with self.assertLogs("cp0_syncd.store", level="WARNING"):
|
||||
self.assertFalse(is_media_file(zero, cfg))
|
||||
|
||||
sha = compute_sha256(img)
|
||||
self.assertEqual(
|
||||
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
|
||||
"enqueued",
|
||||
)
|
||||
self.assertEqual(
|
||||
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
|
||||
"already_queued",
|
||||
)
|
||||
row = db.next_pending()
|
||||
self.assertIsNotNone(row)
|
||||
assert row is not None
|
||||
db.mark_synced(row["id"])
|
||||
self.assertEqual(
|
||||
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
|
||||
"already_synced",
|
||||
)
|
||||
|
||||
# Different content at same path should enqueue cleanly
|
||||
img.write_bytes(b"updated-photo-bytes")
|
||||
sha2 = compute_sha256(img)
|
||||
self.assertEqual(
|
||||
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 19, 2, sha2),
|
||||
"enqueued",
|
||||
)
|
||||
|
||||
for i in range(550):
|
||||
db.log("INFO", f"msg {i}")
|
||||
self.assertEqual(db.log_count(), 500)
|
||||
recent = db.list_logs(limit=10)
|
||||
self.assertEqual(len(recent), 10)
|
||||
self.assertEqual(recent[0]["message"], "msg 549")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def test_wal_mode_and_enqueue_zero_byte_logs_warn(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
db_path = Path(tmp) / "state.db"
|
||||
db = SyncStore(db_path)
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
mode = conn.execute("PRAGMA journal_mode;").fetchone()[0]
|
||||
conn.close()
|
||||
self.assertEqual(mode.lower(), "wal")
|
||||
|
||||
status = db.enqueue_file("cam", "/tmp/zero.jpg", "zero.jpg", 0, 1, "0" * 64)
|
||||
self.assertEqual(status, "skipped_empty")
|
||||
logs = db.list_logs(limit=5)
|
||||
self.assertEqual(len(logs), 1)
|
||||
self.assertEqual(logs[0]["level"], "WARN")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def test_media_filtering_rules(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg = DaemonConfig()
|
||||
for name, expected in [
|
||||
("IMG_1.JPEG", True),
|
||||
("clip.mp4", True),
|
||||
(".hidden.jpg", False),
|
||||
("~temp.png", False),
|
||||
("photo.jpg.part", False),
|
||||
("photo.jpg.tmp", False),
|
||||
("photo.jpg.crdownload", False),
|
||||
("notes.txt", False),
|
||||
]:
|
||||
p = Path(tmp) / name
|
||||
p.write_bytes(b"abc")
|
||||
self.assertEqual(is_media_file(p, cfg), expected, msg=name)
|
||||
|
||||
subdir = Path(tmp) / "sub.jpg"
|
||||
subdir.mkdir()
|
||||
self.assertFalse(is_media_file(subdir, cfg))
|
||||
self.assertFalse(is_media_file(Path(tmp) / "nonexistent.jpg", cfg))
|
||||
|
||||
def test_queue_lifecycle_meta_and_status_snapshot(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cam_dir = Path(tmp) / "Camera"
|
||||
cam_dir.mkdir()
|
||||
missing_dir = Path(tmp) / "SDCard"
|
||||
|
||||
db = SyncStore(Path(tmp) / "state.db")
|
||||
try:
|
||||
db.enqueue_file("cam", str(cam_dir / "a.jpg"), "a.jpg", 100, 10, "a" * 64)
|
||||
db.enqueue_file("cam", str(cam_dir / "b.jpg"), "b.jpg", 200, 20, "b" * 64)
|
||||
db.enqueue_file("sd", str(missing_dir / "c.jpg"), "c.jpg", 300, 30, "c" * 64)
|
||||
|
||||
first = db.next_pending()
|
||||
assert first is not None
|
||||
self.assertEqual(first["rel_path"], "a.jpg")
|
||||
db.mark_uploading(first["id"])
|
||||
db.mark_retry(first["id"], "timeout")
|
||||
|
||||
# Re-fetch after retry: still PENDING with retry_count == 1
|
||||
again = db.next_pending()
|
||||
assert again is not None
|
||||
self.assertEqual(again["id"], first["id"])
|
||||
self.assertEqual(again["retry_count"], 1)
|
||||
self.assertEqual(again["last_error"], "timeout")
|
||||
db.mark_synced(again["id"])
|
||||
|
||||
second = db.next_pending()
|
||||
assert second is not None
|
||||
self.assertEqual(second["rel_path"], "b.jpg")
|
||||
db.mark_failed(second["id"], "413 too large")
|
||||
|
||||
# Meta roundtrip
|
||||
self.assertIsNone(db.get_meta("mac_online"))
|
||||
db.set_meta("mac_online", "true")
|
||||
db.set_meta("mac_online", "false")
|
||||
self.assertEqual(db.get_meta("mac_online"), "false")
|
||||
|
||||
folders = [
|
||||
WatchedFolder(id="cam", path=str(cam_dir), label="Camera"),
|
||||
WatchedFolder(id="sd", path=str(missing_dir), label="SDCard"),
|
||||
]
|
||||
snap = db.get_status_snapshot(folders)
|
||||
self.assertEqual(snap["counts"]["synced"], 1)
|
||||
self.assertEqual(snap["counts"]["failed"], 1)
|
||||
self.assertEqual(snap["counts"]["pending"], 1)
|
||||
self.assertEqual(snap["counts"]["pending_bytes"], 300)
|
||||
self.assertEqual(snap["meta"]["mac_online"], "false")
|
||||
self.assertEqual(len(snap["folders"]), 2)
|
||||
self.assertTrue(snap["folders"][0]["exists"])
|
||||
self.assertFalse(snap["folders"][1]["exists"])
|
||||
self.assertEqual(snap["folders"][0]["synced"], 1)
|
||||
self.assertEqual(snap["folders"][0]["failed"], 1)
|
||||
self.assertEqual(snap["folders"][1]["pending"], 1)
|
||||
|
||||
# retry_all_failed flips FAILED -> PENDING
|
||||
self.assertEqual(db.retry_all_failed(), 1)
|
||||
# purge_pending_for_folder deletes pending for sd without touching cam
|
||||
self.assertEqual(db.purge_pending_for_folder("sd"), 1)
|
||||
# delete_file removes remaining pending b.jpg
|
||||
rem = db.next_pending()
|
||||
assert rem is not None
|
||||
self.assertEqual(rem["rel_path"], "b.jpg")
|
||||
db.delete_file(rem["id"])
|
||||
self.assertIsNone(db.next_pending())
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user