mirror of
https://github.com/cottongin/cardputerzero-sync.git
synced 2026-11-20 22:44:48 -06:00
feat(device-daemon): add daemon config, overlap guard, and SQLite WAL queue store
This commit is contained in:
31
device-daemon/cp0_syncd/__init__.py
Normal file
31
device-daemon/cp0_syncd/__init__.py
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
"""CardputerZero on-device background photo sync daemon package."""
|
||||||
|
|
||||||
|
from .config import (
|
||||||
|
DaemonConfig,
|
||||||
|
WatchedFolder,
|
||||||
|
add_watched_folder,
|
||||||
|
default_config_path,
|
||||||
|
load_daemon_config,
|
||||||
|
remove_watched_folder,
|
||||||
|
save_daemon_config,
|
||||||
|
)
|
||||||
|
from .store import (
|
||||||
|
SyncStore,
|
||||||
|
compute_sha256,
|
||||||
|
default_db_path,
|
||||||
|
is_media_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DaemonConfig",
|
||||||
|
"SyncStore",
|
||||||
|
"WatchedFolder",
|
||||||
|
"add_watched_folder",
|
||||||
|
"compute_sha256",
|
||||||
|
"default_config_path",
|
||||||
|
"default_db_path",
|
||||||
|
"is_media_file",
|
||||||
|
"load_daemon_config",
|
||||||
|
"remove_watched_folder",
|
||||||
|
"save_daemon_config",
|
||||||
|
]
|
||||||
266
device-daemon/cp0_syncd/config.py
Normal file
266
device-daemon/cp0_syncd/config.py
Normal file
@@ -0,0 +1,266 @@
|
|||||||
|
"""Configuration persistence and watched-folder management for cp0-syncd."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import uuid
|
||||||
|
from dataclasses import asdict, dataclass, field, replace
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
ENV_CONFIG_PATH = "CP0_SYNC_CONFIG"
|
||||||
|
DEFAULT_TARGET_PORT = 8765
|
||||||
|
DEFAULT_THEME = "dark"
|
||||||
|
VALID_THEMES = ("dark", "light")
|
||||||
|
DEFAULT_EXTENSIONS = [
|
||||||
|
".jpg",
|
||||||
|
".jpeg",
|
||||||
|
".png",
|
||||||
|
".webp",
|
||||||
|
".gif",
|
||||||
|
".bmp",
|
||||||
|
".dng",
|
||||||
|
".raw",
|
||||||
|
".mp4",
|
||||||
|
".mov",
|
||||||
|
]
|
||||||
|
DEFAULT_IGNORE_PREFIXES = [".", "~"]
|
||||||
|
DEFAULT_IGNORE_SUFFIXES = [".part", ".tmp", ".crdownload"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class WatchedFolder:
|
||||||
|
"""A single local directory watched by cp0-syncd."""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
path: str
|
||||||
|
label: str
|
||||||
|
enabled: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DaemonConfig:
|
||||||
|
"""Runtime configuration for the on-device sync daemon and companion UI."""
|
||||||
|
|
||||||
|
target_host: str = ""
|
||||||
|
target_port: int = DEFAULT_TARGET_PORT
|
||||||
|
theme: str = DEFAULT_THEME
|
||||||
|
watched_folders: list[WatchedFolder] = field(default_factory=list)
|
||||||
|
extensions: list[str] = field(default_factory=lambda: list(DEFAULT_EXTENSIONS))
|
||||||
|
ignore_prefixes: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE_PREFIXES))
|
||||||
|
ignore_suffixes: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORE_SUFFIXES))
|
||||||
|
|
||||||
|
|
||||||
|
def default_config_path() -> Path:
|
||||||
|
"""Return the default daemon config path, honoring CP0_SYNC_CONFIG if set."""
|
||||||
|
override = os.environ.get(ENV_CONFIG_PATH, "").strip()
|
||||||
|
if override:
|
||||||
|
return Path(override).expanduser()
|
||||||
|
return Path.home() / ".config" / "cp0-sync" / "config.json"
|
||||||
|
|
||||||
|
|
||||||
|
def default_watched_folders(home_dir: Path | None = None) -> list[WatchedFolder]:
|
||||||
|
"""Return the first-run default watched folder list (<home>/Pictures/DCIM/Camera)."""
|
||||||
|
base_home = Path(home_dir).expanduser() if home_dir is not None else Path.home()
|
||||||
|
camera_path = base_home / "Pictures" / "DCIM" / "Camera"
|
||||||
|
return [
|
||||||
|
WatchedFolder(
|
||||||
|
id="camera",
|
||||||
|
path=str(camera_path),
|
||||||
|
label="Camera",
|
||||||
|
enabled=True,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_config_path(path: Path | None) -> Path:
|
||||||
|
return Path(path).expanduser() if path is not None else default_config_path()
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_port(value: object) -> bool:
|
||||||
|
return isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_str_list(value: object) -> list[str] | None:
|
||||||
|
if not isinstance(value, list):
|
||||||
|
return None
|
||||||
|
cleaned = [item for item in value if isinstance(item, str) and item]
|
||||||
|
return cleaned if len(cleaned) == len(value) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_watched_folders(raw_folders: object, home_dir: Path | None) -> list[WatchedFolder]:
|
||||||
|
if not isinstance(raw_folders, list):
|
||||||
|
return default_watched_folders(home_dir)
|
||||||
|
|
||||||
|
parsed: list[WatchedFolder] = []
|
||||||
|
for item in raw_folders:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
folder_id = item.get("id")
|
||||||
|
folder_path = item.get("path")
|
||||||
|
label = item.get("label")
|
||||||
|
enabled = item.get("enabled", True)
|
||||||
|
if (
|
||||||
|
isinstance(folder_id, str)
|
||||||
|
and folder_id.strip()
|
||||||
|
and isinstance(folder_path, str)
|
||||||
|
and folder_path.strip()
|
||||||
|
and isinstance(label, str)
|
||||||
|
and label.strip()
|
||||||
|
and isinstance(enabled, bool)
|
||||||
|
):
|
||||||
|
parsed.append(
|
||||||
|
WatchedFolder(
|
||||||
|
id=folder_id.strip(),
|
||||||
|
path=folder_path.strip(),
|
||||||
|
label=label.strip(),
|
||||||
|
enabled=enabled,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def _default_daemon_config(home_dir: Path | None = None) -> DaemonConfig:
|
||||||
|
return DaemonConfig(watched_folders=default_watched_folders(home_dir))
|
||||||
|
|
||||||
|
|
||||||
|
def load_daemon_config(path: Path | None = None, home_dir: Path | None = None) -> DaemonConfig:
|
||||||
|
"""Load DaemonConfig from JSON, creating first-run defaults or safely falling back on errors."""
|
||||||
|
cfg_path = _resolve_config_path(path)
|
||||||
|
defaults = _default_daemon_config(home_dir)
|
||||||
|
|
||||||
|
if not cfg_path.exists():
|
||||||
|
try:
|
||||||
|
save_daemon_config(defaults, cfg_path)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.warning("Could not persist first-run config %s (%s); continuing in memory", cfg_path, exc)
|
||||||
|
return defaults
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw = json.loads(cfg_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
logger.warning("Failed to read config %s (%s); using defaults", cfg_path, exc)
|
||||||
|
return defaults
|
||||||
|
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
logger.warning("Config %s is not a JSON object; using defaults", cfg_path)
|
||||||
|
return defaults
|
||||||
|
|
||||||
|
target_host = raw.get("target_host")
|
||||||
|
target_port = raw.get("target_port")
|
||||||
|
theme = raw.get("theme")
|
||||||
|
extensions = _valid_str_list(raw.get("extensions"))
|
||||||
|
ignore_prefixes = _valid_str_list(raw.get("ignore_prefixes"))
|
||||||
|
ignore_suffixes = _valid_str_list(raw.get("ignore_suffixes"))
|
||||||
|
watched_folders = (
|
||||||
|
_parse_watched_folders(raw.get("watched_folders"), home_dir)
|
||||||
|
if "watched_folders" in raw
|
||||||
|
else defaults.watched_folders
|
||||||
|
)
|
||||||
|
|
||||||
|
return DaemonConfig(
|
||||||
|
target_host=target_host.strip() if isinstance(target_host, str) else defaults.target_host,
|
||||||
|
target_port=target_port if _valid_port(target_port) else defaults.target_port,
|
||||||
|
theme=theme if isinstance(theme, str) and theme in VALID_THEMES else defaults.theme,
|
||||||
|
watched_folders=watched_folders,
|
||||||
|
extensions=extensions if extensions is not None else defaults.extensions,
|
||||||
|
ignore_prefixes=ignore_prefixes if ignore_prefixes is not None else defaults.ignore_prefixes,
|
||||||
|
ignore_suffixes=ignore_suffixes if ignore_suffixes is not None else defaults.ignore_suffixes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def save_daemon_config(cfg: DaemonConfig, path: Path | None = None) -> None:
|
||||||
|
"""Atomically persist DaemonConfig to disk using a temp file, fsync, and os.replace."""
|
||||||
|
cfg_path = _resolve_config_path(path)
|
||||||
|
cfg_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp_path = cfg_path.parent / f".{cfg_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
|
||||||
|
payload = json.dumps(asdict(cfg), indent=2) + "\n"
|
||||||
|
try:
|
||||||
|
with open(tmp_path, "w", encoding="utf-8") as handle:
|
||||||
|
handle.write(payload)
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(tmp_path, cfg_path)
|
||||||
|
except BaseException:
|
||||||
|
try:
|
||||||
|
tmp_path.unlink(missing_ok=True)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_path(raw_path: str) -> Path:
|
||||||
|
expanded = os.path.expanduser(raw_path)
|
||||||
|
return Path(os.path.realpath(expanded))
|
||||||
|
|
||||||
|
|
||||||
|
def _paths_overlap(a: Path, b: Path) -> bool:
|
||||||
|
return a == b or a in b.parents or b in a.parents
|
||||||
|
|
||||||
|
|
||||||
|
def _disambiguate_label(base_label: str, existing_labels: set[str]) -> str:
|
||||||
|
if base_label.lower() not in existing_labels:
|
||||||
|
return base_label
|
||||||
|
suffix = 2
|
||||||
|
while True:
|
||||||
|
candidate = f"{base_label}-{suffix}"
|
||||||
|
if candidate.lower() not in existing_labels:
|
||||||
|
return candidate
|
||||||
|
suffix += 1
|
||||||
|
|
||||||
|
|
||||||
|
def _make_folder_id(label: str, existing_ids: set[str]) -> str:
|
||||||
|
slug = re.sub(r"[^a-z0-9]+", "-", label.lower()).strip("-") or "folder"
|
||||||
|
while True:
|
||||||
|
candidate = f"{slug}-{uuid.uuid4().hex[:8]}"
|
||||||
|
if candidate not in existing_ids:
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
|
def add_watched_folder(
|
||||||
|
cfg: DaemonConfig,
|
||||||
|
folder_path: str,
|
||||||
|
label: str | None = None,
|
||||||
|
) -> tuple[DaemonConfig, WatchedFolder]:
|
||||||
|
"""Validate and add a watched folder, rejecting missing/overlapping paths and disambiguating labels."""
|
||||||
|
if not isinstance(folder_path, str) or not folder_path.strip():
|
||||||
|
raise ValueError("Folder path must be a non-empty string")
|
||||||
|
|
||||||
|
candidate = _canonical_path(folder_path.strip())
|
||||||
|
if not candidate.exists() or not candidate.is_dir():
|
||||||
|
raise ValueError(f"Watched directory does not exist: {folder_path}")
|
||||||
|
|
||||||
|
for existing in cfg.watched_folders:
|
||||||
|
existing_real = _canonical_path(existing.path)
|
||||||
|
if _paths_overlap(candidate, existing_real):
|
||||||
|
raise ValueError(
|
||||||
|
f"Folder {candidate} overlaps already-watched path {existing_real}"
|
||||||
|
)
|
||||||
|
|
||||||
|
base_label = label.strip() if isinstance(label, str) and label.strip() else (candidate.name or "Folder")
|
||||||
|
existing_labels = {wf.label.lower() for wf in cfg.watched_folders}
|
||||||
|
unique_label = _disambiguate_label(base_label, existing_labels)
|
||||||
|
|
||||||
|
existing_ids = {wf.id for wf in cfg.watched_folders}
|
||||||
|
folder_id = _make_folder_id(unique_label, existing_ids)
|
||||||
|
|
||||||
|
new_folder = WatchedFolder(
|
||||||
|
id=folder_id,
|
||||||
|
path=str(candidate),
|
||||||
|
label=unique_label,
|
||||||
|
enabled=True,
|
||||||
|
)
|
||||||
|
updated_cfg = replace(cfg, watched_folders=[*cfg.watched_folders, new_folder])
|
||||||
|
return updated_cfg, new_folder
|
||||||
|
|
||||||
|
|
||||||
|
def remove_watched_folder(cfg: DaemonConfig, folder_id: str) -> DaemonConfig:
|
||||||
|
"""Return a copy of DaemonConfig without the specified watched folder ID."""
|
||||||
|
return replace(
|
||||||
|
cfg,
|
||||||
|
watched_folders=[wf for wf in cfg.watched_folders if wf.id != folder_id],
|
||||||
|
)
|
||||||
439
device-daemon/cp0_syncd/store.py
Normal file
439
device-daemon/cp0_syncd/store.py
Normal file
@@ -0,0 +1,439 @@
|
|||||||
|
"""SQLite WAL queue store and media file filtering for cp0-syncd."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import stat
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .config import DaemonConfig, WatchedFolder
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
ENV_DB_PATH = "CP0_SYNC_DB"
|
||||||
|
MAX_LOG_ROWS = 500
|
||||||
|
CHUNK_SIZE = 65536
|
||||||
|
|
||||||
|
|
||||||
|
def default_db_path() -> Path:
|
||||||
|
"""Return the default SQLite state database path, honoring CP0_SYNC_DB if set."""
|
||||||
|
override = os.environ.get(ENV_DB_PATH, "").strip()
|
||||||
|
if override:
|
||||||
|
return Path(override).expanduser()
|
||||||
|
return Path.home() / ".local" / "share" / "cp0-sync" / "state.db"
|
||||||
|
|
||||||
|
|
||||||
|
def compute_sha256(path: Path, chunk_size: int = CHUNK_SIZE) -> str:
|
||||||
|
"""Compute the lowercase hex SHA-256 digest of a file incrementally."""
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with open(path, "rb") as handle:
|
||||||
|
while True:
|
||||||
|
chunk = handle.read(chunk_size)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def is_media_file(
|
||||||
|
path: Path,
|
||||||
|
cfg: DaemonConfig,
|
||||||
|
store: SyncStore | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Return True if path is a non-empty regular media file permitted by DaemonConfig."""
|
||||||
|
name = path.name
|
||||||
|
if not name:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if any(name.startswith(prefix) for prefix in cfg.ignore_prefixes if prefix):
|
||||||
|
return False
|
||||||
|
|
||||||
|
name_lower = name.lower()
|
||||||
|
if any(name_lower.endswith(suffix.lower()) for suffix in cfg.ignore_suffixes if suffix):
|
||||||
|
return False
|
||||||
|
|
||||||
|
allowed_exts = {
|
||||||
|
ext.lower() if ext.startswith(".") else f".{ext.lower()}"
|
||||||
|
for ext in cfg.extensions
|
||||||
|
if ext
|
||||||
|
}
|
||||||
|
if path.suffix.lower() not in allowed_exts:
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
st = path.stat()
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if not stat.S_ISREG(st.st_mode):
|
||||||
|
return False
|
||||||
|
|
||||||
|
if st.st_size == 0:
|
||||||
|
logger.warning("Skipping 0-byte media file: %s", path)
|
||||||
|
if store is not None:
|
||||||
|
store.log("WARN", f"Skipped 0-byte file: {name}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _now_ms() -> int:
|
||||||
|
return int(time.time() * 1000)
|
||||||
|
|
||||||
|
|
||||||
|
class SyncStore:
|
||||||
|
"""Thread-safe SQLite WAL queue, metadata, and ring-buffered log store."""
|
||||||
|
|
||||||
|
def __init__(self, db_path: Path | None = None) -> None:
|
||||||
|
resolved = Path(db_path).expanduser() if db_path is not None else default_db_path()
|
||||||
|
resolved.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.db_path = resolved
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._conn = sqlite3.connect(str(resolved), timeout=5.0, check_same_thread=False)
|
||||||
|
self._conn.row_factory = sqlite3.Row
|
||||||
|
self._closed = False
|
||||||
|
self._init_schema()
|
||||||
|
|
||||||
|
def _init_schema(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute("PRAGMA journal_mode=WAL;")
|
||||||
|
self._conn.execute("PRAGMA busy_timeout=5000;")
|
||||||
|
self._conn.execute("PRAGMA synchronous=NORMAL;")
|
||||||
|
self._conn.executescript(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS files (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
folder_id TEXT NOT NULL,
|
||||||
|
abs_path TEXT NOT NULL,
|
||||||
|
rel_path TEXT NOT NULL,
|
||||||
|
size_bytes INTEGER NOT NULL,
|
||||||
|
mtime_ns INTEGER NOT NULL,
|
||||||
|
sha256 TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
last_error TEXT,
|
||||||
|
discovered_at INTEGER NOT NULL,
|
||||||
|
synced_at INTEGER
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_files_dedup
|
||||||
|
ON files(folder_id, rel_path, sha256);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_files_status_discovered
|
||||||
|
ON files(status, discovered_at, id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS logs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
ts INTEGER NOT NULL,
|
||||||
|
level TEXT NOT NULL,
|
||||||
|
message TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def enqueue_file(
|
||||||
|
self,
|
||||||
|
folder_id: str,
|
||||||
|
abs_path: str,
|
||||||
|
rel_path: str,
|
||||||
|
size_bytes: int,
|
||||||
|
mtime_ns: int,
|
||||||
|
sha256: str,
|
||||||
|
) -> str:
|
||||||
|
"""Enqueue a file if not already synced/queued; skip 0-byte payloads with a WARN log."""
|
||||||
|
if size_bytes <= 0:
|
||||||
|
self.log("WARN", f"Skipped 0-byte file: {rel_path}")
|
||||||
|
return "skipped_empty"
|
||||||
|
|
||||||
|
now_ms = _now_ms()
|
||||||
|
with self._lock:
|
||||||
|
row = self._conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, status
|
||||||
|
FROM files
|
||||||
|
WHERE folder_id = ? AND rel_path = ? AND sha256 = ?
|
||||||
|
""",
|
||||||
|
(folder_id, rel_path, sha256),
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
if row is not None:
|
||||||
|
status = str(row["status"])
|
||||||
|
if status == "SYNCED":
|
||||||
|
return "already_synced"
|
||||||
|
if status == "FAILED":
|
||||||
|
return "already_failed"
|
||||||
|
return "already_queued"
|
||||||
|
|
||||||
|
self._conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO files (
|
||||||
|
folder_id, abs_path, rel_path, size_bytes, mtime_ns,
|
||||||
|
sha256, status, retry_count, last_error, discovered_at, synced_at
|
||||||
|
)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, 'PENDING', 0, NULL, ?, NULL)
|
||||||
|
""",
|
||||||
|
(folder_id, abs_path, rel_path, size_bytes, mtime_ns, sha256, now_ms),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
return "enqueued"
|
||||||
|
|
||||||
|
def next_pending(self) -> dict[str, Any] | None:
|
||||||
|
"""Return the oldest PENDING file row as a dict, or None if the queue is empty."""
|
||||||
|
with self._lock:
|
||||||
|
row = self._conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, folder_id, abs_path, rel_path, size_bytes, mtime_ns,
|
||||||
|
sha256, status, retry_count, last_error, discovered_at, synced_at
|
||||||
|
FROM files
|
||||||
|
WHERE status = 'PENDING'
|
||||||
|
ORDER BY discovered_at ASC, id ASC
|
||||||
|
LIMIT 1
|
||||||
|
"""
|
||||||
|
).fetchone()
|
||||||
|
return dict(row) if row is not None else None
|
||||||
|
|
||||||
|
def mark_uploading(self, file_id: int) -> None:
|
||||||
|
"""Transition a queued file to UPLOADING."""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"UPDATE files SET status = 'UPLOADING' WHERE id = ?",
|
||||||
|
(file_id,),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def mark_synced(self, file_id: int) -> None:
|
||||||
|
"""Mark a file as SYNCED and clear any prior error."""
|
||||||
|
now_ms = _now_ms()
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE files
|
||||||
|
SET status = 'SYNCED', synced_at = ?, last_error = NULL
|
||||||
|
WHERE id = ?
|
||||||
|
""",
|
||||||
|
(now_ms, file_id),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def mark_retry(self, file_id: int, err: str) -> None:
|
||||||
|
"""Return an item to PENDING while recording the retry count and error."""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE files
|
||||||
|
SET status = 'PENDING', retry_count = retry_count + 1, last_error = ?
|
||||||
|
WHERE id = ?
|
||||||
|
""",
|
||||||
|
(err, file_id),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def mark_failed(self, file_id: int, err: str) -> None:
|
||||||
|
"""Mark a file as FAILED after a permanent per-file rejection."""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE files
|
||||||
|
SET status = 'FAILED', retry_count = retry_count + 1, last_error = ?
|
||||||
|
WHERE id = ?
|
||||||
|
""",
|
||||||
|
(err, file_id),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def delete_file(self, file_id: int) -> None:
|
||||||
|
"""Delete a file record (e.g., when the source file vanished before upload)."""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute("DELETE FROM files WHERE id = ?", (file_id,))
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def purge_pending_for_folder(self, folder_id: str) -> int:
|
||||||
|
"""Delete un-synced PENDING/UPLOADING queue entries for a removed folder."""
|
||||||
|
with self._lock:
|
||||||
|
cur = self._conn.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM files
|
||||||
|
WHERE folder_id = ? AND status IN ('PENDING', 'UPLOADING')
|
||||||
|
""",
|
||||||
|
(folder_id,),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
return cur.rowcount
|
||||||
|
|
||||||
|
def retry_all_failed(self) -> int:
|
||||||
|
"""Flip all FAILED rows back to PENDING and return the count updated."""
|
||||||
|
with self._lock:
|
||||||
|
cur = self._conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE files
|
||||||
|
SET status = 'PENDING', last_error = NULL
|
||||||
|
WHERE status = 'FAILED'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
return cur.rowcount
|
||||||
|
|
||||||
|
def log(self, level: str, message: str) -> None:
|
||||||
|
"""Append a log row and prune the logs table to the newest 500 rows."""
|
||||||
|
now_ms = _now_ms()
|
||||||
|
normalized_level = level.strip().upper() or "INFO"
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"INSERT INTO logs (ts, level, message) VALUES (?, ?, ?)",
|
||||||
|
(now_ms, normalized_level, message),
|
||||||
|
)
|
||||||
|
self._conn.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM logs
|
||||||
|
WHERE id NOT IN (
|
||||||
|
SELECT id FROM logs ORDER BY id DESC LIMIT ?
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
(MAX_LOG_ROWS,),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def log_count(self) -> int:
|
||||||
|
"""Return the number of rows currently stored in the logs table."""
|
||||||
|
with self._lock:
|
||||||
|
row = self._conn.execute("SELECT COUNT(*) FROM logs").fetchone()
|
||||||
|
return int(row[0]) if row is not None else 0
|
||||||
|
|
||||||
|
def list_logs(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||||
|
"""Return up to `limit` newest log rows ordered newest-first."""
|
||||||
|
safe_limit = max(0, int(limit))
|
||||||
|
with self._lock:
|
||||||
|
rows = self._conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, ts, level, message
|
||||||
|
FROM logs
|
||||||
|
ORDER BY id DESC
|
||||||
|
LIMIT ?
|
||||||
|
""",
|
||||||
|
(safe_limit,),
|
||||||
|
).fetchall()
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
def set_meta(self, key: str, value: str) -> None:
|
||||||
|
"""Upsert a metadata key-value pair."""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO meta (key, value) VALUES (?, ?)
|
||||||
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||||
|
""",
|
||||||
|
(key, str(value)),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def get_meta(self, key: str) -> str | None:
|
||||||
|
"""Fetch a metadata value by key, or None if unset."""
|
||||||
|
with self._lock:
|
||||||
|
row = self._conn.execute(
|
||||||
|
"SELECT value FROM meta WHERE key = ?",
|
||||||
|
(key,),
|
||||||
|
).fetchone()
|
||||||
|
return str(row["value"]) if row is not None else None
|
||||||
|
|
||||||
|
def get_status_snapshot(self, watched_folders: list[WatchedFolder]) -> dict[str, Any]:
|
||||||
|
"""Return queue totals, per-folder counts, metadata, and recent logs for status IPC/UI."""
|
||||||
|
with self._lock:
|
||||||
|
folder_stats: dict[str, dict[str, int]] = {}
|
||||||
|
rows = self._conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT folder_id, status, COUNT(*) AS cnt, COALESCE(SUM(size_bytes), 0) AS total_bytes
|
||||||
|
FROM files
|
||||||
|
GROUP BY folder_id, status
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
totals = {
|
||||||
|
"pending": 0,
|
||||||
|
"uploading": 0,
|
||||||
|
"synced": 0,
|
||||||
|
"failed": 0,
|
||||||
|
"pending_bytes": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for row in rows:
|
||||||
|
fid = str(row["folder_id"])
|
||||||
|
status = str(row["status"])
|
||||||
|
cnt = int(row["cnt"])
|
||||||
|
total_bytes = int(row["total_bytes"])
|
||||||
|
bucket = folder_stats.setdefault(
|
||||||
|
fid,
|
||||||
|
{
|
||||||
|
"pending": 0,
|
||||||
|
"uploading": 0,
|
||||||
|
"synced": 0,
|
||||||
|
"failed": 0,
|
||||||
|
"pending_bytes": 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if status == "PENDING":
|
||||||
|
bucket["pending"] += cnt
|
||||||
|
bucket["pending_bytes"] += total_bytes
|
||||||
|
totals["pending"] += cnt
|
||||||
|
totals["pending_bytes"] += total_bytes
|
||||||
|
elif status == "UPLOADING":
|
||||||
|
bucket["pending"] += cnt
|
||||||
|
bucket["uploading"] += cnt
|
||||||
|
bucket["pending_bytes"] += total_bytes
|
||||||
|
totals["pending"] += cnt
|
||||||
|
totals["uploading"] += cnt
|
||||||
|
totals["pending_bytes"] += total_bytes
|
||||||
|
elif status == "SYNCED":
|
||||||
|
bucket["synced"] += cnt
|
||||||
|
totals["synced"] += cnt
|
||||||
|
elif status == "FAILED":
|
||||||
|
bucket["failed"] += cnt
|
||||||
|
totals["failed"] += cnt
|
||||||
|
|
||||||
|
meta_rows = self._conn.execute("SELECT key, value FROM meta").fetchall()
|
||||||
|
meta = {str(r["key"]): str(r["value"]) for r in meta_rows}
|
||||||
|
recent_logs = self.list_logs(limit=50)
|
||||||
|
|
||||||
|
folders_out = []
|
||||||
|
for wf in watched_folders:
|
||||||
|
stats = folder_stats.get(
|
||||||
|
wf.id,
|
||||||
|
{"pending": 0, "uploading": 0, "synced": 0, "failed": 0, "pending_bytes": 0},
|
||||||
|
)
|
||||||
|
folders_out.append(
|
||||||
|
{
|
||||||
|
"id": wf.id,
|
||||||
|
"path": wf.path,
|
||||||
|
"label": wf.label,
|
||||||
|
"enabled": wf.enabled,
|
||||||
|
"exists": os.path.isdir(os.path.expanduser(wf.path)),
|
||||||
|
"pending": stats["pending"],
|
||||||
|
"uploading": stats["uploading"],
|
||||||
|
"synced": stats["synced"],
|
||||||
|
"failed": stats["failed"],
|
||||||
|
"pending_bytes": stats["pending_bytes"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"counts": totals,
|
||||||
|
"folders": folders_out,
|
||||||
|
"meta": meta,
|
||||||
|
"logs": recent_logs,
|
||||||
|
}
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
"""Close the SQLite connection."""
|
||||||
|
with self._lock:
|
||||||
|
if not self._closed:
|
||||||
|
self._conn.close()
|
||||||
|
self._closed = True
|
||||||
303
device-daemon/tests/test_store_and_config.py
Normal file
303
device-daemon/tests/test_store_and_config.py
Normal file
@@ -0,0 +1,303 @@
|
|||||||
|
"""Unit tests for device-daemon configuration and SQLite WAL queue store."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from cp0_syncd.config import (
|
||||||
|
DaemonConfig,
|
||||||
|
WatchedFolder,
|
||||||
|
add_watched_folder,
|
||||||
|
default_config_path,
|
||||||
|
load_daemon_config,
|
||||||
|
remove_watched_folder,
|
||||||
|
save_daemon_config,
|
||||||
|
)
|
||||||
|
from cp0_syncd.store import (
|
||||||
|
SyncStore,
|
||||||
|
compute_sha256,
|
||||||
|
default_db_path,
|
||||||
|
is_media_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDaemonConfig(unittest.TestCase):
|
||||||
|
def test_first_run_creates_default_camera_folder_and_theme(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
home = Path(tmp) / "home"
|
||||||
|
cfg_path = Path(tmp) / "config.json"
|
||||||
|
cfg = load_daemon_config(cfg_path, home_dir=home)
|
||||||
|
self.assertEqual(cfg.theme, "dark")
|
||||||
|
self.assertEqual(cfg.target_port, 8765)
|
||||||
|
self.assertEqual(cfg.target_host, "")
|
||||||
|
self.assertEqual(len(cfg.watched_folders), 1)
|
||||||
|
self.assertEqual(
|
||||||
|
cfg.watched_folders[0].path,
|
||||||
|
str(home / "Pictures" / "DCIM" / "Camera"),
|
||||||
|
)
|
||||||
|
self.assertEqual(cfg.watched_folders[0].label, "Camera")
|
||||||
|
self.assertTrue(cfg.watched_folders[0].enabled)
|
||||||
|
self.assertFalse(hasattr(cfg, "auth_token"))
|
||||||
|
|
||||||
|
def test_corrupt_config_falls_back_to_defaults(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg_path = Path(tmp) / "config.json"
|
||||||
|
cfg_path.write_text("{corrupt", encoding="utf-8")
|
||||||
|
with self.assertLogs("cp0_syncd.config", level="WARNING"):
|
||||||
|
cfg = load_daemon_config(cfg_path, home_dir=Path(tmp))
|
||||||
|
self.assertEqual(cfg.theme, "dark")
|
||||||
|
self.assertEqual(cfg.target_port, 8765)
|
||||||
|
self.assertEqual(len(cfg.watched_folders), 1)
|
||||||
|
self.assertEqual(cfg_path.read_text(encoding="utf-8"), "{corrupt")
|
||||||
|
|
||||||
|
def test_save_and_load_roundtrip_preserves_empty_folders_and_theme(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg_path = Path(tmp) / "nested" / "config.json"
|
||||||
|
original = DaemonConfig(
|
||||||
|
target_host="my-mac.tailnet.ts.net",
|
||||||
|
target_port=9000,
|
||||||
|
theme="light",
|
||||||
|
watched_folders=[],
|
||||||
|
)
|
||||||
|
save_daemon_config(original, cfg_path)
|
||||||
|
raw = json.loads(cfg_path.read_text(encoding="utf-8"))
|
||||||
|
self.assertNotIn("auth_token", raw)
|
||||||
|
|
||||||
|
loaded = load_daemon_config(cfg_path, home_dir=Path(tmp))
|
||||||
|
self.assertEqual(loaded.target_host, "my-mac.tailnet.ts.net")
|
||||||
|
self.assertEqual(loaded.target_port, 9000)
|
||||||
|
self.assertEqual(loaded.theme, "light")
|
||||||
|
self.assertEqual(loaded.watched_folders, [])
|
||||||
|
|
||||||
|
def test_env_overrides_default_paths(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg_override = str(Path(tmp) / "custom-config.json")
|
||||||
|
db_override = str(Path(tmp) / "custom-state.db")
|
||||||
|
with mock.patch.dict(
|
||||||
|
os.environ,
|
||||||
|
{"CP0_SYNC_CONFIG": cfg_override, "CP0_SYNC_DB": db_override},
|
||||||
|
clear=False,
|
||||||
|
):
|
||||||
|
self.assertEqual(default_config_path(), Path(cfg_override))
|
||||||
|
self.assertEqual(default_db_path(), Path(db_override))
|
||||||
|
|
||||||
|
def test_overlap_rejected_and_duplicate_labels_disambiguated(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
d1 = Path(tmp) / "a" / "Camera"
|
||||||
|
d2 = Path(tmp) / "b" / "Camera"
|
||||||
|
d3 = Path(tmp) / "c" / "Camera"
|
||||||
|
child = d1 / "sub"
|
||||||
|
parent = Path(tmp) / "a"
|
||||||
|
d1.mkdir(parents=True)
|
||||||
|
d2.mkdir(parents=True)
|
||||||
|
d3.mkdir(parents=True)
|
||||||
|
child.mkdir()
|
||||||
|
|
||||||
|
cfg = DaemonConfig(watched_folders=[])
|
||||||
|
cfg, wf1 = add_watched_folder(cfg, str(d1))
|
||||||
|
cfg, wf2 = add_watched_folder(cfg, str(d2))
|
||||||
|
cfg, wf3 = add_watched_folder(cfg, str(d3))
|
||||||
|
self.assertEqual(wf1.label, "Camera")
|
||||||
|
self.assertEqual(wf2.label, "Camera-2")
|
||||||
|
self.assertEqual(wf3.label, "Camera-3")
|
||||||
|
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
add_watched_folder(cfg, str(child))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
add_watched_folder(cfg, str(parent))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
add_watched_folder(cfg, str(d1))
|
||||||
|
|
||||||
|
symlink_d1 = Path(tmp) / "cam-link"
|
||||||
|
symlink_d1.symlink_to(d1)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
add_watched_folder(cfg, str(symlink_d1))
|
||||||
|
|
||||||
|
def test_add_nonexistent_or_file_raises_and_remove_folder(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
missing = Path(tmp) / "missing"
|
||||||
|
not_dir = Path(tmp) / "file.txt"
|
||||||
|
not_dir.write_text("hi", encoding="utf-8")
|
||||||
|
valid = Path(tmp) / "Screenshots"
|
||||||
|
valid.mkdir()
|
||||||
|
|
||||||
|
cfg = DaemonConfig(watched_folders=[])
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
add_watched_folder(cfg, str(missing))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
add_watched_folder(cfg, str(not_dir))
|
||||||
|
|
||||||
|
cfg, wf = add_watched_folder(cfg, str(valid), label="Shots")
|
||||||
|
self.assertEqual(wf.label, "Shots")
|
||||||
|
self.assertEqual(len(cfg.watched_folders), 1)
|
||||||
|
|
||||||
|
cfg = remove_watched_folder(cfg, wf.id)
|
||||||
|
self.assertEqual(cfg.watched_folders, [])
|
||||||
|
|
||||||
|
|
||||||
|
class TestSyncStore(unittest.TestCase):
|
||||||
|
def test_dedup_zero_byte_skip_and_log_cap(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db = SyncStore(Path(tmp) / "state.db")
|
||||||
|
try:
|
||||||
|
img = Path(tmp) / "IMG_0001.jpg"
|
||||||
|
img.write_bytes(b"photo-bytes")
|
||||||
|
zero = Path(tmp) / "empty.jpg"
|
||||||
|
zero.write_bytes(b"")
|
||||||
|
cfg = DaemonConfig()
|
||||||
|
|
||||||
|
self.assertTrue(is_media_file(img, cfg))
|
||||||
|
with self.assertLogs("cp0_syncd.store", level="WARNING"):
|
||||||
|
self.assertFalse(is_media_file(zero, cfg))
|
||||||
|
|
||||||
|
sha = compute_sha256(img)
|
||||||
|
self.assertEqual(
|
||||||
|
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
|
||||||
|
"enqueued",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
|
||||||
|
"already_queued",
|
||||||
|
)
|
||||||
|
row = db.next_pending()
|
||||||
|
self.assertIsNotNone(row)
|
||||||
|
assert row is not None
|
||||||
|
db.mark_synced(row["id"])
|
||||||
|
self.assertEqual(
|
||||||
|
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 11, 1, sha),
|
||||||
|
"already_synced",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Different content at same path should enqueue cleanly
|
||||||
|
img.write_bytes(b"updated-photo-bytes")
|
||||||
|
sha2 = compute_sha256(img)
|
||||||
|
self.assertEqual(
|
||||||
|
db.enqueue_file("cam", str(img), "IMG_0001.jpg", 19, 2, sha2),
|
||||||
|
"enqueued",
|
||||||
|
)
|
||||||
|
|
||||||
|
for i in range(550):
|
||||||
|
db.log("INFO", f"msg {i}")
|
||||||
|
self.assertEqual(db.log_count(), 500)
|
||||||
|
recent = db.list_logs(limit=10)
|
||||||
|
self.assertEqual(len(recent), 10)
|
||||||
|
self.assertEqual(recent[0]["message"], "msg 549")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
def test_wal_mode_and_enqueue_zero_byte_logs_warn(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
db_path = Path(tmp) / "state.db"
|
||||||
|
db = SyncStore(db_path)
|
||||||
|
try:
|
||||||
|
conn = sqlite3.connect(str(db_path))
|
||||||
|
mode = conn.execute("PRAGMA journal_mode;").fetchone()[0]
|
||||||
|
conn.close()
|
||||||
|
self.assertEqual(mode.lower(), "wal")
|
||||||
|
|
||||||
|
status = db.enqueue_file("cam", "/tmp/zero.jpg", "zero.jpg", 0, 1, "0" * 64)
|
||||||
|
self.assertEqual(status, "skipped_empty")
|
||||||
|
logs = db.list_logs(limit=5)
|
||||||
|
self.assertEqual(len(logs), 1)
|
||||||
|
self.assertEqual(logs[0]["level"], "WARN")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
def test_media_filtering_rules(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = DaemonConfig()
|
||||||
|
for name, expected in [
|
||||||
|
("IMG_1.JPEG", True),
|
||||||
|
("clip.mp4", True),
|
||||||
|
(".hidden.jpg", False),
|
||||||
|
("~temp.png", False),
|
||||||
|
("photo.jpg.part", False),
|
||||||
|
("photo.jpg.tmp", False),
|
||||||
|
("photo.jpg.crdownload", False),
|
||||||
|
("notes.txt", False),
|
||||||
|
]:
|
||||||
|
p = Path(tmp) / name
|
||||||
|
p.write_bytes(b"abc")
|
||||||
|
self.assertEqual(is_media_file(p, cfg), expected, msg=name)
|
||||||
|
|
||||||
|
subdir = Path(tmp) / "sub.jpg"
|
||||||
|
subdir.mkdir()
|
||||||
|
self.assertFalse(is_media_file(subdir, cfg))
|
||||||
|
self.assertFalse(is_media_file(Path(tmp) / "nonexistent.jpg", cfg))
|
||||||
|
|
||||||
|
def test_queue_lifecycle_meta_and_status_snapshot(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cam_dir = Path(tmp) / "Camera"
|
||||||
|
cam_dir.mkdir()
|
||||||
|
missing_dir = Path(tmp) / "SDCard"
|
||||||
|
|
||||||
|
db = SyncStore(Path(tmp) / "state.db")
|
||||||
|
try:
|
||||||
|
db.enqueue_file("cam", str(cam_dir / "a.jpg"), "a.jpg", 100, 10, "a" * 64)
|
||||||
|
db.enqueue_file("cam", str(cam_dir / "b.jpg"), "b.jpg", 200, 20, "b" * 64)
|
||||||
|
db.enqueue_file("sd", str(missing_dir / "c.jpg"), "c.jpg", 300, 30, "c" * 64)
|
||||||
|
|
||||||
|
first = db.next_pending()
|
||||||
|
assert first is not None
|
||||||
|
self.assertEqual(first["rel_path"], "a.jpg")
|
||||||
|
db.mark_uploading(first["id"])
|
||||||
|
db.mark_retry(first["id"], "timeout")
|
||||||
|
|
||||||
|
# Re-fetch after retry: still PENDING with retry_count == 1
|
||||||
|
again = db.next_pending()
|
||||||
|
assert again is not None
|
||||||
|
self.assertEqual(again["id"], first["id"])
|
||||||
|
self.assertEqual(again["retry_count"], 1)
|
||||||
|
self.assertEqual(again["last_error"], "timeout")
|
||||||
|
db.mark_synced(again["id"])
|
||||||
|
|
||||||
|
second = db.next_pending()
|
||||||
|
assert second is not None
|
||||||
|
self.assertEqual(second["rel_path"], "b.jpg")
|
||||||
|
db.mark_failed(second["id"], "413 too large")
|
||||||
|
|
||||||
|
# Meta roundtrip
|
||||||
|
self.assertIsNone(db.get_meta("mac_online"))
|
||||||
|
db.set_meta("mac_online", "true")
|
||||||
|
db.set_meta("mac_online", "false")
|
||||||
|
self.assertEqual(db.get_meta("mac_online"), "false")
|
||||||
|
|
||||||
|
folders = [
|
||||||
|
WatchedFolder(id="cam", path=str(cam_dir), label="Camera"),
|
||||||
|
WatchedFolder(id="sd", path=str(missing_dir), label="SDCard"),
|
||||||
|
]
|
||||||
|
snap = db.get_status_snapshot(folders)
|
||||||
|
self.assertEqual(snap["counts"]["synced"], 1)
|
||||||
|
self.assertEqual(snap["counts"]["failed"], 1)
|
||||||
|
self.assertEqual(snap["counts"]["pending"], 1)
|
||||||
|
self.assertEqual(snap["counts"]["pending_bytes"], 300)
|
||||||
|
self.assertEqual(snap["meta"]["mac_online"], "false")
|
||||||
|
self.assertEqual(len(snap["folders"]), 2)
|
||||||
|
self.assertTrue(snap["folders"][0]["exists"])
|
||||||
|
self.assertFalse(snap["folders"][1]["exists"])
|
||||||
|
self.assertEqual(snap["folders"][0]["synced"], 1)
|
||||||
|
self.assertEqual(snap["folders"][0]["failed"], 1)
|
||||||
|
self.assertEqual(snap["folders"][1]["pending"], 1)
|
||||||
|
|
||||||
|
# retry_all_failed flips FAILED -> PENDING
|
||||||
|
self.assertEqual(db.retry_all_failed(), 1)
|
||||||
|
# purge_pending_for_folder deletes pending for sd without touching cam
|
||||||
|
self.assertEqual(db.purge_pending_for_folder("sd"), 1)
|
||||||
|
# delete_file removes remaining pending b.jpg
|
||||||
|
rem = db.next_pending()
|
||||||
|
assert rem is not None
|
||||||
|
self.assertEqual(rem["rel_path"], "b.jpg")
|
||||||
|
db.delete_file(rem["id"])
|
||||||
|
self.assertIsNone(db.next_pending())
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user