mirror of
https://github.com/cottongin/cardputerzero-sync.git
synced 2026-11-20 14:44:47 -06:00
fix(device-daemon): recover interrupted uploads, fix wake race, and filter ignored subdirs
This commit is contained in:
@@ -23,7 +23,7 @@ from .config import (
|
|||||||
from .ipc import IpcServer, default_sock_path
|
from .ipc import IpcServer, default_sock_path
|
||||||
from .store import SyncStore, compute_sha256, default_db_path, is_media_file
|
from .store import SyncStore, compute_sha256, default_db_path, is_media_file
|
||||||
from .uploader import Uploader
|
from .uploader import Uploader
|
||||||
from .watcher import InotifyWatcher, scan_folder
|
from .watcher import InotifyWatcher, is_ignored_name, scan_folder
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -37,6 +37,10 @@ def _safe_int(value: str | None, default: int = 0) -> int:
|
|||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _has_ignored_dir_part(parts: tuple[str, ...], ignore_prefixes: list[str]) -> bool:
|
||||||
|
return any(is_ignored_name(part, ignore_prefixes) for part in parts)
|
||||||
|
|
||||||
|
|
||||||
class SyncDaemon:
|
class SyncDaemon:
|
||||||
"""On-device CardputerZero photo sync daemon."""
|
"""On-device CardputerZero photo sync daemon."""
|
||||||
|
|
||||||
@@ -55,6 +59,7 @@ class SyncDaemon:
|
|||||||
self._config_lock = threading.RLock()
|
self._config_lock = threading.RLock()
|
||||||
self.config: DaemonConfig = load_daemon_config(self.config_path)
|
self.config: DaemonConfig = load_daemon_config(self.config_path)
|
||||||
self.store = SyncStore(self.db_path)
|
self.store = SyncStore(self.db_path)
|
||||||
|
self._recover_interrupted_uploads()
|
||||||
self.wake_event = threading.Event()
|
self.wake_event = threading.Event()
|
||||||
|
|
||||||
self.uploader = Uploader(
|
self.uploader = Uploader(
|
||||||
@@ -67,10 +72,18 @@ class SyncDaemon:
|
|||||||
on_file_ready=self._on_file_ready,
|
on_file_ready=self._on_file_ready,
|
||||||
on_dir_created=self._on_dir_created,
|
on_dir_created=self._on_dir_created,
|
||||||
poll_interval=watcher_poll_interval,
|
poll_interval=watcher_poll_interval,
|
||||||
|
ignore_prefixes=self.config.ignore_prefixes,
|
||||||
|
get_ignore_prefixes=lambda: self.get_config().ignore_prefixes,
|
||||||
)
|
)
|
||||||
self.ipc_server = IpcServer(self.sock_path, self.handle_ipc)
|
self.ipc_server = IpcServer(self.sock_path, self.handle_ipc)
|
||||||
self._started = False
|
self._started = False
|
||||||
|
|
||||||
|
def _recover_interrupted_uploads(self) -> int:
|
||||||
|
recovered = self.store.recover_interrupted_uploads()
|
||||||
|
if recovered > 0:
|
||||||
|
self.store.log("WARN", f"Recovered {recovered} interrupted upload(s) to PENDING")
|
||||||
|
return recovered
|
||||||
|
|
||||||
def get_config(self) -> DaemonConfig:
|
def get_config(self) -> DaemonConfig:
|
||||||
"""Return the current DaemonConfig safely across threads."""
|
"""Return the current DaemonConfig safely across threads."""
|
||||||
with self._config_lock:
|
with self._config_lock:
|
||||||
@@ -91,17 +104,25 @@ class SyncDaemon:
|
|||||||
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
|
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
|
||||||
if active is None:
|
if active is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
root = Path(os.path.expanduser(active.path))
|
||||||
|
try:
|
||||||
|
rel_obj = file_path.relative_to(root)
|
||||||
|
except ValueError:
|
||||||
|
return
|
||||||
|
|
||||||
|
if _has_ignored_dir_part(rel_obj.parts[:-1], cfg.ignore_prefixes):
|
||||||
|
return
|
||||||
if not is_media_file(file_path, cfg, store=self.store):
|
if not is_media_file(file_path, cfg, store=self.store):
|
||||||
return
|
return
|
||||||
|
|
||||||
root = Path(os.path.expanduser(active.path))
|
|
||||||
try:
|
try:
|
||||||
st = file_path.stat()
|
st = file_path.stat()
|
||||||
if st.st_size <= 0:
|
if st.st_size <= 0:
|
||||||
return
|
return
|
||||||
sha256 = compute_sha256(file_path)
|
sha256 = compute_sha256(file_path)
|
||||||
rel_path = file_path.relative_to(root).as_posix()
|
rel_path = rel_obj.as_posix()
|
||||||
except (OSError, ValueError):
|
except OSError:
|
||||||
return
|
return
|
||||||
|
|
||||||
outcome = self.store.enqueue_file(
|
outcome = self.store.enqueue_file(
|
||||||
@@ -115,11 +136,18 @@ class SyncDaemon:
|
|||||||
if outcome == "enqueued":
|
if outcome == "enqueued":
|
||||||
self.uploader.notify_new_file()
|
self.uploader.notify_new_file()
|
||||||
|
|
||||||
def _on_dir_created(self, folder: WatchedFolder, _dir_path: Path) -> None:
|
def _on_dir_created(self, folder: WatchedFolder, dir_path: Path) -> None:
|
||||||
cfg = self.get_config()
|
cfg = self.get_config()
|
||||||
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
|
active = next((wf for wf in cfg.watched_folders if wf.id == folder.id and wf.enabled), None)
|
||||||
if active is None:
|
if active is None:
|
||||||
return
|
return
|
||||||
|
root = Path(os.path.expanduser(active.path))
|
||||||
|
try:
|
||||||
|
rel_parts = dir_path.relative_to(root).parts
|
||||||
|
except ValueError:
|
||||||
|
return
|
||||||
|
if rel_parts and _has_ignored_dir_part(rel_parts, cfg.ignore_prefixes):
|
||||||
|
return
|
||||||
added = scan_folder(active, cfg, self.store)
|
added = scan_folder(active, cfg, self.store)
|
||||||
if added > 0:
|
if added > 0:
|
||||||
self.uploader.notify_new_file()
|
self.uploader.notify_new_file()
|
||||||
@@ -130,6 +158,7 @@ class SyncDaemon:
|
|||||||
return
|
return
|
||||||
self._started = True
|
self._started = True
|
||||||
self.store.log("INFO", "cp0-syncd started")
|
self.store.log("INFO", "cp0-syncd started")
|
||||||
|
self._recover_interrupted_uploads()
|
||||||
self.watcher.sync_folders(self.get_config().watched_folders)
|
self.watcher.sync_folders(self.get_config().watched_folders)
|
||||||
self.watcher.start()
|
self.watcher.start()
|
||||||
self.scan_all()
|
self.scan_all()
|
||||||
|
|||||||
@@ -211,6 +211,19 @@ class SyncStore:
|
|||||||
)
|
)
|
||||||
self._conn.commit()
|
self._conn.commit()
|
||||||
|
|
||||||
|
def recover_interrupted_uploads(self) -> int:
|
||||||
|
"""Reset stale UPLOADING rows back to PENDING after an unclean restart."""
|
||||||
|
with self._lock:
|
||||||
|
cur = self._conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE files
|
||||||
|
SET status = 'PENDING'
|
||||||
|
WHERE status = 'UPLOADING'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
return cur.rowcount
|
||||||
|
|
||||||
def mark_synced(self, file_id: int) -> None:
|
def mark_synced(self, file_id: int) -> None:
|
||||||
"""Mark a file as SYNCED and clear any prior error."""
|
"""Mark a file as SYNCED and clear any prior error."""
|
||||||
now_ms = _now_ms()
|
now_ms = _now_ms()
|
||||||
|
|||||||
@@ -105,6 +105,8 @@ class Uploader:
|
|||||||
|
|
||||||
def wake(self, reset_backoff: bool = True, probe: bool = False) -> None:
|
def wake(self, reset_backoff: bool = True, probe: bool = False) -> None:
|
||||||
"""Interrupt backoff sleep immediately and optionally trigger a /health probe."""
|
"""Interrupt backoff sleep immediately and optionally trigger a /health probe."""
|
||||||
|
if reset_backoff:
|
||||||
|
self._store.set_meta("next_retry_ts", "0")
|
||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
if reset_backoff:
|
if reset_backoff:
|
||||||
self._backoff_index = 0
|
self._backoff_index = 0
|
||||||
@@ -112,27 +114,26 @@ class Uploader:
|
|||||||
self._next_retry_deadline = 0.0
|
self._next_retry_deadline = 0.0
|
||||||
if probe:
|
if probe:
|
||||||
self._probe_requested = True
|
self._probe_requested = True
|
||||||
if reset_backoff:
|
self._wake_event.set()
|
||||||
self._store.set_meta("next_retry_ts", "0")
|
|
||||||
self._wake_event.set()
|
|
||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
"""Start the background upload loop thread."""
|
"""Start the background upload loop thread."""
|
||||||
if self._thread is not None and self._thread.is_alive():
|
if self._thread is not None and self._thread.is_alive():
|
||||||
return
|
return
|
||||||
self._stop_event.clear()
|
|
||||||
cfg = self._get_config()
|
cfg = self._get_config()
|
||||||
if cfg.target_host.strip():
|
with self._state_lock:
|
||||||
with self._state_lock:
|
self._stop_event.clear()
|
||||||
|
if cfg.target_host.strip():
|
||||||
self._probe_requested = True
|
self._probe_requested = True
|
||||||
self._wake_event.set()
|
self._wake_event.set()
|
||||||
self._thread = threading.Thread(target=self._run_loop, name="cp0-uploader", daemon=True)
|
self._thread = threading.Thread(target=self._run_loop, name="cp0-uploader", daemon=True)
|
||||||
self._thread.start()
|
self._thread.start()
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Stop the background upload loop cleanly."""
|
"""Stop the background upload loop cleanly."""
|
||||||
self._stop_event.set()
|
with self._state_lock:
|
||||||
self._wake_event.set()
|
self._stop_event.set()
|
||||||
|
self._wake_event.set()
|
||||||
if self._thread is not None and self._thread is not threading.current_thread():
|
if self._thread is not None and self._thread is not threading.current_thread():
|
||||||
self._thread.join(timeout=5.0)
|
self._thread.join(timeout=5.0)
|
||||||
self._thread = None
|
self._thread = None
|
||||||
@@ -142,7 +143,7 @@ class Uploader:
|
|||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
probe = self._probe_requested
|
probe = self._probe_requested
|
||||||
self._probe_requested = False
|
self._probe_requested = False
|
||||||
self._wake_event.clear()
|
self._wake_event.clear()
|
||||||
|
|
||||||
has_pending = self._store.next_pending() is not None
|
has_pending = self._store.next_pending() is not None
|
||||||
if has_pending or probe:
|
if has_pending or probe:
|
||||||
@@ -150,17 +151,15 @@ class Uploader:
|
|||||||
else:
|
else:
|
||||||
result = "idle"
|
result = "idle"
|
||||||
|
|
||||||
if self._stop_event.is_set():
|
with self._state_lock:
|
||||||
break
|
if self._stop_event.is_set():
|
||||||
if self._wake_event.is_set():
|
break
|
||||||
|
woken = self._wake_event.is_set() or self._probe_requested
|
||||||
|
wait_sec = max(1, self._current_delay) if result in ("offline", "paused") else 60.0
|
||||||
|
if woken:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if result in ("offline", "paused"):
|
self._wake_event.wait(timeout=wait_sec)
|
||||||
with self._state_lock:
|
|
||||||
wait_sec = max(1, self._current_delay)
|
|
||||||
self._wake_event.wait(timeout=wait_sec)
|
|
||||||
else:
|
|
||||||
self._wake_event.wait(timeout=60.0)
|
|
||||||
|
|
||||||
def _reset_backoff(self) -> None:
|
def _reset_backoff(self) -> None:
|
||||||
with self._state_lock:
|
with self._state_lock:
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import sys
|
|||||||
import threading
|
import threading
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable
|
from typing import Any, Callable, Sequence
|
||||||
|
|
||||||
from .config import DaemonConfig, WatchedFolder
|
from .config import DEFAULT_IGNORE_PREFIXES, DaemonConfig, WatchedFolder
|
||||||
from .store import SyncStore, compute_sha256, is_media_file
|
from .store import SyncStore, compute_sha256, is_media_file
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -67,6 +67,11 @@ def parse_inotify_events(data: bytes) -> list[InotifyEvent]:
|
|||||||
return events
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
def is_ignored_name(name: str, ignore_prefixes: Sequence[str]) -> bool:
|
||||||
|
"""Return True if a file or directory basename matches any configured ignore prefix."""
|
||||||
|
return bool(name) and any(name.startswith(prefix) for prefix in ignore_prefixes if prefix)
|
||||||
|
|
||||||
|
|
||||||
def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> int:
|
def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> int:
|
||||||
"""Recursively scan an enabled watched folder and enqueue new media files."""
|
"""Recursively scan an enabled watched folder and enqueue new media files."""
|
||||||
if not folder.enabled:
|
if not folder.enabled:
|
||||||
@@ -77,13 +82,10 @@ def scan_folder(folder: WatchedFolder, cfg: DaemonConfig, store: SyncStore) -> i
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
enqueued = 0
|
enqueued = 0
|
||||||
ignore_prefixes = [prefix for prefix in cfg.ignore_prefixes if prefix]
|
|
||||||
|
|
||||||
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
||||||
dirnames[:] = sorted(
|
dirnames[:] = sorted(
|
||||||
name
|
name for name in dirnames if not is_ignored_name(name, cfg.ignore_prefixes)
|
||||||
for name in dirnames
|
|
||||||
if not any(name.startswith(prefix) for prefix in ignore_prefixes)
|
|
||||||
)
|
)
|
||||||
current_dir = Path(dirpath)
|
current_dir = Path(dirpath)
|
||||||
for filename in sorted(filenames):
|
for filename in sorted(filenames):
|
||||||
@@ -159,13 +161,16 @@ def _nearest_existing_ancestor(target: Path) -> Path:
|
|||||||
candidate = parent
|
candidate = parent
|
||||||
|
|
||||||
|
|
||||||
def _iter_subdirs(root: Path) -> list[Path]:
|
def _iter_subdirs(
|
||||||
"""Yield root and all non-symlink subdirectories beneath root."""
|
root: Path,
|
||||||
|
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
|
||||||
|
) -> list[Path]:
|
||||||
|
"""Yield root and all non-symlink, non-ignored subdirectories beneath root."""
|
||||||
dirs: list[Path] = []
|
dirs: list[Path] = []
|
||||||
if not root.is_dir():
|
if not root.is_dir():
|
||||||
return dirs
|
return dirs
|
||||||
for dirpath, dirnames, _ in os.walk(root, followlinks=False):
|
for dirpath, dirnames, _ in os.walk(root, followlinks=False):
|
||||||
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
|
dirnames[:] = sorted(d for d in dirnames if not is_ignored_name(d, ignore_prefixes))
|
||||||
dirs.append(Path(dirpath))
|
dirs.append(Path(dirpath))
|
||||||
return dirs
|
return dirs
|
||||||
|
|
||||||
@@ -178,16 +183,25 @@ class PollingWatcher:
|
|||||||
on_file_ready: Callable[[WatchedFolder, Path], None],
|
on_file_ready: Callable[[WatchedFolder, Path], None],
|
||||||
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
|
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
|
||||||
poll_interval: float = 0.5,
|
poll_interval: float = 0.5,
|
||||||
|
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
|
||||||
|
get_ignore_prefixes: Callable[[], Sequence[str]] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._on_file_ready = on_file_ready
|
self._on_file_ready = on_file_ready
|
||||||
self._on_dir_created = on_dir_created
|
self._on_dir_created = on_dir_created
|
||||||
self._poll_interval = max(0.02, float(poll_interval))
|
self._poll_interval = max(0.02, float(poll_interval))
|
||||||
|
self._ignore_prefixes = tuple(ignore_prefixes)
|
||||||
|
self._get_ignore_prefixes = get_ignore_prefixes
|
||||||
self._lock = threading.RLock()
|
self._lock = threading.RLock()
|
||||||
self._stop_event = threading.Event()
|
self._stop_event = threading.Event()
|
||||||
self._thread: threading.Thread | None = None
|
self._thread: threading.Thread | None = None
|
||||||
self._folders: dict[str, WatchedFolder] = {}
|
self._folders: dict[str, WatchedFolder] = {}
|
||||||
self._snapshots: dict[str, dict[str, Any]] = {}
|
self._snapshots: dict[str, dict[str, Any]] = {}
|
||||||
|
|
||||||
|
def _current_ignore_prefixes(self) -> Sequence[str]:
|
||||||
|
if self._get_ignore_prefixes is not None:
|
||||||
|
return self._get_ignore_prefixes()
|
||||||
|
return self._ignore_prefixes
|
||||||
|
|
||||||
def sync_folders(self, folders: list[WatchedFolder]) -> None:
|
def sync_folders(self, folders: list[WatchedFolder]) -> None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
target_ids = {wf.id for wf in folders if wf.enabled}
|
target_ids = {wf.id for wf in folders if wf.enabled}
|
||||||
@@ -219,15 +233,18 @@ class PollingWatcher:
|
|||||||
root = Path(os.path.expanduser(folder.path))
|
root = Path(os.path.expanduser(folder.path))
|
||||||
if not root.is_dir():
|
if not root.is_dir():
|
||||||
return {"present": False, "dirs": set(), "files": {}}
|
return {"present": False, "dirs": set(), "files": {}}
|
||||||
dirs, files = self._scan_tree(root)
|
dirs, files = self._scan_tree(root, self._current_ignore_prefixes())
|
||||||
return {"present": True, "dirs": dirs, "files": files}
|
return {"present": True, "dirs": dirs, "files": files}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _scan_tree(root: Path) -> tuple[set[Path], dict[Path, tuple[int, int]]]:
|
def _scan_tree(
|
||||||
|
root: Path,
|
||||||
|
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
|
||||||
|
) -> tuple[set[Path], dict[Path, tuple[int, int]]]:
|
||||||
dirs: set[Path] = set()
|
dirs: set[Path] = set()
|
||||||
files: dict[Path, tuple[int, int]] = {}
|
files: dict[Path, tuple[int, int]] = {}
|
||||||
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
||||||
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
|
dirnames[:] = sorted(d for d in dirnames if not is_ignored_name(d, ignore_prefixes))
|
||||||
cur = Path(dirpath)
|
cur = Path(dirpath)
|
||||||
dirs.add(cur)
|
dirs.add(cur)
|
||||||
for name in filenames:
|
for name in filenames:
|
||||||
@@ -242,6 +259,7 @@ class PollingWatcher:
|
|||||||
def poll_once(self) -> None:
|
def poll_once(self) -> None:
|
||||||
events_dirs: list[tuple[WatchedFolder, Path]] = []
|
events_dirs: list[tuple[WatchedFolder, Path]] = []
|
||||||
events_files: list[tuple[WatchedFolder, Path]] = []
|
events_files: list[tuple[WatchedFolder, Path]] = []
|
||||||
|
ignore_prefixes = self._current_ignore_prefixes()
|
||||||
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
items = list(self._folders.items())
|
items = list(self._folders.items())
|
||||||
@@ -260,7 +278,7 @@ class PollingWatcher:
|
|||||||
snap["files"] = {}
|
snap["files"] = {}
|
||||||
continue
|
continue
|
||||||
|
|
||||||
dirs, files = self._scan_tree(root)
|
dirs, files = self._scan_tree(root, ignore_prefixes)
|
||||||
prev_dirs: set[Path] = snap["dirs"]
|
prev_dirs: set[Path] = snap["dirs"]
|
||||||
prev_files: dict[Path, tuple[int, int]] = snap["files"]
|
prev_files: dict[Path, tuple[int, int]] = snap["files"]
|
||||||
|
|
||||||
@@ -315,10 +333,14 @@ class InotifyWatcher:
|
|||||||
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
|
on_dir_created: Callable[[WatchedFolder, Path], None] | None = None,
|
||||||
poll_interval: float = 0.5,
|
poll_interval: float = 0.5,
|
||||||
libc: Any | None = None,
|
libc: Any | None = None,
|
||||||
|
ignore_prefixes: Sequence[str] = DEFAULT_IGNORE_PREFIXES,
|
||||||
|
get_ignore_prefixes: Callable[[], Sequence[str]] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._on_file_ready = on_file_ready
|
self._on_file_ready = on_file_ready
|
||||||
self._on_dir_created = on_dir_created
|
self._on_dir_created = on_dir_created
|
||||||
self._poll_interval = max(0.05, float(poll_interval))
|
self._poll_interval = max(0.05, float(poll_interval))
|
||||||
|
self._ignore_prefixes = tuple(ignore_prefixes)
|
||||||
|
self._get_ignore_prefixes = get_ignore_prefixes
|
||||||
self._lock = threading.RLock()
|
self._lock = threading.RLock()
|
||||||
self._stop_event = threading.Event()
|
self._stop_event = threading.Event()
|
||||||
self._thread: threading.Thread | None = None
|
self._thread: threading.Thread | None = None
|
||||||
@@ -353,8 +375,15 @@ class InotifyWatcher:
|
|||||||
on_file_ready=on_file_ready,
|
on_file_ready=on_file_ready,
|
||||||
on_dir_created=on_dir_created,
|
on_dir_created=on_dir_created,
|
||||||
poll_interval=self._poll_interval,
|
poll_interval=self._poll_interval,
|
||||||
|
ignore_prefixes=self._ignore_prefixes,
|
||||||
|
get_ignore_prefixes=self._get_ignore_prefixes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _current_ignore_prefixes(self) -> Sequence[str]:
|
||||||
|
if self._get_ignore_prefixes is not None:
|
||||||
|
return self._get_ignore_prefixes()
|
||||||
|
return self._ignore_prefixes
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_using_polling(self) -> bool:
|
def is_using_polling(self) -> bool:
|
||||||
"""Return True if running with the PollingWatcher fallback."""
|
"""Return True if running with the PollingWatcher fallback."""
|
||||||
@@ -438,7 +467,7 @@ class InotifyWatcher:
|
|||||||
|
|
||||||
def _attach_tree_locked(self, folder: WatchedFolder, root: Path) -> None:
|
def _attach_tree_locked(self, folder: WatchedFolder, root: Path) -> None:
|
||||||
wds = self._folder_wds.setdefault(folder.id, set())
|
wds = self._folder_wds.setdefault(folder.id, set())
|
||||||
for subdir in _iter_subdirs(root):
|
for subdir in _iter_subdirs(root, self._current_ignore_prefixes()):
|
||||||
wd = self._add_watch_locked(subdir, _DIR_WATCH_MASK)
|
wd = self._add_watch_locked(subdir, _DIR_WATCH_MASK)
|
||||||
if wd is not None:
|
if wd is not None:
|
||||||
self._active_by_wd[wd] = (folder, subdir)
|
self._active_by_wd[wd] = (folder, subdir)
|
||||||
@@ -584,9 +613,10 @@ class InotifyWatcher:
|
|||||||
active = self._active_by_wd.get(wd)
|
active = self._active_by_wd.get(wd)
|
||||||
if active is not None:
|
if active is not None:
|
||||||
folder, _ = active
|
folder, _ = active
|
||||||
if created_dir.is_dir():
|
if not is_ignored_name(created_dir.name, self._current_ignore_prefixes()):
|
||||||
self._attach_tree_locked(folder, created_dir)
|
if created_dir.is_dir():
|
||||||
dir_callbacks.append((folder, created_dir))
|
self._attach_tree_locked(folder, created_dir)
|
||||||
|
dir_callbacks.append((folder, created_dir))
|
||||||
|
|
||||||
anc = self._ancestors_by_wd.get(wd)
|
anc = self._ancestors_by_wd.get(wd)
|
||||||
if anc is not None:
|
if anc is not None:
|
||||||
|
|||||||
@@ -711,6 +711,209 @@ class TestWatchersAndLinuxInotify(unittest.TestCase):
|
|||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
def test_recovers_interrupted_uploading_rows_on_startup(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cam_dir = Path(tmp) / "Camera"
|
||||||
|
dest_root = Path(tmp) / "MacDest"
|
||||||
|
cam_dir.mkdir()
|
||||||
|
photo = cam_dir / "crashed_mid_upload.jpg"
|
||||||
|
photo.write_bytes(b"recovered-photo-payload")
|
||||||
|
|
||||||
|
mac_cfg = Path(tmp) / "mac.json"
|
||||||
|
save_mac_config(
|
||||||
|
MacConfig(
|
||||||
|
dest_root=str(dest_root),
|
||||||
|
listen_host="127.0.0.1",
|
||||||
|
listen_port=0,
|
||||||
|
notify_on_batch=False,
|
||||||
|
tailscale_only=True,
|
||||||
|
),
|
||||||
|
mac_cfg,
|
||||||
|
)
|
||||||
|
mac_srv = MacReceiverServer(mac_cfg)
|
||||||
|
mac_srv.start()
|
||||||
|
try:
|
||||||
|
cfg_path = Path(tmp) / "daemon.json"
|
||||||
|
db_path = Path(tmp) / "state.db"
|
||||||
|
sock_path = Path(tmp) / "cp0.sock"
|
||||||
|
save_daemon_config(
|
||||||
|
DaemonConfig(
|
||||||
|
target_host="127.0.0.1",
|
||||||
|
target_port=mac_srv.server_port,
|
||||||
|
watched_folders=[WatchedFolder(id="cam", path=str(cam_dir), label="Camera")],
|
||||||
|
),
|
||||||
|
cfg_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Simulate crash leaving row stuck in UPLOADING in state.db
|
||||||
|
pre_store = SyncStore(db_path)
|
||||||
|
try:
|
||||||
|
pre_store.enqueue_file(
|
||||||
|
"cam",
|
||||||
|
str(photo),
|
||||||
|
"crashed_mid_upload.jpg",
|
||||||
|
len(b"recovered-photo-payload"),
|
||||||
|
photo.stat().st_mtime_ns,
|
||||||
|
compute_sha256(photo),
|
||||||
|
)
|
||||||
|
row = pre_store.next_pending()
|
||||||
|
assert row is not None
|
||||||
|
pre_store.mark_uploading(row["id"])
|
||||||
|
self.assertIsNone(pre_store.next_pending())
|
||||||
|
finally:
|
||||||
|
pre_store.close()
|
||||||
|
|
||||||
|
daemon = SyncDaemon(cfg_path, db_path, sock_path, watcher_poll_interval=0.05)
|
||||||
|
daemon.start()
|
||||||
|
try:
|
||||||
|
self.assertTrue(_wait_until(lambda: (dest_root / "Camera" / "crashed_mid_upload.jpg").exists()))
|
||||||
|
status = _send_ipc(sock_path, {"cmd": "status"})
|
||||||
|
self.assertEqual(status["synced_count"], 1)
|
||||||
|
self.assertEqual(status["pending_count"], 0)
|
||||||
|
finally:
|
||||||
|
daemon.stop()
|
||||||
|
finally:
|
||||||
|
mac_srv.shutdown()
|
||||||
|
mac_srv.server_close()
|
||||||
|
|
||||||
|
def test_concurrent_wake_probe_not_lost_by_run_loop(self) -> None:
|
||||||
|
class _SlowClearEvent(threading.Event):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.clear_entered = threading.Event()
|
||||||
|
self.delay_clear = True
|
||||||
|
|
||||||
|
def clear(self) -> None:
|
||||||
|
self.clear_entered.set()
|
||||||
|
if self.delay_clear:
|
||||||
|
time.sleep(0.05)
|
||||||
|
super().clear()
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
dest_root = Path(tmp) / "MacDest"
|
||||||
|
mac_cfg = Path(tmp) / "mac.json"
|
||||||
|
save_mac_config(
|
||||||
|
MacConfig(
|
||||||
|
dest_root=str(dest_root),
|
||||||
|
listen_host="127.0.0.1",
|
||||||
|
listen_port=0,
|
||||||
|
notify_on_batch=False,
|
||||||
|
tailscale_only=True,
|
||||||
|
),
|
||||||
|
mac_cfg,
|
||||||
|
)
|
||||||
|
mac_srv = MacReceiverServer(mac_cfg)
|
||||||
|
mac_srv.start()
|
||||||
|
try:
|
||||||
|
cfg_holder = [DaemonConfig(target_host="", target_port=mac_srv.server_port, watched_folders=[])]
|
||||||
|
store = SyncStore(Path(tmp) / "state.db")
|
||||||
|
wake_event = _SlowClearEvent()
|
||||||
|
from cp0_syncd.uploader import Uploader
|
||||||
|
|
||||||
|
uploader = Uploader(store=store, get_config=lambda: cfg_holder[0], wake_event=wake_event)
|
||||||
|
uploader.start()
|
||||||
|
try:
|
||||||
|
self.assertTrue(wake_event.clear_entered.wait(timeout=2.0))
|
||||||
|
# Configure target and fire wake(probe=True) while _run_loop is inside clear().
|
||||||
|
cfg_holder[0] = DaemonConfig(
|
||||||
|
target_host="127.0.0.1",
|
||||||
|
target_port=mac_srv.server_port,
|
||||||
|
watched_folders=[],
|
||||||
|
)
|
||||||
|
wake_event.delay_clear = False
|
||||||
|
uploader.wake(reset_backoff=True, probe=True)
|
||||||
|
self.assertTrue(
|
||||||
|
_wait_until(
|
||||||
|
lambda: store.get_meta("mac_online") == "true"
|
||||||
|
and store.get_meta("mac_dest_root") == str(dest_root),
|
||||||
|
timeout=1.5,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
uploader.stop()
|
||||||
|
store.close()
|
||||||
|
finally:
|
||||||
|
mac_srv.shutdown()
|
||||||
|
mac_srv.server_close()
|
||||||
|
|
||||||
|
def test_ignored_subdirectories_skipped_by_watchers_and_on_file_ready(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
cam_dir = root / "Camera"
|
||||||
|
(cam_dir / ".hidden").mkdir(parents=True)
|
||||||
|
(cam_dir / "~tmp").mkdir()
|
||||||
|
(cam_dir / "2026").mkdir()
|
||||||
|
wf = WatchedFolder(id="cam", path=str(cam_dir), label="Camera")
|
||||||
|
|
||||||
|
# 1. PollingWatcher ignores files inside .hidden/ and ~tmp/
|
||||||
|
polled_files: list[Path] = []
|
||||||
|
poller = PollingWatcher(
|
||||||
|
on_file_ready=lambda _wf, p: polled_files.append(p),
|
||||||
|
)
|
||||||
|
poller.sync_folders([wf])
|
||||||
|
(cam_dir / ".hidden" / "a.jpg").write_bytes(b"hidden")
|
||||||
|
(cam_dir / "~tmp" / "b.jpg").write_bytes(b"tilde")
|
||||||
|
(cam_dir / "2026" / "ok.jpg").write_bytes(b"real")
|
||||||
|
poller.poll_once()
|
||||||
|
self.assertEqual(polled_files, [cam_dir / "2026" / "ok.jpg"])
|
||||||
|
|
||||||
|
# 2. InotifyWatcher ignores .hidden/ and ~tmp/ on initial attach and dynamic IN_CREATE | IN_ISDIR
|
||||||
|
fake_libc = _FakeLibc()
|
||||||
|
inotify_dirs: list[Path] = []
|
||||||
|
watcher = InotifyWatcher(
|
||||||
|
on_file_ready=lambda _wf, _p: None,
|
||||||
|
on_dir_created=lambda _wf, p: inotify_dirs.append(p),
|
||||||
|
libc=fake_libc,
|
||||||
|
)
|
||||||
|
watcher.sync_folders([wf])
|
||||||
|
watched_paths = {p for p, _ in fake_libc.watches.values()}
|
||||||
|
self.assertIn(str(cam_dir), watched_paths)
|
||||||
|
self.assertIn(str(cam_dir / "2026"), watched_paths)
|
||||||
|
self.assertNotIn(str(cam_dir / ".hidden"), watched_paths)
|
||||||
|
self.assertNotIn(str(cam_dir / "~tmp"), watched_paths)
|
||||||
|
|
||||||
|
cam_wd = next(wd for wd, (p, _) in fake_libc.watches.items() if p == str(cam_dir))
|
||||||
|
(cam_dir / "~newtmp").mkdir()
|
||||||
|
(cam_dir / ".newdot").mkdir()
|
||||||
|
watcher.process_inotify_bytes(
|
||||||
|
_pack_inotify_event(cam_wd, IN_CREATE | IN_ISDIR, 0, b"~newtmp")
|
||||||
|
+ _pack_inotify_event(cam_wd, IN_CREATE | IN_ISDIR, 0, b".newdot")
|
||||||
|
)
|
||||||
|
watched_after = {p for p, _ in fake_libc.watches.values()}
|
||||||
|
self.assertNotIn(str(cam_dir / "~newtmp"), watched_after)
|
||||||
|
self.assertNotIn(str(cam_dir / ".newdot"), watched_after)
|
||||||
|
self.assertEqual(inotify_dirs, [])
|
||||||
|
|
||||||
|
# 3. SyncDaemon._on_file_ready skips files whose relative parent parts start with ignore_prefixes
|
||||||
|
cfg_path = root / "daemon.json"
|
||||||
|
db_path = root / "state.db"
|
||||||
|
sock_path = root / "cp0.sock"
|
||||||
|
save_daemon_config(
|
||||||
|
DaemonConfig(
|
||||||
|
watched_folders=[wf],
|
||||||
|
ignore_prefixes=[".", "~", "SKIP_"],
|
||||||
|
),
|
||||||
|
cfg_path,
|
||||||
|
)
|
||||||
|
daemon = SyncDaemon(cfg_path, db_path, sock_path)
|
||||||
|
try:
|
||||||
|
(cam_dir / "SKIP_dir").mkdir()
|
||||||
|
skip_file = cam_dir / "SKIP_dir" / "ignored.jpg"
|
||||||
|
tilde_file = cam_dir / "~tmp" / "b.jpg"
|
||||||
|
dot_file = cam_dir / ".hidden" / "a.jpg"
|
||||||
|
skip_file.write_bytes(b"skip-me")
|
||||||
|
daemon._on_file_ready(wf, dot_file)
|
||||||
|
daemon._on_file_ready(wf, tilde_file)
|
||||||
|
daemon._on_file_ready(wf, skip_file)
|
||||||
|
self.assertIsNone(daemon.store.next_pending())
|
||||||
|
|
||||||
|
daemon._on_file_ready(wf, cam_dir / "2026" / "ok.jpg")
|
||||||
|
pending = daemon.store.next_pending()
|
||||||
|
assert pending is not None
|
||||||
|
self.assertEqual(pending["rel_path"], "2026/ok.jpg")
|
||||||
|
finally:
|
||||||
|
daemon.store.close()
|
||||||
|
|
||||||
|
|
||||||
class TestDaemonBinaryEntrypoint(unittest.TestCase):
|
class TestDaemonBinaryEntrypoint(unittest.TestCase):
|
||||||
def test_cp0_syncd_binary_runs_via_symlink_and_stops_on_sigterm(self) -> None:
|
def test_cp0_syncd_binary_runs_via_symlink_and_stops_on_sigterm(self) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user