Files
cardputerzero-sync/device-daemon/cp0_syncd/store.py

453 lines
15 KiB
Python

"""SQLite WAL queue store and media file filtering for cp0-syncd."""
from __future__ import annotations
import hashlib
import logging
import os
import sqlite3
import stat
import threading
import time
from pathlib import Path
from typing import Any
from .config import DaemonConfig, WatchedFolder
logger = logging.getLogger(__name__)
ENV_DB_PATH = "CP0_SYNC_DB"
MAX_LOG_ROWS = 500
CHUNK_SIZE = 65536
def default_db_path() -> Path:
"""Return the default SQLite state database path, honoring CP0_SYNC_DB if set."""
override = os.environ.get(ENV_DB_PATH, "").strip()
if override:
return Path(override).expanduser()
return Path.home() / ".local" / "share" / "cp0-sync" / "state.db"
def compute_sha256(path: Path, chunk_size: int = CHUNK_SIZE) -> str:
"""Compute the lowercase hex SHA-256 digest of a file incrementally."""
digest = hashlib.sha256()
with open(path, "rb") as handle:
while True:
chunk = handle.read(chunk_size)
if not chunk:
break
digest.update(chunk)
return digest.hexdigest()
def is_media_file(
path: Path,
cfg: DaemonConfig,
store: SyncStore | None = None,
) -> bool:
"""Return True if path is a non-empty regular media file permitted by DaemonConfig."""
name = path.name
if not name:
return False
if any(name.startswith(prefix) for prefix in cfg.ignore_prefixes if prefix):
return False
name_lower = name.lower()
if any(name_lower.endswith(suffix.lower()) for suffix in cfg.ignore_suffixes if suffix):
return False
allowed_exts = {
ext.lower() if ext.startswith(".") else f".{ext.lower()}"
for ext in cfg.extensions
if ext
}
if path.suffix.lower() not in allowed_exts:
return False
try:
st = path.stat()
except OSError:
return False
if not stat.S_ISREG(st.st_mode):
return False
if st.st_size == 0:
logger.warning("Skipping 0-byte media file: %s", path)
if store is not None:
store.log("WARN", f"Skipped 0-byte file: {name}")
return False
return True
def _now_ms() -> int:
return int(time.time() * 1000)
class SyncStore:
"""Thread-safe SQLite WAL queue, metadata, and ring-buffered log store."""
def __init__(self, db_path: Path | None = None) -> None:
resolved = Path(db_path).expanduser() if db_path is not None else default_db_path()
resolved.parent.mkdir(parents=True, exist_ok=True)
self.db_path = resolved
self._lock = threading.RLock()
self._conn = sqlite3.connect(str(resolved), timeout=5.0, check_same_thread=False)
self._conn.row_factory = sqlite3.Row
self._closed = False
self._init_schema()
def _init_schema(self) -> None:
with self._lock:
self._conn.execute("PRAGMA journal_mode=WAL;")
self._conn.execute("PRAGMA busy_timeout=5000;")
self._conn.execute("PRAGMA synchronous=NORMAL;")
self._conn.executescript(
"""
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
folder_id TEXT NOT NULL,
abs_path TEXT NOT NULL,
rel_path TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
mtime_ns INTEGER NOT NULL,
sha256 TEXT NOT NULL,
status TEXT NOT NULL,
retry_count INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
discovered_at INTEGER NOT NULL,
synced_at INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_files_dedup
ON files(folder_id, rel_path, sha256);
CREATE INDEX IF NOT EXISTS idx_files_status_discovered
ON files(status, discovered_at, id);
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL,
level TEXT NOT NULL,
message TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"""
)
self._conn.commit()
def enqueue_file(
self,
folder_id: str,
abs_path: str,
rel_path: str,
size_bytes: int,
mtime_ns: int,
sha256: str,
) -> str:
"""Enqueue a file if not already synced/queued; skip 0-byte payloads with a WARN log."""
if size_bytes <= 0:
self.log("WARN", f"Skipped 0-byte file: {rel_path}")
return "skipped_empty"
now_ms = _now_ms()
with self._lock:
row = self._conn.execute(
"""
SELECT id, status
FROM files
WHERE folder_id = ? AND rel_path = ? AND sha256 = ?
""",
(folder_id, rel_path, sha256),
).fetchone()
if row is not None:
status = str(row["status"])
if status == "SYNCED":
return "already_synced"
if status == "FAILED":
return "already_failed"
return "already_queued"
self._conn.execute(
"""
INSERT INTO files (
folder_id, abs_path, rel_path, size_bytes, mtime_ns,
sha256, status, retry_count, last_error, discovered_at, synced_at
)
VALUES (?, ?, ?, ?, ?, ?, 'PENDING', 0, NULL, ?, NULL)
""",
(folder_id, abs_path, rel_path, size_bytes, mtime_ns, sha256, now_ms),
)
self._conn.commit()
return "enqueued"
def next_pending(self) -> dict[str, Any] | None:
"""Return the oldest PENDING file row as a dict, or None if the queue is empty."""
with self._lock:
row = self._conn.execute(
"""
SELECT id, folder_id, abs_path, rel_path, size_bytes, mtime_ns,
sha256, status, retry_count, last_error, discovered_at, synced_at
FROM files
WHERE status = 'PENDING'
ORDER BY discovered_at ASC, id ASC
LIMIT 1
"""
).fetchone()
return dict(row) if row is not None else None
def mark_uploading(self, file_id: int) -> None:
"""Transition a queued file to UPLOADING."""
with self._lock:
self._conn.execute(
"UPDATE files SET status = 'UPLOADING' WHERE id = ?",
(file_id,),
)
self._conn.commit()
def recover_interrupted_uploads(self) -> int:
"""Reset stale UPLOADING rows back to PENDING after an unclean restart."""
with self._lock:
cur = self._conn.execute(
"""
UPDATE files
SET status = 'PENDING'
WHERE status = 'UPLOADING'
"""
)
self._conn.commit()
return cur.rowcount
def mark_synced(self, file_id: int) -> None:
"""Mark a file as SYNCED and clear any prior error."""
now_ms = _now_ms()
with self._lock:
self._conn.execute(
"""
UPDATE files
SET status = 'SYNCED', synced_at = ?, last_error = NULL
WHERE id = ?
""",
(now_ms, file_id),
)
self._conn.commit()
def mark_retry(self, file_id: int, err: str) -> None:
"""Return an item to PENDING while recording the retry count and error."""
with self._lock:
self._conn.execute(
"""
UPDATE files
SET status = 'PENDING', retry_count = retry_count + 1, last_error = ?
WHERE id = ?
""",
(err, file_id),
)
self._conn.commit()
def mark_failed(self, file_id: int, err: str) -> None:
"""Mark a file as FAILED after a permanent per-file rejection."""
with self._lock:
self._conn.execute(
"""
UPDATE files
SET status = 'FAILED', retry_count = retry_count + 1, last_error = ?
WHERE id = ?
""",
(err, file_id),
)
self._conn.commit()
def delete_file(self, file_id: int) -> None:
"""Delete a file record (e.g., when the source file vanished before upload)."""
with self._lock:
self._conn.execute("DELETE FROM files WHERE id = ?", (file_id,))
self._conn.commit()
def purge_pending_for_folder(self, folder_id: str) -> int:
"""Delete un-synced PENDING/UPLOADING queue entries for a removed folder."""
with self._lock:
cur = self._conn.execute(
"""
DELETE FROM files
WHERE folder_id = ? AND status IN ('PENDING', 'UPLOADING')
""",
(folder_id,),
)
self._conn.commit()
return cur.rowcount
def retry_all_failed(self) -> int:
"""Flip all FAILED rows back to PENDING and return the count updated."""
with self._lock:
cur = self._conn.execute(
"""
UPDATE files
SET status = 'PENDING', last_error = NULL
WHERE status = 'FAILED'
"""
)
self._conn.commit()
return cur.rowcount
def log(self, level: str, message: str) -> None:
"""Append a log row and prune the logs table to the newest 500 rows."""
now_ms = _now_ms()
normalized_level = level.strip().upper() or "INFO"
with self._lock:
self._conn.execute(
"INSERT INTO logs (ts, level, message) VALUES (?, ?, ?)",
(now_ms, normalized_level, message),
)
self._conn.execute(
"""
DELETE FROM logs
WHERE id NOT IN (
SELECT id FROM logs ORDER BY id DESC LIMIT ?
)
""",
(MAX_LOG_ROWS,),
)
self._conn.commit()
def log_count(self) -> int:
"""Return the number of rows currently stored in the logs table."""
with self._lock:
row = self._conn.execute("SELECT COUNT(*) FROM logs").fetchone()
return int(row[0]) if row is not None else 0
def list_logs(self, limit: int = 100) -> list[dict[str, Any]]:
"""Return up to `limit` newest log rows ordered newest-first."""
safe_limit = max(0, int(limit))
with self._lock:
rows = self._conn.execute(
"""
SELECT id, ts, level, message
FROM logs
ORDER BY id DESC
LIMIT ?
""",
(safe_limit,),
).fetchall()
return [dict(row) for row in rows]
def set_meta(self, key: str, value: str) -> None:
"""Upsert a metadata key-value pair."""
with self._lock:
self._conn.execute(
"""
INSERT INTO meta (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
""",
(key, str(value)),
)
self._conn.commit()
def get_meta(self, key: str) -> str | None:
"""Fetch a metadata value by key, or None if unset."""
with self._lock:
row = self._conn.execute(
"SELECT value FROM meta WHERE key = ?",
(key,),
).fetchone()
return str(row["value"]) if row is not None else None
def get_status_snapshot(self, watched_folders: list[WatchedFolder]) -> dict[str, Any]:
"""Return queue totals, per-folder counts, metadata, and recent logs for status IPC/UI."""
with self._lock:
folder_stats: dict[str, dict[str, int]] = {}
rows = self._conn.execute(
"""
SELECT folder_id, status, COUNT(*) AS cnt, COALESCE(SUM(size_bytes), 0) AS total_bytes
FROM files
GROUP BY folder_id, status
"""
).fetchall()
totals = {
"pending": 0,
"uploading": 0,
"synced": 0,
"failed": 0,
"pending_bytes": 0,
}
for row in rows:
fid = str(row["folder_id"])
status = str(row["status"])
cnt = int(row["cnt"])
total_bytes = int(row["total_bytes"])
bucket = folder_stats.setdefault(
fid,
{
"pending": 0,
"uploading": 0,
"synced": 0,
"failed": 0,
"pending_bytes": 0,
},
)
if status == "PENDING":
bucket["pending"] += cnt
bucket["pending_bytes"] += total_bytes
totals["pending"] += cnt
totals["pending_bytes"] += total_bytes
elif status == "UPLOADING":
bucket["pending"] += cnt
bucket["uploading"] += cnt
bucket["pending_bytes"] += total_bytes
totals["pending"] += cnt
totals["uploading"] += cnt
totals["pending_bytes"] += total_bytes
elif status == "SYNCED":
bucket["synced"] += cnt
totals["synced"] += cnt
elif status == "FAILED":
bucket["failed"] += cnt
totals["failed"] += cnt
meta_rows = self._conn.execute("SELECT key, value FROM meta").fetchall()
meta = {str(r["key"]): str(r["value"]) for r in meta_rows}
recent_logs = self.list_logs(limit=50)
folders_out = []
for wf in watched_folders:
stats = folder_stats.get(
wf.id,
{"pending": 0, "uploading": 0, "synced": 0, "failed": 0, "pending_bytes": 0},
)
folders_out.append(
{
"id": wf.id,
"path": wf.path,
"label": wf.label,
"enabled": wf.enabled,
"exists": os.path.isdir(os.path.expanduser(wf.path)),
"pending": stats["pending"],
"uploading": stats["uploading"],
"synced": stats["synced"],
"failed": stats["failed"],
"pending_bytes": stats["pending_bytes"],
}
)
return {
"counts": totals,
"folders": folders_out,
"meta": meta,
"logs": recent_logs,
}
def close(self) -> None:
"""Close the SQLite connection."""
with self._lock:
if not self._closed:
self._conn.close()
self._closed = True