356 lines
14 KiB
Python
356 lines
14 KiB
Python
"""Dateisystem-Synchronisation ohne Abhängigkeit von der Benutzeroberfläche."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
from collections.abc import Callable
|
|
from contextlib import suppress
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from syncvito.file_locks import LockingProcess, find_locking_processes, is_file_locked
|
|
from syncvito.models import ConflictMode, DeleteMode, SyncDirection, SyncTask
|
|
from syncvito.state_store import StateStore
|
|
|
|
__all__ = [
|
|
"BlockedFile",
|
|
"ProgressCallback",
|
|
"SyncEngine",
|
|
"SyncResult",
|
|
"UnavailablePathError",
|
|
]
|
|
|
|
ProgressCallback = Callable[[int, str], None]
|
|
|
|
|
|
class UnavailablePathError(RuntimeError):
|
|
"""Ein für die Synchronisation benötigter Pfad ist nicht erreichbar."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class FileRecord:
|
|
"""Kompakte Dateisystemmetadaten für Vergleiche."""
|
|
|
|
path: Path
|
|
size: int
|
|
mtime_ns: int
|
|
|
|
@property
|
|
def signature(self) -> str:
|
|
"""Liefert eine persistierbare Signatur."""
|
|
return f"{self.size}:{self.mtime_ns}"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class BlockedFile:
|
|
"""Eine zurückgestellte Datei einschließlich bekannter Sperrprozesse."""
|
|
|
|
path: Path
|
|
processes: tuple[LockingProcess, ...] = ()
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class SyncResult:
|
|
"""Zusammenfassung eines vollständigen Aufgabenlaufs."""
|
|
|
|
copied: int = 0
|
|
deleted: int = 0
|
|
conflicts: int = 0
|
|
skipped: int = 0
|
|
blocked: list[BlockedFile] = field(default_factory=list)
|
|
errors: list[str] = field(default_factory=list)
|
|
inaccessible: bool = False
|
|
|
|
@property
|
|
def successful(self) -> bool:
|
|
"""Zeigt an, ob der Lauf ohne Fehler und Sperren abgeschlossen wurde."""
|
|
return not self.errors and not self.blocked and not self.inaccessible
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _Operation:
|
|
kind: str
|
|
source: Path | None
|
|
target: Path
|
|
relative: str
|
|
conflict_copy: bool = False
|
|
|
|
|
|
class SyncEngine:
|
|
"""Plant und verarbeitet Einweg- sowie bidirektionale Aufgaben."""
|
|
|
|
def __init__(self, state_store: StateStore | None = None) -> None:
|
|
self.state_store = state_store or StateStore()
|
|
|
|
def synchronize(self, task: SyncTask, progress: ProgressCallback | None = None) -> SyncResult:
|
|
"""Führt eine Aufgabe aus und isoliert Fehler auf einzelne Dateien."""
|
|
task.validate()
|
|
source_root = Path(task.source)
|
|
target_root = Path(task.target)
|
|
self._validate_roots(source_root, target_root)
|
|
source_files, source_dirs = self._inventory(source_root, task.selected_paths)
|
|
target_files, target_dirs = self._inventory(target_root, task.selected_paths)
|
|
previous = self.state_store.get(task.id)
|
|
|
|
if task.direction is SyncDirection.ONE_WAY:
|
|
operations = self._plan_one_way(
|
|
task, source_root, target_root, source_files, target_files, source_dirs, target_dirs
|
|
)
|
|
else:
|
|
operations = self._plan_bidirectional(
|
|
task,
|
|
source_root,
|
|
target_root,
|
|
source_files,
|
|
target_files,
|
|
source_dirs,
|
|
target_dirs,
|
|
previous,
|
|
)
|
|
|
|
result = SyncResult()
|
|
total = max(1, len(operations))
|
|
if progress:
|
|
progress(0, "Änderungen werden vorbereitet …")
|
|
for index, operation in enumerate(operations, start=1):
|
|
try:
|
|
self._execute(operation, result)
|
|
except (OSError, shutil.Error) as exc:
|
|
failed_path = (
|
|
Path(exc.filename2 or exc.filename)
|
|
if isinstance(exc, OSError) and (exc.filename2 or exc.filename)
|
|
else operation.target
|
|
)
|
|
blocked_path = _blocked_path(operation, exc)
|
|
if blocked_path is not None:
|
|
processes = tuple(find_locking_processes(blocked_path))
|
|
result.blocked.append(BlockedFile(blocked_path, processes))
|
|
elif isinstance(exc, PermissionError):
|
|
result.errors.append(f"Zugriff verweigert: {failed_path} ({exc})")
|
|
else:
|
|
result.errors.append(f"{failed_path}: {exc}")
|
|
if progress:
|
|
detail = f"{operation.kind}: {operation.relative}"
|
|
progress(round(index / total * 100), detail)
|
|
|
|
if not operations and progress:
|
|
progress(100, "Keine Änderungen gefunden")
|
|
|
|
if task.direction is SyncDirection.BIDIRECTIONAL and result.successful:
|
|
refreshed_source, _ = self._inventory(source_root, task.selected_paths)
|
|
refreshed_target, _ = self._inventory(target_root, task.selected_paths)
|
|
self.state_store.set(task.id, self._state_from_inventories(refreshed_source, refreshed_target))
|
|
return result
|
|
|
|
@staticmethod
|
|
def _validate_roots(source_root: Path, target_root: Path) -> None:
|
|
if not source_root.is_dir():
|
|
raise UnavailablePathError(f"Quellordner nicht erreichbar: {source_root}")
|
|
if not target_root.exists():
|
|
try:
|
|
target_root.mkdir(parents=True, exist_ok=True)
|
|
except OSError as exc:
|
|
raise UnavailablePathError(f"Zielordner nicht erreichbar: {target_root}") from exc
|
|
if not target_root.is_dir():
|
|
raise UnavailablePathError(f"Ziel ist kein Ordner: {target_root}")
|
|
|
|
@staticmethod
|
|
def _inventory(root: Path, selected_paths: list[str]) -> tuple[dict[str, FileRecord], set[str]]:
|
|
files: dict[str, FileRecord] = {}
|
|
directories: set[str] = set()
|
|
selections = tuple(Path(item).parts for item in selected_paths)
|
|
try:
|
|
for current, dir_names, file_names in os.walk(root):
|
|
current_path = Path(current)
|
|
relative_dir = current_path.relative_to(root)
|
|
dir_names[:] = [name for name in dir_names if not (current_path / name).is_symlink()]
|
|
for directory_name in dir_names:
|
|
relative = relative_dir / directory_name
|
|
if _is_selected(relative.parts, selections):
|
|
directories.add(str(relative))
|
|
for file_name in file_names:
|
|
path = current_path / file_name
|
|
if path.is_symlink():
|
|
continue
|
|
relative = path.relative_to(root)
|
|
if not _is_selected(relative.parts, selections):
|
|
continue
|
|
try:
|
|
file_stat = path.stat()
|
|
except OSError:
|
|
continue
|
|
files[str(relative)] = FileRecord(path, file_stat.st_size, file_stat.st_mtime_ns)
|
|
except OSError as exc:
|
|
raise UnavailablePathError(f"Ordner kann nicht gelesen werden: {root}") from exc
|
|
return files, directories
|
|
|
|
@staticmethod
|
|
def _plan_one_way(
|
|
task: SyncTask,
|
|
source_root: Path,
|
|
target_root: Path,
|
|
source_files: dict[str, FileRecord],
|
|
target_files: dict[str, FileRecord],
|
|
source_dirs: set[str],
|
|
target_dirs: set[str],
|
|
) -> list[_Operation]:
|
|
operations: list[_Operation] = []
|
|
for relative in sorted(source_dirs - target_dirs):
|
|
operations.append(_Operation("Ordner anlegen", None, target_root / relative, relative))
|
|
for relative, source in sorted(source_files.items()):
|
|
target = target_files.get(relative)
|
|
if target is None or target.signature != source.signature:
|
|
operations.append(_Operation("Kopieren", source.path, target_root / relative, relative))
|
|
if task.delete_mode is DeleteMode.MIRROR:
|
|
for relative in sorted(target_files.keys() - source_files.keys()):
|
|
operations.append(_Operation("Löschen", None, target_root / relative, relative))
|
|
for relative in sorted(target_dirs - source_dirs, key=lambda item: len(Path(item).parts), reverse=True):
|
|
operations.append(_Operation("Ordner löschen", None, target_root / relative, relative))
|
|
return operations
|
|
|
|
@staticmethod
|
|
def _plan_bidirectional(
|
|
task: SyncTask,
|
|
source_root: Path,
|
|
target_root: Path,
|
|
source_files: dict[str, FileRecord],
|
|
target_files: dict[str, FileRecord],
|
|
source_dirs: set[str],
|
|
target_dirs: set[str],
|
|
previous: dict[str, dict[str, str | None]],
|
|
) -> list[_Operation]:
|
|
operations = [
|
|
_Operation("Ordner anlegen", None, target_root / relative, relative)
|
|
for relative in sorted(source_dirs - target_dirs)
|
|
]
|
|
operations.extend(
|
|
_Operation("Ordner anlegen", None, source_root / relative, relative)
|
|
for relative in sorted(target_dirs - source_dirs)
|
|
)
|
|
all_paths = sorted(source_files.keys() | target_files.keys() | previous.keys())
|
|
for relative in all_paths:
|
|
source = source_files.get(relative)
|
|
target = target_files.get(relative)
|
|
old = previous.get(relative, {})
|
|
old_source = old.get("source")
|
|
old_target = old.get("target")
|
|
if source and target:
|
|
if source.signature == target.signature:
|
|
continue
|
|
winner, destination = _newer_pair(source, target)
|
|
conflict = task.conflict_mode is ConflictMode.CONFLICT_COPY
|
|
operations.append(
|
|
_Operation(
|
|
"Konflikt lösen" if conflict else "Aktualisieren",
|
|
winner.path,
|
|
destination.path,
|
|
relative,
|
|
conflict,
|
|
)
|
|
)
|
|
elif source:
|
|
target_was_deleted = old_target is not None and source.signature == old_source
|
|
if task.delete_mode is DeleteMode.MIRROR and target_was_deleted:
|
|
operations.append(_Operation("Löschung spiegeln", None, source.path, relative))
|
|
else:
|
|
operations.append(_Operation("Kopieren", source.path, target_root / relative, relative))
|
|
elif target:
|
|
source_was_deleted = old_source is not None and target.signature == old_target
|
|
if task.delete_mode is DeleteMode.MIRROR and source_was_deleted:
|
|
operations.append(_Operation("Löschung spiegeln", None, target.path, relative))
|
|
else:
|
|
operations.append(_Operation("Kopieren", target.path, source_root / relative, relative))
|
|
return operations
|
|
|
|
@staticmethod
|
|
def _execute(operation: _Operation, result: SyncResult) -> None:
|
|
if operation.kind == "Ordner anlegen":
|
|
operation.target.mkdir(parents=True, exist_ok=True)
|
|
return
|
|
if operation.kind in {"Löschen", "Löschung spiegeln"}:
|
|
operation.target.unlink(missing_ok=True)
|
|
result.deleted += 1
|
|
return
|
|
if operation.kind == "Ordner löschen":
|
|
try:
|
|
operation.target.rmdir()
|
|
result.deleted += 1
|
|
except OSError:
|
|
result.skipped += 1
|
|
return
|
|
if operation.source is None:
|
|
return
|
|
operation.target.parent.mkdir(parents=True, exist_ok=True)
|
|
if operation.conflict_copy and operation.target.exists():
|
|
conflict_path = _conflict_path(operation.target)
|
|
shutil.copy2(operation.target, conflict_path)
|
|
result.conflicts += 1
|
|
_atomic_copy(operation.source, operation.target)
|
|
result.copied += 1
|
|
|
|
@staticmethod
|
|
def _state_from_inventories(
|
|
source_files: dict[str, FileRecord], target_files: dict[str, FileRecord]
|
|
) -> dict[str, dict[str, str | None]]:
|
|
return {
|
|
relative: {
|
|
"source": source_files[relative].signature if relative in source_files else None,
|
|
"target": target_files[relative].signature if relative in target_files else None,
|
|
}
|
|
for relative in source_files.keys() | target_files.keys()
|
|
}
|
|
|
|
|
|
def _blocked_path(operation: _Operation, exc: OSError | shutil.Error) -> Path | None:
|
|
"""Ordnet auch Windows-Kopierfehler ohne Dateinamen der gesperrten Seite zu."""
|
|
winerror = getattr(exc, "winerror", None)
|
|
if winerror not in {5, 32, 33}:
|
|
return None
|
|
reported = (exc.filename2 or exc.filename) if isinstance(exc, OSError) else None
|
|
if reported and winerror in {32, 33}:
|
|
return Path(reported)
|
|
candidates = [Path(reported)] if reported else []
|
|
candidates.extend(path for path in (operation.source, operation.target) if path is not None)
|
|
for path in dict.fromkeys(candidates):
|
|
if is_file_locked(path):
|
|
return path
|
|
# Eine inzwischen freigegebene Datei beim nächsten Lauf erneut versuchen.
|
|
if winerror in {32, 33}:
|
|
return operation.source or operation.target
|
|
return None
|
|
|
|
|
|
def _atomic_copy(source: Path, target: Path) -> None:
|
|
temporary = target.with_name(f".{target.name}.syncvito.tmp")
|
|
try:
|
|
shutil.copy2(source, temporary)
|
|
os.replace(temporary, target)
|
|
except BaseException:
|
|
with suppress(OSError):
|
|
temporary.unlink(missing_ok=True)
|
|
raise
|
|
|
|
|
|
def _conflict_path(path: Path) -> Path:
|
|
timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
candidate = path.with_name(f"{path.stem}_conflict_{timestamp}{path.suffix}")
|
|
counter = 2
|
|
while candidate.exists():
|
|
candidate = path.with_name(f"{path.stem}_conflict_{timestamp}_{counter}{path.suffix}")
|
|
counter += 1
|
|
return candidate
|
|
|
|
|
|
def _newer_pair(left: FileRecord, right: FileRecord) -> tuple[FileRecord, FileRecord]:
|
|
if left.mtime_ns >= right.mtime_ns:
|
|
return left, right
|
|
return right, left
|
|
|
|
|
|
def _is_selected(parts: tuple[str, ...], selections: tuple[tuple[str, ...], ...]) -> bool:
|
|
if not selections:
|
|
return True
|
|
return any(parts[: len(selection)] == selection or selection[: len(parts)] == parts for selection in selections)
|