Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
136 lines
5.3 KiB
Python
136 lines
5.3 KiB
Python
"""Verified, rotated SQLite backups and online restore.
|
|
|
|
* Backups are written with ``VACUUM INTO`` (a consistent, compacted copy
|
|
taken inside a read transaction, safe while ServiceMap runs) to
|
|
``<data dir>/backups`` and checked with ``PRAGMA quick_check`` before
|
|
they count. A copy that fails the check is deleted, never kept.
|
|
* Rotation keeps the newest ``keep`` scheduled/manual backups; pre-restore
|
|
safety copies are rotated the same way.
|
|
* Restore uses SQLite's online backup API from the chosen file into the
|
|
live database: it is transactional and needs no file swapping under open
|
|
WAL connections. A fresh ``pre-restore`` backup is taken first.
|
|
* Legacy pre-upgrade copies (``servicemap-v*-pre-*.sqlite3`` next to the
|
|
database) are pruned to the newest ``legacy_keep`` only after a new
|
|
backup passed its check.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
from contextlib import closing
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from .store import Store
|
|
|
|
_NAME = re.compile(r"^servicemap-(?P<kind>backup|pre-restore)-(?P<stamp>\d{8}T\d{6}Z)\.db$")
|
|
_LEGACY = re.compile(r"^servicemap-v\d+\.\d+\.\d+-pre-[A-Za-z0-9T._-]+\.sqlite3$")
|
|
|
|
|
|
class BackupInfo(BaseModel):
|
|
name: str
|
|
kind: str
|
|
created_at: datetime
|
|
size_bytes: int
|
|
|
|
|
|
class BackupManager:
|
|
def __init__(self, store: Store, *, keep: int = 14, legacy_keep: int = 2) -> None:
|
|
self.store = store
|
|
self.keep = max(1, keep)
|
|
self.legacy_keep = max(0, legacy_keep)
|
|
|
|
@property
|
|
def directory(self) -> Path:
|
|
return self.store.path.parent / "backups"
|
|
|
|
def list(self) -> list[BackupInfo]:
|
|
if not self.directory.exists():
|
|
return []
|
|
items: list[BackupInfo] = []
|
|
for path in self.directory.iterdir():
|
|
match = _NAME.match(path.name)
|
|
if not match or not path.is_file():
|
|
continue
|
|
created = datetime.strptime(match["stamp"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
|
|
items.append(
|
|
BackupInfo(name=path.name, kind=match["kind"], created_at=created, size_bytes=path.stat().st_size)
|
|
)
|
|
return sorted(items, key=lambda item: item.created_at, reverse=True)
|
|
|
|
def path_for(self, name: str) -> Path:
|
|
"""Resolve a listed backup name; anything else is refused."""
|
|
|
|
if not _NAME.match(name):
|
|
raise ValueError("Onbekende back-up")
|
|
path = self.directory / name
|
|
if not path.is_file():
|
|
raise ValueError("Onbekende back-up")
|
|
return path
|
|
|
|
def create(self, kind: str = "backup", now: datetime | None = None) -> BackupInfo:
|
|
now = now or datetime.now(UTC)
|
|
self.directory.mkdir(parents=True, exist_ok=True)
|
|
name = f"servicemap-{kind}-{now.strftime('%Y%m%dT%H%M%SZ')}.db"
|
|
target = self.directory / name
|
|
if target.exists():
|
|
raise ValueError("Er bestaat al een back-up met dit tijdstip")
|
|
with self.store._connect() as db:
|
|
db.execute("VACUUM INTO ?", (str(target),))
|
|
try:
|
|
os.chmod(target, 0o600)
|
|
except OSError:
|
|
pass
|
|
if not self._verify(target):
|
|
target.unlink(missing_ok=True)
|
|
raise RuntimeError("De nieuwe back-up faalde de integriteitscontrole en is verwijderd")
|
|
self._rotate()
|
|
self._prune_legacy()
|
|
return next(item for item in self.list() if item.name == name)
|
|
|
|
def restore(self, name: str) -> BackupInfo:
|
|
source = self.path_for(name)
|
|
if not self._verify(source):
|
|
raise RuntimeError("De gekozen back-up faalde de integriteitscontrole; er is niets teruggezet")
|
|
if self.store.demo_active():
|
|
raise ValueError("Sluit de demo af voor je een back-up terugzet")
|
|
safety = self.create(kind="pre-restore")
|
|
# closing(): a sqlite3 connection's own context manager only commits.
|
|
with closing(sqlite3.connect(source)) as src, closing(sqlite3.connect(self.store.path, timeout=30)) as dst:
|
|
src.backup(dst)
|
|
return safety
|
|
|
|
@staticmethod
|
|
def _verify(path: Path) -> bool:
|
|
try:
|
|
with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True)) as db:
|
|
result = db.execute("PRAGMA quick_check").fetchone()
|
|
tables = db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='services'").fetchone()
|
|
except sqlite3.Error:
|
|
return False
|
|
return bool(result and result[0] == "ok" and tables)
|
|
|
|
def _rotate(self) -> None:
|
|
for kind in ("backup", "pre-restore"):
|
|
for stale in [item for item in self.list() if item.kind == kind][self.keep:]:
|
|
(self.directory / stale.name).unlink(missing_ok=True)
|
|
|
|
def _prune_legacy(self) -> list[str]:
|
|
folder = self.store.path.parent
|
|
legacy = sorted(
|
|
(path for path in folder.iterdir() if path.is_file() and _LEGACY.match(path.name)),
|
|
key=lambda path: path.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
removed: list[str] = []
|
|
for path in legacy[self.legacy_keep:]:
|
|
for candidate in (path, Path(f"{path}-wal"), Path(f"{path}-shm"), Path(f"{path}-journal")):
|
|
if candidate.exists():
|
|
candidate.unlink()
|
|
removed.append(path.name)
|
|
return removed
|