ServiceMap: update tests and backend/app (+7 areas)
Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
This commit is contained in:
1 parent
d7319250e8
commit
adc2c70c21
91 files changed
+9205
-1691
No files matched your search
@@ -0,0 +1,135 @@
|
||||
"""Verified, rotated SQLite backups and online restore.
|
||||
|
||||
* Backups are written with ``VACUUM INTO`` (a consistent, compacted copy
|
||||
taken inside a read transaction, safe while ServiceMap runs) to
|
||||
``<data dir>/backups`` and checked with ``PRAGMA quick_check`` before
|
||||
they count. A copy that fails the check is deleted, never kept.
|
||||
* Rotation keeps the newest ``keep`` scheduled/manual backups; pre-restore
|
||||
safety copies are rotated the same way.
|
||||
* Restore uses SQLite's online backup API from the chosen file into the
|
||||
live database: it is transactional and needs no file swapping under open
|
||||
WAL connections. A fresh ``pre-restore`` backup is taken first.
|
||||
* Legacy pre-upgrade copies (``servicemap-v*-pre-*.sqlite3`` next to the
|
||||
database) are pruned to the newest ``legacy_keep`` only after a new
|
||||
backup passed its check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .store import Store
|
||||
|
||||
_NAME = re.compile(r"^servicemap-(?P<kind>backup|pre-restore)-(?P<stamp>\d{8}T\d{6}Z)\.db$")
|
||||
_LEGACY = re.compile(r"^servicemap-v\d+\.\d+\.\d+-pre-[A-Za-z0-9T._-]+\.sqlite3$")
|
||||
|
||||
|
||||
class BackupInfo(BaseModel):
|
||||
name: str
|
||||
kind: str
|
||||
created_at: datetime
|
||||
size_bytes: int
|
||||
|
||||
|
||||
class BackupManager:
|
||||
def __init__(self, store: Store, *, keep: int = 14, legacy_keep: int = 2) -> None:
|
||||
self.store = store
|
||||
self.keep = max(1, keep)
|
||||
self.legacy_keep = max(0, legacy_keep)
|
||||
|
||||
@property
|
||||
def directory(self) -> Path:
|
||||
return self.store.path.parent / "backups"
|
||||
|
||||
def list(self) -> list[BackupInfo]:
|
||||
if not self.directory.exists():
|
||||
return []
|
||||
items: list[BackupInfo] = []
|
||||
for path in self.directory.iterdir():
|
||||
match = _NAME.match(path.name)
|
||||
if not match or not path.is_file():
|
||||
continue
|
||||
created = datetime.strptime(match["stamp"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
|
||||
items.append(
|
||||
BackupInfo(name=path.name, kind=match["kind"], created_at=created, size_bytes=path.stat().st_size)
|
||||
)
|
||||
return sorted(items, key=lambda item: item.created_at, reverse=True)
|
||||
|
||||
def path_for(self, name: str) -> Path:
|
||||
"""Resolve a listed backup name; anything else is refused."""
|
||||
|
||||
if not _NAME.match(name):
|
||||
raise ValueError("Onbekende back-up")
|
||||
path = self.directory / name
|
||||
if not path.is_file():
|
||||
raise ValueError("Onbekende back-up")
|
||||
return path
|
||||
|
||||
def create(self, kind: str = "backup", now: datetime | None = None) -> BackupInfo:
|
||||
now = now or datetime.now(UTC)
|
||||
self.directory.mkdir(parents=True, exist_ok=True)
|
||||
name = f"servicemap-{kind}-{now.strftime('%Y%m%dT%H%M%SZ')}.db"
|
||||
target = self.directory / name
|
||||
if target.exists():
|
||||
raise ValueError("Er bestaat al een back-up met dit tijdstip")
|
||||
with self.store._connect() as db:
|
||||
db.execute("VACUUM INTO ?", (str(target),))
|
||||
try:
|
||||
os.chmod(target, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
if not self._verify(target):
|
||||
target.unlink(missing_ok=True)
|
||||
raise RuntimeError("De nieuwe back-up faalde de integriteitscontrole en is verwijderd")
|
||||
self._rotate()
|
||||
self._prune_legacy()
|
||||
return next(item for item in self.list() if item.name == name)
|
||||
|
||||
def restore(self, name: str) -> BackupInfo:
|
||||
source = self.path_for(name)
|
||||
if not self._verify(source):
|
||||
raise RuntimeError("De gekozen back-up faalde de integriteitscontrole; er is niets teruggezet")
|
||||
if self.store.demo_active():
|
||||
raise ValueError("Sluit de demo af voor je een back-up terugzet")
|
||||
safety = self.create(kind="pre-restore")
|
||||
# closing(): a sqlite3 connection's own context manager only commits.
|
||||
with closing(sqlite3.connect(source)) as src, closing(sqlite3.connect(self.store.path, timeout=30)) as dst:
|
||||
src.backup(dst)
|
||||
return safety
|
||||
|
||||
@staticmethod
|
||||
def _verify(path: Path) -> bool:
|
||||
try:
|
||||
with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True)) as db:
|
||||
result = db.execute("PRAGMA quick_check").fetchone()
|
||||
tables = db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='services'").fetchone()
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
return bool(result and result[0] == "ok" and tables)
|
||||
|
||||
def _rotate(self) -> None:
|
||||
for kind in ("backup", "pre-restore"):
|
||||
for stale in [item for item in self.list() if item.kind == kind][self.keep:]:
|
||||
(self.directory / stale.name).unlink(missing_ok=True)
|
||||
|
||||
def _prune_legacy(self) -> list[str]:
|
||||
folder = self.store.path.parent
|
||||
legacy = sorted(
|
||||
(path for path in folder.iterdir() if path.is_file() and _LEGACY.match(path.name)),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
removed: list[str] = []
|
||||
for path in legacy[self.legacy_keep:]:
|
||||
for candidate in (path, Path(f"{path}-wal"), Path(f"{path}-shm"), Path(f"{path}-journal")):
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
removed.append(path.name)
|
||||
return removed
|
||||
Reference in new issue
Block a user