ServiceMap: update tests and backend/app (+7 areas)

Public changes compared with the previous published export:
89 files changed.

Affected areas:
- backend/app: 21 files
- dependencies: 5 files
- deployment and CI: 6 files
- documentation: 2 files
- frontend: 3 files
- frontend/src: 20 files
- project files: 1 file
- tests: 30 files
- unraid: 1 file

Changed files:
- Updated: ".env.example"
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Added: "backend/app/alerts.py"
- Updated: "backend/app/auth.py"
- Added: "backend/app/backups.py"
- Updated: "backend/app/config_io.py"
- Updated: "backend/app/demo.py"
- Updated: "backend/app/discovery.py"
- Updated: "backend/app/discovery_store.py"
- Updated: "backend/app/docker_discovery.py"
- Updated: "backend/app/events.py"
- Updated: "backend/app/evidence.py"
- Updated: "backend/app/health.py"
- Updated: "backend/app/incidents.py"
- Added: "backend/app/loops.py"
- Updated: "backend/app/main.py"
- Updated: "backend/app/models.py"
- Updated: "backend/app/observation.py"
- Updated: "backend/app/probes.py"
- Added: "backend/app/response_cache.py"
- Updated: "backend/app/scheduler.py"
- Updated: "backend/app/semantic.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Updated: "backend/tests/test_api.py"
- Updated: "backend/tests/test_config_io.py"
- Updated: "backend/tests/test_demo.py"
- Updated: "backend/tests/test_discovery_v120.py"
- Updated: "backend/tests/test_health.py"
- Updated: "backend/tests/test_probes.py"
- Updated: "backend/tests/test_scheduler.py"
- Updated: "backend/tests/test_v130_probe_reachability.py"
- Updated: "backend/tests/test_v160_evidence_changes.py"
- Added: "backend/tests/test_v163_alerts.py"
- Added: "backend/tests/test_v163_backups.py"
- Added: "backend/tests/test_v163_data_integrity.py"
- Added: "backend/tests/test_v163_efficiency.py"
- Added: "backend/tests/test_v163_health_performance.py"
- Added: "backend/tests/test_v163_loops.py"
- Added: "backend/tests/test_v163_management.py"
- Added: "backend/tests/test_v163_realtime.py"
- Added: "backend/tests/test_v163_security.py"
- Added: "backend/tests/test_v163_semantic_etag.py"
- Added: "compose.demo.yaml"
- Updated: "compose.yaml"
- Added: "docker-compose.broker.yml"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Added: "frontend/e2e/a11y.spec.ts"
- Updated: "frontend/e2e/demo.spec.ts"
- Updated: "frontend/index.html"
- Updated: "frontend/nginx.conf"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "frontend/playwright.config.ts"
- Updated: "frontend/src/App.tsx"
- Updated: "frontend/src/LivingEdge.tsx"
- Added: "frontend/src/activityView.test.ts"
- Added: "frontend/src/activityView.ts"
- Updated: "frontend/src/api.ts"
- Added: "frontend/src/apiAuth.test.ts"
- Added: "frontend/src/components/ActionsMenu.tsx"
- Added: "frontend/src/components/ConfirmDialog.test.tsx"
- Added: "frontend/src/components/ConfirmDialog.tsx"
- Added: "frontend/src/components/StatusIcon.tsx"
- Updated: "frontend/src/demo/DemoWalkthrough.test.ts"
- Updated: "frontend/src/demo/DemoWalkthrough.tsx"
- Added: "frontend/src/elk-worker-url.d.ts"
- Added: "frontend/src/labels.ts"
- Updated: "frontend/src/main.tsx"
- Added: "frontend/src/realtime/refreshPlan.test.ts"
- Added: "frontend/src/realtime/refreshPlan.ts"
- Updated: "frontend/src/realtime/useRealtimeEvents.ts"
- Added: "frontend/src/routing/urlState.test.ts"
- Added: "frontend/src/routing/urlState.ts"
- Updated: "frontend/src/styles.css"
- Added: "frontend/src/topology/GraphListView.test.tsx"
- Added: "frontend/src/topology/GraphListView.tsx"
- Updated: "frontend/src/topology/ServiceNode.tsx"
- Updated: "frontend/src/topology/TopologyCommandBar.tsx"
- Added: "frontend/src/topology/semanticGraph.test.ts"
- Updated: "frontend/src/topology/semanticGraph.ts"
- Added: "frontend/src/topologyLayout.test.ts"
- Updated: "frontend/src/topologyLayout.ts"
- Updated: "package-lock.json"
- Updated: "package.json"
- Updated: "unraid/my-servicemap.xml"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
This commit is contained in:
NuklearRabbit committed 2026-11-20 13:51:46 +01:00
1 parent d7319250e8
commit adc2c70c21
91 files changed
+9205 -1691

No files matched your search

+243
View File
@@ -0,0 +1,243 @@
"""Incident notifications via ntfy.
The notifier is deliberately small and idempotent:
* settings live in the ``settings`` table (key ``alerts``); the optional ntfy
access token is write-only and never returned by the API;
* ``alert_log`` records which notification (``opened``, ``resolved``,
``reminder:<n>``) was sent for which incident, so restarts and repeated
cycles never send duplicates;
* an incident is announced only after it has stayed open for
``delay_seconds`` (short flaps stay quiet) and when its severity reaches
``min_severity``. Maintenance never opens incidents, so it is quiet too.
"""
from __future__ import annotations
import json
import logging
import sqlite3
from datetime import UTC, datetime, timedelta
from typing import Literal
from urllib.parse import quote, urlparse
import httpx
from pydantic import BaseModel, Field, field_validator
from .models import Criticality, IncidentRecord, IncidentStatus
from .store import Store
logger = logging.getLogger("servicemap.alerts")
_SEVERITY_RANK = {Criticality.LOW: 0, Criticality.MEDIUM: 1, Criticality.HIGH: 2, Criticality.CRITICAL: 3}
_SETTINGS_KEY = "alerts"
class AlertSettings(BaseModel):
enabled: bool = False
ntfy_server: str = Field(default="https://ntfy.sh", max_length=300)
ntfy_topic: str = Field(default="", max_length=120)
min_severity: Criticality = Criticality.MEDIUM
delay_seconds: int = Field(default=60, ge=0, le=3600)
notify_resolved: bool = True
reminder_minutes: int = Field(default=0, ge=0, le=1440)
public_url: str = Field(default="", max_length=300)
@field_validator("ntfy_server", "public_url")
@classmethod
def _http_url(cls, value: str) -> str:
value = value.strip().rstrip("/")
if value and urlparse(value).scheme not in {"http", "https"}:
raise ValueError("moet een http(s)-URL zijn")
return value
@field_validator("ntfy_topic")
@classmethod
def _topic(cls, value: str) -> str:
value = value.strip()
if value and not all(char.isalnum() or char in "-_" for char in value):
raise ValueError("mag alleen letters, cijfers, - en _ bevatten")
return value
class AlertSettingsView(AlertSettings):
"""What the API returns: the access token is never echoed."""
ntfy_token_set: bool = False
class AlertSettingsUpdate(AlertSettings):
# None keeps the stored token, "" removes it.
ntfy_token: str | None = Field(default=None, max_length=200)
class AlertTestResult(BaseModel):
ok: bool
detail: str
class AlertManager:
def __init__(self, store: Store) -> None:
self.store = store
self._init_schema()
def _connect(self) -> sqlite3.Connection:
return self.store._connect()
def _init_schema(self) -> None:
with self._connect() as db:
db.executescript(
"""
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS alert_log (
incident_id TEXT NOT NULL,
kind TEXT NOT NULL,
sent_at TEXT NOT NULL,
PRIMARY KEY (incident_id, kind)
);
"""
)
# -- settings -----------------------------------------------------------
def _raw(self) -> dict[str, object]:
with self._connect() as db:
row = db.execute("SELECT value FROM settings WHERE key = ?", (_SETTINGS_KEY,)).fetchone()
if row is None:
return {}
try:
value = json.loads(row["value"])
except json.JSONDecodeError:
return {}
return value if isinstance(value, dict) else {}
def settings(self) -> AlertSettings:
raw = self._raw()
return AlertSettings.model_validate({key: value for key, value in raw.items() if key != "ntfy_token"})
def view(self) -> AlertSettingsView:
raw = self._raw()
return AlertSettingsView(**self.settings().model_dump(), ntfy_token_set=bool(raw.get("ntfy_token")))
def update(self, update: AlertSettingsUpdate) -> AlertSettingsView:
raw = self._raw()
stored = update.model_dump(exclude={"ntfy_token"})
token = raw.get("ntfy_token") if update.ntfy_token is None else update.ntfy_token.strip()
if token:
stored["ntfy_token"] = token
if stored["enabled"] and not stored["ntfy_topic"]:
raise ValueError("Stel een ntfy-topic in voor je meldingen aanzet")
with self._connect() as db:
db.execute(
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at",
(_SETTINGS_KEY, json.dumps(stored), datetime.now(UTC).isoformat()),
)
return self.view()
# -- delivery -------------------------------------------------------------
def _send(self, title: str, message: str, *, priority: str, tags: str, click: str | None) -> None:
settings = self.settings()
token = self._raw().get("ntfy_token")
headers = {"Title": title, "Priority": priority, "Tags": tags}
if click:
headers["Click"] = click
if token:
headers["Authorization"] = f"Bearer {token}"
url = f"{settings.ntfy_server}/{quote(settings.ntfy_topic)}"
with httpx.Client(timeout=10.0, trust_env=False, follow_redirects=False) as client:
response = client.post(url, content=message.encode("utf-8"), headers=headers)
if response.status_code >= 300:
raise RuntimeError(f"ntfy antwoordde met HTTP {response.status_code}")
def send_test(self) -> AlertTestResult:
settings = self.settings()
if not settings.ntfy_topic:
return AlertTestResult(ok=False, detail="Er is geen ntfy-topic ingesteld")
try:
self._send(
"Testmelding van ServiceMap",
"Meldingen van ServiceMap komen aan op dit topic.",
priority="default",
tags="white_check_mark",
click=settings.public_url or None,
)
except (httpx.HTTPError, RuntimeError) as exc:
return AlertTestResult(ok=False, detail=str(exc))
return AlertTestResult(ok=True, detail=f"Verstuurd naar {settings.ntfy_server}/{settings.ntfy_topic}")
def _sent(self, incident_id: str) -> set[str]:
with self._connect() as db:
rows = db.execute("SELECT kind FROM alert_log WHERE incident_id = ?", (incident_id,)).fetchall()
return {row["kind"] for row in rows}
def _mark(self, incident_id: str, kind: str, at: datetime) -> None:
with self._connect() as db:
db.execute(
"INSERT OR IGNORE INTO alert_log (incident_id, kind, sent_at) VALUES (?, ?, ?)",
(incident_id, kind, at.isoformat()),
)
def _link(self, settings: AlertSettings, incident: IncidentRecord) -> str | None:
if not settings.public_url:
return None
return f"{settings.public_url}/#/incidents?incident={quote(incident.id)}"
def plan(self, incidents: list[IncidentRecord], now: datetime) -> list[tuple[IncidentRecord, Literal["opened", "resolved"] | str]]:
"""Notifications that are due now (pure apart from reading the log)."""
settings = self.settings()
due: list[tuple[IncidentRecord, str]] = []
for incident in incidents:
if _SEVERITY_RANK[incident.severity] < _SEVERITY_RANK[settings.min_severity]:
continue
sent = self._sent(incident.id)
if incident.status == IncidentStatus.RESOLVED:
if settings.notify_resolved and "opened" in sent and "resolved" not in sent:
due.append((incident, "resolved"))
continue
open_for = now - incident.detected_at
if "opened" not in sent:
if open_for >= timedelta(seconds=settings.delay_seconds):
due.append((incident, "opened"))
continue
if settings.reminder_minutes and incident.status == IncidentStatus.OPEN:
count = int(open_for.total_seconds() // (settings.reminder_minutes * 60))
if count >= 1 and f"reminder:{count}" not in sent:
due.append((incident, f"reminder:{count}"))
return due
def run_cycle(self, now: datetime | None = None) -> int:
settings = self.settings()
if not settings.enabled or not settings.ntfy_topic:
return 0
now = now or datetime.now(UTC)
# Recently resolved incidents still need their "resolved" message.
incidents = self.store.list_incidents(active_only=False, limit=200)
sent = 0
for incident, kind in self.plan(incidents, now):
critical = incident.severity in {Criticality.HIGH, Criticality.CRITICAL}
if kind == "resolved":
title = f"Opgelost: {incident.title}"
message = f"{incident.root_service_name} is hersteld."
priority, tags = "default", "white_check_mark"
else:
reminder = kind.startswith("reminder")
title = f"{'Nog steeds uitgevallen' if reminder else 'Uitgevallen'}: {incident.title}"
affected = len(incident.affected_service_ids)
message = (
f"{incident.root_cause_reason}"
+ (f"\n{affected} afhankelijke service(s) geraakt." if affected else "")
)
priority, tags = ("urgent" if critical else "high"), "rotating_light"
try:
self._send(title, message, priority=priority, tags=tags, click=self._link(settings, incident))
except (httpx.HTTPError, RuntimeError):
logger.exception("Sending the %s notification for incident %s failed", kind, incident.id)
continue
self._mark(incident.id, kind, now)
sent += 1
return sent
+151 -10
View File
@@ -1,10 +1,27 @@
"""Optional, fail-closed OIDC bearer authorization for ServiceMap APIs."""
"""ServiceMap API authorization.
Modes (``SERVICEMAP_AUTH_MODE``):
* ``disabled`` - no authentication (v1.5 behaviour). Cross-site mutations are
still refused by the Origin check below.
* ``token`` - reads stay anonymous on the LAN (integrations such as ITWorx
Forge and monitors keep working); every mutating request needs
``Authorization: Bearer <admin token>``. The token lives in a 0600 file
next to the database and is generated on first start.
* ``oidc`` - fail-closed bearer JWT validation with viewer/operator/admin
roles mapped from claims or groups.
"""
from __future__ import annotations
import hmac
import logging
import os
import secrets
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from threading import Lock
from urllib.parse import urlparse
import httpx
@@ -36,10 +53,10 @@ class AuthConfig:
@classmethod
def from_environment(cls) -> AuthConfig:
mode = os.getenv("SERVICEMAP_AUTH_MODE", "disabled").strip().casefold() or "disabled"
if mode == "disabled":
return cls(mode="disabled")
if mode in {"disabled", "token"}:
return cls(mode=mode)
if mode != "oidc":
raise AuthorizationError("SERVICEMAP_AUTH_MODE must be 'disabled' or 'oidc'")
raise AuthorizationError("SERVICEMAP_AUTH_MODE must be 'disabled', 'token' or 'oidc'")
issuer = os.getenv("SERVICEMAP_OIDC_ISSUER", "").strip().rstrip("/")
audience = os.getenv("SERVICEMAP_OIDC_AUDIENCE", "").strip()
parsed = urlparse(issuer)
@@ -131,17 +148,123 @@ def verify_bearer(token: str, config: AuthConfig) -> set[str]:
return _roles_for(claims, config)
SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
# Reads that expose secrets need write access even where reads are open
# (an ntfy topic is effectively a secret on a public server).
SECRET_READ_PREFIXES = ("/api/v1/config", "/api/v1/settings", "/api/v1/backups")
# A backup contains every record, so listing and downloading need the token too.
TOKEN_READ_PREFIXES = ("/api/v1/settings", "/api/v1/backups")
_MIN_TOKEN_LENGTH = 24
_logger = logging.getLogger("servicemap.auth")
_token_lock = Lock()
_token_cache: dict[str, str] = {}
def admin_token_path() -> Path:
configured = os.getenv("SERVICEMAP_ADMIN_TOKEN_FILE", "").strip()
if configured:
return Path(configured)
database = Path(os.getenv("SERVICEMAP_DB", "data/servicemap.db"))
return database.parent / "admin-token"
def admin_token() -> str:
"""Read the admin token, generating it on first use.
The value is never logged; only its location is. Read it on the host
with e.g. ``docker exec servicemap cat /data/admin-token``.
"""
path = admin_token_path()
key = str(path)
with _token_lock:
cached = _token_cache.get(key)
if cached:
return cached
if not path.exists():
path.parent.mkdir(parents=True, exist_ok=True)
generated = secrets.token_urlsafe(32)
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
handle.write(generated + "\n")
_logger.warning("Generated a new ServiceMap admin token at %s", path)
token = path.read_text(encoding="utf-8").strip()
if len(token) < _MIN_TOKEN_LENGTH:
raise AuthorizationError(
f"The admin token in {path} must be at least {_MIN_TOKEN_LENGTH} characters"
)
_token_cache[key] = token
return token
def reset_token_cache() -> None:
with _token_lock:
_token_cache.clear()
def required_role(request: Request) -> str:
if request.method in {"GET", "HEAD", "OPTIONS"}:
return "viewer"
path = request.url.path
if request.method in SAFE_METHODS:
# A full configuration export contains every endpoint and target;
# settings hold notification targets.
return "admin" if path.startswith(SECRET_READ_PREFIXES) else "viewer"
if path.startswith(
("/api/v1/config", "/api/v1/discovery", "/api/v1/docker", "/api/v1/maintenance")
(
"/api/v1/config",
"/api/v1/discovery",
"/api/v1/docker",
"/api/v1/maintenance",
# Demo replaces the whole live topology.
"/api/v1/demo",
"/api/v1/settings",
"/api/v1/backups",
)
):
return "admin"
return "operator"
_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
def _host_name(value: str) -> str:
value = value.strip().casefold()
if value.startswith("["): # IPv6 literal with optional port
return value[1:].split("]", 1)[0]
return value.rsplit(":", 1)[0] if value.count(":") == 1 else value
def check_request_origin(request: Request) -> None:
"""Refuse DNS-rebinding hosts and cross-site mutations.
* ``SERVICEMAP_ALLOWED_HOSTS`` (comma separated, optional): when set,
requests for any other Host are refused. Loopback is always allowed
for the container healthcheck.
* A mutating request whose ``Origin`` is not this server, or that a
browser marks ``Sec-Fetch-Site: cross-site``, is refused. Requests
without these headers (curl, integrations) are unaffected.
"""
host_header = request.headers.get("host", "")
# Behind a reverse proxy the public name arrives as X-Forwarded-Host.
forwarded = request.headers.get("x-forwarded-host", "").split(",")[0].strip()
public_host = _host_name(forwarded or host_header)
allowed = _csv("SERVICEMAP_ALLOWED_HOSTS")
if allowed and public_host not in allowed | _LOOPBACK_HOSTS and _host_name(host_header) not in _LOOPBACK_HOSTS:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Deze host is niet toegestaan")
if request.method in SAFE_METHODS:
return
if request.headers.get("sec-fetch-site", "").casefold() == "cross-site":
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Verzoek van een andere site geweigerd")
origin = request.headers.get("origin")
if origin is not None:
parsed = urlparse(origin)
# Hostnames are compared without ports: proxies (nginx ``$host``,
# NPM) commonly drop the port from Host while Origin keeps it.
if origin == "null" or not parsed.hostname or parsed.hostname.casefold() != public_host:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Verzoek van een andere origin geweigerd")
def authorize_request(request: Request) -> None:
"""Authorize API requests; health and static content remain reachable.
@@ -150,7 +273,8 @@ def authorize_request(request: Request) -> None:
available for recovery and deployment diagnostics.
"""
if request.url.path == "/health" or not request.url.path.startswith("/api/"):
check_request_origin(request)
if request.url.path in {"/health", "/ready"} or not request.url.path.startswith("/api/"):
return
try:
config = AuthConfig.from_environment()
@@ -161,9 +285,26 @@ def authorize_request(request: Request) -> None:
if config.mode == "disabled":
return
header = request.headers.get("Authorization", "")
if config.mode == "token":
if request.method in SAFE_METHODS and not request.url.path.startswith(TOKEN_READ_PREFIXES):
return
try:
expected = admin_token()
except (AuthorizationError, OSError) as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Het beheerderstoken is niet beschikbaar"
) from exc
supplied = header.removeprefix("Bearer ").strip() if header.startswith("Bearer ") else ""
if not supplied or not hmac.compare_digest(supplied.encode(), expected.encode()):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Bewerken vraagt het ServiceMap-beheerderstoken",
headers={"WWW-Authenticate": "Bearer"},
)
return
if not header.startswith("Bearer "):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, detail="Bearer authentication is required"
status_code=status.HTTP_401_UNAUTHORIZED, detail="Bearer-authenticatie is vereist"
)
try:
roles = verify_bearer(header.removeprefix("Bearer ").strip(), config)
@@ -174,5 +315,5 @@ def authorize_request(request: Request) -> None:
< _ROLE_ORDER[required_role(request)]
):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="Insufficient ServiceMap role"
status_code=status.HTTP_403_FORBIDDEN, detail="Onvoldoende ServiceMap-rol"
)
+135
View File
@@ -0,0 +1,135 @@
"""Verified, rotated SQLite backups and online restore.
* Backups are written with ``VACUUM INTO`` (a consistent, compacted copy
taken inside a read transaction, safe while ServiceMap runs) to
``<data dir>/backups`` and checked with ``PRAGMA quick_check`` before
they count. A copy that fails the check is deleted, never kept.
* Rotation keeps the newest ``keep`` scheduled/manual backups; pre-restore
safety copies are rotated the same way.
* Restore uses SQLite's online backup API from the chosen file into the
live database: it is transactional and needs no file swapping under open
WAL connections. A fresh ``pre-restore`` backup is taken first.
* Legacy pre-upgrade copies (``servicemap-v*-pre-*.sqlite3`` next to the
database) are pruned to the newest ``legacy_keep`` only after a new
backup passed its check.
"""
from __future__ import annotations
import os
import re
import sqlite3
from contextlib import closing
from datetime import UTC, datetime
from pathlib import Path
from pydantic import BaseModel
from .store import Store
_NAME = re.compile(r"^servicemap-(?P<kind>backup|pre-restore)-(?P<stamp>\d{8}T\d{6}Z)\.db$")
_LEGACY = re.compile(r"^servicemap-v\d+\.\d+\.\d+-pre-[A-Za-z0-9T._-]+\.sqlite3$")
class BackupInfo(BaseModel):
name: str
kind: str
created_at: datetime
size_bytes: int
class BackupManager:
def __init__(self, store: Store, *, keep: int = 14, legacy_keep: int = 2) -> None:
self.store = store
self.keep = max(1, keep)
self.legacy_keep = max(0, legacy_keep)
@property
def directory(self) -> Path:
return self.store.path.parent / "backups"
def list(self) -> list[BackupInfo]:
if not self.directory.exists():
return []
items: list[BackupInfo] = []
for path in self.directory.iterdir():
match = _NAME.match(path.name)
if not match or not path.is_file():
continue
created = datetime.strptime(match["stamp"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
items.append(
BackupInfo(name=path.name, kind=match["kind"], created_at=created, size_bytes=path.stat().st_size)
)
return sorted(items, key=lambda item: item.created_at, reverse=True)
def path_for(self, name: str) -> Path:
"""Resolve a listed backup name; anything else is refused."""
if not _NAME.match(name):
raise ValueError("Onbekende back-up")
path = self.directory / name
if not path.is_file():
raise ValueError("Onbekende back-up")
return path
def create(self, kind: str = "backup", now: datetime | None = None) -> BackupInfo:
now = now or datetime.now(UTC)
self.directory.mkdir(parents=True, exist_ok=True)
name = f"servicemap-{kind}-{now.strftime('%Y%m%dT%H%M%SZ')}.db"
target = self.directory / name
if target.exists():
raise ValueError("Er bestaat al een back-up met dit tijdstip")
with self.store._connect() as db:
db.execute("VACUUM INTO ?", (str(target),))
try:
os.chmod(target, 0o600)
except OSError:
pass
if not self._verify(target):
target.unlink(missing_ok=True)
raise RuntimeError("De nieuwe back-up faalde de integriteitscontrole en is verwijderd")
self._rotate()
self._prune_legacy()
return next(item for item in self.list() if item.name == name)
def restore(self, name: str) -> BackupInfo:
source = self.path_for(name)
if not self._verify(source):
raise RuntimeError("De gekozen back-up faalde de integriteitscontrole; er is niets teruggezet")
if self.store.demo_active():
raise ValueError("Sluit de demo af voor je een back-up terugzet")
safety = self.create(kind="pre-restore")
# closing(): a sqlite3 connection's own context manager only commits.
with closing(sqlite3.connect(source)) as src, closing(sqlite3.connect(self.store.path, timeout=30)) as dst:
src.backup(dst)
return safety
@staticmethod
def _verify(path: Path) -> bool:
try:
with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True)) as db:
result = db.execute("PRAGMA quick_check").fetchone()
tables = db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='services'").fetchone()
except sqlite3.Error:
return False
return bool(result and result[0] == "ok" and tables)
def _rotate(self) -> None:
for kind in ("backup", "pre-restore"):
for stale in [item for item in self.list() if item.kind == kind][self.keep:]:
(self.directory / stale.name).unlink(missing_ok=True)
def _prune_legacy(self) -> list[str]:
folder = self.store.path.parent
legacy = sorted(
(path for path in folder.iterdir() if path.is_file() and _LEGACY.match(path.name)),
key=lambda path: path.stat().st_mtime,
reverse=True,
)
removed: list[str] = []
for path in legacy[self.legacy_keep:]:
for candidate in (path, Path(f"{path}-wal"), Path(f"{path}-shm"), Path(f"{path}-journal")):
if candidate.exists():
candidate.unlink()
removed.append(path.name)
return removed
+50 -73
View File
@@ -5,12 +5,44 @@ import json
import yaml
from pydantic import ValidationError
from .models import ConfigDocument, ConfigImportResult, ConfigService
from .models import ConfigDocument, ConfigImportResult, ConfigService, Service, ServiceCreate
from .store import Store
# Observed runtime state is not configuration: it is re-learned from Docker
# and probes and must not be frozen into an exported document.
OBSERVED_FIELDS = frozenset(
{"runtime_state", "runtime_health", "first_seen_at", "last_seen_at", "last_discovery_status"}
)
CONFIG_FIELDS = frozenset(ServiceCreate.model_fields) - OBSERVED_FIELDS - {"depends_on"}
def config_service(service: Service, names_by_id: dict[str, str]) -> ConfigService:
"""Every operator-owned field of a service, dependencies by name.
The previous field-by-field copy omitted application/domain/role, their
locks, category, visibility, lifecycle, monitoring and probe metadata and
layout, so an export/import round trip silently lost them.
"""
return ConfigService(
**service.model_dump(include=CONFIG_FIELDS),
depends_on=[names_by_id[item] for item in service.depends_on if item in names_by_id],
)
def _reject_yaml_aliases(content: str) -> None:
"""Refuse YAML anchors/aliases: nested aliases expand exponentially
("billion laughs") and a configuration document never needs them."""
for event in yaml.parse(content, Loader=yaml.SafeLoader):
if isinstance(event, yaml.AliasEvent) or getattr(event, "anchor", None):
raise ValueError("De configuratie mag geen YAML-anchors of -aliassen gebruiken")
def parse_document(content: str, format_name: str) -> ConfigDocument:
try:
if format_name != "json":
_reject_yaml_aliases(content)
raw = json.loads(content) if format_name == "json" else yaml.safe_load(content)
except (json.JSONDecodeError, yaml.YAMLError) as exc:
raise ValueError(f"Invalid {format_name.upper()} configuration: {exc}") from exc
@@ -19,22 +51,22 @@ def parse_document(content: str, format_name: str) -> ConfigDocument:
try:
document = ConfigDocument.model_validate(raw)
except ValidationError as exc:
raise ValueError(f"Configuration schema validation failed: {exc}") from exc
raise ValueError(f"De configuratie voldoet niet aan het schema: {exc}") from exc
if document.version == 2 and not document.services:
# Preserve the v1 contract's explicit unsupported-empty-v2 guard while
# accepting real v2 documents that carry the richer service schema.
raise ValueError("Configuration version must be less than or equal to 1 for an empty document")
raise ValueError("Voor een leeg document moet de configuratieversie 1 of lager zijn")
names = [service.name.casefold() for service in document.services]
if len(names) != len(set(names)):
raise ValueError("Service names must be unique in the configuration")
raise ValueError("Servicenamen moeten uniek zijn in de configuratie")
for service in document.services:
dependencies = [dependency.casefold() for dependency in service.depends_on]
if len(dependencies) != len(set(dependencies)):
raise ValueError(f"Service {service.name} has duplicate dependencies")
raise ValueError(f"Service {service.name} heeft dubbele afhankelijkheden")
if service.name.casefold() in dependencies:
raise ValueError(f"Service {service.name} cannot depend on itself")
raise ValueError(f"Service {service.name} kan niet van zichzelf afhangen")
# A merge document may reference services that already exist in the store,
# so missing-reference validation belongs to import_document(). Cycles that
@@ -53,7 +85,7 @@ def _validate_acyclic(services: list[ConfigService]) -> None:
def visit(node: str) -> None:
if node in visiting:
raise ValueError("Dependency graph contains a cycle")
raise ValueError("De afhankelijkheidsgrafiek bevat een kring")
if node in visited:
return
visiting.add(node)
@@ -68,44 +100,22 @@ def _validate_acyclic(services: list[ConfigService]) -> None:
def _validate_import_graph(store: Store, document: ConfigDocument, mode: str) -> None:
if mode not in {"merge", "replace"}:
raise ValueError("Import mode must be merge or replace")
raise ValueError("Importmodus moet merge of replace zijn")
projected: dict[str, ConfigService] = {}
if mode == "merge":
existing = store.list_services()
names_by_id = {service.id: service.name for service in existing}
for service in existing:
projected[service.name.casefold()] = ConfigService(
name=service.name,
kind=service.kind,
url=service.url,
host=service.host,
port=service.port,
depends_on=[names_by_id[item] for item in service.depends_on if item in names_by_id],
description=service.description,
tags=service.tags,
group=service.group,
environment=service.environment,
owner=service.owner,
enabled=service.enabled,
monitoring_enabled=service.monitoring_enabled,
criticality=service.criticality,
source_type=service.source_type,
source_ref=service.source_ref,
interval_seconds=service.interval_seconds,
timeout_seconds=service.timeout_seconds,
retries=service.retries,
failure_threshold=service.failure_threshold,
recovery_threshold=service.recovery_threshold,
expected_status_min=service.expected_status_min,
expected_status_max=service.expected_status_max,
availability_target=service.availability_target,
maintenance_enabled=service.maintenance_enabled,
maintenance_reason=service.maintenance_reason,
)
projected[service.name.casefold()] = config_service(service, names_by_id)
for service in document.services:
projected[service.name.casefold()] = service
key = service.name.casefold()
previous = projected.get(key)
if previous is not None and "depends_on" not in service.model_fields_set:
# A merge keeps dependencies the document does not mention.
service = service.model_copy(update={"depends_on": previous.depends_on})
projected[key] = service
available = set(projected)
for service in projected.values():
@@ -116,7 +126,7 @@ def _validate_import_graph(store: Store, document: ConfigDocument, mode: str) ->
]
if missing:
raise ValueError(
f"Service {service.name} references unknown dependencies: {', '.join(missing)}"
f"Service {service.name} verwijst naar onbekende afhankelijkheden: {', '.join(missing)}"
)
_validate_acyclic(list(projected.values()))
@@ -125,44 +135,11 @@ def _validate_import_graph(store: Store, document: ConfigDocument, mode: str) ->
def export_document(store: Store) -> ConfigDocument:
services = store.list_services()
names = {service.id: service.name for service in services}
return ConfigDocument(
version=2,
services=[
ConfigService(
name=service.name,
kind=service.kind,
url=service.url,
host=service.host,
port=service.port,
depends_on=[names[item] for item in service.depends_on if item in names],
description=service.description,
tags=service.tags,
group=service.group,
environment=service.environment,
owner=service.owner,
enabled=service.enabled,
monitoring_enabled=service.monitoring_enabled,
criticality=service.criticality,
source_type=service.source_type,
source_ref=service.source_ref,
interval_seconds=service.interval_seconds,
timeout_seconds=service.timeout_seconds,
retries=service.retries,
failure_threshold=service.failure_threshold,
recovery_threshold=service.recovery_threshold,
expected_status_min=service.expected_status_min,
expected_status_max=service.expected_status_max,
availability_target=service.availability_target,
maintenance_enabled=service.maintenance_enabled,
maintenance_reason=service.maintenance_reason,
)
for service in services
],
)
return ConfigDocument(version=2, services=[config_service(service, names) for service in services])
def serialize_document(document: ConfigDocument, format_name: str) -> str:
payload = document.model_dump(mode="json")
payload = document.model_dump(mode="json", exclude={"services": {"__all__": set(OBSERVED_FIELDS)}})
if format_name == "json":
return json.dumps(payload, indent=2, sort_keys=False)
return yaml.safe_dump(payload, sort_keys=False, allow_unicode=True)
+13 -13
View File
@@ -39,15 +39,15 @@ def build_demo_records(now: datetime | None = None) -> tuple[list[Service], list
]
ids_by_name = {name: _stable_id(f"service:{name}") for name, *_ in definitions}
metadata = {
"Public Endpoint": ("Edge", "Public Gateway", "Edge", ComponentRole.FRONTEND, DiscoveryCategory.PROXY, Criticality.CRITICAL, "Customer-facing availability check", 100.0),
"Reverse Proxy": ("Edge", "Public Gateway", "Edge", ComponentRole.PROXY, DiscoveryCategory.PROXY, Criticality.HIGH, "Northstar ingress gateway", 100.0),
"Web Frontend": ("Application", "Northstar App", "Application", ComponentRole.FRONTEND, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Browser application", 99.0),
"Auth Service": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Identity and session service", 100.0),
"API Gateway": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.CRITICAL, "Application API entry point", 99.0),
"PostgreSQL": ("Data", "Data Platform", "Data", ComponentRole.DATABASE, DiscoveryCategory.DATABASE, Criticality.CRITICAL, "Primary relational datastore", 99.9),
"Redis": ("Data", "Data Platform", "Data", ComponentRole.CACHE, DiscoveryCategory.CACHE, Criticality.HIGH, "Cache and queue backend", 100.0),
"Background Worker": ("Application", "Worker Platform", "Application", ComponentRole.WORKER, DiscoveryCategory.BACKGROUND_WORKER, Criticality.HIGH, "Asynchronous job processor", 99.0),
"Object Storage": ("Data", "Worker Platform", "Application", ComponentRole.STORAGE, DiscoveryCategory.STORAGE, Criticality.MEDIUM, "Durable object store", 100.0),
"Public Endpoint": ("Edge", "Public Gateway", "Edge", ComponentRole.FRONTEND, DiscoveryCategory.PROXY, Criticality.CRITICAL, "Beschikbaarheidscontrole voor klanten", 100.0),
"Reverse Proxy": ("Edge", "Public Gateway", "Edge", ComponentRole.PROXY, DiscoveryCategory.PROXY, Criticality.HIGH, "Northstar-ingangsgateway", 100.0),
"Web Frontend": ("Application", "Northstar App", "Application", ComponentRole.FRONTEND, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Browserapplicatie", 99.0),
"Auth Service": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Identiteits- en sessieservice", 100.0),
"API Gateway": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.CRITICAL, "API-toegangspunt van de applicatie", 99.0),
"PostgreSQL": ("Data", "Data Platform", "Data", ComponentRole.DATABASE, DiscoveryCategory.DATABASE, Criticality.CRITICAL, "Primaire relationele database", 99.9),
"Redis": ("Data", "Data Platform", "Data", ComponentRole.CACHE, DiscoveryCategory.CACHE, Criticality.HIGH, "Cache- en wachtrijbackend", 100.0),
"Background Worker": ("Application", "Worker Platform", "Application", ComponentRole.WORKER, DiscoveryCategory.BACKGROUND_WORKER, Criticality.HIGH, "Verwerker van asynchrone taken", 99.0),
"Object Storage": ("Data", "Worker Platform", "Application", ComponentRole.STORAGE, DiscoveryCategory.STORAGE, Criticality.MEDIUM, "Duurzame objectopslag", 100.0),
}
services = [
Service(
@@ -113,9 +113,9 @@ def _history_status(name: str, index: int) -> HealthStatus:
def _probe_detail(name: str, status: HealthStatus, index: int) -> str:
if name == DEMO_ROOT_CAUSE_NAME and status == HealthStatus.DOWN:
return "Synthetic outage: PostgreSQL connection refused"
return "Synthetische storing: PostgreSQL weigert de verbinding"
if status == HealthStatus.DOWN:
return f"Synthetic dependency impact observed at history point {index + 1}"
return f"Synthetische impact via afhankelijkheid op historiekpunt {index + 1}"
if status == HealthStatus.DEGRADED:
return f"Synthetic elevated latency observed at history point {index + 1}"
return "Synthetic monitoring observation"
return f"Synthetisch verhoogde latentie op historiekpunt {index + 1}"
return "Synthetische bewakingswaarneming"
+32 -32
View File
@@ -185,7 +185,7 @@ class DiscoveryManager:
source_service_id=service_by_ref.get(source.stable_source_ref).id if source.stable_source_ref in service_by_ref else None,
target_ref=target.stable_source_ref,
target_service_id=service_by_ref.get(target.stable_source_ref).id if target.stable_source_ref in service_by_ref else None,
proposed_name=f"{source.name} depends on {target.name}",
proposed_name=f"{source.name} hangt af van {target.name}",
relation_type=relation_type,
confidence=confidence,
evidence=[evidence],
@@ -222,10 +222,10 @@ class DiscoveryManager:
suggestion_type="relation",
source_ref=source.stable_source_ref,
source_service_id=service_by_ref.get(source.stable_source_ref).id if source.stable_source_ref in service_by_ref else None,
proposed_name=f"{source.name} has an ambiguous reference to {reference}",
proposed_name=f"{source.name} verwijst dubbelzinnig naar {reference}",
relation_type=RelationType.DEPENDS_ON,
confidence=DiscoveryConfidence.LOW,
evidence=[f"{evidence}; candidates: {candidates_label}", "Manual review required; no canonical dependency was created"],
evidence=[f"{evidence}; kandidaten: {candidates_label}", "Handmatige controle nodig; er is geen canonieke afhankelijkheid aangemaakt"],
created_at=now,
updated_at=now,
),
@@ -240,14 +240,14 @@ class DiscoveryManager:
explicit = [item.strip() for item in labels.get("servicemap.depends_on", "").split(",") if item.strip()]
compose = [item.split(":", 1)[0].strip() for item in labels.get("com.docker.compose.depends_on", "").split(",") if item.strip()]
for name in sorted(set(explicit)):
resolve_and_add(source, name, DiscoveryConfidence.CONFIRMED, "Explicit servicemap.depends_on label")
resolve_and_add(source, name, DiscoveryConfidence.CONFIRMED, "Expliciet servicemap.depends_on-label")
for name in sorted(set(compose)):
resolve_and_add(source, name, DiscoveryConfidence.HIGH, "Docker Compose com.docker.compose.depends_on label", compose_only=True)
resolve_and_add(source, name, DiscoveryConfidence.HIGH, "Docker Compose-label com.docker.compose.depends_on", compose_only=True)
for item in source.environment_evidence:
if "hostname=" not in item:
continue
hostname = item.split("hostname=", 1)[1].split(":", 1)[0].casefold()
resolve_and_add(source, hostname, DiscoveryConfidence.MEDIUM, f"Environment endpoint reference: {item}")
resolve_and_add(source, hostname, DiscoveryConfidence.MEDIUM, f"Endpointverwijzing in omgevingsvariabele: {item}")
for item in source.proxy_route_evidence:
proxies = [proxy for proxy in candidates if proxy.category == DiscoveryCategory.PROXY and set(proxy.networks).intersection(source.networks)]
if len(proxies) == 1:
@@ -255,12 +255,12 @@ class DiscoveryManager:
# Shared networks are review-only hints. Cap them to avoid flooding
# a large host with O(n²) low-confidence suggestions.
for name in source.dependency_suggestions[:8]:
resolve_and_add(source, name, DiscoveryConfidence.LOW, f"Shared Docker network hint: {', '.join(source.networks)}")
resolve_and_add(source, name, DiscoveryConfidence.LOW, f"Hint via gedeeld Docker-netwerk: {', '.join(source.networks)}")
return result
def run(self, request: DiscoveryRunRequest) -> DiscoveryResult:
if not self._lock.acquire(blocking=False):
raise ValueError("A discovery run is already in progress")
raise ValueError("Er loopt al een detectierun")
run = self.discovery_store.start_run()
try:
docker_host = self._docker_host(request.docker_host)
@@ -278,7 +278,7 @@ class DiscoveryManager:
status=ProviderHealth.HEALTHY,
observed_at=observed_at,
valid_until=observed_at + evidence_ttl,
detail=f"Docker inventory returned {len(result.candidates)} candidates",
detail=f"Docker-inventaris gaf {len(result.candidates)} kandidaten",
metadata={"docker_host": docker_host},
)
)
@@ -473,7 +473,7 @@ class DiscoveryManager:
url=url,
host=host,
port=port,
description=f"Discovered Docker service from {candidate.image}",
description=f"Gevonden Docker-service op basis van {candidate.image}",
tags=["discovered", "docker"],
group=group,
environment="production",
@@ -559,7 +559,7 @@ class DiscoveryManager:
observed_at=now,
valid_until=now + evidence_ttl,
received_at=now,
summary="Docker healthcheck" if candidate.healthcheck_present else "Docker runtime state",
summary="Docker-healthcheck" if candidate.healthcheck_present else "Docker-runtimestatus",
metadata={"container_id": candidate.container_id, "state": candidate.state, "health": candidate.health},
)
)
@@ -579,7 +579,7 @@ class DiscoveryManager:
observed_at=now,
valid_until=now + evidence_ttl,
received_at=now,
summary="AppOps application health; not used as component direct health",
summary="Applicatiegezondheid via AppOps; niet gebruikt als directe componentgezondheid",
metadata={
"app_id": candidate.appops_app_id,
"category": candidate.appops_category,
@@ -597,7 +597,7 @@ class DiscoveryManager:
def reconcile(self, request: DiscoveryReconcileRequest) -> ReconcileResult:
run = self.discovery_store.get_run(request.run_id) if request.run_id else self.discovery_store.last_completed()
if run is None or run.status != DiscoveryRunStatus.COMPLETED:
raise ValueError("A completed discovery run is required before reconciliation")
raise ValueError("Voor verwerking is een afgeronde detectierun nodig")
self.discovery_store.set_run_status(run.id, DiscoveryRunStatus.RECONCILING)
candidates = [DockerCandidate.model_validate(item.get("candidate", item)) for item in self.discovery_store.candidates_for_run(run.id)]
by_ref = self._service_by_ref()
@@ -727,7 +727,7 @@ class DiscoveryManager:
payload.depends_on = [*source.depends_on, target.id]
self.store.update_service(source.id, payload)
self.discovery_store.set_suggestion_status(suggestion.id, SuggestionStatus.ACCEPTED)
self.discovery_store.record_change("dependency_added", f"{source.name} depends on {target.name}", source_service_id=source.id, target_service_id=target.id)
self.discovery_store.record_change("dependency_added", f"{source.name} hangt af van {target.name}", source_service_id=source.id, target_service_id=target.id)
accepted_relations += 1
final_run = self.discovery_store.finish_run(
@@ -767,7 +767,7 @@ class DiscoveryManager:
run = self.discovery_store.get_run(run_id) if run_id else self.discovery_store.last_completed()
if run is None or run.status != DiscoveryRunStatus.COMPLETED:
raise ValueError("A completed discovery run is required before repairing probe reachability")
raise ValueError("Voor het herstellen van controlebereikbaarheid is een afgeronde detectierun nodig")
candidates = [DockerCandidate.model_validate(item.get("candidate", item)) for item in self.discovery_store.candidates_for_run(run.id)]
services = self._service_by_ref()
changes: list[ProbeRepairChange] = []
@@ -799,19 +799,19 @@ class DiscoveryManager:
)
continue
if current.monitoring_enabled and not desired.monitoring_enabled:
reason = "Internal container endpoint is not reachable from ServiceMap; active probe disabled"
reason = "Intern container-endpoint is niet bereikbaar vanuit ServiceMap; actieve controle uitgeschakeld"
disabled_unsafe += 1
elif desired.monitoring_enabled:
reason = "Host-published endpoint selected using the configured probe host"
reason = "Gepubliceerd host-endpoint gekozen via de ingestelde controlehost"
enabled_published += 1
elif candidate.healthcheck_present:
reason = "Docker healthcheck retained as runtime observation without a network probe"
reason = "Docker-healthcheck behouden als runtimewaarneming zonder netwerkcontrole"
docker_health_observed += 1
elif desired.monitoring_state == MonitoringState.OBSERVED:
reason = "No verified reachable endpoint; Docker runtime observation is retained without a network probe"
reason = "Geen gecontroleerd bereikbaar endpoint; Docker-runtimewaarneming blijft behouden zonder netwerkcontrole"
runtime_observed += 1
else:
reason = "No verified reachable endpoint; service remains observed as unmonitored"
reason = "Geen gecontroleerd bereikbaar endpoint; service blijft waargenomen als niet bewaakt"
unmonitored += 1
changes.append(
ProbeRepairChange(
@@ -842,7 +842,7 @@ class DiscoveryManager:
from .incidents import sync_incidents
before = len([item for item in self.store.list_incidents(active_only=True)])
sync_incidents(self.store, resolution_reason="Monitoring endpoint superseded; active probe was replaced by reachability-safe observation")
sync_incidents(self.store, resolution_reason="Bewakings-endpoint vervangen; de actieve controle is vervangen door een veilige waarneming")
after = len([item for item in self.store.list_incidents(active_only=True)])
resolved = max(0, before - after)
return ProbeRepairReport(
@@ -864,7 +864,7 @@ class DiscoveryManager:
source = self.store.get_service(request.source_service_id)
target = self.store.get_service(request.target_service_id)
if source is None or target is None:
raise ValueError("Both edge services must exist")
raise ValueError("Beide services van de relatie moeten bestaan")
if request.relation_type == RelationType.DEPENDS_ON and target.id not in source.depends_on:
validate_dependencies([*source.depends_on, target.id], self.store, service_id=source.id)
payload = ServiceUpdate.model_validate(source.model_dump())
@@ -892,7 +892,7 @@ class DiscoveryManager:
def suggestion_action(self, suggestion_id: str, action: str) -> TopologySuggestion:
suggestion = self.discovery_store.get_suggestion(suggestion_id)
if suggestion is None:
raise ValueError("Suggestion not found")
raise ValueError("Voorstel niet gevonden")
if action == "reject":
if suggestion.suggestion_type == "relation" and suggestion.relation_type == RelationType.DEPENDS_ON:
source = self._service_by_ref().get(suggestion.source_ref or "")
@@ -908,7 +908,7 @@ class DiscoveryManager:
if action in {"confirm", "lock"}:
if suggestion.suggestion_type == "service":
if not suggestion.source_ref:
raise ValueError("Service suggestion has no stable source reference")
raise ValueError("Het servicevoorstel heeft geen stabiele bronverwijzing")
existing = self._service_by_ref().get(suggestion.source_ref)
if existing is None:
candidate = next(
@@ -920,18 +920,18 @@ class DiscoveryManager:
None,
)
if candidate is None:
raise ValueError("Discovery observation for service suggestion is no longer available")
raise ValueError("De detectiewaarneming voor dit servicevoorstel is niet meer beschikbaar")
existing = self.store.create_service(self._payload_for_candidate(candidate))
source = self.discovery_store.get_source("docker", suggestion.source_ref)
if source:
self.discovery_store.attach_source(existing.id, source.id)
self.discovery_store.record_change("accepted", f"Accepted discovered service {existing.name}", service_id=existing.id)
self.discovery_store.record_change("accepted", f"Gevonden service {existing.name} aanvaard", service_id=existing.id)
self.discovery_store.save_override("service", existing.id, "source", suggestion.source_ref, True)
elif suggestion.suggestion_type == "relation":
source = self._service_by_ref().get(suggestion.source_ref or "")
target = self._service_by_ref().get(suggestion.target_ref or "")
if source is None or target is None:
raise ValueError("Accept the related services before accepting this relationship")
raise ValueError("Aanvaard eerst de betrokken services voor je deze relatie aanvaardt")
if suggestion.relation_type == RelationType.DEPENDS_ON and target.id not in source.depends_on:
validate_dependencies([*source.depends_on, target.id], self.store, service_id=source.id)
payload = ServiceUpdate.model_validate(source.model_dump())
@@ -954,9 +954,9 @@ class DiscoveryManager:
last_seen_at=now,
)
)
self.discovery_store.record_change("dependency_added", f"Accepted relationship {source.name} → {target.name}", source_service_id=source.id, target_service_id=target.id)
self.discovery_store.record_change("dependency_added", f"Relatie {source.name} → {target.name} aanvaard", source_service_id=source.id, target_service_id=target.id)
return self.discovery_store.set_suggestion_status(suggestion_id, SuggestionStatus.ACCEPTED) or suggestion
raise ValueError("Unsupported suggestion action")
raise ValueError("Niet-ondersteunde actie op voorstel")
def record_service_override(self, before: Service, after: Service) -> None:
for field in ("name", "group", "category", "criticality", "environment", "visibility"):
@@ -975,16 +975,16 @@ class DiscoveryManager:
def edge_action(self, evidence_id: str, action: str) -> RelationEvidence:
evidence = self.discovery_store.get_evidence(evidence_id)
if evidence is None or evidence.target_service_id is None:
raise ValueError("Edge evidence not found")
raise ValueError("Bewijs voor de relatie niet gevonden")
source = self.store.get_service(evidence.source_service_id)
if source is None:
raise ValueError("Edge source service not found")
raise ValueError("Bronservice van de relatie niet gevonden")
target_id = evidence.target_service_id
if action in {"remove", "reject"} and evidence.relation_type == RelationType.DEPENDS_ON:
payload = ServiceUpdate.model_validate(source.model_dump())
payload.depends_on = [item for item in source.depends_on if item != target_id]
self.store.update_service(source.id, payload)
self.discovery_store.record_change("dependency_removed", f"Removed relationship from {source.name}", source_service_id=source.id, target_service_id=target_id)
self.discovery_store.record_change("dependency_removed", f"Relatie van {source.name} verwijderd", source_service_id=source.id, target_service_id=target_id)
return self.discovery_store.set_evidence_state(evidence_id, active=False) or evidence
if action == "confirm" and evidence.relation_type == RelationType.DEPENDS_ON and target_id not in source.depends_on:
validate_dependencies([*source.depends_on, target_id], self.store, service_id=source.id)
+93 -4
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import json
import sqlite3
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from hashlib import sha256
from typing import Any
from uuid import uuid4
@@ -354,7 +354,7 @@ class DiscoveryStore:
(
DiscoveryRunStatus.FAILED.value,
run.started_at.isoformat(),
"Superseded by a newer discovery run",
"Vervangen door een nieuwere detectierun",
DiscoveryRunStatus.DISCOVERING.value,
DiscoveryRunStatus.RECONCILING.value,
),
@@ -502,7 +502,7 @@ class DiscoveryStore:
"INSERT INTO topology_snapshot_relations VALUES (?, ?, ?, ?, ?, ?, ?)",
[(snapshot.id, relation_id, source_id, target_id, relation_type, fingerprint, _json(payload)) for relation_id, source_id, target_id, relation_type, fingerprint, payload in relation_rows],
)
self.record_change("topology_snapshot", f"Topology snapshot captured ({snapshot.service_count} services)")
self.record_change("topology_snapshot", f"Topologiesnapshot gemaakt ({snapshot.service_count} services)")
return snapshot
def list_snapshots(self, *, limit: int = 50) -> list[TopologySnapshot]:
@@ -514,7 +514,7 @@ class DiscoveryStore:
with self._connect() as db:
after = db.execute("SELECT * FROM topology_snapshots WHERE id=?", (after_snapshot_id,)).fetchone()
if after is None:
raise ValueError("Topology snapshot not found")
raise ValueError("Topologiesnapshot niet gevonden")
before = (
db.execute("SELECT * FROM topology_snapshots WHERE id=?", (before_snapshot_id,)).fetchone()
if before_snapshot_id
@@ -758,6 +758,7 @@ class DiscoveryStore:
if existing:
previous = self._evidence(existing)
evidence = evidence.model_copy(update={
"id": previous.id,
"confidence": _higher_confidence(previous.confidence, evidence.confidence),
"explanation": "; ".join(dict.fromkeys(part for part in (previous.explanation, evidence.explanation) if part)),
"source_ref": evidence.source_ref or previous.source_ref,
@@ -765,6 +766,19 @@ class DiscoveryStore:
"active": evidence.active,
"first_seen_at": min(previous.first_seen_at, evidence.first_seen_at),
})
# Update the found row by id. The UNIQUE constraint includes
# nullable columns (target_ref is NULL for manual edges), so
# ON CONFLICT never matched and each save inserted a duplicate.
db.execute(
"""
UPDATE relation_evidence SET target_service_id=?, confidence=?, explanation=?,
source_ref=?, locked=MAX(locked, ?), active=?, last_seen_at=?
WHERE id=?
""",
(evidence.target_service_id, evidence.confidence.value, evidence.explanation, evidence.source_ref, int(evidence.locked), int(evidence.active), evidence.last_seen_at.isoformat(), evidence.id),
)
row = db.execute("SELECT * FROM relation_evidence WHERE id=?", (evidence.id,)).fetchone()
return self._evidence(row)
db.execute(
"""
INSERT INTO relation_evidence
@@ -884,6 +898,53 @@ class DiscoveryStore:
def evidence_for_service(self, service_id: str) -> list[RelationEvidence]:
return self.list_evidence(service_id, active_only=True)
def evidence_for_services(self, service_ids: list[str]) -> dict[str, list[RelationEvidence]]:
"""Active relation evidence for many services in one read (no N+1)."""
ids = sorted(set(service_ids))
result: dict[str, list[RelationEvidence]] = {service_id: [] for service_id in ids}
if not ids:
return result
placeholders = ", ".join("?" for _ in ids)
with self._connect() as db:
rows = db.execute(
f"""
SELECT * FROM relation_evidence
WHERE active=1
AND (source_service_id IN ({placeholders}) OR target_service_id IN ({placeholders}))
ORDER BY last_seen_at DESC
""",
[*ids, *ids],
).fetchall()
for row in rows:
evidence = self._evidence(row)
for service_id in {row["source_service_id"], row["target_service_id"]}:
if service_id in result:
result[service_id].append(evidence)
return result
def sources_for_services(self, service_ids: list[str]) -> dict[str, list[ServiceSource]]:
"""Discovery sources for many services in one read (no N+1)."""
ids = sorted(set(service_ids))
result: dict[str, list[ServiceSource]] = {service_id: [] for service_id in ids}
if not ids:
return result
placeholders = ", ".join("?" for _ in ids)
with self._connect() as db:
rows = db.execute(
f"""
SELECT s.*, ss.service_id AS linked_service_id
FROM discovery_sources s JOIN service_sources ss ON ss.source_id=s.id
WHERE ss.service_id IN ({placeholders})
ORDER BY s.source_type, s.source_name
""",
ids,
).fetchall()
for row in rows:
result[row["linked_service_id"]].append(self._source(row))
return result
def sources_for_service(self, service_id: str) -> list[ServiceSource]:
with self._connect() as db:
rows = db.execute(
@@ -925,6 +986,34 @@ class DiscoveryStore:
with self._connect() as db:
db.execute("UPDATE discovery_runs SET status=? WHERE id=?", (status.value, run_id))
def prune_observations(self, keep_runs: int = 10) -> int:
"""Keep per-container observations only for the most recent runs.
Each run stores a full candidate dump per container. Runs themselves
are kept because review decisions (suggestions) reference them.
"""
if keep_runs < 1:
return 0
with self._connect() as db:
cursor = db.execute(
"""
DELETE FROM discovery_observations WHERE run_id NOT IN (
SELECT id FROM discovery_runs ORDER BY started_at DESC LIMIT ?
)
""",
(keep_runs,),
)
return cursor.rowcount
def prune_change_events(self, retention_days: int) -> int:
if retention_days <= 0:
return 0
cutoff = (_now() - timedelta(days=retention_days)).isoformat()
with self._connect() as db:
cursor = db.execute("DELETE FROM topology_change_events WHERE created_at < ?", (cutoff,))
return cursor.rowcount
def record_change(
self,
event_type: str,
+89 -20
View File
@@ -4,6 +4,7 @@ import os
import re
import socket
import time
from contextlib import suppress
from dataclasses import dataclass
from urllib.parse import urlparse
@@ -233,7 +234,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
hostname = parsed.hostname if parsed else candidate.host
port = parsed.port if parsed else candidate.port
if not hostname or not port:
return candidate.model_copy(update={"preflight_detail": "No host-published endpoint to preflight"})
return candidate.model_copy(update={"preflight_detail": "Geen gepubliceerd host-endpoint om vooraf te testen"})
started = time.perf_counter()
try:
with socket.create_connection((hostname, port), timeout=0.75):
@@ -243,7 +244,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
return candidate.model_copy(update={
"reachability": ProbeReachability.VERIFIED,
"primary": True,
"preflight_detail": "TCP connect succeeded",
"preflight_detail": "TCP-verbinding gelukt",
"preflight_latency_ms": elapsed,
})
with httpx.Client(timeout=1.25, follow_redirects=False) as client:
@@ -260,7 +261,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
return candidate.model_copy(update={
"reachability": ProbeReachability.UNVERIFIED,
"primary": False,
"preflight_detail": f"Preflight failed: {type(exc).__name__}",
"preflight_detail": f"Voorafgaande test mislukt: {type(exc).__name__}",
})
@@ -292,7 +293,7 @@ def _probe_candidates(
kind=suggestion.kind if suggestion else ServiceKind.TCP,
reachability=ProbeReachability.VERIFIED,
confidence=DiscoveryConfidence.HIGH,
evidence="Docker healthcheck present (observed without a network probe)",
evidence="Docker-healthcheck aanwezig (waargenomen zonder netwerkcontrole)",
)
)
return candidates
@@ -327,7 +328,7 @@ def safe_environment_evidence(env: list[str]) -> list[str]:
def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
if not appops_url:
return AppOpsInventory({}, False, "AppOps URL is not configured")
return AppOpsInventory({}, False, "Er is geen AppOps-URL ingesteld")
base = appops_url.rstrip("/")
try:
mappings_response = httpx.get(f"{base}/api/inventory/mappings", timeout=5)
@@ -337,7 +338,7 @@ def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
mappings = mappings_response.json()
apps = apps_response.json()
except (httpx.HTTPError, ValueError) as exc:
return AppOpsInventory({}, False, f"AppOps inventory unavailable: {type(exc).__name__}")
return AppOpsInventory({}, False, f"AppOps-inventaris niet beschikbaar: {type(exc).__name__}")
app_by_id = {str(item.get("id")): item for item in apps if isinstance(item, dict)}
result: dict[str, AppOpsInfo] = {}
for mapping in mappings if isinstance(mappings, list) else []:
@@ -362,7 +363,7 @@ def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
app_type=str(app.get("app_type")) if app.get("app_type") else None,
reasons=tuple(str(item) for item in (app.get("discovery_reasons") or [])[:3]),
)
return AppOpsInventory(result, True, f"{len(result)} container mappings loaded")
return AppOpsInventory(result, True, f"{len(result)} containerkoppelingen geladen")
def appops_inventory(appops_url: str | None) -> AppOpsInventory:
@@ -371,6 +372,38 @@ def appops_inventory(appops_url: str | None) -> AppOpsInventory:
return _appops_inventory(appops_url)
_HEALTH_IN_STATUS = re.compile(r"\((healthy|unhealthy|health: starting)\)")
_observer_clients: dict[str, object] = {}
def _observer_client(docker_host: str):
"""One long-lived socket client per host (no /version + /_ping per cycle)."""
client = _observer_clients.get(docker_host)
if client is None:
client = docker.DockerClient(base_url=docker_host, timeout=5, version="auto")
_observer_clients[docker_host] = client
return client
def _drop_observer_client(docker_host: str) -> None:
client = _observer_clients.pop(docker_host, None)
close = getattr(client, "close", None)
if callable(close):
with suppress(Exception): # best effort on an already failed client
close()
def health_from_status(status_text: str) -> str | None:
"""Docker's list summary embeds health: "Up 3 hours (healthy)"."""
match = _HEALTH_IN_STATUS.search(status_text or "")
if not match:
return None
value = match.group(1)
return "starting" if value == "health: starting" else value
def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
"""Read bounded Docker runtime/health facts without discovery side effects.
@@ -378,18 +411,48 @@ def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
suggestion persistence, reconciliation and snapshot capture. It is safe
to run on the observation cadence because it cannot rewrite topology or
operator-owned configuration.
Socket mode uses a single low-level ``containers(all=True)`` call on a
long-lived client: the list summary already carries id, name, state,
labels and health. The high-level ``containers.list()`` inspected every
container individually (1 + N API calls per minute).
"""
parsed = urlparse(docker_host)
if parsed.scheme not in ALLOWED_SCHEMES:
raise ValueError("docker_host must use unix://, tcp://, http:// or https://")
if docker_provider_mode() == "socket":
try:
summaries = _observer_client(docker_host).api.containers(all=True)
except (DockerException, OSError) as exc:
_drop_observer_client(docker_host)
raise RuntimeError(f"Docker-waarneming niet beschikbaar: {type(exc).__name__}") from exc
observations: list[DockerHealthObservation] = []
for item in summaries:
names = item.get("Names") or []
name = str(names[0] if names else item.get("Id", "unknown")[:12]).lstrip("/")
labels = {str(key): str(value) for key, value in (item.get("Labels") or {}).items()}
health = health_from_status(str(item.get("Status", "")))
observations.append(
DockerHealthObservation(
container_id=str(item.get("Id", ""))[:12],
name=name,
stable_source_ref=stable_source_ref(name, labels),
state=str(item.get("State", "unknown")),
health=health,
healthcheck_present=health is not None,
)
)
return observations
return _observe_container_health_full(docker_host)
def _observe_container_health_full(docker_host: str) -> list[DockerHealthObservation]:
"""Broker mode: the broker client exposes only the high-level listing."""
client = None
try:
client = (
docker.DockerClient(base_url=docker_host, timeout=5)
if docker_provider_mode() == "socket"
else docker_client_for_inventory(docker_host)
)
client = docker_client_for_inventory(docker_host)
client.ping()
observations: list[DockerHealthObservation] = []
for container in client.containers.list(all=True):
@@ -414,7 +477,7 @@ def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
)
return observations
except (DockerException, DockerProviderError) as exc:
raise RuntimeError(f"Docker observation unavailable: {type(exc).__name__}") from exc
raise RuntimeError(f"Docker-waarneming niet beschikbaar: {type(exc).__name__}") from exc
finally:
close = getattr(client, "close", None)
if callable(close):
@@ -509,7 +572,11 @@ def discover_containers(
client = docker.DockerClient(base_url=docker_host, timeout=5) if docker_provider_mode() == "socket" else docker_client_for_inventory(docker_host)
try:
client.ping()
containers = client.containers.list(all=include_stopped)
# Always list stopped containers: they still exist. Excluding them here
# made every stopped service look "missing" and, after three runs,
# retired with monitoring disabled. ``include_stopped`` only decides
# whether a stopped container is offered as a candidate below.
containers = client.containers.list(all=True)
existing_services = existing_services or []
appops_inventory = _appops_inventory(appops_url)
appops = appops_inventory.by_container
@@ -521,6 +588,13 @@ def discover_containers(
attrs = container.attrs or {}
state = attrs.get("State", {}) or {}
config = attrs.get("Config", {}) or {}
name = str(getattr(container, "name", "unknown")).lstrip("/")
state_name = str(state.get("Status", getattr(container, "status", "unknown")))
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
stable_ref = stable_source_ref(name, raw_labels)
discovered_refs.add(stable_ref)
if not include_stopped and state_name != "running":
continue
network_settings = attrs.get("NetworkSettings", {}) or {}
raw_ports = network_settings.get("Ports", {}) or {}
ports: list[DockerPort] = []
@@ -541,15 +615,10 @@ def discover_containers(
image_tags = getattr(container.image, "tags", []) or []
image_name = image_tags[0] if image_tags else str(config.get("Image", "unknown"))
name = str(getattr(container, "name", "unknown")).lstrip("/")
health = state.get("Health", {}).get("Status") if isinstance(state.get("Health"), dict) else None
state_name = str(state.get("Status", getattr(container, "status", "unknown")))
networks = sorted((network_settings.get("Networks", {}) or {}).keys())
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
labels = _safe_labels(raw_labels)
stable_ref = stable_source_ref(name, raw_labels)
legacy_ref = _legacy_source_ref(container.id, image_name)
discovered_refs.add(stable_ref)
matching_source = next(
(
service
@@ -677,6 +746,6 @@ def discover_containers(
appops_detail=appops_inventory.detail,
)
except (DockerException, DockerProviderError) as exc:
raise ValueError(f"Docker discovery failed: {exc}") from exc
raise ValueError(f"Docker-detectie mislukt: {exc}") from exc
finally:
client.close()
+68 -28
View File
@@ -2,52 +2,90 @@ from __future__ import annotations
import asyncio
import json
from queue import Empty, Full, Queue
from threading import Lock
from typing import Any
# Clients reconnect after this delay if the stream drops (SSE ``retry:``).
SSE_RETRY_MS = 5_000
# A comment line keeps proxies and browsers from timing out a quiet stream.
SSE_KEEPALIVE_SECONDS = 15.0
class Subscription:
"""One connected SSE client: an asyncio queue bound to its event loop."""
__slots__ = ("loop", "queue")
def __init__(self, loop: asyncio.AbstractEventLoop, max_queue_size: int) -> None:
self.loop = loop
self.queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=max_queue_size)
def offer(self, payload: dict[str, Any]) -> None:
"""Enqueue on the owning loop; drop the oldest wakeup when full."""
if self.queue.full():
try:
self.queue.get_nowait()
except asyncio.QueueEmpty:
pass
try:
self.queue.put_nowait(payload)
except asyncio.QueueFull:
pass
class EventBus:
"""Small bounded fan-out bus used by the SSE endpoint.
Persistence is handled by SQLite; this bus only wakes currently connected
clients. A queue per client keeps one slow browser from blocking probes or
other subscribers.
clients. Each client owns an ``asyncio.Queue`` so a waiting stream costs no
thread: previously every open browser tab parked one default-executor
thread in ``queue.get``, which starved ``asyncio.to_thread`` work such as
the observation loop. ``publish`` is safe from worker threads and from
the event loop; delivery is scheduled with ``call_soon_threadsafe``.
"""
def __init__(self, max_queue_size: int = 100) -> None:
self.max_queue_size = max_queue_size
self._subscribers: set[Queue[dict[str, Any]]] = set()
self._subscribers: set[Subscription] = set()
self._lock = Lock()
self._revision = 0
def subscribe(self) -> Queue[dict[str, Any]]:
queue: Queue[dict[str, Any]] = Queue(maxsize=self.max_queue_size)
with self._lock:
self._subscribers.add(queue)
return queue
@property
def revision(self) -> int:
"""Monotonic count of published events; read caches compare it."""
def unsubscribe(self, queue: Queue[dict[str, Any]]) -> None:
with self._lock:
self._subscribers.discard(queue)
return self._revision
def subscribe(self) -> Subscription:
subscription = Subscription(asyncio.get_running_loop(), self.max_queue_size)
with self._lock:
self._subscribers.add(subscription)
return subscription
def unsubscribe(self, subscription: Subscription) -> None:
with self._lock:
self._subscribers.discard(subscription)
@property
def subscriber_count(self) -> int:
with self._lock:
return len(self._subscribers)
def publish(self, payload: dict[str, Any]) -> None:
with self._lock:
self._revision += 1
subscribers = tuple(self._subscribers)
for queue in subscribers:
for subscription in subscribers:
try:
queue.put_nowait(payload)
except Full:
# Drop the oldest event for a lagging client, then retain the
# newest state notification. The durable event journal remains
# the source of truth if a client reconnects.
try:
queue.get_nowait()
except Empty:
pass
try:
queue.put_nowait(payload)
except Full:
pass
subscription.loop.call_soon_threadsafe(subscription.offer, payload)
except RuntimeError:
# The client's loop has closed (shutdown); it can no longer
# receive anything, so forget it.
self.unsubscribe(subscription)
event_bus = EventBus()
@@ -56,8 +94,10 @@ def sse_payload(payload: dict[str, Any]) -> str:
return f"event: {event_type}\ndata: {json.dumps(payload, default=str)}\n\n"
async def next_event(queue: Queue[dict[str, Any]], timeout: float = 15.0) -> dict[str, Any] | None:
async def next_event(
subscription: Subscription, timeout: float = SSE_KEEPALIVE_SECONDS
) -> dict[str, Any] | None:
try:
return await asyncio.to_thread(queue.get, True, timeout)
except Empty:
return await asyncio.wait_for(subscription.queue.get(), timeout)
except TimeoutError:
return None
+6 -6
View File
@@ -98,10 +98,10 @@ def resolve_health(
else:
reason_code = HealthResolutionReason.NO_CURRENT_COMPONENT_EVIDENCE
explanation = (
"Only application-scoped evidence is current; it is shown for provenance and is not "
"used as component health."
"Alleen bewijs op applicatieniveau is actueel; het wordt getoond als herkomst en niet "
"gebruikt als componentgezondheid."
if app_claims
else "No current component-scoped health evidence is available."
else "Er is geen actueel gezondheidsbewijs op componentniveau."
)
return HealthResolution(reason_code=reason_code, explanation=explanation, evidence=annotated)
@@ -118,7 +118,7 @@ def resolve_health(
)
selected = ordered[0]
conflict = len({item.status for item in eligible}) > 1
scope_note = " Conflicting current component signals are retained below." if conflict else ""
scope_note = " Tegenstrijdige actuele componentsignalen worden hieronder bewaard." if conflict else ""
return HealthResolution(
status=selected.status,
selected_evidence_id=selected.id,
@@ -126,8 +126,8 @@ def resolve_health(
confidence=selected.confidence,
reason_code=HealthResolutionReason.SELECTED_COMPONENT_EVIDENCE,
explanation=(
f"{selected.provider.value} selected from current component-scoped evidence "
f"(confidence {selected.confidence:.2f}).{scope_note}"
f"{selected.provider.value} gekozen uit actueel bewijs op componentniveau "
f"(zekerheid {selected.confidence:.2f}).{scope_note}"
),
conflict=conflict,
evidence=annotated,
+27 -38
View File
@@ -1,5 +1,7 @@
from __future__ import annotations
from typing import NamedTuple
from .models import HealthStatus, Service
from .store import Store
@@ -14,51 +16,38 @@ SEVERITY = {
}
def _direct_status(service: Service, store: Store) -> HealthStatus:
return store.service_health(service.id)
class TopologyHealth(NamedTuple):
services: list[Service]
direct: dict[str, HealthStatus]
effective: dict[str, HealthStatus]
dependency: dict[str, HealthStatus]
def _dependency_status(service: Service, store: Store, visiting: set[str]) -> HealthStatus:
if not service.depends_on:
return HealthStatus.HEALTHY
def topology_health(store: Store, *, services: list[Service] | None = None) -> TopologyHealth:
"""Resolve direct, dependency and effective health for the whole graph.
states: list[HealthStatus] = []
for dependency_id in service.depends_on:
dependency = store.get_service(dependency_id)
if dependency is None:
states.append(HealthStatus.UNKNOWN)
continue
states.append(_effective_status(dependency, store, visiting))
return max(states, key=lambda status: SEVERITY[status])
One bulk evidence read plus one in-memory traversal. The former
per-service recursive walk re-queried SQLite for every node on every
dependency path and dominated server CPU.
"""
services = store.list_services() if services is None else services
direct, _ = store.service_healths([service.id for service in services])
effective, dependency = status_maps(services, direct_statuses=direct)
return TopologyHealth(services, direct, effective, dependency)
def _effective_status(service: Service, store: Store, visiting: set[str]) -> HealthStatus:
if service.id in visiting:
return HealthStatus.UNKNOWN
next_visiting = set(visiting)
next_visiting.add(service.id)
own = _direct_status(service, store)
if own == HealthStatus.MAINTENANCE:
return own
dependency = _dependency_status(service, store, next_visiting)
if own == HealthStatus.DOWN:
return own
if dependency == HealthStatus.DOWN:
return HealthStatus.DEGRADED
if dependency in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} and own in {
HealthStatus.HEALTHY,
HealthStatus.RECOVERING,
}:
return HealthStatus.DEGRADED
return own
def _single(service: Service, store: Store) -> TopologyHealth:
services = [item for item in store.list_services() if item.id != service.id]
return topology_health(store, services=[*services, service])
def dependency_status(service: Service, store: Store) -> HealthStatus:
return _dependency_status(service, store, {service.id})
return _single(service, store).dependency.get(service.id, HealthStatus.UNKNOWN)
def effective_status(service: Service, store: Store) -> HealthStatus:
return _effective_status(service, store, set())
return _single(service, store).effective.get(service.id, HealthStatus.UNKNOWN)
def status_maps(
@@ -134,15 +123,15 @@ def validate_dependencies(
service_id: str | None = None,
) -> None:
if len(depends_on) != len(set(depends_on)):
raise ValueError("Dependencies must be unique")
raise ValueError("Afhankelijkheden moeten uniek zijn")
if service_id and service_id in depends_on:
raise ValueError("A service cannot depend on itself")
raise ValueError("Een service kan niet van zichzelf afhangen")
services = store.list_services()
known_ids = {service.id for service in services}
missing = [dependency_id for dependency_id in depends_on if dependency_id not in known_ids]
if missing:
raise ValueError(f"Unknown dependency ids: {', '.join(missing)}")
raise ValueError(f"Onbekende afhankelijkheids-id's: {', '.join(missing)}")
if service_id is None:
return
@@ -154,7 +143,7 @@ def validate_dependencies(
def visit(node: str) -> None:
if node in visiting:
raise ValueError("Dependency graph contains a cycle")
raise ValueError("De afhankelijkheidsgrafiek bevat een kring")
if node in visited:
return
visiting.add(node)
+29 -15
View File
@@ -1,9 +1,10 @@
from __future__ import annotations
import threading
from datetime import UTC, datetime
from uuid import uuid4
from .health import dependency_status, effective_status
from .health import TopologyHealth, topology_health
from .models import (
Criticality,
HealthStatus,
@@ -37,11 +38,12 @@ def affected_service_ids(root_id: str, services: list[Service]) -> list[str]:
return sorted(seen)
def incident_overview(store: Store) -> IncidentOverview:
services = store.list_services()
def incident_overview(store: Store, *, health: TopologyHealth | None = None) -> IncidentOverview:
health = health or topology_health(store)
services = health.services
by_id = {service.id: service for service in services}
effective = {service.id: effective_status(service, store) for service in services}
dependency = {service.id: dependency_status(service, store) for service in services}
effective = health.effective
dependency = health.dependency
down_ids = sorted(service_id for service_id, state in effective.items() if state == HealthStatus.DOWN)
degraded_ids = sorted(
@@ -62,9 +64,9 @@ def incident_overview(store: Store) -> IncidentOverview:
continue
affected_ids = affected_service_ids(service.id, services)
affected_names = [by_id[item].name for item in affected_ids if item in by_id]
reason = "Service is down and no configured dependency is also down"
reason = "Service is uitgevallen en geen enkele ingestelde afhankelijkheid is ook uitgevallen"
if dependency[service.id] == HealthStatus.UNKNOWN:
reason += "; at least one dependency has unknown health"
reason += "; minstens één afhankelijkheid heeft een onbekende gezondheid"
candidates.append(
IncidentCandidate(
service_id=service.id,
@@ -91,15 +93,27 @@ def _severity(service: Service) -> Criticality:
return service.criticality
def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recovered; incident resolved") -> list[IncidentRecord]:
# Probes, observation and API handlers reconcile from different threads. Two
# concurrent passes could both see "no active incident" for a root and each
# open one (the later duplicate cleanup then left two journal entries).
_SYNC_LOCK = threading.RLock()
def sync_incidents(store: Store, *, resolution_reason: str = "Hoofdoorzaak hersteld; incident opgelost") -> list[IncidentRecord]:
with _SYNC_LOCK:
return _sync_incidents_locked(store, resolution_reason=resolution_reason)
def _sync_incidents_locked(store: Store, *, resolution_reason: str) -> list[IncidentRecord]:
"""Reconcile the current graph with the durable incident journal.
A root service has at most one active incident. Repeated probe cycles only
update its affected set; a later independent outage gets a new id.
"""
overview = incident_overview(store)
services = {service.id: service for service in store.list_services()}
health = topology_health(store)
overview = incident_overview(store, health=health)
services = {service.id: service for service in health.services}
now = datetime.now(UTC)
active: dict[str, IncidentRecord] = {}
for incident in sorted(
@@ -115,7 +129,7 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
store.append_incident_event(
incident.id,
"incident.duplicate_resolved",
"Duplicate active incident closed; the earliest incident remains active",
"Dubbel actief incident gesloten; het oudste incident blijft actief",
incident.root_service_id,
)
continue
@@ -133,7 +147,7 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
root_service_name=candidate.service_name,
started_at=now,
detected_at=now,
title=f"{candidate.service_name} is down",
title=f"{candidate.service_name} is uitgevallen",
root_cause_reason=candidate.reason,
affected_service_ids=candidate.affected_service_ids,
affected_service_names=candidate.affected_service_names,
@@ -144,7 +158,7 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
store.append_incident_event(
incident.id,
"incident.opened",
f"Incident opened: {candidate.service_name} is the likely root cause",
f"Incident geopend: {candidate.service_name} is de waarschijnlijke hoofdoorzaak",
candidate.service_id,
)
active[candidate.service_id] = incident
@@ -158,14 +172,14 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
}
)
store.upsert_incident(updated)
store.append_incident_event(incident.id, "incident.updated", "Downstream impact recalculated", candidate.service_id)
store.append_incident_event(incident.id, "incident.updated", "Impact op afhankelijke services herberekend", candidate.service_id)
for root_id, incident in list(active.items()):
if any(candidate.service_id == root_id for candidate in overview.candidates):
continue
root = services.get(root_id)
if (
(root is None or store.service_health(root_id) not in {
(root is None or health.direct.get(root_id, HealthStatus.UNKNOWN) not in {
HealthStatus.DOWN,
HealthStatus.DEGRADED,
HealthStatus.RECOVERING,
+150
View File
@@ -0,0 +1,150 @@
"""Background loops that never die silently, and their readiness record.
Before v1.6.3 the observation and discovery loops caught only ``OSError``,
``RuntimeError`` and ``ValueError``. Any other exception, for example
``sqlite3.OperationalError: database is locked``, ended the task without a
log line; ServiceMap then looked healthy while it no longer observed
anything. ``run_periodic`` logs every failure, retries with bounded
backoff and records the last success so ``/ready`` can report a stalled
loop.
"""
from __future__ import annotations
import asyncio
import logging
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from threading import Lock
logger = logging.getLogger("servicemap.loops")
@dataclass
class LoopStatus:
name: str
interval_seconds: float
started_at: float = field(default_factory=time.time)
last_success_at: float | None = None
last_failure_at: float | None = None
last_error: str | None = None
consecutive_failures: int = 0
paused: bool = False
def stale_after_seconds(self) -> float:
# Three missed cycles, with a floor for very short intervals.
return max(3 * self.interval_seconds, 180.0)
def healthy(self, now: float) -> bool:
if self.paused:
return True
reference = self.last_success_at if self.last_success_at is not None else self.started_at
return now - reference <= self.stale_after_seconds()
def as_dict(self, now: float) -> dict[str, object]:
def iso(value: float | None) -> str | None:
return None if value is None else time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(value))
return {
"name": self.name,
"healthy": self.healthy(now),
"interval_seconds": self.interval_seconds,
"last_success_at": iso(self.last_success_at),
"last_failure_at": iso(self.last_failure_at),
"last_error": self.last_error,
"consecutive_failures": self.consecutive_failures,
"paused": self.paused,
}
class LoopRegistry:
def __init__(self) -> None:
self._loops: dict[str, LoopStatus] = {}
self._lock = Lock()
def register(self, name: str, interval_seconds: float) -> LoopStatus:
with self._lock:
status = LoopStatus(name=name, interval_seconds=interval_seconds)
self._loops[name] = status
return status
def unregister(self, name: str) -> None:
with self._lock:
self._loops.pop(name, None)
def snapshot(self) -> list[dict[str, object]]:
now = time.time()
with self._lock:
return [status.as_dict(now) for status in self._loops.values()]
def all_healthy(self) -> bool:
now = time.time()
with self._lock:
return all(status.healthy(now) for status in self._loops.values())
loop_registry = LoopRegistry()
def record_success(status: LoopStatus) -> None:
status.last_success_at = time.time()
status.consecutive_failures = 0
status.paused = False
def record_failure(status: LoopStatus, error: BaseException) -> None:
status.last_failure_at = time.time()
status.consecutive_failures += 1
status.last_error = f"{type(error).__name__}: {str(error)[:300]}"
async def run_periodic(
name: str,
work: Callable[[], Awaitable[object]],
*,
interval_seconds: float,
initial_delay_seconds: float = 0.0,
paused: Callable[[], bool] | None = None,
registry: LoopRegistry = loop_registry,
max_backoff_seconds: float = 300.0,
) -> None:
"""Run ``work`` every ``interval_seconds`` until cancelled.
A failure is logged with its traceback and retried after an exponential
backoff capped at ``max_backoff_seconds``. ``paused`` lets a loop
idle deliberately (for example while the synthetic demo is active)
without being reported as stalled.
"""
status = registry.register(name, interval_seconds)
try:
if initial_delay_seconds > 0:
await asyncio.sleep(initial_delay_seconds)
while True:
delay = interval_seconds
try:
if paused is not None and paused():
status.paused = True
else:
await work()
record_success(status)
except asyncio.CancelledError:
raise
except Exception as exc:
record_failure(status, exc)
# interval, 2x, 4x … capped: a persistent fault does not spin,
# a long interval still retries within max_backoff_seconds.
delay = min(
max_backoff_seconds,
max(5.0, interval_seconds) * 2 ** min(status.consecutive_failures - 1, 5),
)
logger.exception(
"Background loop %s failed (%s consecutive); retrying in %.0fs",
name,
status.consecutive_failures,
delay,
)
await asyncio.sleep(delay)
finally:
registry.unregister(name)
+430 -100
View File
@@ -1,26 +1,30 @@
from __future__ import annotations
import asyncio
import logging
import os
import sqlite3
from contextlib import asynccontextmanager, suppress
from datetime import timedelta
from datetime import datetime, timedelta
from pathlib import Path
from fastapi import FastAPI, HTTPException, Query, Response, status
from fastapi.responses import JSONResponse, StreamingResponse
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from .auth import AuthConfig, AuthorizationError, authorize_request
from .alerts import AlertManager, AlertSettingsUpdate, AlertSettingsView, AlertTestResult
from .auth import AuthConfig, AuthorizationError, admin_token, admin_token_path, authorize_request
from .backups import BackupInfo, BackupManager
from .config_io import export_document, import_document, parse_document, serialize_document
from .demo import DEMO_ROOT_CAUSE_NAME, DEMO_SCENARIO_NAME, build_demo_records
from .discovery import DiscoveryManager
from .discovery_store import DiscoveryStore
from .docker_discovery import discover_containers
from .docker_provider import docker_provider_mode
from .events import event_bus, next_event, sse_payload
from .health import dependency_status, effective_status, status_maps, validate_dependencies
from .events import SSE_RETRY_MS, event_bus, next_event, sse_payload
from .health import status_maps, topology_health, validate_dependencies
from .incidents import incident_overview, sync_incidents
from .loops import loop_registry, run_periodic
from .models import (
AvailabilitySummary,
BulkProbeResult,
@@ -39,7 +43,10 @@ from .models import (
EdgeMutationRequest,
EventRecord,
HealthResolution,
HealthStatus,
IncidentNoteRequest,
IncidentOverview,
IncidentResolveRequest,
LogicalRelation,
MaintenanceRequest,
OverviewSummary,
@@ -60,6 +67,14 @@ from .models import (
)
from .observation import ObservationManager
from .probes import probe_many, probe_service
from .response_cache import (
GENERATED_AT_HEADER,
CachedRepresentation,
RepresentationCache,
etag_matches,
next_freshness_boundary,
representation,
)
from .scheduler import scheduled_probe_loop
from .semantic import build_semantic_topology
from .store import Store
@@ -83,6 +98,16 @@ def _env_float(name: str, default: float, minimum: float = 0.1) -> float:
return default
logger = logging.getLogger("servicemap")
if not logger.handlers:
# Uvicorn configures only its own loggers; without a handler our INFO
# lines (maintenance results, loop recovery) were never emitted.
_handler = logging.StreamHandler()
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
logger.addHandler(_handler)
logger.setLevel(logging.INFO)
logger.propagate = False
PROBE_INTERVAL = _env_int("SERVICEMAP_PROBE_INTERVAL", 0)
PROBE_TIMEOUT = _env_float("SERVICEMAP_PROBE_TIMEOUT", 5.0)
PROBE_RETRIES = _env_int("SERVICEMAP_PROBE_RETRIES", 1)
@@ -90,8 +115,48 @@ PROBE_CONCURRENCY = _env_int("SERVICEMAP_PROBE_CONCURRENCY", 10, minimum=1)
RETENTION_DAYS = _env_int("SERVICEMAP_RETENTION_DAYS", 30, minimum=1)
EVENT_RETENTION_DAYS = _env_int("SERVICEMAP_EVENT_RETENTION_DAYS", 90, minimum=1)
DISCOVERY_INTERVAL = _env_int("SERVICEMAP_DISCOVERY_INTERVAL_SECONDS", 0, minimum=0)
OBSERVATION_INTERVAL = _env_int("SERVICEMAP_OBSERVATION_INTERVAL_SECONDS", 60, minimum=10)
OBSERVATION_INTERVAL = _env_int("SERVICEMAP_OBSERVATION_INTERVAL_SECONDS", 60)
if OBSERVATION_INTERVAL > 0:
OBSERVATION_INTERVAL = max(10, OBSERVATION_INTERVAL)
# Superseded health-evidence history; current claims are never pruned. 0 disables.
EVIDENCE_RETENTION_DAYS = _env_int("SERVICEMAP_EVIDENCE_RETENTION_DAYS", 7, minimum=0)
MAINTENANCE_INTERVAL = _env_int("SERVICEMAP_MAINTENANCE_INTERVAL_SECONDS", 3600, minimum=60)
# Discovery runs whose per-container observations are kept.
DISCOVERY_OBSERVATION_RUNS = _env_int("SERVICEMAP_DISCOVERY_OBSERVATION_RUNS", 10, minimum=1)
# Verified database backups in <data>/backups; 0 hours disables the schedule.
BACKUP_INTERVAL_HOURS = _env_int("SERVICEMAP_BACKUP_INTERVAL_HOURS", 24, minimum=0)
BACKUP_KEEP = _env_int("SERVICEMAP_BACKUP_KEEP", 14, minimum=1)
observation_manager = ObservationManager(store, interval_seconds=OBSERVATION_INTERVAL)
semantic_cache = RepresentationCache(max_age_seconds=10.0)
_alert_manager: AlertManager | None = None
def backup_manager() -> BackupManager:
return BackupManager(store, keep=BACKUP_KEEP)
def alert_manager() -> AlertManager:
"""Bound to the current store (tests swap ``store``)."""
global _alert_manager
if _alert_manager is None or _alert_manager.store is not store:
_alert_manager = AlertManager(store)
return _alert_manager
def run_maintenance() -> dict[str, int]:
"""Bounded retention for tables that otherwise grow with uptime."""
if store.demo_active():
return {}
pruned = {
"health_evidence": store.prune_health_evidence(EVIDENCE_RETENTION_DAYS),
"probe_results": store.prune_probes(RETENTION_DAYS, EVENT_RETENTION_DAYS),
"discovery_observations": discovery_store.prune_observations(DISCOVERY_OBSERVATION_RUNS),
"topology_change_events": discovery_store.prune_change_events(EVENT_RETENTION_DAYS),
}
store.checkpoint()
return pruned
def _require_configured_discovery_targets(
@@ -102,16 +167,22 @@ def _require_configured_discovery_targets(
configured_docker = os.getenv("SERVICEMAP_DOCKER_HOST", "unix:///var/run/docker.sock")
configured_appops = os.getenv("SERVICEMAP_APPOPS_URL", "")
if docker_host and docker_host != configured_docker:
raise HTTPException(status_code=403, detail="Docker discovery target is not configured")
raise HTTPException(status_code=403, detail="Er is geen Docker-detectiedoel ingesteld")
if appops_url and appops_url.rstrip("/") != configured_appops.rstrip("/"):
raise HTTPException(status_code=403, detail="AppOps discovery target is not configured")
raise HTTPException(status_code=403, detail="Er is geen AppOps-detectiedoel ingesteld")
@asynccontextmanager
async def lifespan(_: FastAPI):
try:
if AuthConfig.from_environment().mode == "token":
# Create the token on first start so its location is logged
# before anyone needs it.
admin_token()
logger.info("Editing requires the admin token in %s", admin_token_path())
except (AuthorizationError, OSError):
logger.exception("ServiceMap admin token is unavailable; edits are refused")
task: asyncio.Task[None] | None = None
discovery_task: asyncio.Task[None] | None = None
observation_task: asyncio.Task[None] | None = None
if PROBE_INTERVAL > 0:
task = asyncio.create_task(
scheduled_probe_loop(
@@ -121,37 +192,88 @@ async def lifespan(_: FastAPI):
retries=PROBE_RETRIES,
concurrency=PROBE_CONCURRENCY,
retention_days=RETENTION_DAYS,
event_retention_days=EVENT_RETENTION_DAYS,
)
)
background: list[asyncio.Task[None]] = []
if DISCOVERY_INTERVAL > 0:
async def discovery_loop() -> None:
while True:
await asyncio.sleep(DISCOVERY_INTERVAL)
if store.demo_active():
continue
try:
request = DiscoveryRunRequest()
result = await asyncio.to_thread(discovery_manager.run, request)
await asyncio.to_thread(
discovery_manager.reconcile,
DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=False),
)
except (OSError, RuntimeError, ValueError):
# Discovery is review-first; a failed background run is
# persisted and must not take down the monitoring loop.
continue
discovery_task = asyncio.create_task(discovery_loop())
async def observation_loop() -> None:
while True:
if not store.demo_active():
try:
await asyncio.to_thread(observation_manager.refresh)
except (OSError, RuntimeError, ValueError):
# Provider availability is persisted by the observer; an
# unavailable provider must not stop scheduled probing.
pass
await asyncio.sleep(OBSERVATION_INTERVAL)
observation_task = asyncio.create_task(observation_loop())
async def discovery_cycle() -> None:
result = await asyncio.to_thread(discovery_manager.run, DiscoveryRunRequest())
await asyncio.to_thread(
discovery_manager.reconcile,
DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=False),
)
background.append(asyncio.create_task(run_periodic(
"discovery",
discovery_cycle,
interval_seconds=DISCOVERY_INTERVAL,
initial_delay_seconds=DISCOVERY_INTERVAL,
paused=store.demo_active,
)))
if OBSERVATION_INTERVAL > 0:
async def observation_cycle() -> None:
await asyncio.to_thread(observation_manager.refresh)
background.append(asyncio.create_task(run_periodic(
"observation",
observation_cycle,
interval_seconds=OBSERVATION_INTERVAL,
paused=store.demo_active,
)))
async def alert_cycle() -> None:
sent = await asyncio.to_thread(alert_manager().run_cycle)
if sent:
logger.info("Sent %s incident notification(s)", sent)
background.append(asyncio.create_task(run_periodic(
"alerts",
alert_cycle,
interval_seconds=30,
paused=store.demo_active,
)))
if BACKUP_INTERVAL_HOURS > 0:
async def backup_cycle() -> None:
info = await asyncio.to_thread(backup_manager().create)
logger.info("Backup %s written (%s bytes, verified)", info.name, info.size_bytes)
background.append(asyncio.create_task(run_periodic(
"backups",
backup_cycle,
interval_seconds=BACKUP_INTERVAL_HOURS * 3600,
# First backup shortly after start, then on the schedule.
initial_delay_seconds=120,
paused=store.demo_active,
)))
async def maintenance_window_cycle() -> None:
ended = await asyncio.to_thread(store.expire_maintenance_windows)
if ended:
logger.info("Maintenance windows ended for %s service(s)", len(ended))
background.append(asyncio.create_task(run_periodic(
"maintenance-windows",
maintenance_window_cycle,
interval_seconds=60,
paused=store.demo_active,
)))
async def maintenance_cycle() -> None:
pruned = await asyncio.to_thread(run_maintenance)
if any(pruned.values()):
logger.info("Maintenance pruned %s", pruned)
# Start shortly after boot so a long-running deployment sheds its backlog
# without delaying startup, then repeat on a slow cadence.
background.append(asyncio.create_task(run_periodic(
"maintenance",
maintenance_cycle,
interval_seconds=MAINTENANCE_INTERVAL,
initial_delay_seconds=30,
)))
try:
yield
finally:
@@ -159,38 +281,74 @@ async def lifespan(_: FastAPI):
task.cancel()
with suppress(asyncio.CancelledError):
await task
if discovery_task:
discovery_task.cancel()
for background_task in background:
background_task.cancel()
for background_task in background:
with suppress(asyncio.CancelledError):
await discovery_task
if observation_task:
observation_task.cancel()
with suppress(asyncio.CancelledError):
await observation_task
await background_task
app = FastAPI(title="ServiceMap API", version="1.6.2", lifespan=lifespan)
app = FastAPI(title="ServiceMap API", version="1.6.3", lifespan=lifespan)
# Bounded request bodies: the largest legitimate payload is a config import.
MAX_REQUEST_BYTES = 4 * 1024 * 1024
SECURITY_HEADERS = {
"X-Content-Type-Options": "nosniff",
"X-Frame-Options": "DENY",
"Referrer-Policy": "no-referrer",
"Cross-Origin-Opener-Policy": "same-origin",
# The SPA ships no inline scripts or third-party assets; React Flow and
# the layout use inline style attributes.
"Content-Security-Policy": (
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
"img-src 'self' data: blob:; font-src 'self'; connect-src 'self'; "
"object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'"
),
}
@app.middleware("http")
async def authorization_boundary(request, call_next):
try:
authorize_request(request)
except HTTPException as exc:
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
return await call_next(request)
declared = request.headers.get("content-length", "")
if declared.isdigit() and int(declared) > MAX_REQUEST_BYTES:
response = JSONResponse(status_code=413, content={"detail": "De aanvraag is te groot"})
else:
try:
authorize_request(request)
except HTTPException as exc:
response = JSONResponse(
status_code=exc.status_code,
content={"detail": exc.detail},
headers=dict(exc.headers or {}),
)
else:
response = await call_next(request)
for name, value in SECURITY_HEADERS.items():
response.headers.setdefault(name, value)
path = request.url.path
if path.startswith("/assets/"):
# Vite fingerprints asset names, so they never change in place.
response.headers.setdefault("Cache-Control", "public, max-age=31536000, immutable")
elif not path.startswith("/api/") and path not in {"/health", "/ready"}:
# index.html (and other unhashed files) must be revalidated, or a
# browser keeps running the previous release after a deploy.
response.headers.setdefault("Cache-Control", "no-cache")
return response
def _snapshot(service: Service) -> ServiceSnapshot:
services = store.list_services()
services = [item for item in store.list_services() if item.id != service.id]
health = topology_health(store, services=[*services, service])
dependents = [item.name for item in services if service.id in item.depends_on]
return ServiceSnapshot(
service=service,
latest_probe=store.latest_probe(service.id),
availability_24h=store.availability_summary(service.id, 24),
dependency_status=dependency_status(service, store),
effective_status=effective_status(service, store),
direct_status=store.service_health(service.id),
dependency_status=health.dependency.get(service.id, HealthStatus.UNKNOWN),
effective_status=health.effective.get(service.id, HealthStatus.UNKNOWN),
direct_status=health.direct.get(service.id, HealthStatus.UNKNOWN),
flapping=service.flapping,
flap_count=service.flap_count,
next_probe_at=store.next_probe_at(service, PROBE_INTERVAL),
@@ -199,6 +357,7 @@ def _snapshot(service: Service) -> ServiceSnapshot:
evidence=discovery_store.evidence_for_service(service.id),
health_evidence=store.health_evidence_for_service(service.id),
health_resolution=store.health_resolution(service.id),
maintenance_until=store.maintenance_windows().get(service.id),
)
@@ -209,9 +368,34 @@ def _name_conflict(name: str, *, exclude_id: str | None = None) -> bool:
@app.get("/health")
def health() -> dict[str, str]:
"""Liveness: the process answers. Used by the container healthcheck."""
return {"status": "ok"}
@app.get("/ready")
def ready() -> JSONResponse:
"""Readiness: the database takes writes and no background loop stalled.
A loop is stalled when it has not completed a cycle for three intervals
(at least three minutes); its last error is included for diagnosis.
"""
checks: dict[str, object] = {"loops": loop_registry.snapshot()}
database_ok = True
try:
store.ready_check()
except sqlite3.Error as exc:
database_ok = False
checks["database_error"] = f"{type(exc).__name__}: {exc}"
checks["database"] = "ok" if database_ok else "unavailable"
ok = database_ok and loop_registry.all_healthy()
return JSONResponse(
status_code=status.HTTP_200_OK if ok else status.HTTP_503_SERVICE_UNAVAILABLE,
content={"status": "ready" if ok else "not_ready", **checks},
)
@app.get("/api/v1/meta")
def meta() -> dict[str, str | int | float | bool]:
try:
@@ -220,7 +404,7 @@ def meta() -> dict[str, str | int | float | bool]:
auth_mode = "misconfigured"
return {
"name": "ServiceMap",
"version": "1.6.2",
"version": "1.6.3",
"stage": "v1",
"scheduled_probing": PROBE_INTERVAL > 0,
"probe_interval_seconds": PROBE_INTERVAL,
@@ -237,9 +421,73 @@ def meta() -> dict[str, str | int | float | bool]:
"docker_provider": docker_provider_mode(),
"appops_configured": bool(os.getenv("SERVICEMAP_APPOPS_URL")),
"auth_mode": auth_mode,
# What a browser needs before it may change anything.
"write_auth": {"token": "admin_token", "oidc": "oidc"}.get(auth_mode, "none"),
}
@app.get("/api/v1/settings/alerts", response_model=AlertSettingsView)
def get_alert_settings() -> AlertSettingsView:
return alert_manager().view()
@app.put("/api/v1/settings/alerts", response_model=AlertSettingsView)
def put_alert_settings(payload: AlertSettingsUpdate) -> AlertSettingsView:
try:
return alert_manager().update(payload)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@app.post("/api/v1/settings/alerts/test", response_model=AlertTestResult)
def test_alert() -> AlertTestResult:
return alert_manager().send_test()
@app.get("/api/v1/backups", response_model=list[BackupInfo])
def list_backups() -> list[BackupInfo]:
return backup_manager().list()
@app.post("/api/v1/backups", response_model=BackupInfo, status_code=status.HTTP_201_CREATED)
def create_backup() -> BackupInfo:
try:
return backup_manager().create()
except (ValueError, RuntimeError, sqlite3.Error) as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@app.get("/api/v1/backups/{name}")
def download_backup(name: str) -> FileResponse:
try:
path = backup_manager().path_for(name)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
return FileResponse(path, media_type="application/vnd.sqlite3", filename=name)
@app.post("/api/v1/backups/{name}/restore", response_model=BackupInfo)
def restore_backup(name: str) -> BackupInfo:
"""Restore ``name`` into the live database; returns the safety copy taken first."""
try:
safety = backup_manager().restore(name)
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except (RuntimeError, sqlite3.Error) as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
semantic_cache.clear()
event_bus.publish({"event_type": "topology.changed", "message": f"Database teruggezet vanaf {name}"})
return safety
@app.post("/api/v1/auth/verify")
def verify_write_access() -> dict[str, bool]:
"""Succeeds only for a caller allowed to mutate (checked by the middleware)."""
return {"ok": True}
def _service_snapshots(
*, include_hidden: bool = False, include_context: bool = True
) -> list[ServiceSnapshot]:
@@ -268,6 +516,10 @@ def _service_snapshots(
service.id: [item.name for item in services if service.id in item.depends_on]
for service in services
}
visible_ids = [service.id for service in services]
windows = store.maintenance_windows()
sources_by_service = discovery_store.sources_for_services(visible_ids) if include_context else {}
relations_by_service = discovery_store.evidence_for_services(visible_ids) if include_context else {}
return [
ServiceSnapshot(
service=service,
@@ -284,10 +536,11 @@ def _service_snapshots(
else None
),
dependents=dependent_names.get(service.id, []),
sources=discovery_store.sources_for_service(service.id) if include_context else [],
evidence=discovery_store.evidence_for_service(service.id) if include_context else [],
sources=sources_by_service.get(service.id, []),
evidence=relations_by_service.get(service.id, []),
health_evidence=evidence_by_service.get(service.id, []),
health_resolution=resolutions.get(service.id, HealthResolution()),
maintenance_until=windows.get(service.id),
)
for service in services
]
@@ -301,21 +554,21 @@ def list_services(include_hidden: bool = Query(default=False)) -> list[ServiceSn
@app.post("/api/v1/services", response_model=Service, status_code=status.HTTP_201_CREATED)
def create_service(payload: ServiceCreate) -> Service:
if _name_conflict(payload.name):
raise HTTPException(status_code=409, detail="Service name already exists")
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam")
try:
validate_dependencies(payload.depends_on, store)
return store.create_service(payload)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except sqlite3.IntegrityError as exc:
raise HTTPException(status_code=409, detail="Service name already exists") from exc
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam") from exc
@app.get("/api/v1/services/{service_id}", response_model=ServiceSnapshot)
def get_service(service_id: str) -> ServiceSnapshot:
service = store.get_service(service_id)
if service is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return _snapshot(service)
@@ -323,18 +576,18 @@ def get_service(service_id: str) -> ServiceSnapshot:
def update_service(service_id: str, payload: ServiceUpdate) -> Service:
before = store.get_service(service_id)
if before is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
if _name_conflict(payload.name, exclude_id=service_id):
raise HTTPException(status_code=409, detail="Service name already exists")
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam")
try:
validate_dependencies(payload.depends_on, store, service_id=service_id)
updated = store.update_service(service_id, payload)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except sqlite3.IntegrityError as exc:
raise HTTPException(status_code=409, detail="Service name already exists") from exc
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam") from exc
if updated is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
if before.source_type:
discovery_manager.record_service_override(before, updated)
return updated
@@ -343,7 +596,7 @@ def update_service(service_id: str, payload: ServiceUpdate) -> Service:
@app.delete("/api/v1/services/{service_id}", status_code=status.HTTP_204_NO_CONTENT)
def delete_service(service_id: str) -> None:
if not store.delete_service(service_id):
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
@app.post("/api/v1/services/{service_id}/probe", response_model=ProbeResult)
@@ -353,10 +606,10 @@ async def run_probe(
retries: int | None = Query(default=None, ge=0, le=5),
) -> ProbeResult:
if store.demo_active():
raise HTTPException(status_code=409, detail="Explore demo uses synthetic probe data; live probing is paused")
raise HTTPException(status_code=409, detail="De demo gebruikt synthetische controlegegevens; live controles zijn gepauzeerd")
service = store.get_service(service_id)
if service is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
result = await probe_service(
service,
timeout_seconds=timeout_seconds,
@@ -394,21 +647,21 @@ async def run_all_probes(
@app.get("/api/v1/services/{service_id}/history", response_model=list[ProbeResult])
def probe_history(service_id: str, limit: int = Query(default=50, ge=1, le=500)) -> list[ProbeResult]:
if store.get_service(service_id) is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return store.probe_history(service_id, limit)
@app.get("/api/v1/services/{service_id}/availability", response_model=AvailabilitySummary)
def availability(service_id: str, hours: int = Query(default=24, ge=1, le=24 * 90)) -> AvailabilitySummary:
if store.get_service(service_id) is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return store.availability_summary(service_id, hours)
@app.get("/api/v1/services/{service_id}/evidence", response_model=HealthResolution)
def service_evidence(service_id: str) -> HealthResolution:
if store.get_service(service_id) is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return store.health_resolution(service_id)
@@ -419,9 +672,12 @@ def providers() -> list[ProviderStatus]:
@app.post("/api/v1/services/{service_id}/maintenance", response_model=Service)
def maintenance(service_id: str, payload: MaintenanceRequest) -> Service:
updated = store.set_maintenance(service_id, payload.enabled, payload.reason)
try:
updated = store.set_maintenance(service_id, payload.enabled, payload.reason, payload.until)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
if updated is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return updated
@@ -437,7 +693,7 @@ def incidents() -> IncidentOverview:
def get_incident(incident_id: str) -> dict[str, object]:
incident = store.get_incident(incident_id)
if incident is None:
raise HTTPException(status_code=404, detail="Incident not found")
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return {"incident": incident.model_dump(mode="json"), "timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)]}
@@ -448,30 +704,60 @@ def acknowledge_incident(incident_id: str) -> dict[str, object]:
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
if incident is None:
raise HTTPException(status_code=404, detail="Incident not found")
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return {"incident": incident.model_dump(mode="json"), "timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)]}
def _incident_payload(incident_id: str) -> dict[str, object]:
incident = store.get_incident(incident_id)
if incident is None:
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return {
"incident": incident.model_dump(mode="json"),
"timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)],
}
@app.post("/api/v1/incidents/{incident_id}/resolve")
def resolve_incident(incident_id: str, payload: IncidentResolveRequest | None = None) -> dict[str, object]:
if store.resolve_incident(incident_id, payload.note if payload else None) is None:
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return _incident_payload(incident_id)
@app.post("/api/v1/incidents/{incident_id}/notes")
def add_incident_note(incident_id: str, payload: IncidentNoteRequest) -> dict[str, object]:
if store.add_incident_note(incident_id, payload.text.strip()) is None:
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return _incident_payload(incident_id)
@app.get("/api/v1/events", response_model=list[EventRecord])
def events(limit: int = Query(default=50, ge=1, le=500), service_id: str | None = None) -> list[EventRecord]:
return store.list_events(limit=limit, service_id=service_id)
def events(
limit: int = Query(default=50, ge=1, le=500),
service_id: str | None = None,
event_type: str | None = Query(default=None, max_length=80),
before: datetime | None = None,
) -> list[EventRecord]:
return store.list_events(limit=limit, service_id=service_id, event_type=event_type, before=before)
@app.get("/api/v1/events/stream")
async def event_stream() -> StreamingResponse:
queue = event_bus.subscribe()
subscription = event_bus.subscribe()
async def stream():
try:
yield ": connected\n\n"
# ``retry`` tells EventSource how long to wait before reconnecting.
yield f"retry: {SSE_RETRY_MS}\n: connected\n\n"
while True:
payload = await next_event(queue)
payload = await next_event(subscription)
if payload is None:
yield ": keepalive\n\n"
else:
yield sse_payload(payload)
finally:
event_bus.unsubscribe(queue)
event_bus.unsubscribe(subscription)
return StreamingResponse(
stream(),
@@ -483,7 +769,8 @@ async def event_stream() -> StreamingResponse:
@app.get("/api/v1/overview", response_model=OverviewSummary)
def overview() -> OverviewSummary:
sync_incidents(store)
snapshots = _service_snapshots(include_hidden=False)
# Only status and coverage are counted; discovery context is not needed.
snapshots = _service_snapshots(include_hidden=False, include_context=False)
overview_state = incident_overview(store)
counts: dict[str, int] = {}
coverage: dict[str, int] = {}
@@ -521,14 +808,20 @@ def import_config(payload: ConfigImportRequest) -> ConfigImportResult:
raise HTTPException(status_code=422, detail=str(exc)) from exc
def _demo_enabled() -> bool:
return os.getenv("SERVICEMAP_DEMO_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"}
def _demo_state() -> DemoState:
enabled = _demo_enabled()
if not store.demo_active():
return DemoState(active=False)
return DemoState(active=False, enabled=enabled)
sync_incidents(store)
overview_state = incident_overview(store)
candidate = overview_state.candidates[0] if overview_state.candidates else None
return DemoState(
active=True,
enabled=enabled,
synthetic=True,
scenario_name=DEMO_SCENARIO_NAME,
service_count=len(store.list_services()),
@@ -557,15 +850,10 @@ def demo_state() -> DemoState:
@app.post("/api/v1/demo", response_model=DemoMutationResult)
def load_demo() -> DemoMutationResult:
services, probes = build_demo_records()
result = store.activate_demo(services, probes)
sync_incidents(store)
return _demo_result(result)
@app.post("/api/v1/demo/reset", response_model=DemoMutationResult)
def reset_demo() -> DemoMutationResult:
def load_demo() -> DemoMutationResult:
if not _demo_enabled():
raise HTTPException(status_code=409, detail="Start de demo in een aparte ServiceMap-instantie")
services, probes = build_demo_records()
result = store.activate_demo(services, probes)
sync_incidents(store)
@@ -578,7 +866,7 @@ async def exit_demo() -> DemoMutationResult:
# Refresh live provider evidence immediately after restoring the saved
# production topology. Test stores intentionally do not inherit the
# process-global observer, but production always does.
if restored and observation_manager.store is store:
if restored and OBSERVATION_INTERVAL > 0 and observation_manager.store is store:
await asyncio.to_thread(observation_manager.refresh)
return _demo_result(restored=restored)
@@ -586,7 +874,7 @@ async def exit_demo() -> DemoMutationResult:
@app.post("/api/v1/discovery/run", response_model=DiscoveryResult)
def run_discovery(payload: DiscoveryRunRequest | None = None) -> DiscoveryResult:
if store.demo_active():
raise HTTPException(status_code=409, detail="Exit Explore demo before discovering production infrastructure")
raise HTTPException(status_code=409, detail="Sluit de demo af voor je de productie-infrastructuur detecteert")
request = payload or DiscoveryRunRequest()
_require_configured_discovery_targets(request.docker_host, request.appops_url)
try:
@@ -613,7 +901,7 @@ def discovery_suggestions(
try:
suggestion_status = SuggestionStatus(status_filter) if status_filter else None
except ValueError as exc:
raise HTTPException(status_code=422, detail="Unknown suggestion status") from exc
raise HTTPException(status_code=422, detail="Onbekende status van het voorstel") from exc
return discovery_store.list_suggestions(status=suggestion_status)
@@ -636,7 +924,7 @@ def reject_discovery_suggestion(suggestion_id: str) -> TopologySuggestion:
@app.post("/api/v1/discovery/reconcile", response_model=ReconcileResult)
def reconcile_discovery(payload: DiscoveryReconcileRequest | None = None) -> ReconcileResult:
if store.demo_active():
raise HTTPException(status_code=409, detail="Exit Explore demo before reconciling production infrastructure")
raise HTTPException(status_code=409, detail="Sluit de demo af voor je de productie-infrastructuur verwerkt")
try:
return discovery_manager.reconcile(payload or DiscoveryReconcileRequest())
except ValueError as exc:
@@ -649,7 +937,7 @@ def repair_discovered_probes(
run_id: str | None = Query(default=None),
) -> ProbeRepairReport:
if store.demo_active():
raise HTTPException(status_code=409, detail="Exit Explore demo before repairing production probe endpoints")
raise HTTPException(status_code=409, detail="Sluit de demo af voor je controle-endpoints in productie herstelt")
try:
return discovery_manager.repair_probe_reachability(run_id=run_id, dry_run=dry_run)
except ValueError as exc:
@@ -686,8 +974,14 @@ def topology_diff(
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/api/v1/topology/semantic", response_model=SemanticTopology)
def semantic_topology(include_hidden: bool = Query(default=False)) -> SemanticTopology:
def _semantic_representation(include_hidden: bool) -> CachedRepresentation:
revision = event_bus.revision
# The store identity keeps a swapped store (tests, demo restore paths)
# from ever reading another database's cached topology.
cache_key = (id(store), include_hidden)
cached = semantic_cache.get(cache_key, revision)
if cached is not None:
return cached
snapshots = _service_snapshots(include_hidden=include_hidden, include_context=False)
incident_state = incident_overview(store)
root_ids = {candidate.service_id for candidate in incident_state.candidates}
@@ -696,12 +990,48 @@ def semantic_topology(include_hidden: bool = Query(default=False)) -> SemanticTo
for candidate in incident_state.candidates
for service_id in candidate.affected_service_ids
}
return build_semantic_topology(
topology = build_semantic_topology(
snapshots,
discovery_store.list_logical_relations(),
root_service_ids=root_ids,
affected_service_ids=affected_ids,
)
value = representation(topology)
boundary = next_freshness_boundary(
[item for snapshot in snapshots for item in snapshot.health_evidence],
topology.generated_at,
)
semantic_cache.put(
cache_key,
revision,
value,
valid_for_seconds=(
(boundary - topology.generated_at).total_seconds() if boundary is not None else None
),
)
return value
@app.get("/api/v1/topology/semantic", response_model=SemanticTopology)
def semantic_topology(
request: Request, include_hidden: bool = Query(default=False)
) -> Response:
"""Semantic topology with a content ETag and conditional GET support.
Clients that send ``If-None-Match`` receive ``304`` without a body while
the topology is unchanged; ``X-ServiceMap-Generated-At`` then states when
the server last verified it. Browsers revalidate automatically.
"""
value = _semantic_representation(include_hidden)
headers = {
"ETag": value.etag,
"Cache-Control": "no-cache",
GENERATED_AT_HEADER: value.generated_at,
}
if etag_matches(request.headers.get("if-none-match"), value.etag):
return Response(status_code=status.HTTP_304_NOT_MODIFIED, headers=headers)
return Response(content=value.body, media_type="application/json", headers=headers)
@app.post("/api/v1/topology/edges", response_model=RelationEvidence, status_code=status.HTTP_201_CREATED)
+28 -14
View File
@@ -190,8 +190,8 @@ class RelationType(StrEnum):
class ServiceCreate(BaseModel):
name: str = Field(min_length=1, max_length=120)
kind: ServiceKind
url: str | None = None
host: str | None = Field(default=None, validation_alias=AliasChoices("host", "hostname"))
url: str | None = Field(default=None, max_length=2048)
host: str | None = Field(default=None, max_length=253, validation_alias=AliasChoices("host", "hostname"))
port: int | None = Field(default=None, ge=1, le=65535)
depends_on: list[str] = Field(default_factory=list)
description: str | None = Field(default=None, max_length=500)
@@ -243,27 +243,27 @@ class ServiceCreate(BaseModel):
if not self.monitoring_enabled and not self.url and not self.host and self.port is None:
return self
if self.kind == ServiceKind.HTTP and not self.url:
raise ValueError("HTTP services require url")
raise ValueError("HTTP-services vragen een url")
if self.kind == ServiceKind.HTTP:
parsed = urlparse(self.url or "")
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("HTTP services require an absolute http(s) url")
raise ValueError("HTTP-services vragen een absolute http(s)-url")
if self.host or self.port:
raise ValueError("HTTP services must use url instead of host/port")
raise ValueError("HTTP-services gebruiken url in plaats van host/poort")
if self.kind in {ServiceKind.TCP, ServiceKind.TLS} and (not self.host or self.port is None):
raise ValueError(f"{self.kind.value.upper()} services require host and port")
raise ValueError(f"{self.kind.value.upper()} services vragen een host en poort")
if self.kind == ServiceKind.DNS and not self.host:
raise ValueError("DNS services require hostname in host")
raise ValueError("DNS-services vragen een hostnaam in host")
if self.kind in {ServiceKind.TCP, ServiceKind.TLS, ServiceKind.DNS} and self.url:
raise ValueError(f"{self.kind.value.upper()} services must use host/port instead of url")
raise ValueError(f"{self.kind.value.upper()} services gebruiken host/poort in plaats van url")
if self.kind == ServiceKind.DNS and self.port is not None:
raise ValueError("DNS services use a hostname without a port")
raise ValueError("DNS-services gebruiken een hostnaam zonder poort")
if (
self.expected_status_min is not None
and self.expected_status_max is not None
and self.expected_status_min > self.expected_status_max
):
raise ValueError("expected_status_min cannot exceed expected_status_max")
raise ValueError("expected_status_min mag niet groter zijn dan expected_status_max")
return self
@@ -327,7 +327,7 @@ class HealthResolution(BaseModel):
selected_provider: EvidenceProvider | None = None
confidence: float | None = Field(default=None, ge=0, le=1)
reason_code: HealthResolutionReason = HealthResolutionReason.NO_CURRENT_COMPONENT_EVIDENCE
explanation: str = "No current component-scoped health evidence is available."
explanation: str = "Er is geen actueel gezondheidsbewijs op componentniveau."
conflict: bool = False
evidence: list[HealthEvidence] = Field(default_factory=list)
@@ -337,7 +337,7 @@ class ApplicationHealthResolution(BaseModel):
status: HealthStatus = HealthStatus.UNKNOWN
confidence: float | None = Field(default=None, ge=0, le=1)
explanation: str = "No current application or component health evidence is available."
explanation: str = "Er is geen actueel gezondheidsbewijs voor applicatie of componenten."
reason_code: HealthResolutionReason = HealthResolutionReason.NO_APPLICATION_EVIDENCE
selected_evidence_id: str | None = None
selected_provider: EvidenceProvider | None = None
@@ -372,6 +372,8 @@ class ServiceSnapshot(BaseModel):
evidence: list[RelationEvidence] = Field(default_factory=list)
health_evidence: list[HealthEvidence] = Field(default_factory=list)
health_resolution: HealthResolution = Field(default_factory=HealthResolution)
# End of a scheduled maintenance window, if one is set.
maintenance_until: datetime | None = None
class AvailabilitySummary(BaseModel):
@@ -658,7 +660,7 @@ class EdgeMutationRequest(BaseModel):
target_service_id: str
relation_type: RelationType = RelationType.DEPENDS_ON
lock: bool = True
explanation: str = Field(default="Manual topology relationship", max_length=500)
explanation: str = Field(default="Handmatige topologierelatie", max_length=500)
class EdgeActionRequest(BaseModel):
@@ -678,7 +680,8 @@ class ConfigDocument(BaseModel):
class ConfigImportRequest(BaseModel):
content: str
# About 2 MB of YAML/JSON is far beyond any real topology.
content: str = Field(max_length=2_000_000)
format: str = Field(pattern="^(yaml|json)$")
mode: str = Field(default="merge", pattern="^(merge|replace)$")
@@ -691,6 +694,7 @@ class ConfigImportResult(BaseModel):
class DemoState(BaseModel):
active: bool
enabled: bool = False
synthetic: bool = False
scenario_name: str | None = None
service_count: int = 0
@@ -860,3 +864,13 @@ class OverviewSummary(BaseModel):
class MaintenanceRequest(BaseModel):
enabled: bool
reason: str | None = Field(default=None, max_length=240)
# Optional end of the window; maintenance ends automatically afterwards.
until: datetime | None = None
class IncidentNoteRequest(BaseModel):
text: str = Field(min_length=1, max_length=2000)
class IncidentResolveRequest(BaseModel):
note: str | None = Field(default=None, max_length=2000)
+4 -4
View File
@@ -111,14 +111,14 @@ class ObservationManager:
and not docker_observations
and any(ref.startswith("docker:") for ref in services_by_ref)
):
raise ValueError("Docker broker returned no containers for configured services")
raise ValueError("De Docker-broker gaf geen containers terug voor de ingestelde services")
self.store.set_provider_status(
ProviderStatus(
provider=EvidenceProvider.DOCKER,
status=ProviderHealth.HEALTHY,
observed_at=now,
valid_until=valid_until,
detail=f"{len(docker_observations)} containers observed",
detail=f"{len(docker_observations)} containers waargenomen",
metadata={"container_count": len(docker_observations)},
)
)
@@ -147,7 +147,7 @@ class ObservationManager:
observed_at=now,
valid_until=valid_until,
received_at=now,
summary="Docker healthcheck" if observation.healthcheck_present else "Docker runtime state",
summary="Docker-healthcheck" if observation.healthcheck_present else "Docker-runtimestatus",
metadata={
"container_id": observation.container_id,
"state": observation.state,
@@ -190,7 +190,7 @@ class ObservationManager:
observed_at=now,
valid_until=valid_until,
received_at=now,
summary="AppOps application health; never used as component direct health",
summary="Applicatiegezondheid via AppOps; nooit gebruikt als directe componentgezondheid",
metadata={
"app_id": app_id,
"category": info.category,
+41 -8
View File
@@ -41,10 +41,36 @@ def _http_status(status_code: int, service: Service) -> HealthStatus:
return HealthStatus.DOWN
_MAX_BODY_BYTES = 64 * 1024
_SSL_CONTEXT: ssl.SSLContext | None = None
def _ssl_context() -> ssl.SSLContext:
"""Build the CA bundle context once instead of on every HTTP probe."""
global _SSL_CONTEXT
if _SSL_CONTEXT is None:
import certifi
_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
return _SSL_CONTEXT
async def _probe_once_with_metadata(service: Service, timeout_seconds: float) -> tuple[HealthStatus, str, dict[str, object]]:
if service.kind == ServiceKind.HTTP:
async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True) as client:
response = await client.get(service.url or "")
# One overall deadline (httpx timeouts are per phase, so a slow-drip
# body could run for many times the timeout) and a bounded body read:
# the status line answers the health question.
async with asyncio.timeout(timeout_seconds):
async with httpx.AsyncClient(
timeout=timeout_seconds, follow_redirects=True, max_redirects=5, verify=_ssl_context()
) as client:
async with client.stream("GET", service.url or "") as response:
received = 0
async for chunk in response.aiter_raw():
received += len(chunk)
if received >= _MAX_BODY_BYTES:
break
status = _http_status(response.status_code, service)
return status, f"HTTP {response.status_code}", {"status_code": response.status_code}
@@ -72,7 +98,7 @@ async def _probe_once_with_metadata(service: Service, timeout_seconds: float) ->
del reader
ssl_object = writer.get_extra_info("ssl_object")
metadata: dict[str, object] = {"hostname": service.host, "port": service.port}
detail = f"TLS {service.host}:{service.port} certificate reachable"
detail = f"TLS-certificaat van {service.host}:{service.port} bereikbaar"
status = HealthStatus.HEALTHY
if ssl_object is not None:
certificate = ssl_object.getpeercert() or {}
@@ -84,10 +110,10 @@ async def _probe_once_with_metadata(service: Service, timeout_seconds: float) ->
warning_days = max(1, int(os.getenv("SERVICEMAP_TLS_WARNING_DAYS", "30")))
if days_remaining <= 0:
status = HealthStatus.DOWN
detail = f"TLS certificate expired on {expires_at.date().isoformat()}"
detail = f"TLS-certificaat verlopen op {expires_at.date().isoformat()}"
elif days_remaining <= warning_days:
status = HealthStatus.DEGRADED
detail = f"TLS certificate expires in {round(days_remaining, 1)} days"
detail = f"TLS-certificaat verloopt over {round(days_remaining, 1)} dagen"
writer.close()
await writer.wait_closed()
return status, detail, metadata
@@ -122,7 +148,7 @@ async def probe_service(
service_id=service.id,
status=HealthStatus.UNKNOWN,
latency_ms=None,
detail="Monitoring disabled for service",
detail="Bewaking uitgeschakeld voor deze service",
checked_at=datetime.now(UTC),
trigger=trigger,
)
@@ -132,7 +158,7 @@ async def probe_service(
retry_count = retry_count if retry_count is not None else default_retries
attempts = max(1, retry_count + 1)
status = HealthStatus.DOWN
detail = "Probe failed"
detail = "Controle mislukt"
latency_ms: float | None = None
metadata: dict[str, object] = {}
@@ -152,9 +178,16 @@ async def probe_service(
detail = f"{type(exc).__name__}: {exc}"
if isinstance(exc, ssl.CertificateError):
status = HealthStatus.DOWN
detail = f"TLS hostname mismatch: {exc}"
detail = f"TLS-hostnaam komt niet overeen: {exc}"
if attempt < attempts:
await asyncio.sleep(min(0.2 * attempt, 1.0))
except Exception as exc: # noqa: BLE001 - one bad target must not fail a whole cycle
# e.g. httpx.InvalidURL or an IDNA UnicodeError: a permanent
# configuration error, reported as down without retrying.
latency_ms = round((perf_counter() - started) * 1000, 2)
status = HealthStatus.DOWN
detail = f"Controlefout ({type(exc).__name__}): {exc}"
break
return ProbeResult(
id=str(uuid4()),
+124
View File
@@ -0,0 +1,124 @@
"""Validated, briefly cached JSON representations for hot read endpoints.
The semantic topology is read by the UI after every event burst and by
integrations such as ITWorx Forge on a fixed poll. Its content changes far
less often than it is read: in production only ``generated_at`` differed
between polls. Two mechanisms make an unchanged read nearly free:
* A strong ``ETag`` computed over the content *excluding* ``generated_at``.
A client that sends ``If-None-Match`` gets ``304 Not Modified`` with no
body. Because the tag is derived from the content itself it is always
correct, including after writes that publish no event.
* A short-lived computation cache. It is reused only while no event has
been published since it was built, no evidence freshness boundary
(fresh→stale→expired) has been crossed, and it is younger than
``max_age_seconds``. The age cap bounds staleness for the few write paths
that publish no event (for example discovery relation evidence).
"""
from __future__ import annotations
import hashlib
import time
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timedelta
from threading import Lock
from pydantic import BaseModel
from .models import HealthEvidence
GENERATED_AT_HEADER = "X-ServiceMap-Generated-At"
@dataclass(frozen=True)
class CachedRepresentation:
body: bytes
etag: str
generated_at: str
def representation(model: BaseModel, *, timestamp_field: str = "generated_at") -> CachedRepresentation:
"""Serialize once and tag the content independent of its timestamp."""
content = model.model_dump_json(exclude={timestamp_field})
stamp = model.model_dump_json(include={timestamp_field})
# Splice ``{"generated_at": ...}`` in front of the remaining fields: one
# serialization for the body, the hash covers everything but the stamp.
body = stamp if content == "{}" else f"{stamp[:-1]},{content[1:]}"
digest = hashlib.sha256(content.encode()).hexdigest()[:32]
generated_at = str(getattr(model, timestamp_field).isoformat())
return CachedRepresentation(body=body.encode(), etag=f'"{digest}"', generated_at=generated_at)
def etag_matches(if_none_match: str | None, etag: str) -> bool:
if not if_none_match:
return False
for candidate in if_none_match.split(","):
value = candidate.strip()
if value == "*" or value.removeprefix("W/") == etag:
return True
return False
def next_freshness_boundary(evidence: list[HealthEvidence], now: datetime) -> datetime | None:
"""Earliest future instant at which any claim changes freshness."""
boundary: datetime | None = None
for item in evidence:
ttl = max(item.valid_until - item.observed_at, timedelta(seconds=1))
for instant in (item.valid_until, item.valid_until + ttl):
if instant > now and (boundary is None or instant < boundary):
boundary = instant
return boundary
@dataclass
class _Entry:
value: CachedRepresentation
revision: int
expires_at: float
class RepresentationCache:
"""Thread-safe cache keyed by request variant and event revision."""
def __init__(
self,
*,
max_age_seconds: float = 10.0,
clock: Callable[[], float] = time.monotonic,
) -> None:
self.max_age_seconds = max_age_seconds
self._clock = clock
self._entries: dict[object, _Entry] = {}
self._lock = Lock()
def get(self, key: object, revision: int) -> CachedRepresentation | None:
with self._lock:
entry = self._entries.get(key)
if entry is None:
return None
if entry.revision != revision or self._clock() >= entry.expires_at:
del self._entries[key]
return None
return entry.value
def put(
self,
key: object,
revision: int,
value: CachedRepresentation,
*,
valid_for_seconds: float | None = None,
) -> None:
ttl = self.max_age_seconds if valid_for_seconds is None else min(self.max_age_seconds, valid_for_seconds)
if ttl <= 0:
return
with self._lock:
self._entries[key] = _Entry(value=value, revision=revision, expires_at=self._clock() + ttl)
def clear(self) -> None:
with self._lock:
self._entries.clear()
+8 -2
View File
@@ -7,7 +7,7 @@ from time import monotonic
from .probes import probe_many
from .store import Store
logger = logging.getLogger(__name__)
logger = logging.getLogger("servicemap.scheduler")
async def scheduled_probe_loop(
@@ -18,6 +18,7 @@ async def scheduled_probe_loop(
retries: int,
concurrency: int,
retention_days: int,
event_retention_days: int | None = None,
) -> None:
next_due: dict[str, float] = {}
while True:
@@ -58,7 +59,12 @@ async def scheduled_probe_loop(
sleep_seconds = min([interval_seconds, *next_intervals]) if next_intervals else interval_seconds
else:
sleep_seconds = interval_seconds
store.prune_probes(retention_days)
if event_retention_days is None:
store.prune_probes(retention_days)
else:
# Events have their own (longer) retention; pruning them at the
# probe retention silently shortened the journal.
store.prune_probes(retention_days, event_retention_days)
except asyncio.CancelledError:
raise
except Exception: # pragma: no cover - defensive daemon boundary
+8 -8
View File
@@ -214,7 +214,7 @@ def _application_resolution(
return ApplicationHealthResolution(
status=status,
confidence=max((item.health_resolution.confidence or 1.0 for item in direct_down), default=None),
explanation=f"{names} has current component failure evidence; aggregate health does not override it.",
explanation=f"{names} heeft actueel bewijs van een componentstoring; de totale gezondheid overschrijft dat niet.",
reason_code=HealthResolutionReason.COMPONENT_FAILURE,
**common,
)
@@ -223,7 +223,7 @@ def _application_resolution(
return ApplicationHealthResolution(
status=HealthStatus.DEGRADED,
confidence=selected.confidence if selected else None,
explanation=f"{names} is impacted through a dependency health signal.",
explanation=f"{names} wordt geraakt via het gezondheidssignaal van een afhankelijkheid.",
reason_code=HealthResolutionReason.COMPONENT_IMPACT,
**common,
)
@@ -232,8 +232,8 @@ def _application_resolution(
status=selected.status,
confidence=selected.confidence,
explanation=(
f"{selected.provider.value} application observation selected "
f"(confidence {selected.confidence:.2f}); component coverage is {coverage:.0f}% ."
f"{selected.provider.value} applicatiewaarneming gekozen "
f"(zekerheid {selected.confidence:.2f}); componentdekking is {coverage:.0f}%."
),
reason_code=HealthResolutionReason.SELECTED_APPLICATION_EVIDENCE,
**common,
@@ -244,7 +244,7 @@ def _application_resolution(
status = HealthStatus.DEGRADED
return ApplicationHealthResolution(
status=status,
explanation=f"{len(resolved)} of {len(members)} components have current direct health evidence.",
explanation=f"{len(resolved)} van {len(members)} componenten hebben actueel direct gezondheidsbewijs.",
reason_code=HealthResolutionReason.COMPONENT_COVERAGE,
**common,
)
@@ -257,20 +257,20 @@ def _application_resolution(
)
return ApplicationHealthResolution(
status=HealthStatus.UNKNOWN,
explanation="Application evidence is no longer current; waiting for a provider refresh.",
explanation="Het applicatiebewijs is niet meer actueel; wachten op een nieuwe meting van de bron.",
reason_code=reason,
**common,
)
if members and all(item.direct_status == HealthStatus.UNMONITORED for item in members):
return ApplicationHealthResolution(
status=HealthStatus.UNMONITORED,
explanation="No active health signal is configured for this application.",
explanation="Voor deze applicatie is geen actief gezondheidssignaal ingesteld.",
reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE,
**common,
)
return ApplicationHealthResolution(
status=HealthStatus.UNKNOWN,
explanation="No current application or component health evidence is available.",
explanation="Er is geen actueel gezondheidsbewijs voor applicatie of componenten.",
reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE,
**common,
)
+273 -52
View File
@@ -52,6 +52,16 @@ def _iso(value: datetime | None) -> str | None:
return value.isoformat() if value is not None else None
# Per-observation bookkeeping that changes on every probe. It is stored, but a
# change in only these keys is not a material evidence change and must not
# publish an SSE wakeup (that turned every probe into a full UI refresh).
VOLATILE_EVIDENCE_METADATA = frozenset({"probe_id", "latency_ms", "trigger"})
def _stable_metadata(metadata: dict[str, object]) -> dict[str, object]:
return {key: value for key, value in metadata.items() if key not in VOLATILE_EVIDENCE_METADATA}
def _bool(value: object, default: bool = False) -> bool:
if value is None:
return default
@@ -70,14 +80,24 @@ class Store:
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
connection.execute("PRAGMA busy_timeout = 5000")
# Per connection, not persistent: without this every commit ran at
# the FULL default (an fsync per write). NORMAL is durable under WAL.
connection.execute("PRAGMA synchronous = NORMAL")
return connection
def _init_schema(self) -> None:
with self._connect() as db:
db.execute("PRAGMA journal_mode = WAL")
db.execute("PRAGMA synchronous = NORMAL")
# Bound the WAL file after checkpoints (64 MiB).
db.execute("PRAGMA journal_size_limit = 67108864")
db.executescript(
"""
CREATE TABLE IF NOT EXISTS maintenance_windows (
service_id TEXT PRIMARY KEY,
ends_at TEXT NOT NULL,
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS services (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
@@ -770,7 +790,7 @@ class Store:
with self._connect() as db:
self._upsert_service(db, updated)
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {updated.name} updated", {})
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service updated"})
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service bijgewerkt"})
return updated
def delete_service(self, service_id: str) -> bool:
@@ -787,14 +807,24 @@ class Store:
"UPDATE services SET depends_on = ?, updated_at = ? WHERE id = ?",
(json.dumps([value for value in dependencies if value != service_id]), _now().isoformat(), item["id"]),
)
db.execute("UPDATE incidents SET affected_service_ids = replace(affected_service_ids, ?, '') WHERE status != 'resolved'", (service_id,))
# Remove the id as a JSON element. A textual replace() left an
# empty-string entry behind (["a","b"] -> ["","b"]).
for incident in db.execute(
"SELECT id, affected_service_ids FROM incidents WHERE status != 'resolved'"
).fetchall():
affected = self._json_list(incident["affected_service_ids"])
if service_id in affected:
db.execute(
"UPDATE incidents SET affected_service_ids = ? WHERE id = ?",
(json.dumps([value for value in affected if value != service_id]), incident["id"]),
)
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {row['name']} deleted", {})
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service deleted"})
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service verwijderd"})
return True
def import_config_services(self, definitions: list[ConfigService], mode: str) -> ConfigImportResult:
if mode not in {"merge", "replace"}:
raise ValueError("Import mode must be merge or replace")
raise ValueError("Importmodus moet merge of replace zijn")
with self._connect() as db:
db.execute("BEGIN IMMEDIATE")
@@ -813,7 +843,7 @@ class Store:
missing = [item for item in definition.depends_on if item.casefold() not in ids_by_name]
if missing:
raise ValueError(
f"Service {definition.name} references unknown dependencies: {', '.join(missing)}"
f"Service {definition.name} verwijst naar onbekende afhankelijkheden: {', '.join(missing)}"
)
created = 0
@@ -822,8 +852,16 @@ class Store:
for definition in definitions:
key = definition.name.casefold()
current = existing.get(key)
payload = definition.model_dump()
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
if current is None:
payload = definition.model_dump()
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
else:
# Merge: only fields the document states change. Rebuilding
# from model_dump() reset every omitted field to its default.
payload = current.model_dump(include=set(ServiceCreate.model_fields))
payload.update(definition.model_dump(exclude_unset=True, exclude={"depends_on"}))
if "depends_on" in definition.model_fields_set:
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
if current is None:
service = Service(id=ids_by_name[key], created_at=now, updated_at=now, **payload)
created += 1
@@ -845,8 +883,8 @@ class Store:
self._insert_service(db, service)
else:
self._upsert_service(db, service)
self._insert_event_db(db, "topology.changed", None, None, None, None, "Configuration imported", {"mode": mode})
event_bus.publish({"event_type": "topology.changed", "message": "Configuration imported"})
self._insert_event_db(db, "topology.changed", None, None, None, None, "Configuratie geïmporteerd", {"mode": mode})
event_bus.publish({"event_type": "topology.changed", "message": "Configuratie geïmporteerd"})
return ConfigImportResult(created=created, updated=updated, total=len(definitions))
def replace_all_services(self, definitions: list[ServiceCreate]) -> list[Service]:
@@ -916,12 +954,29 @@ class Store:
("status", evidence.status.value),
("confidence", evidence.confidence),
("summary", evidence.summary),
("metadata", json.dumps(evidence.metadata, sort_keys=True)),
)
)
)
) or _stable_metadata(self._json_object(prior["metadata"])) != _stable_metadata(evidence.metadata)
if prior is not None and prior["status"] == evidence.status.value:
saved = evidence.model_copy(update={"id": prior["id"], "active": True, "superseded_by": None})
ttl = evidence.valid_until - evidence.observed_at
prior_valid_until = datetime.fromisoformat(prior["valid_until"])
if (
not materially_changed
and prior["service_id"] == evidence.service_id
and prior_valid_until - _now() > ttl / 2
):
# Unchanged and comfortably fresh: skip the timestamp-only
# rewrite. It is renewed once less than half the TTL is
# left, so the claim never goes stale, with about half
# the writes of renewing on every observation cycle.
saved = self._row_to_health_evidence(prior)
for service_id in links:
db.execute(
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
(saved.id, service_id),
)
return saved
db.execute(
"""
UPDATE health_evidence SET service_id = ?, confidence = ?, observed_at = ?, valid_until = ?,
@@ -980,18 +1035,24 @@ class Store:
"service_id": evidence.service_id,
"provider": evidence.provider.value,
"status": evidence.status.value,
"message": "Health evidence changed",
"message": "Gezondheidsbewijs gewijzigd",
}
)
return saved
def health_evidence_for_service(self, service_id: str, *, limit: int = 100) -> list[HealthEvidence]:
# A direct claim and a linked claim are matched by two independently
# indexed lookups. An ``OR`` across the LEFT JOIN cannot use either
# index and forced a full ``health_evidence`` scan on every call.
with self._connect() as db:
rows = db.execute(
"""
SELECT DISTINCT e.* FROM health_evidence e
LEFT JOIN health_evidence_service_links link ON link.evidence_id = e.id
WHERE e.service_id = ? OR link.service_id = ?
SELECT e.* FROM health_evidence e
WHERE e.id IN (
SELECT id FROM health_evidence WHERE service_id = ?
UNION
SELECT evidence_id FROM health_evidence_service_links WHERE service_id = ?
)
ORDER BY e.observed_at DESC, e.rowid DESC LIMIT ?
""",
(service_id, service_id, limit),
@@ -1015,27 +1076,41 @@ class Store:
return result
placeholders = ", ".join("?" for _ in ids)
with self._connect() as db:
# Both match paths use their own index; the per-service limit is
# applied in SQL so superseded history is not materialised.
rows = db.execute(
f"""
SELECT e.*, e.service_id AS direct_service_id, link.service_id AS linked_service_id
FROM health_evidence e
LEFT JOIN health_evidence_service_links link ON link.evidence_id = e.id
WHERE e.service_id IN ({placeholders}) OR link.service_id IN ({placeholders})
ORDER BY e.observed_at DESC, e.rowid DESC
WITH matched(evidence_id, matched_service_id) AS (
SELECT id, service_id FROM health_evidence WHERE service_id IN ({placeholders})
UNION
SELECT evidence_id, service_id FROM health_evidence_service_links
WHERE service_id IN ({placeholders})
),
ranked AS (
SELECT e.*, m.matched_service_id,
ROW_NUMBER() OVER (
PARTITION BY m.matched_service_id
ORDER BY e.observed_at DESC, e.rowid DESC
) AS rank_in_service
FROM matched m
JOIN health_evidence e ON e.id = m.evidence_id
)
SELECT * FROM ranked
WHERE rank_in_service <= ?
ORDER BY matched_service_id, rank_in_service
""",
[*ids, *ids],
[*ids, *ids, limit_per_service],
).fetchall()
seen: dict[str, set[str]] = {service_id: set() for service_id in ids}
parsed: dict[str, HealthEvidence] = {}
for row in rows:
evidence = self._row_to_health_evidence(row)
for service_id in {row["direct_service_id"], row["linked_service_id"]}:
if (
service_id in result
and evidence.id not in seen[service_id]
and len(result[service_id]) < limit_per_service
):
result[service_id].append(evidence)
seen[service_id].add(evidence.id)
service_id = row["matched_service_id"]
if service_id not in result:
continue
evidence = parsed.get(row["id"])
if evidence is None:
evidence = self._row_to_health_evidence(row)
parsed[row["id"]] = evidence
result[service_id].append(evidence)
return result
def application_evidence_links(self, provider: EvidenceProvider) -> dict[str, set[str]]:
@@ -1098,7 +1173,7 @@ class Store:
"event_type": "provider.status_changed",
"provider": value.provider.value,
"status": value.status.value,
"message": "Provider availability changed",
"message": "Beschikbaarheid van de bron gewijzigd",
}
)
return value
@@ -1205,10 +1280,37 @@ class Store:
status_by_id[service_id] = self._legacy_health_from_row(row)
return status_by_id, resolutions
def set_maintenance(self, service_id: str, enabled: bool, reason: str | None = None) -> Service | None:
def maintenance_windows(self) -> dict[str, datetime]:
with self._connect() as db:
rows = db.execute("SELECT service_id, ends_at FROM maintenance_windows").fetchall()
return {row["service_id"]: datetime.fromisoformat(row["ends_at"]) for row in rows}
def expire_maintenance_windows(self, now: datetime | None = None) -> list[str]:
"""End maintenance for every window whose end time has passed."""
now = now or _now()
with self._connect() as db:
rows = db.execute(
"SELECT service_id FROM maintenance_windows WHERE ends_at <= ?", (now.isoformat(),)
).fetchall()
ended: list[str] = []
for row in rows:
if self.set_maintenance(row["service_id"], False, "Maintenance window ended") is not None:
ended.append(row["service_id"])
return ended
def set_maintenance(
self,
service_id: str,
enabled: bool,
reason: str | None = None,
until: datetime | None = None,
) -> Service | None:
service = self.get_service(service_id)
if service is None:
return None
if enabled and until is not None and until <= _now():
raise ValueError("Het einde van het onderhoud moet in de toekomst liggen")
updated = service.model_copy(
update={
"maintenance_enabled": enabled,
@@ -1221,6 +1323,14 @@ class Store:
)
with self._connect() as db:
self._upsert_service(db, updated)
if enabled and until is not None:
db.execute(
"INSERT INTO maintenance_windows (service_id, ends_at) VALUES (?, ?) "
"ON CONFLICT(service_id) DO UPDATE SET ends_at = excluded.ends_at",
(service_id, until.isoformat()),
)
else:
db.execute("DELETE FROM maintenance_windows WHERE service_id = ?", (service_id,))
self._insert_event_db(
db,
"service.maintenance_changed",
@@ -1228,15 +1338,18 @@ class Store:
None,
service.state.value,
HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
f"Maintenance {'enabled' if enabled else 'disabled'} for {service.name}",
{"reason": reason} if reason else {},
f"Onderhoud {'gestart' if enabled else 'beëindigd'} voor {service.name}",
{
**({"reason": reason} if reason else {}),
**({"until": until.isoformat()} if enabled and until else {}),
},
)
event_bus.publish(
{
"event_type": "service.maintenance_changed",
"service_id": service_id,
"status": HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
"message": f"Maintenance {'enabled' if enabled else 'disabled'}",
"message": f"Onderhoud {'gestart' if enabled else 'beëindigd'}",
}
)
from .incidents import sync_incidents
@@ -1330,11 +1443,11 @@ class Store:
None,
service.state.value,
status.value,
f"Docker runtime observation updated for {service.name}",
f"Docker-runtimewaarneming bijgewerkt voor {service.name}",
{"runtime_state": runtime_state, "runtime_health": runtime_health, "monitoring_state": state_kind.value},
)
if service.state != status:
event_bus.publish({"event_type": "service.runtime_observed", "service_id": service_id, "status": status.value, "message": "Docker runtime observation updated"})
event_bus.publish({"event_type": "service.runtime_observed", "service_id": service_id, "status": status.value, "message": "Docker-runtimewaarneming bijgewerkt"})
observed_at = _now()
self.record_health_evidence(
HealthEvidence(
@@ -1350,7 +1463,7 @@ class Store:
observed_at=observed_at,
valid_until=observed_at + timedelta(seconds=max(60, int(service.interval_seconds or 60) * 2)),
received_at=observed_at,
summary="Docker runtime observation" if provider == EvidenceProvider.DOCKER else "Runtime observation",
summary="Docker-runtimewaarneming" if provider == EvidenceProvider.DOCKER else "Runtimewaarneming",
metadata={"runtime_state": runtime_state, "runtime_health": runtime_health},
)
)
@@ -1389,7 +1502,7 @@ class Store:
"service_id": probe.service_id,
"from_status": previous.value,
"to_status": next_state.value,
"message": f"Service changed from {previous.value} to {next_state.value}",
"message": f"Service gewijzigd van {previous.value} to {next_state.value}",
"metadata": {"trigger": probe.trigger, "probe_id": probe.id},
}
self._insert_event_db(
@@ -1420,11 +1533,11 @@ class Store:
None,
None,
None,
f"Flapping detected: {flap_count} state changes in 30 minutes",
f"Wisselvalligheid vastgesteld: {flap_count} statuswijzigingen in 30 minuten",
{"state_changes": flap_count, "window_minutes": 30},
created_at=now,
)
events.append({"event_type": "flapping.detected", "service_id": probe.service_id, "message": "Flapping detected"})
events.append({"event_type": "flapping.detected", "service_id": probe.service_id, "message": "Wisselvalligheid vastgesteld"})
db.execute(
"""
UPDATE services
@@ -1449,7 +1562,7 @@ class Store:
"service_id": probe.service_id,
"status": probe.status.value,
"trigger": probe.trigger,
"message": "Probe completed",
"message": "Controle voltooid",
}
)
for event in events:
@@ -1618,6 +1731,52 @@ class Store:
db.execute("DELETE FROM events WHERE created_at < ? AND event_type NOT LIKE 'incident.%'", (event_cutoff.isoformat(),))
return cursor.rowcount
def checkpoint(self) -> None:
"""Fold the WAL back into the database and refresh planner statistics.
Without a TRUNCATE checkpoint the WAL file keeps its high-water size.
"""
with self._connect() as db:
db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
db.execute("PRAGMA optimize")
def ready_check(self) -> None:
"""Raise if the database cannot take a write lock right now."""
connection = self._connect()
try:
connection.execute("BEGIN IMMEDIATE")
connection.execute("ROLLBACK")
finally:
connection.close()
def prune_health_evidence(self, retention_days: int) -> int:
"""Delete superseded evidence history older than the retention window.
Only inactive rows are removed. A row becomes inactive exclusively
when a newer claim for the same provider/subject supersedes it, so
every subject keeps its current (active) claim and health resolution
is unaffected. Without this, every status flip leaves a permanent row
and evidence reads grow linearly with uptime.
"""
if retention_days <= 0:
return 0
cutoff = _now() - timedelta(days=retention_days)
with self._connect() as db:
cursor = db.execute(
"DELETE FROM health_evidence WHERE active = 0 AND superseded_by IS NOT NULL AND received_at < ?",
(cutoff.isoformat(),),
)
# Links cascade through the foreign key; also remove any orphan
# links left by databases that predate enforced foreign keys.
db.execute(
"DELETE FROM health_evidence_service_links "
"WHERE evidence_id NOT IN (SELECT id FROM health_evidence)"
)
return cursor.rowcount
def _insert_event_db(
self,
db: sqlite3.Connection,
@@ -1669,12 +1828,37 @@ class Store:
event_bus.publish(result.model_dump(mode="json"))
return result
def list_events(self, limit: int = 50, service_id: str | None = None) -> list[EventRecord]:
with self._connect() as db:
if service_id:
rows = db.execute("SELECT * FROM events WHERE service_id = ? ORDER BY created_at DESC LIMIT ?", (service_id, limit)).fetchall()
def list_events(
self,
limit: int = 50,
service_id: str | None = None,
event_type: str | None = None,
before: datetime | None = None,
) -> list[EventRecord]:
"""Newest first. ``event_type`` ending in ``.`` matches a family
(``incident.``); ``before`` pages to older events."""
clauses: list[str] = []
params: list[object] = []
if service_id:
clauses.append("service_id = ?")
params.append(service_id)
if event_type:
if event_type.endswith("."):
# Exact prefix match without LIKE wildcards.
clauses.append("substr(event_type, 1, ?) = ?")
params.extend([len(event_type), event_type])
else:
rows = db.execute("SELECT * FROM events ORDER BY created_at DESC LIMIT ?", (limit,)).fetchall()
clauses.append("event_type = ?")
params.append(event_type)
if before is not None:
clauses.append("created_at < ?")
params.append(before.isoformat())
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
with self._connect() as db:
rows = db.execute(
f"SELECT * FROM events {where} ORDER BY created_at DESC LIMIT ?", [*params, limit]
).fetchall()
return [self._row_to_event(row) for row in rows]
def demo_active(self) -> bool:
@@ -1928,7 +2112,7 @@ class Store:
""",
(snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, json.dumps([service.id for service in services]), started_at),
)
event_bus.publish({"event_type": "topology.changed", "message": "Synthetic demo loaded"})
event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo geladen"})
return ConfigImportResult(created=len(services), updated=0, total=len(services))
def exit_demo(self) -> bool:
@@ -1947,7 +2131,7 @@ class Store:
session["snapshot_health_evidence"] or "{}",
)
db.execute("DELETE FROM demo_sessions WHERE id = 1")
event_bus.publish({"event_type": "topology.changed", "message": "Synthetic demo exited"})
event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo afgesloten"})
return True
def list_incidents(self, active_only: bool = False, limit: int = 100) -> list[IncidentRecord]:
@@ -1999,14 +2183,51 @@ class Store:
event_bus.publish(payload)
return IncidentEvent(id=event_id, incident_id=incident_id, event_type=event_type, message=message, created_at=created_at, service_id=service_id)
def resolve_incident(self, incident_id: str, note: str | None = None) -> IncidentRecord | None:
"""Close an incident by hand (e.g. a stale one).
If its root is still down, the next incident sync opens a new one.
"""
incident = self.get_incident(incident_id)
if incident is None:
return None
if incident.status == IncidentStatus.RESOLVED:
return incident
now = _now()
updated = incident.model_copy(
update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now}
)
self.upsert_incident(updated)
self.append_incident_event(
incident_id,
"incident.resolved",
f"Incident handmatig opgelost{f': {note}' if note else ''}",
incident.root_service_id,
)
return updated
def add_incident_note(self, incident_id: str, text: str) -> IncidentRecord | None:
incident = self.get_incident(incident_id)
if incident is None:
return None
self.append_incident_event(incident_id, "incident.note", text, incident.root_service_id)
updated = incident.model_copy(update={"latest_update_at": _now()})
self.upsert_incident(updated)
return updated
def acknowledge_incident(self, incident_id: str) -> IncidentRecord | None:
incident = self.get_incident(incident_id)
if incident is None:
return None
if incident.status == IncidentStatus.RESOLVED:
raise ValueError("Resolved incidents cannot be acknowledged")
raise ValueError("Opgeloste incidenten kunnen niet bevestigd worden")
if incident.status == IncidentStatus.ACKNOWLEDGED:
# Idempotent: a repeated acknowledge keeps the original time and
# adds no duplicate timeline entry.
return incident
now = _now()
updated = incident.model_copy(update={"status": IncidentStatus.ACKNOWLEDGED, "acknowledged_at": now, "latest_update_at": now})
self.upsert_incident(updated)
self.append_incident_event(incident_id, "incident.acknowledged", "Incident acknowledged", incident.root_service_id)
self.append_incident_event(incident_id, "incident.acknowledged", "Incident bevestigd", incident.root_service_id)
return updated