ServiceMap: update tests and backend/app (+7 areas)
Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
This commit is contained in:
1 parent
d7319250e8
commit
adc2c70c21
91 files changed
+9205
-1691
No files matched your search
@@ -0,0 +1,243 @@
|
||||
"""Incident notifications via ntfy.
|
||||
|
||||
The notifier is deliberately small and idempotent:
|
||||
|
||||
* settings live in the ``settings`` table (key ``alerts``); the optional ntfy
|
||||
access token is write-only and never returned by the API;
|
||||
* ``alert_log`` records which notification (``opened``, ``resolved``,
|
||||
``reminder:<n>``) was sent for which incident, so restarts and repeated
|
||||
cycles never send duplicates;
|
||||
* an incident is announced only after it has stayed open for
|
||||
``delay_seconds`` (short flaps stay quiet) and when its severity reaches
|
||||
``min_severity``. Maintenance never opens incidents, so it is quiet too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from .models import Criticality, IncidentRecord, IncidentStatus
|
||||
from .store import Store
|
||||
|
||||
logger = logging.getLogger("servicemap.alerts")
|
||||
|
||||
_SEVERITY_RANK = {Criticality.LOW: 0, Criticality.MEDIUM: 1, Criticality.HIGH: 2, Criticality.CRITICAL: 3}
|
||||
_SETTINGS_KEY = "alerts"
|
||||
|
||||
|
||||
class AlertSettings(BaseModel):
|
||||
enabled: bool = False
|
||||
ntfy_server: str = Field(default="https://ntfy.sh", max_length=300)
|
||||
ntfy_topic: str = Field(default="", max_length=120)
|
||||
min_severity: Criticality = Criticality.MEDIUM
|
||||
delay_seconds: int = Field(default=60, ge=0, le=3600)
|
||||
notify_resolved: bool = True
|
||||
reminder_minutes: int = Field(default=0, ge=0, le=1440)
|
||||
public_url: str = Field(default="", max_length=300)
|
||||
|
||||
@field_validator("ntfy_server", "public_url")
|
||||
@classmethod
|
||||
def _http_url(cls, value: str) -> str:
|
||||
value = value.strip().rstrip("/")
|
||||
if value and urlparse(value).scheme not in {"http", "https"}:
|
||||
raise ValueError("moet een http(s)-URL zijn")
|
||||
return value
|
||||
|
||||
@field_validator("ntfy_topic")
|
||||
@classmethod
|
||||
def _topic(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if value and not all(char.isalnum() or char in "-_" for char in value):
|
||||
raise ValueError("mag alleen letters, cijfers, - en _ bevatten")
|
||||
return value
|
||||
|
||||
|
||||
class AlertSettingsView(AlertSettings):
|
||||
"""What the API returns: the access token is never echoed."""
|
||||
|
||||
ntfy_token_set: bool = False
|
||||
|
||||
|
||||
class AlertSettingsUpdate(AlertSettings):
|
||||
# None keeps the stored token, "" removes it.
|
||||
ntfy_token: str | None = Field(default=None, max_length=200)
|
||||
|
||||
|
||||
class AlertTestResult(BaseModel):
|
||||
ok: bool
|
||||
detail: str
|
||||
|
||||
|
||||
class AlertManager:
|
||||
def __init__(self, store: Store) -> None:
|
||||
self.store = store
|
||||
self._init_schema()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
return self.store._connect()
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._connect() as db:
|
||||
db.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS alert_log (
|
||||
incident_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
sent_at TEXT NOT NULL,
|
||||
PRIMARY KEY (incident_id, kind)
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
# -- settings -----------------------------------------------------------
|
||||
def _raw(self) -> dict[str, object]:
|
||||
with self._connect() as db:
|
||||
row = db.execute("SELECT value FROM settings WHERE key = ?", (_SETTINGS_KEY,)).fetchone()
|
||||
if row is None:
|
||||
return {}
|
||||
try:
|
||||
value = json.loads(row["value"])
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
def settings(self) -> AlertSettings:
|
||||
raw = self._raw()
|
||||
return AlertSettings.model_validate({key: value for key, value in raw.items() if key != "ntfy_token"})
|
||||
|
||||
def view(self) -> AlertSettingsView:
|
||||
raw = self._raw()
|
||||
return AlertSettingsView(**self.settings().model_dump(), ntfy_token_set=bool(raw.get("ntfy_token")))
|
||||
|
||||
def update(self, update: AlertSettingsUpdate) -> AlertSettingsView:
|
||||
raw = self._raw()
|
||||
stored = update.model_dump(exclude={"ntfy_token"})
|
||||
token = raw.get("ntfy_token") if update.ntfy_token is None else update.ntfy_token.strip()
|
||||
if token:
|
||||
stored["ntfy_token"] = token
|
||||
if stored["enabled"] and not stored["ntfy_topic"]:
|
||||
raise ValueError("Stel een ntfy-topic in voor je meldingen aanzet")
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at",
|
||||
(_SETTINGS_KEY, json.dumps(stored), datetime.now(UTC).isoformat()),
|
||||
)
|
||||
return self.view()
|
||||
|
||||
# -- delivery -------------------------------------------------------------
|
||||
def _send(self, title: str, message: str, *, priority: str, tags: str, click: str | None) -> None:
|
||||
settings = self.settings()
|
||||
token = self._raw().get("ntfy_token")
|
||||
headers = {"Title": title, "Priority": priority, "Tags": tags}
|
||||
if click:
|
||||
headers["Click"] = click
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
url = f"{settings.ntfy_server}/{quote(settings.ntfy_topic)}"
|
||||
with httpx.Client(timeout=10.0, trust_env=False, follow_redirects=False) as client:
|
||||
response = client.post(url, content=message.encode("utf-8"), headers=headers)
|
||||
if response.status_code >= 300:
|
||||
raise RuntimeError(f"ntfy antwoordde met HTTP {response.status_code}")
|
||||
|
||||
def send_test(self) -> AlertTestResult:
|
||||
settings = self.settings()
|
||||
if not settings.ntfy_topic:
|
||||
return AlertTestResult(ok=False, detail="Er is geen ntfy-topic ingesteld")
|
||||
try:
|
||||
self._send(
|
||||
"Testmelding van ServiceMap",
|
||||
"Meldingen van ServiceMap komen aan op dit topic.",
|
||||
priority="default",
|
||||
tags="white_check_mark",
|
||||
click=settings.public_url or None,
|
||||
)
|
||||
except (httpx.HTTPError, RuntimeError) as exc:
|
||||
return AlertTestResult(ok=False, detail=str(exc))
|
||||
return AlertTestResult(ok=True, detail=f"Verstuurd naar {settings.ntfy_server}/{settings.ntfy_topic}")
|
||||
|
||||
def _sent(self, incident_id: str) -> set[str]:
|
||||
with self._connect() as db:
|
||||
rows = db.execute("SELECT kind FROM alert_log WHERE incident_id = ?", (incident_id,)).fetchall()
|
||||
return {row["kind"] for row in rows}
|
||||
|
||||
def _mark(self, incident_id: str, kind: str, at: datetime) -> None:
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO alert_log (incident_id, kind, sent_at) VALUES (?, ?, ?)",
|
||||
(incident_id, kind, at.isoformat()),
|
||||
)
|
||||
|
||||
def _link(self, settings: AlertSettings, incident: IncidentRecord) -> str | None:
|
||||
if not settings.public_url:
|
||||
return None
|
||||
return f"{settings.public_url}/#/incidents?incident={quote(incident.id)}"
|
||||
|
||||
def plan(self, incidents: list[IncidentRecord], now: datetime) -> list[tuple[IncidentRecord, Literal["opened", "resolved"] | str]]:
|
||||
"""Notifications that are due now (pure apart from reading the log)."""
|
||||
|
||||
settings = self.settings()
|
||||
due: list[tuple[IncidentRecord, str]] = []
|
||||
for incident in incidents:
|
||||
if _SEVERITY_RANK[incident.severity] < _SEVERITY_RANK[settings.min_severity]:
|
||||
continue
|
||||
sent = self._sent(incident.id)
|
||||
if incident.status == IncidentStatus.RESOLVED:
|
||||
if settings.notify_resolved and "opened" in sent and "resolved" not in sent:
|
||||
due.append((incident, "resolved"))
|
||||
continue
|
||||
open_for = now - incident.detected_at
|
||||
if "opened" not in sent:
|
||||
if open_for >= timedelta(seconds=settings.delay_seconds):
|
||||
due.append((incident, "opened"))
|
||||
continue
|
||||
if settings.reminder_minutes and incident.status == IncidentStatus.OPEN:
|
||||
count = int(open_for.total_seconds() // (settings.reminder_minutes * 60))
|
||||
if count >= 1 and f"reminder:{count}" not in sent:
|
||||
due.append((incident, f"reminder:{count}"))
|
||||
return due
|
||||
|
||||
def run_cycle(self, now: datetime | None = None) -> int:
|
||||
settings = self.settings()
|
||||
if not settings.enabled or not settings.ntfy_topic:
|
||||
return 0
|
||||
now = now or datetime.now(UTC)
|
||||
# Recently resolved incidents still need their "resolved" message.
|
||||
incidents = self.store.list_incidents(active_only=False, limit=200)
|
||||
sent = 0
|
||||
for incident, kind in self.plan(incidents, now):
|
||||
critical = incident.severity in {Criticality.HIGH, Criticality.CRITICAL}
|
||||
if kind == "resolved":
|
||||
title = f"Opgelost: {incident.title}"
|
||||
message = f"{incident.root_service_name} is hersteld."
|
||||
priority, tags = "default", "white_check_mark"
|
||||
else:
|
||||
reminder = kind.startswith("reminder")
|
||||
title = f"{'Nog steeds uitgevallen' if reminder else 'Uitgevallen'}: {incident.title}"
|
||||
affected = len(incident.affected_service_ids)
|
||||
message = (
|
||||
f"{incident.root_cause_reason}"
|
||||
+ (f"\n{affected} afhankelijke service(s) geraakt." if affected else "")
|
||||
)
|
||||
priority, tags = ("urgent" if critical else "high"), "rotating_light"
|
||||
try:
|
||||
self._send(title, message, priority=priority, tags=tags, click=self._link(settings, incident))
|
||||
except (httpx.HTTPError, RuntimeError):
|
||||
logger.exception("Sending the %s notification for incident %s failed", kind, incident.id)
|
||||
continue
|
||||
self._mark(incident.id, kind, now)
|
||||
sent += 1
|
||||
return sent
|
||||
+151
-10
@@ -1,10 +1,27 @@
|
||||
"""Optional, fail-closed OIDC bearer authorization for ServiceMap APIs."""
|
||||
"""ServiceMap API authorization.
|
||||
|
||||
Modes (``SERVICEMAP_AUTH_MODE``):
|
||||
|
||||
* ``disabled`` - no authentication (v1.5 behaviour). Cross-site mutations are
|
||||
still refused by the Origin check below.
|
||||
* ``token`` - reads stay anonymous on the LAN (integrations such as ITWorx
|
||||
Forge and monitors keep working); every mutating request needs
|
||||
``Authorization: Bearer <admin token>``. The token lives in a 0600 file
|
||||
next to the database and is generated on first start.
|
||||
* ``oidc`` - fail-closed bearer JWT validation with viewer/operator/admin
|
||||
roles mapped from claims or groups.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
@@ -36,10 +53,10 @@ class AuthConfig:
|
||||
@classmethod
|
||||
def from_environment(cls) -> AuthConfig:
|
||||
mode = os.getenv("SERVICEMAP_AUTH_MODE", "disabled").strip().casefold() or "disabled"
|
||||
if mode == "disabled":
|
||||
return cls(mode="disabled")
|
||||
if mode in {"disabled", "token"}:
|
||||
return cls(mode=mode)
|
||||
if mode != "oidc":
|
||||
raise AuthorizationError("SERVICEMAP_AUTH_MODE must be 'disabled' or 'oidc'")
|
||||
raise AuthorizationError("SERVICEMAP_AUTH_MODE must be 'disabled', 'token' or 'oidc'")
|
||||
issuer = os.getenv("SERVICEMAP_OIDC_ISSUER", "").strip().rstrip("/")
|
||||
audience = os.getenv("SERVICEMAP_OIDC_AUDIENCE", "").strip()
|
||||
parsed = urlparse(issuer)
|
||||
@@ -131,17 +148,123 @@ def verify_bearer(token: str, config: AuthConfig) -> set[str]:
|
||||
return _roles_for(claims, config)
|
||||
|
||||
|
||||
SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
||||
# Reads that expose secrets need write access even where reads are open
|
||||
# (an ntfy topic is effectively a secret on a public server).
|
||||
SECRET_READ_PREFIXES = ("/api/v1/config", "/api/v1/settings", "/api/v1/backups")
|
||||
# A backup contains every record, so listing and downloading need the token too.
|
||||
TOKEN_READ_PREFIXES = ("/api/v1/settings", "/api/v1/backups")
|
||||
_MIN_TOKEN_LENGTH = 24
|
||||
_logger = logging.getLogger("servicemap.auth")
|
||||
_token_lock = Lock()
|
||||
_token_cache: dict[str, str] = {}
|
||||
|
||||
|
||||
def admin_token_path() -> Path:
|
||||
configured = os.getenv("SERVICEMAP_ADMIN_TOKEN_FILE", "").strip()
|
||||
if configured:
|
||||
return Path(configured)
|
||||
database = Path(os.getenv("SERVICEMAP_DB", "data/servicemap.db"))
|
||||
return database.parent / "admin-token"
|
||||
|
||||
|
||||
def admin_token() -> str:
|
||||
"""Read the admin token, generating it on first use.
|
||||
|
||||
The value is never logged; only its location is. Read it on the host
|
||||
with e.g. ``docker exec servicemap cat /data/admin-token``.
|
||||
"""
|
||||
|
||||
path = admin_token_path()
|
||||
key = str(path)
|
||||
with _token_lock:
|
||||
cached = _token_cache.get(key)
|
||||
if cached:
|
||||
return cached
|
||||
if not path.exists():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
generated = secrets.token_urlsafe(32)
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(generated + "\n")
|
||||
_logger.warning("Generated a new ServiceMap admin token at %s", path)
|
||||
token = path.read_text(encoding="utf-8").strip()
|
||||
if len(token) < _MIN_TOKEN_LENGTH:
|
||||
raise AuthorizationError(
|
||||
f"The admin token in {path} must be at least {_MIN_TOKEN_LENGTH} characters"
|
||||
)
|
||||
_token_cache[key] = token
|
||||
return token
|
||||
|
||||
|
||||
def reset_token_cache() -> None:
|
||||
with _token_lock:
|
||||
_token_cache.clear()
|
||||
|
||||
|
||||
def required_role(request: Request) -> str:
|
||||
if request.method in {"GET", "HEAD", "OPTIONS"}:
|
||||
return "viewer"
|
||||
path = request.url.path
|
||||
if request.method in SAFE_METHODS:
|
||||
# A full configuration export contains every endpoint and target;
|
||||
# settings hold notification targets.
|
||||
return "admin" if path.startswith(SECRET_READ_PREFIXES) else "viewer"
|
||||
if path.startswith(
|
||||
("/api/v1/config", "/api/v1/discovery", "/api/v1/docker", "/api/v1/maintenance")
|
||||
(
|
||||
"/api/v1/config",
|
||||
"/api/v1/discovery",
|
||||
"/api/v1/docker",
|
||||
"/api/v1/maintenance",
|
||||
# Demo replaces the whole live topology.
|
||||
"/api/v1/demo",
|
||||
"/api/v1/settings",
|
||||
"/api/v1/backups",
|
||||
)
|
||||
):
|
||||
return "admin"
|
||||
return "operator"
|
||||
|
||||
|
||||
_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
|
||||
|
||||
|
||||
def _host_name(value: str) -> str:
|
||||
value = value.strip().casefold()
|
||||
if value.startswith("["): # IPv6 literal with optional port
|
||||
return value[1:].split("]", 1)[0]
|
||||
return value.rsplit(":", 1)[0] if value.count(":") == 1 else value
|
||||
|
||||
|
||||
def check_request_origin(request: Request) -> None:
|
||||
"""Refuse DNS-rebinding hosts and cross-site mutations.
|
||||
|
||||
* ``SERVICEMAP_ALLOWED_HOSTS`` (comma separated, optional): when set,
|
||||
requests for any other Host are refused. Loopback is always allowed
|
||||
for the container healthcheck.
|
||||
* A mutating request whose ``Origin`` is not this server, or that a
|
||||
browser marks ``Sec-Fetch-Site: cross-site``, is refused. Requests
|
||||
without these headers (curl, integrations) are unaffected.
|
||||
"""
|
||||
|
||||
host_header = request.headers.get("host", "")
|
||||
# Behind a reverse proxy the public name arrives as X-Forwarded-Host.
|
||||
forwarded = request.headers.get("x-forwarded-host", "").split(",")[0].strip()
|
||||
public_host = _host_name(forwarded or host_header)
|
||||
allowed = _csv("SERVICEMAP_ALLOWED_HOSTS")
|
||||
if allowed and public_host not in allowed | _LOOPBACK_HOSTS and _host_name(host_header) not in _LOOPBACK_HOSTS:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Deze host is niet toegestaan")
|
||||
if request.method in SAFE_METHODS:
|
||||
return
|
||||
if request.headers.get("sec-fetch-site", "").casefold() == "cross-site":
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Verzoek van een andere site geweigerd")
|
||||
origin = request.headers.get("origin")
|
||||
if origin is not None:
|
||||
parsed = urlparse(origin)
|
||||
# Hostnames are compared without ports: proxies (nginx ``$host``,
|
||||
# NPM) commonly drop the port from Host while Origin keeps it.
|
||||
if origin == "null" or not parsed.hostname or parsed.hostname.casefold() != public_host:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Verzoek van een andere origin geweigerd")
|
||||
|
||||
|
||||
def authorize_request(request: Request) -> None:
|
||||
"""Authorize API requests; health and static content remain reachable.
|
||||
|
||||
@@ -150,7 +273,8 @@ def authorize_request(request: Request) -> None:
|
||||
available for recovery and deployment diagnostics.
|
||||
"""
|
||||
|
||||
if request.url.path == "/health" or not request.url.path.startswith("/api/"):
|
||||
check_request_origin(request)
|
||||
if request.url.path in {"/health", "/ready"} or not request.url.path.startswith("/api/"):
|
||||
return
|
||||
try:
|
||||
config = AuthConfig.from_environment()
|
||||
@@ -161,9 +285,26 @@ def authorize_request(request: Request) -> None:
|
||||
if config.mode == "disabled":
|
||||
return
|
||||
header = request.headers.get("Authorization", "")
|
||||
if config.mode == "token":
|
||||
if request.method in SAFE_METHODS and not request.url.path.startswith(TOKEN_READ_PREFIXES):
|
||||
return
|
||||
try:
|
||||
expected = admin_token()
|
||||
except (AuthorizationError, OSError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Het beheerderstoken is niet beschikbaar"
|
||||
) from exc
|
||||
supplied = header.removeprefix("Bearer ").strip() if header.startswith("Bearer ") else ""
|
||||
if not supplied or not hmac.compare_digest(supplied.encode(), expected.encode()):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Bewerken vraagt het ServiceMap-beheerderstoken",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
return
|
||||
if not header.startswith("Bearer "):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail="Bearer authentication is required"
|
||||
status_code=status.HTTP_401_UNAUTHORIZED, detail="Bearer-authenticatie is vereist"
|
||||
)
|
||||
try:
|
||||
roles = verify_bearer(header.removeprefix("Bearer ").strip(), config)
|
||||
@@ -174,5 +315,5 @@ def authorize_request(request: Request) -> None:
|
||||
< _ROLE_ORDER[required_role(request)]
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN, detail="Insufficient ServiceMap role"
|
||||
status_code=status.HTTP_403_FORBIDDEN, detail="Onvoldoende ServiceMap-rol"
|
||||
)
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Verified, rotated SQLite backups and online restore.
|
||||
|
||||
* Backups are written with ``VACUUM INTO`` (a consistent, compacted copy
|
||||
taken inside a read transaction, safe while ServiceMap runs) to
|
||||
``<data dir>/backups`` and checked with ``PRAGMA quick_check`` before
|
||||
they count. A copy that fails the check is deleted, never kept.
|
||||
* Rotation keeps the newest ``keep`` scheduled/manual backups; pre-restore
|
||||
safety copies are rotated the same way.
|
||||
* Restore uses SQLite's online backup API from the chosen file into the
|
||||
live database: it is transactional and needs no file swapping under open
|
||||
WAL connections. A fresh ``pre-restore`` backup is taken first.
|
||||
* Legacy pre-upgrade copies (``servicemap-v*-pre-*.sqlite3`` next to the
|
||||
database) are pruned to the newest ``legacy_keep`` only after a new
|
||||
backup passed its check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .store import Store
|
||||
|
||||
_NAME = re.compile(r"^servicemap-(?P<kind>backup|pre-restore)-(?P<stamp>\d{8}T\d{6}Z)\.db$")
|
||||
_LEGACY = re.compile(r"^servicemap-v\d+\.\d+\.\d+-pre-[A-Za-z0-9T._-]+\.sqlite3$")
|
||||
|
||||
|
||||
class BackupInfo(BaseModel):
|
||||
name: str
|
||||
kind: str
|
||||
created_at: datetime
|
||||
size_bytes: int
|
||||
|
||||
|
||||
class BackupManager:
|
||||
def __init__(self, store: Store, *, keep: int = 14, legacy_keep: int = 2) -> None:
|
||||
self.store = store
|
||||
self.keep = max(1, keep)
|
||||
self.legacy_keep = max(0, legacy_keep)
|
||||
|
||||
@property
|
||||
def directory(self) -> Path:
|
||||
return self.store.path.parent / "backups"
|
||||
|
||||
def list(self) -> list[BackupInfo]:
|
||||
if not self.directory.exists():
|
||||
return []
|
||||
items: list[BackupInfo] = []
|
||||
for path in self.directory.iterdir():
|
||||
match = _NAME.match(path.name)
|
||||
if not match or not path.is_file():
|
||||
continue
|
||||
created = datetime.strptime(match["stamp"], "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
|
||||
items.append(
|
||||
BackupInfo(name=path.name, kind=match["kind"], created_at=created, size_bytes=path.stat().st_size)
|
||||
)
|
||||
return sorted(items, key=lambda item: item.created_at, reverse=True)
|
||||
|
||||
def path_for(self, name: str) -> Path:
|
||||
"""Resolve a listed backup name; anything else is refused."""
|
||||
|
||||
if not _NAME.match(name):
|
||||
raise ValueError("Onbekende back-up")
|
||||
path = self.directory / name
|
||||
if not path.is_file():
|
||||
raise ValueError("Onbekende back-up")
|
||||
return path
|
||||
|
||||
def create(self, kind: str = "backup", now: datetime | None = None) -> BackupInfo:
|
||||
now = now or datetime.now(UTC)
|
||||
self.directory.mkdir(parents=True, exist_ok=True)
|
||||
name = f"servicemap-{kind}-{now.strftime('%Y%m%dT%H%M%SZ')}.db"
|
||||
target = self.directory / name
|
||||
if target.exists():
|
||||
raise ValueError("Er bestaat al een back-up met dit tijdstip")
|
||||
with self.store._connect() as db:
|
||||
db.execute("VACUUM INTO ?", (str(target),))
|
||||
try:
|
||||
os.chmod(target, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
if not self._verify(target):
|
||||
target.unlink(missing_ok=True)
|
||||
raise RuntimeError("De nieuwe back-up faalde de integriteitscontrole en is verwijderd")
|
||||
self._rotate()
|
||||
self._prune_legacy()
|
||||
return next(item for item in self.list() if item.name == name)
|
||||
|
||||
def restore(self, name: str) -> BackupInfo:
|
||||
source = self.path_for(name)
|
||||
if not self._verify(source):
|
||||
raise RuntimeError("De gekozen back-up faalde de integriteitscontrole; er is niets teruggezet")
|
||||
if self.store.demo_active():
|
||||
raise ValueError("Sluit de demo af voor je een back-up terugzet")
|
||||
safety = self.create(kind="pre-restore")
|
||||
# closing(): a sqlite3 connection's own context manager only commits.
|
||||
with closing(sqlite3.connect(source)) as src, closing(sqlite3.connect(self.store.path, timeout=30)) as dst:
|
||||
src.backup(dst)
|
||||
return safety
|
||||
|
||||
@staticmethod
|
||||
def _verify(path: Path) -> bool:
|
||||
try:
|
||||
with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True)) as db:
|
||||
result = db.execute("PRAGMA quick_check").fetchone()
|
||||
tables = db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='services'").fetchone()
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
return bool(result and result[0] == "ok" and tables)
|
||||
|
||||
def _rotate(self) -> None:
|
||||
for kind in ("backup", "pre-restore"):
|
||||
for stale in [item for item in self.list() if item.kind == kind][self.keep:]:
|
||||
(self.directory / stale.name).unlink(missing_ok=True)
|
||||
|
||||
def _prune_legacy(self) -> list[str]:
|
||||
folder = self.store.path.parent
|
||||
legacy = sorted(
|
||||
(path for path in folder.iterdir() if path.is_file() and _LEGACY.match(path.name)),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
removed: list[str] = []
|
||||
for path in legacy[self.legacy_keep:]:
|
||||
for candidate in (path, Path(f"{path}-wal"), Path(f"{path}-shm"), Path(f"{path}-journal")):
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
removed.append(path.name)
|
||||
return removed
|
||||
+50
-73
@@ -5,12 +5,44 @@ import json
|
||||
import yaml
|
||||
from pydantic import ValidationError
|
||||
|
||||
from .models import ConfigDocument, ConfigImportResult, ConfigService
|
||||
from .models import ConfigDocument, ConfigImportResult, ConfigService, Service, ServiceCreate
|
||||
from .store import Store
|
||||
|
||||
# Observed runtime state is not configuration: it is re-learned from Docker
|
||||
# and probes and must not be frozen into an exported document.
|
||||
OBSERVED_FIELDS = frozenset(
|
||||
{"runtime_state", "runtime_health", "first_seen_at", "last_seen_at", "last_discovery_status"}
|
||||
)
|
||||
CONFIG_FIELDS = frozenset(ServiceCreate.model_fields) - OBSERVED_FIELDS - {"depends_on"}
|
||||
|
||||
|
||||
def config_service(service: Service, names_by_id: dict[str, str]) -> ConfigService:
|
||||
"""Every operator-owned field of a service, dependencies by name.
|
||||
|
||||
The previous field-by-field copy omitted application/domain/role, their
|
||||
locks, category, visibility, lifecycle, monitoring and probe metadata and
|
||||
layout, so an export/import round trip silently lost them.
|
||||
"""
|
||||
|
||||
return ConfigService(
|
||||
**service.model_dump(include=CONFIG_FIELDS),
|
||||
depends_on=[names_by_id[item] for item in service.depends_on if item in names_by_id],
|
||||
)
|
||||
|
||||
|
||||
def _reject_yaml_aliases(content: str) -> None:
|
||||
"""Refuse YAML anchors/aliases: nested aliases expand exponentially
|
||||
("billion laughs") and a configuration document never needs them."""
|
||||
|
||||
for event in yaml.parse(content, Loader=yaml.SafeLoader):
|
||||
if isinstance(event, yaml.AliasEvent) or getattr(event, "anchor", None):
|
||||
raise ValueError("De configuratie mag geen YAML-anchors of -aliassen gebruiken")
|
||||
|
||||
|
||||
def parse_document(content: str, format_name: str) -> ConfigDocument:
|
||||
try:
|
||||
if format_name != "json":
|
||||
_reject_yaml_aliases(content)
|
||||
raw = json.loads(content) if format_name == "json" else yaml.safe_load(content)
|
||||
except (json.JSONDecodeError, yaml.YAMLError) as exc:
|
||||
raise ValueError(f"Invalid {format_name.upper()} configuration: {exc}") from exc
|
||||
@@ -19,22 +51,22 @@ def parse_document(content: str, format_name: str) -> ConfigDocument:
|
||||
try:
|
||||
document = ConfigDocument.model_validate(raw)
|
||||
except ValidationError as exc:
|
||||
raise ValueError(f"Configuration schema validation failed: {exc}") from exc
|
||||
raise ValueError(f"De configuratie voldoet niet aan het schema: {exc}") from exc
|
||||
if document.version == 2 and not document.services:
|
||||
# Preserve the v1 contract's explicit unsupported-empty-v2 guard while
|
||||
# accepting real v2 documents that carry the richer service schema.
|
||||
raise ValueError("Configuration version must be less than or equal to 1 for an empty document")
|
||||
raise ValueError("Voor een leeg document moet de configuratieversie 1 of lager zijn")
|
||||
|
||||
names = [service.name.casefold() for service in document.services]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError("Service names must be unique in the configuration")
|
||||
raise ValueError("Servicenamen moeten uniek zijn in de configuratie")
|
||||
|
||||
for service in document.services:
|
||||
dependencies = [dependency.casefold() for dependency in service.depends_on]
|
||||
if len(dependencies) != len(set(dependencies)):
|
||||
raise ValueError(f"Service {service.name} has duplicate dependencies")
|
||||
raise ValueError(f"Service {service.name} heeft dubbele afhankelijkheden")
|
||||
if service.name.casefold() in dependencies:
|
||||
raise ValueError(f"Service {service.name} cannot depend on itself")
|
||||
raise ValueError(f"Service {service.name} kan niet van zichzelf afhangen")
|
||||
|
||||
# A merge document may reference services that already exist in the store,
|
||||
# so missing-reference validation belongs to import_document(). Cycles that
|
||||
@@ -53,7 +85,7 @@ def _validate_acyclic(services: list[ConfigService]) -> None:
|
||||
|
||||
def visit(node: str) -> None:
|
||||
if node in visiting:
|
||||
raise ValueError("Dependency graph contains a cycle")
|
||||
raise ValueError("De afhankelijkheidsgrafiek bevat een kring")
|
||||
if node in visited:
|
||||
return
|
||||
visiting.add(node)
|
||||
@@ -68,44 +100,22 @@ def _validate_acyclic(services: list[ConfigService]) -> None:
|
||||
|
||||
def _validate_import_graph(store: Store, document: ConfigDocument, mode: str) -> None:
|
||||
if mode not in {"merge", "replace"}:
|
||||
raise ValueError("Import mode must be merge or replace")
|
||||
raise ValueError("Importmodus moet merge of replace zijn")
|
||||
|
||||
projected: dict[str, ConfigService] = {}
|
||||
if mode == "merge":
|
||||
existing = store.list_services()
|
||||
names_by_id = {service.id: service.name for service in existing}
|
||||
for service in existing:
|
||||
projected[service.name.casefold()] = ConfigService(
|
||||
name=service.name,
|
||||
kind=service.kind,
|
||||
url=service.url,
|
||||
host=service.host,
|
||||
port=service.port,
|
||||
depends_on=[names_by_id[item] for item in service.depends_on if item in names_by_id],
|
||||
description=service.description,
|
||||
tags=service.tags,
|
||||
group=service.group,
|
||||
environment=service.environment,
|
||||
owner=service.owner,
|
||||
enabled=service.enabled,
|
||||
monitoring_enabled=service.monitoring_enabled,
|
||||
criticality=service.criticality,
|
||||
source_type=service.source_type,
|
||||
source_ref=service.source_ref,
|
||||
interval_seconds=service.interval_seconds,
|
||||
timeout_seconds=service.timeout_seconds,
|
||||
retries=service.retries,
|
||||
failure_threshold=service.failure_threshold,
|
||||
recovery_threshold=service.recovery_threshold,
|
||||
expected_status_min=service.expected_status_min,
|
||||
expected_status_max=service.expected_status_max,
|
||||
availability_target=service.availability_target,
|
||||
maintenance_enabled=service.maintenance_enabled,
|
||||
maintenance_reason=service.maintenance_reason,
|
||||
)
|
||||
projected[service.name.casefold()] = config_service(service, names_by_id)
|
||||
|
||||
for service in document.services:
|
||||
projected[service.name.casefold()] = service
|
||||
key = service.name.casefold()
|
||||
previous = projected.get(key)
|
||||
if previous is not None and "depends_on" not in service.model_fields_set:
|
||||
# A merge keeps dependencies the document does not mention.
|
||||
service = service.model_copy(update={"depends_on": previous.depends_on})
|
||||
projected[key] = service
|
||||
|
||||
available = set(projected)
|
||||
for service in projected.values():
|
||||
@@ -116,7 +126,7 @@ def _validate_import_graph(store: Store, document: ConfigDocument, mode: str) ->
|
||||
]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Service {service.name} references unknown dependencies: {', '.join(missing)}"
|
||||
f"Service {service.name} verwijst naar onbekende afhankelijkheden: {', '.join(missing)}"
|
||||
)
|
||||
|
||||
_validate_acyclic(list(projected.values()))
|
||||
@@ -125,44 +135,11 @@ def _validate_import_graph(store: Store, document: ConfigDocument, mode: str) ->
|
||||
def export_document(store: Store) -> ConfigDocument:
|
||||
services = store.list_services()
|
||||
names = {service.id: service.name for service in services}
|
||||
return ConfigDocument(
|
||||
version=2,
|
||||
services=[
|
||||
ConfigService(
|
||||
name=service.name,
|
||||
kind=service.kind,
|
||||
url=service.url,
|
||||
host=service.host,
|
||||
port=service.port,
|
||||
depends_on=[names[item] for item in service.depends_on if item in names],
|
||||
description=service.description,
|
||||
tags=service.tags,
|
||||
group=service.group,
|
||||
environment=service.environment,
|
||||
owner=service.owner,
|
||||
enabled=service.enabled,
|
||||
monitoring_enabled=service.monitoring_enabled,
|
||||
criticality=service.criticality,
|
||||
source_type=service.source_type,
|
||||
source_ref=service.source_ref,
|
||||
interval_seconds=service.interval_seconds,
|
||||
timeout_seconds=service.timeout_seconds,
|
||||
retries=service.retries,
|
||||
failure_threshold=service.failure_threshold,
|
||||
recovery_threshold=service.recovery_threshold,
|
||||
expected_status_min=service.expected_status_min,
|
||||
expected_status_max=service.expected_status_max,
|
||||
availability_target=service.availability_target,
|
||||
maintenance_enabled=service.maintenance_enabled,
|
||||
maintenance_reason=service.maintenance_reason,
|
||||
)
|
||||
for service in services
|
||||
],
|
||||
)
|
||||
return ConfigDocument(version=2, services=[config_service(service, names) for service in services])
|
||||
|
||||
|
||||
def serialize_document(document: ConfigDocument, format_name: str) -> str:
|
||||
payload = document.model_dump(mode="json")
|
||||
payload = document.model_dump(mode="json", exclude={"services": {"__all__": set(OBSERVED_FIELDS)}})
|
||||
if format_name == "json":
|
||||
return json.dumps(payload, indent=2, sort_keys=False)
|
||||
return yaml.safe_dump(payload, sort_keys=False, allow_unicode=True)
|
||||
|
||||
+13
-13
@@ -39,15 +39,15 @@ def build_demo_records(now: datetime | None = None) -> tuple[list[Service], list
|
||||
]
|
||||
ids_by_name = {name: _stable_id(f"service:{name}") for name, *_ in definitions}
|
||||
metadata = {
|
||||
"Public Endpoint": ("Edge", "Public Gateway", "Edge", ComponentRole.FRONTEND, DiscoveryCategory.PROXY, Criticality.CRITICAL, "Customer-facing availability check", 100.0),
|
||||
"Reverse Proxy": ("Edge", "Public Gateway", "Edge", ComponentRole.PROXY, DiscoveryCategory.PROXY, Criticality.HIGH, "Northstar ingress gateway", 100.0),
|
||||
"Web Frontend": ("Application", "Northstar App", "Application", ComponentRole.FRONTEND, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Browser application", 99.0),
|
||||
"Auth Service": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Identity and session service", 100.0),
|
||||
"API Gateway": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.CRITICAL, "Application API entry point", 99.0),
|
||||
"PostgreSQL": ("Data", "Data Platform", "Data", ComponentRole.DATABASE, DiscoveryCategory.DATABASE, Criticality.CRITICAL, "Primary relational datastore", 99.9),
|
||||
"Redis": ("Data", "Data Platform", "Data", ComponentRole.CACHE, DiscoveryCategory.CACHE, Criticality.HIGH, "Cache and queue backend", 100.0),
|
||||
"Background Worker": ("Application", "Worker Platform", "Application", ComponentRole.WORKER, DiscoveryCategory.BACKGROUND_WORKER, Criticality.HIGH, "Asynchronous job processor", 99.0),
|
||||
"Object Storage": ("Data", "Worker Platform", "Application", ComponentRole.STORAGE, DiscoveryCategory.STORAGE, Criticality.MEDIUM, "Durable object store", 100.0),
|
||||
"Public Endpoint": ("Edge", "Public Gateway", "Edge", ComponentRole.FRONTEND, DiscoveryCategory.PROXY, Criticality.CRITICAL, "Beschikbaarheidscontrole voor klanten", 100.0),
|
||||
"Reverse Proxy": ("Edge", "Public Gateway", "Edge", ComponentRole.PROXY, DiscoveryCategory.PROXY, Criticality.HIGH, "Northstar-ingangsgateway", 100.0),
|
||||
"Web Frontend": ("Application", "Northstar App", "Application", ComponentRole.FRONTEND, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Browserapplicatie", 99.0),
|
||||
"Auth Service": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.HIGH, "Identiteits- en sessieservice", 100.0),
|
||||
"API Gateway": ("Application", "Northstar App", "Application", ComponentRole.API, DiscoveryCategory.APPLICATION, Criticality.CRITICAL, "API-toegangspunt van de applicatie", 99.0),
|
||||
"PostgreSQL": ("Data", "Data Platform", "Data", ComponentRole.DATABASE, DiscoveryCategory.DATABASE, Criticality.CRITICAL, "Primaire relationele database", 99.9),
|
||||
"Redis": ("Data", "Data Platform", "Data", ComponentRole.CACHE, DiscoveryCategory.CACHE, Criticality.HIGH, "Cache- en wachtrijbackend", 100.0),
|
||||
"Background Worker": ("Application", "Worker Platform", "Application", ComponentRole.WORKER, DiscoveryCategory.BACKGROUND_WORKER, Criticality.HIGH, "Verwerker van asynchrone taken", 99.0),
|
||||
"Object Storage": ("Data", "Worker Platform", "Application", ComponentRole.STORAGE, DiscoveryCategory.STORAGE, Criticality.MEDIUM, "Duurzame objectopslag", 100.0),
|
||||
}
|
||||
services = [
|
||||
Service(
|
||||
@@ -113,9 +113,9 @@ def _history_status(name: str, index: int) -> HealthStatus:
|
||||
|
||||
def _probe_detail(name: str, status: HealthStatus, index: int) -> str:
|
||||
if name == DEMO_ROOT_CAUSE_NAME and status == HealthStatus.DOWN:
|
||||
return "Synthetic outage: PostgreSQL connection refused"
|
||||
return "Synthetische storing: PostgreSQL weigert de verbinding"
|
||||
if status == HealthStatus.DOWN:
|
||||
return f"Synthetic dependency impact observed at history point {index + 1}"
|
||||
return f"Synthetische impact via afhankelijkheid op historiekpunt {index + 1}"
|
||||
if status == HealthStatus.DEGRADED:
|
||||
return f"Synthetic elevated latency observed at history point {index + 1}"
|
||||
return "Synthetic monitoring observation"
|
||||
return f"Synthetisch verhoogde latentie op historiekpunt {index + 1}"
|
||||
return "Synthetische bewakingswaarneming"
|
||||
+32
-32
@@ -185,7 +185,7 @@ class DiscoveryManager:
|
||||
source_service_id=service_by_ref.get(source.stable_source_ref).id if source.stable_source_ref in service_by_ref else None,
|
||||
target_ref=target.stable_source_ref,
|
||||
target_service_id=service_by_ref.get(target.stable_source_ref).id if target.stable_source_ref in service_by_ref else None,
|
||||
proposed_name=f"{source.name} depends on {target.name}",
|
||||
proposed_name=f"{source.name} hangt af van {target.name}",
|
||||
relation_type=relation_type,
|
||||
confidence=confidence,
|
||||
evidence=[evidence],
|
||||
@@ -222,10 +222,10 @@ class DiscoveryManager:
|
||||
suggestion_type="relation",
|
||||
source_ref=source.stable_source_ref,
|
||||
source_service_id=service_by_ref.get(source.stable_source_ref).id if source.stable_source_ref in service_by_ref else None,
|
||||
proposed_name=f"{source.name} has an ambiguous reference to {reference}",
|
||||
proposed_name=f"{source.name} verwijst dubbelzinnig naar {reference}",
|
||||
relation_type=RelationType.DEPENDS_ON,
|
||||
confidence=DiscoveryConfidence.LOW,
|
||||
evidence=[f"{evidence}; candidates: {candidates_label}", "Manual review required; no canonical dependency was created"],
|
||||
evidence=[f"{evidence}; kandidaten: {candidates_label}", "Handmatige controle nodig; er is geen canonieke afhankelijkheid aangemaakt"],
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
),
|
||||
@@ -240,14 +240,14 @@ class DiscoveryManager:
|
||||
explicit = [item.strip() for item in labels.get("servicemap.depends_on", "").split(",") if item.strip()]
|
||||
compose = [item.split(":", 1)[0].strip() for item in labels.get("com.docker.compose.depends_on", "").split(",") if item.strip()]
|
||||
for name in sorted(set(explicit)):
|
||||
resolve_and_add(source, name, DiscoveryConfidence.CONFIRMED, "Explicit servicemap.depends_on label")
|
||||
resolve_and_add(source, name, DiscoveryConfidence.CONFIRMED, "Expliciet servicemap.depends_on-label")
|
||||
for name in sorted(set(compose)):
|
||||
resolve_and_add(source, name, DiscoveryConfidence.HIGH, "Docker Compose com.docker.compose.depends_on label", compose_only=True)
|
||||
resolve_and_add(source, name, DiscoveryConfidence.HIGH, "Docker Compose-label com.docker.compose.depends_on", compose_only=True)
|
||||
for item in source.environment_evidence:
|
||||
if "hostname=" not in item:
|
||||
continue
|
||||
hostname = item.split("hostname=", 1)[1].split(":", 1)[0].casefold()
|
||||
resolve_and_add(source, hostname, DiscoveryConfidence.MEDIUM, f"Environment endpoint reference: {item}")
|
||||
resolve_and_add(source, hostname, DiscoveryConfidence.MEDIUM, f"Endpointverwijzing in omgevingsvariabele: {item}")
|
||||
for item in source.proxy_route_evidence:
|
||||
proxies = [proxy for proxy in candidates if proxy.category == DiscoveryCategory.PROXY and set(proxy.networks).intersection(source.networks)]
|
||||
if len(proxies) == 1:
|
||||
@@ -255,12 +255,12 @@ class DiscoveryManager:
|
||||
# Shared networks are review-only hints. Cap them to avoid flooding
|
||||
# a large host with O(n²) low-confidence suggestions.
|
||||
for name in source.dependency_suggestions[:8]:
|
||||
resolve_and_add(source, name, DiscoveryConfidence.LOW, f"Shared Docker network hint: {', '.join(source.networks)}")
|
||||
resolve_and_add(source, name, DiscoveryConfidence.LOW, f"Hint via gedeeld Docker-netwerk: {', '.join(source.networks)}")
|
||||
return result
|
||||
|
||||
def run(self, request: DiscoveryRunRequest) -> DiscoveryResult:
|
||||
if not self._lock.acquire(blocking=False):
|
||||
raise ValueError("A discovery run is already in progress")
|
||||
raise ValueError("Er loopt al een detectierun")
|
||||
run = self.discovery_store.start_run()
|
||||
try:
|
||||
docker_host = self._docker_host(request.docker_host)
|
||||
@@ -278,7 +278,7 @@ class DiscoveryManager:
|
||||
status=ProviderHealth.HEALTHY,
|
||||
observed_at=observed_at,
|
||||
valid_until=observed_at + evidence_ttl,
|
||||
detail=f"Docker inventory returned {len(result.candidates)} candidates",
|
||||
detail=f"Docker-inventaris gaf {len(result.candidates)} kandidaten",
|
||||
metadata={"docker_host": docker_host},
|
||||
)
|
||||
)
|
||||
@@ -473,7 +473,7 @@ class DiscoveryManager:
|
||||
url=url,
|
||||
host=host,
|
||||
port=port,
|
||||
description=f"Discovered Docker service from {candidate.image}",
|
||||
description=f"Gevonden Docker-service op basis van {candidate.image}",
|
||||
tags=["discovered", "docker"],
|
||||
group=group,
|
||||
environment="production",
|
||||
@@ -559,7 +559,7 @@ class DiscoveryManager:
|
||||
observed_at=now,
|
||||
valid_until=now + evidence_ttl,
|
||||
received_at=now,
|
||||
summary="Docker healthcheck" if candidate.healthcheck_present else "Docker runtime state",
|
||||
summary="Docker-healthcheck" if candidate.healthcheck_present else "Docker-runtimestatus",
|
||||
metadata={"container_id": candidate.container_id, "state": candidate.state, "health": candidate.health},
|
||||
)
|
||||
)
|
||||
@@ -579,7 +579,7 @@ class DiscoveryManager:
|
||||
observed_at=now,
|
||||
valid_until=now + evidence_ttl,
|
||||
received_at=now,
|
||||
summary="AppOps application health; not used as component direct health",
|
||||
summary="Applicatiegezondheid via AppOps; niet gebruikt als directe componentgezondheid",
|
||||
metadata={
|
||||
"app_id": candidate.appops_app_id,
|
||||
"category": candidate.appops_category,
|
||||
@@ -597,7 +597,7 @@ class DiscoveryManager:
|
||||
def reconcile(self, request: DiscoveryReconcileRequest) -> ReconcileResult:
|
||||
run = self.discovery_store.get_run(request.run_id) if request.run_id else self.discovery_store.last_completed()
|
||||
if run is None or run.status != DiscoveryRunStatus.COMPLETED:
|
||||
raise ValueError("A completed discovery run is required before reconciliation")
|
||||
raise ValueError("Voor verwerking is een afgeronde detectierun nodig")
|
||||
self.discovery_store.set_run_status(run.id, DiscoveryRunStatus.RECONCILING)
|
||||
candidates = [DockerCandidate.model_validate(item.get("candidate", item)) for item in self.discovery_store.candidates_for_run(run.id)]
|
||||
by_ref = self._service_by_ref()
|
||||
@@ -727,7 +727,7 @@ class DiscoveryManager:
|
||||
payload.depends_on = [*source.depends_on, target.id]
|
||||
self.store.update_service(source.id, payload)
|
||||
self.discovery_store.set_suggestion_status(suggestion.id, SuggestionStatus.ACCEPTED)
|
||||
self.discovery_store.record_change("dependency_added", f"{source.name} depends on {target.name}", source_service_id=source.id, target_service_id=target.id)
|
||||
self.discovery_store.record_change("dependency_added", f"{source.name} hangt af van {target.name}", source_service_id=source.id, target_service_id=target.id)
|
||||
accepted_relations += 1
|
||||
|
||||
final_run = self.discovery_store.finish_run(
|
||||
@@ -767,7 +767,7 @@ class DiscoveryManager:
|
||||
|
||||
run = self.discovery_store.get_run(run_id) if run_id else self.discovery_store.last_completed()
|
||||
if run is None or run.status != DiscoveryRunStatus.COMPLETED:
|
||||
raise ValueError("A completed discovery run is required before repairing probe reachability")
|
||||
raise ValueError("Voor het herstellen van controlebereikbaarheid is een afgeronde detectierun nodig")
|
||||
candidates = [DockerCandidate.model_validate(item.get("candidate", item)) for item in self.discovery_store.candidates_for_run(run.id)]
|
||||
services = self._service_by_ref()
|
||||
changes: list[ProbeRepairChange] = []
|
||||
@@ -799,19 +799,19 @@ class DiscoveryManager:
|
||||
)
|
||||
continue
|
||||
if current.monitoring_enabled and not desired.monitoring_enabled:
|
||||
reason = "Internal container endpoint is not reachable from ServiceMap; active probe disabled"
|
||||
reason = "Intern container-endpoint is niet bereikbaar vanuit ServiceMap; actieve controle uitgeschakeld"
|
||||
disabled_unsafe += 1
|
||||
elif desired.monitoring_enabled:
|
||||
reason = "Host-published endpoint selected using the configured probe host"
|
||||
reason = "Gepubliceerd host-endpoint gekozen via de ingestelde controlehost"
|
||||
enabled_published += 1
|
||||
elif candidate.healthcheck_present:
|
||||
reason = "Docker healthcheck retained as runtime observation without a network probe"
|
||||
reason = "Docker-healthcheck behouden als runtimewaarneming zonder netwerkcontrole"
|
||||
docker_health_observed += 1
|
||||
elif desired.monitoring_state == MonitoringState.OBSERVED:
|
||||
reason = "No verified reachable endpoint; Docker runtime observation is retained without a network probe"
|
||||
reason = "Geen gecontroleerd bereikbaar endpoint; Docker-runtimewaarneming blijft behouden zonder netwerkcontrole"
|
||||
runtime_observed += 1
|
||||
else:
|
||||
reason = "No verified reachable endpoint; service remains observed as unmonitored"
|
||||
reason = "Geen gecontroleerd bereikbaar endpoint; service blijft waargenomen als niet bewaakt"
|
||||
unmonitored += 1
|
||||
changes.append(
|
||||
ProbeRepairChange(
|
||||
@@ -842,7 +842,7 @@ class DiscoveryManager:
|
||||
from .incidents import sync_incidents
|
||||
|
||||
before = len([item for item in self.store.list_incidents(active_only=True)])
|
||||
sync_incidents(self.store, resolution_reason="Monitoring endpoint superseded; active probe was replaced by reachability-safe observation")
|
||||
sync_incidents(self.store, resolution_reason="Bewakings-endpoint vervangen; de actieve controle is vervangen door een veilige waarneming")
|
||||
after = len([item for item in self.store.list_incidents(active_only=True)])
|
||||
resolved = max(0, before - after)
|
||||
return ProbeRepairReport(
|
||||
@@ -864,7 +864,7 @@ class DiscoveryManager:
|
||||
source = self.store.get_service(request.source_service_id)
|
||||
target = self.store.get_service(request.target_service_id)
|
||||
if source is None or target is None:
|
||||
raise ValueError("Both edge services must exist")
|
||||
raise ValueError("Beide services van de relatie moeten bestaan")
|
||||
if request.relation_type == RelationType.DEPENDS_ON and target.id not in source.depends_on:
|
||||
validate_dependencies([*source.depends_on, target.id], self.store, service_id=source.id)
|
||||
payload = ServiceUpdate.model_validate(source.model_dump())
|
||||
@@ -892,7 +892,7 @@ class DiscoveryManager:
|
||||
def suggestion_action(self, suggestion_id: str, action: str) -> TopologySuggestion:
|
||||
suggestion = self.discovery_store.get_suggestion(suggestion_id)
|
||||
if suggestion is None:
|
||||
raise ValueError("Suggestion not found")
|
||||
raise ValueError("Voorstel niet gevonden")
|
||||
if action == "reject":
|
||||
if suggestion.suggestion_type == "relation" and suggestion.relation_type == RelationType.DEPENDS_ON:
|
||||
source = self._service_by_ref().get(suggestion.source_ref or "")
|
||||
@@ -908,7 +908,7 @@ class DiscoveryManager:
|
||||
if action in {"confirm", "lock"}:
|
||||
if suggestion.suggestion_type == "service":
|
||||
if not suggestion.source_ref:
|
||||
raise ValueError("Service suggestion has no stable source reference")
|
||||
raise ValueError("Het servicevoorstel heeft geen stabiele bronverwijzing")
|
||||
existing = self._service_by_ref().get(suggestion.source_ref)
|
||||
if existing is None:
|
||||
candidate = next(
|
||||
@@ -920,18 +920,18 @@ class DiscoveryManager:
|
||||
None,
|
||||
)
|
||||
if candidate is None:
|
||||
raise ValueError("Discovery observation for service suggestion is no longer available")
|
||||
raise ValueError("De detectiewaarneming voor dit servicevoorstel is niet meer beschikbaar")
|
||||
existing = self.store.create_service(self._payload_for_candidate(candidate))
|
||||
source = self.discovery_store.get_source("docker", suggestion.source_ref)
|
||||
if source:
|
||||
self.discovery_store.attach_source(existing.id, source.id)
|
||||
self.discovery_store.record_change("accepted", f"Accepted discovered service {existing.name}", service_id=existing.id)
|
||||
self.discovery_store.record_change("accepted", f"Gevonden service {existing.name} aanvaard", service_id=existing.id)
|
||||
self.discovery_store.save_override("service", existing.id, "source", suggestion.source_ref, True)
|
||||
elif suggestion.suggestion_type == "relation":
|
||||
source = self._service_by_ref().get(suggestion.source_ref or "")
|
||||
target = self._service_by_ref().get(suggestion.target_ref or "")
|
||||
if source is None or target is None:
|
||||
raise ValueError("Accept the related services before accepting this relationship")
|
||||
raise ValueError("Aanvaard eerst de betrokken services voor je deze relatie aanvaardt")
|
||||
if suggestion.relation_type == RelationType.DEPENDS_ON and target.id not in source.depends_on:
|
||||
validate_dependencies([*source.depends_on, target.id], self.store, service_id=source.id)
|
||||
payload = ServiceUpdate.model_validate(source.model_dump())
|
||||
@@ -954,9 +954,9 @@ class DiscoveryManager:
|
||||
last_seen_at=now,
|
||||
)
|
||||
)
|
||||
self.discovery_store.record_change("dependency_added", f"Accepted relationship {source.name} → {target.name}", source_service_id=source.id, target_service_id=target.id)
|
||||
self.discovery_store.record_change("dependency_added", f"Relatie {source.name} → {target.name} aanvaard", source_service_id=source.id, target_service_id=target.id)
|
||||
return self.discovery_store.set_suggestion_status(suggestion_id, SuggestionStatus.ACCEPTED) or suggestion
|
||||
raise ValueError("Unsupported suggestion action")
|
||||
raise ValueError("Niet-ondersteunde actie op voorstel")
|
||||
|
||||
def record_service_override(self, before: Service, after: Service) -> None:
|
||||
for field in ("name", "group", "category", "criticality", "environment", "visibility"):
|
||||
@@ -975,16 +975,16 @@ class DiscoveryManager:
|
||||
def edge_action(self, evidence_id: str, action: str) -> RelationEvidence:
|
||||
evidence = self.discovery_store.get_evidence(evidence_id)
|
||||
if evidence is None or evidence.target_service_id is None:
|
||||
raise ValueError("Edge evidence not found")
|
||||
raise ValueError("Bewijs voor de relatie niet gevonden")
|
||||
source = self.store.get_service(evidence.source_service_id)
|
||||
if source is None:
|
||||
raise ValueError("Edge source service not found")
|
||||
raise ValueError("Bronservice van de relatie niet gevonden")
|
||||
target_id = evidence.target_service_id
|
||||
if action in {"remove", "reject"} and evidence.relation_type == RelationType.DEPENDS_ON:
|
||||
payload = ServiceUpdate.model_validate(source.model_dump())
|
||||
payload.depends_on = [item for item in source.depends_on if item != target_id]
|
||||
self.store.update_service(source.id, payload)
|
||||
self.discovery_store.record_change("dependency_removed", f"Removed relationship from {source.name}", source_service_id=source.id, target_service_id=target_id)
|
||||
self.discovery_store.record_change("dependency_removed", f"Relatie van {source.name} verwijderd", source_service_id=source.id, target_service_id=target_id)
|
||||
return self.discovery_store.set_evidence_state(evidence_id, active=False) or evidence
|
||||
if action == "confirm" and evidence.relation_type == RelationType.DEPENDS_ON and target_id not in source.depends_on:
|
||||
validate_dependencies([*source.depends_on, target_id], self.store, service_id=source.id)
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
@@ -354,7 +354,7 @@ class DiscoveryStore:
|
||||
(
|
||||
DiscoveryRunStatus.FAILED.value,
|
||||
run.started_at.isoformat(),
|
||||
"Superseded by a newer discovery run",
|
||||
"Vervangen door een nieuwere detectierun",
|
||||
DiscoveryRunStatus.DISCOVERING.value,
|
||||
DiscoveryRunStatus.RECONCILING.value,
|
||||
),
|
||||
@@ -502,7 +502,7 @@ class DiscoveryStore:
|
||||
"INSERT INTO topology_snapshot_relations VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
[(snapshot.id, relation_id, source_id, target_id, relation_type, fingerprint, _json(payload)) for relation_id, source_id, target_id, relation_type, fingerprint, payload in relation_rows],
|
||||
)
|
||||
self.record_change("topology_snapshot", f"Topology snapshot captured ({snapshot.service_count} services)")
|
||||
self.record_change("topology_snapshot", f"Topologiesnapshot gemaakt ({snapshot.service_count} services)")
|
||||
return snapshot
|
||||
|
||||
def list_snapshots(self, *, limit: int = 50) -> list[TopologySnapshot]:
|
||||
@@ -514,7 +514,7 @@ class DiscoveryStore:
|
||||
with self._connect() as db:
|
||||
after = db.execute("SELECT * FROM topology_snapshots WHERE id=?", (after_snapshot_id,)).fetchone()
|
||||
if after is None:
|
||||
raise ValueError("Topology snapshot not found")
|
||||
raise ValueError("Topologiesnapshot niet gevonden")
|
||||
before = (
|
||||
db.execute("SELECT * FROM topology_snapshots WHERE id=?", (before_snapshot_id,)).fetchone()
|
||||
if before_snapshot_id
|
||||
@@ -758,6 +758,7 @@ class DiscoveryStore:
|
||||
if existing:
|
||||
previous = self._evidence(existing)
|
||||
evidence = evidence.model_copy(update={
|
||||
"id": previous.id,
|
||||
"confidence": _higher_confidence(previous.confidence, evidence.confidence),
|
||||
"explanation": "; ".join(dict.fromkeys(part for part in (previous.explanation, evidence.explanation) if part)),
|
||||
"source_ref": evidence.source_ref or previous.source_ref,
|
||||
@@ -765,6 +766,19 @@ class DiscoveryStore:
|
||||
"active": evidence.active,
|
||||
"first_seen_at": min(previous.first_seen_at, evidence.first_seen_at),
|
||||
})
|
||||
# Update the found row by id. The UNIQUE constraint includes
|
||||
# nullable columns (target_ref is NULL for manual edges), so
|
||||
# ON CONFLICT never matched and each save inserted a duplicate.
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE relation_evidence SET target_service_id=?, confidence=?, explanation=?,
|
||||
source_ref=?, locked=MAX(locked, ?), active=?, last_seen_at=?
|
||||
WHERE id=?
|
||||
""",
|
||||
(evidence.target_service_id, evidence.confidence.value, evidence.explanation, evidence.source_ref, int(evidence.locked), int(evidence.active), evidence.last_seen_at.isoformat(), evidence.id),
|
||||
)
|
||||
row = db.execute("SELECT * FROM relation_evidence WHERE id=?", (evidence.id,)).fetchone()
|
||||
return self._evidence(row)
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO relation_evidence
|
||||
@@ -884,6 +898,53 @@ class DiscoveryStore:
|
||||
def evidence_for_service(self, service_id: str) -> list[RelationEvidence]:
|
||||
return self.list_evidence(service_id, active_only=True)
|
||||
|
||||
def evidence_for_services(self, service_ids: list[str]) -> dict[str, list[RelationEvidence]]:
|
||||
"""Active relation evidence for many services in one read (no N+1)."""
|
||||
|
||||
ids = sorted(set(service_ids))
|
||||
result: dict[str, list[RelationEvidence]] = {service_id: [] for service_id in ids}
|
||||
if not ids:
|
||||
return result
|
||||
placeholders = ", ".join("?" for _ in ids)
|
||||
with self._connect() as db:
|
||||
rows = db.execute(
|
||||
f"""
|
||||
SELECT * FROM relation_evidence
|
||||
WHERE active=1
|
||||
AND (source_service_id IN ({placeholders}) OR target_service_id IN ({placeholders}))
|
||||
ORDER BY last_seen_at DESC
|
||||
""",
|
||||
[*ids, *ids],
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
evidence = self._evidence(row)
|
||||
for service_id in {row["source_service_id"], row["target_service_id"]}:
|
||||
if service_id in result:
|
||||
result[service_id].append(evidence)
|
||||
return result
|
||||
|
||||
def sources_for_services(self, service_ids: list[str]) -> dict[str, list[ServiceSource]]:
|
||||
"""Discovery sources for many services in one read (no N+1)."""
|
||||
|
||||
ids = sorted(set(service_ids))
|
||||
result: dict[str, list[ServiceSource]] = {service_id: [] for service_id in ids}
|
||||
if not ids:
|
||||
return result
|
||||
placeholders = ", ".join("?" for _ in ids)
|
||||
with self._connect() as db:
|
||||
rows = db.execute(
|
||||
f"""
|
||||
SELECT s.*, ss.service_id AS linked_service_id
|
||||
FROM discovery_sources s JOIN service_sources ss ON ss.source_id=s.id
|
||||
WHERE ss.service_id IN ({placeholders})
|
||||
ORDER BY s.source_type, s.source_name
|
||||
""",
|
||||
ids,
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
result[row["linked_service_id"]].append(self._source(row))
|
||||
return result
|
||||
|
||||
def sources_for_service(self, service_id: str) -> list[ServiceSource]:
|
||||
with self._connect() as db:
|
||||
rows = db.execute(
|
||||
@@ -925,6 +986,34 @@ class DiscoveryStore:
|
||||
with self._connect() as db:
|
||||
db.execute("UPDATE discovery_runs SET status=? WHERE id=?", (status.value, run_id))
|
||||
|
||||
def prune_observations(self, keep_runs: int = 10) -> int:
|
||||
"""Keep per-container observations only for the most recent runs.
|
||||
|
||||
Each run stores a full candidate dump per container. Runs themselves
|
||||
are kept because review decisions (suggestions) reference them.
|
||||
"""
|
||||
|
||||
if keep_runs < 1:
|
||||
return 0
|
||||
with self._connect() as db:
|
||||
cursor = db.execute(
|
||||
"""
|
||||
DELETE FROM discovery_observations WHERE run_id NOT IN (
|
||||
SELECT id FROM discovery_runs ORDER BY started_at DESC LIMIT ?
|
||||
)
|
||||
""",
|
||||
(keep_runs,),
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
def prune_change_events(self, retention_days: int) -> int:
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
cutoff = (_now() - timedelta(days=retention_days)).isoformat()
|
||||
with self._connect() as db:
|
||||
cursor = db.execute("DELETE FROM topology_change_events WHERE created_at < ?", (cutoff,))
|
||||
return cursor.rowcount
|
||||
|
||||
def record_change(
|
||||
self,
|
||||
event_type: str,
|
||||
|
||||
@@ -4,6 +4,7 @@ import os
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -233,7 +234,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
|
||||
hostname = parsed.hostname if parsed else candidate.host
|
||||
port = parsed.port if parsed else candidate.port
|
||||
if not hostname or not port:
|
||||
return candidate.model_copy(update={"preflight_detail": "No host-published endpoint to preflight"})
|
||||
return candidate.model_copy(update={"preflight_detail": "Geen gepubliceerd host-endpoint om vooraf te testen"})
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with socket.create_connection((hostname, port), timeout=0.75):
|
||||
@@ -243,7 +244,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
|
||||
return candidate.model_copy(update={
|
||||
"reachability": ProbeReachability.VERIFIED,
|
||||
"primary": True,
|
||||
"preflight_detail": "TCP connect succeeded",
|
||||
"preflight_detail": "TCP-verbinding gelukt",
|
||||
"preflight_latency_ms": elapsed,
|
||||
})
|
||||
with httpx.Client(timeout=1.25, follow_redirects=False) as client:
|
||||
@@ -260,7 +261,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
|
||||
return candidate.model_copy(update={
|
||||
"reachability": ProbeReachability.UNVERIFIED,
|
||||
"primary": False,
|
||||
"preflight_detail": f"Preflight failed: {type(exc).__name__}",
|
||||
"preflight_detail": f"Voorafgaande test mislukt: {type(exc).__name__}",
|
||||
})
|
||||
|
||||
|
||||
@@ -292,7 +293,7 @@ def _probe_candidates(
|
||||
kind=suggestion.kind if suggestion else ServiceKind.TCP,
|
||||
reachability=ProbeReachability.VERIFIED,
|
||||
confidence=DiscoveryConfidence.HIGH,
|
||||
evidence="Docker healthcheck present (observed without a network probe)",
|
||||
evidence="Docker-healthcheck aanwezig (waargenomen zonder netwerkcontrole)",
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
@@ -327,7 +328,7 @@ def safe_environment_evidence(env: list[str]) -> list[str]:
|
||||
|
||||
def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
|
||||
if not appops_url:
|
||||
return AppOpsInventory({}, False, "AppOps URL is not configured")
|
||||
return AppOpsInventory({}, False, "Er is geen AppOps-URL ingesteld")
|
||||
base = appops_url.rstrip("/")
|
||||
try:
|
||||
mappings_response = httpx.get(f"{base}/api/inventory/mappings", timeout=5)
|
||||
@@ -337,7 +338,7 @@ def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
|
||||
mappings = mappings_response.json()
|
||||
apps = apps_response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
return AppOpsInventory({}, False, f"AppOps inventory unavailable: {type(exc).__name__}")
|
||||
return AppOpsInventory({}, False, f"AppOps-inventaris niet beschikbaar: {type(exc).__name__}")
|
||||
app_by_id = {str(item.get("id")): item for item in apps if isinstance(item, dict)}
|
||||
result: dict[str, AppOpsInfo] = {}
|
||||
for mapping in mappings if isinstance(mappings, list) else []:
|
||||
@@ -362,7 +363,7 @@ def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
|
||||
app_type=str(app.get("app_type")) if app.get("app_type") else None,
|
||||
reasons=tuple(str(item) for item in (app.get("discovery_reasons") or [])[:3]),
|
||||
)
|
||||
return AppOpsInventory(result, True, f"{len(result)} container mappings loaded")
|
||||
return AppOpsInventory(result, True, f"{len(result)} containerkoppelingen geladen")
|
||||
|
||||
|
||||
def appops_inventory(appops_url: str | None) -> AppOpsInventory:
|
||||
@@ -371,6 +372,38 @@ def appops_inventory(appops_url: str | None) -> AppOpsInventory:
|
||||
return _appops_inventory(appops_url)
|
||||
|
||||
|
||||
_HEALTH_IN_STATUS = re.compile(r"\((healthy|unhealthy|health: starting)\)")
|
||||
_observer_clients: dict[str, object] = {}
|
||||
|
||||
|
||||
def _observer_client(docker_host: str):
|
||||
"""One long-lived socket client per host (no /version + /_ping per cycle)."""
|
||||
|
||||
client = _observer_clients.get(docker_host)
|
||||
if client is None:
|
||||
client = docker.DockerClient(base_url=docker_host, timeout=5, version="auto")
|
||||
_observer_clients[docker_host] = client
|
||||
return client
|
||||
|
||||
|
||||
def _drop_observer_client(docker_host: str) -> None:
|
||||
client = _observer_clients.pop(docker_host, None)
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
with suppress(Exception): # best effort on an already failed client
|
||||
close()
|
||||
|
||||
|
||||
def health_from_status(status_text: str) -> str | None:
|
||||
"""Docker's list summary embeds health: "Up 3 hours (healthy)"."""
|
||||
|
||||
match = _HEALTH_IN_STATUS.search(status_text or "")
|
||||
if not match:
|
||||
return None
|
||||
value = match.group(1)
|
||||
return "starting" if value == "health: starting" else value
|
||||
|
||||
|
||||
def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
|
||||
"""Read bounded Docker runtime/health facts without discovery side effects.
|
||||
|
||||
@@ -378,18 +411,48 @@ def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
|
||||
suggestion persistence, reconciliation and snapshot capture. It is safe
|
||||
to run on the observation cadence because it cannot rewrite topology or
|
||||
operator-owned configuration.
|
||||
|
||||
Socket mode uses a single low-level ``containers(all=True)`` call on a
|
||||
long-lived client: the list summary already carries id, name, state,
|
||||
labels and health. The high-level ``containers.list()`` inspected every
|
||||
container individually (1 + N API calls per minute).
|
||||
"""
|
||||
|
||||
parsed = urlparse(docker_host)
|
||||
if parsed.scheme not in ALLOWED_SCHEMES:
|
||||
raise ValueError("docker_host must use unix://, tcp://, http:// or https://")
|
||||
if docker_provider_mode() == "socket":
|
||||
try:
|
||||
summaries = _observer_client(docker_host).api.containers(all=True)
|
||||
except (DockerException, OSError) as exc:
|
||||
_drop_observer_client(docker_host)
|
||||
raise RuntimeError(f"Docker-waarneming niet beschikbaar: {type(exc).__name__}") from exc
|
||||
observations: list[DockerHealthObservation] = []
|
||||
for item in summaries:
|
||||
names = item.get("Names") or []
|
||||
name = str(names[0] if names else item.get("Id", "unknown")[:12]).lstrip("/")
|
||||
labels = {str(key): str(value) for key, value in (item.get("Labels") or {}).items()}
|
||||
health = health_from_status(str(item.get("Status", "")))
|
||||
observations.append(
|
||||
DockerHealthObservation(
|
||||
container_id=str(item.get("Id", ""))[:12],
|
||||
name=name,
|
||||
stable_source_ref=stable_source_ref(name, labels),
|
||||
state=str(item.get("State", "unknown")),
|
||||
health=health,
|
||||
healthcheck_present=health is not None,
|
||||
)
|
||||
)
|
||||
return observations
|
||||
return _observe_container_health_full(docker_host)
|
||||
|
||||
|
||||
def _observe_container_health_full(docker_host: str) -> list[DockerHealthObservation]:
|
||||
"""Broker mode: the broker client exposes only the high-level listing."""
|
||||
|
||||
client = None
|
||||
try:
|
||||
client = (
|
||||
docker.DockerClient(base_url=docker_host, timeout=5)
|
||||
if docker_provider_mode() == "socket"
|
||||
else docker_client_for_inventory(docker_host)
|
||||
)
|
||||
client = docker_client_for_inventory(docker_host)
|
||||
client.ping()
|
||||
observations: list[DockerHealthObservation] = []
|
||||
for container in client.containers.list(all=True):
|
||||
@@ -414,7 +477,7 @@ def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
|
||||
)
|
||||
return observations
|
||||
except (DockerException, DockerProviderError) as exc:
|
||||
raise RuntimeError(f"Docker observation unavailable: {type(exc).__name__}") from exc
|
||||
raise RuntimeError(f"Docker-waarneming niet beschikbaar: {type(exc).__name__}") from exc
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
@@ -509,7 +572,11 @@ def discover_containers(
|
||||
client = docker.DockerClient(base_url=docker_host, timeout=5) if docker_provider_mode() == "socket" else docker_client_for_inventory(docker_host)
|
||||
try:
|
||||
client.ping()
|
||||
containers = client.containers.list(all=include_stopped)
|
||||
# Always list stopped containers: they still exist. Excluding them here
|
||||
# made every stopped service look "missing" and, after three runs,
|
||||
# retired with monitoring disabled. ``include_stopped`` only decides
|
||||
# whether a stopped container is offered as a candidate below.
|
||||
containers = client.containers.list(all=True)
|
||||
existing_services = existing_services or []
|
||||
appops_inventory = _appops_inventory(appops_url)
|
||||
appops = appops_inventory.by_container
|
||||
@@ -521,6 +588,13 @@ def discover_containers(
|
||||
attrs = container.attrs or {}
|
||||
state = attrs.get("State", {}) or {}
|
||||
config = attrs.get("Config", {}) or {}
|
||||
name = str(getattr(container, "name", "unknown")).lstrip("/")
|
||||
state_name = str(state.get("Status", getattr(container, "status", "unknown")))
|
||||
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
|
||||
stable_ref = stable_source_ref(name, raw_labels)
|
||||
discovered_refs.add(stable_ref)
|
||||
if not include_stopped and state_name != "running":
|
||||
continue
|
||||
network_settings = attrs.get("NetworkSettings", {}) or {}
|
||||
raw_ports = network_settings.get("Ports", {}) or {}
|
||||
ports: list[DockerPort] = []
|
||||
@@ -541,15 +615,10 @@ def discover_containers(
|
||||
|
||||
image_tags = getattr(container.image, "tags", []) or []
|
||||
image_name = image_tags[0] if image_tags else str(config.get("Image", "unknown"))
|
||||
name = str(getattr(container, "name", "unknown")).lstrip("/")
|
||||
health = state.get("Health", {}).get("Status") if isinstance(state.get("Health"), dict) else None
|
||||
state_name = str(state.get("Status", getattr(container, "status", "unknown")))
|
||||
networks = sorted((network_settings.get("Networks", {}) or {}).keys())
|
||||
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
|
||||
labels = _safe_labels(raw_labels)
|
||||
stable_ref = stable_source_ref(name, raw_labels)
|
||||
legacy_ref = _legacy_source_ref(container.id, image_name)
|
||||
discovered_refs.add(stable_ref)
|
||||
matching_source = next(
|
||||
(
|
||||
service
|
||||
@@ -677,6 +746,6 @@ def discover_containers(
|
||||
appops_detail=appops_inventory.detail,
|
||||
)
|
||||
except (DockerException, DockerProviderError) as exc:
|
||||
raise ValueError(f"Docker discovery failed: {exc}") from exc
|
||||
raise ValueError(f"Docker-detectie mislukt: {exc}") from exc
|
||||
finally:
|
||||
client.close()
|
||||
+68
-28
@@ -2,52 +2,90 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from queue import Empty, Full, Queue
|
||||
from threading import Lock
|
||||
from typing import Any
|
||||
|
||||
# Clients reconnect after this delay if the stream drops (SSE ``retry:``).
|
||||
SSE_RETRY_MS = 5_000
|
||||
# A comment line keeps proxies and browsers from timing out a quiet stream.
|
||||
SSE_KEEPALIVE_SECONDS = 15.0
|
||||
|
||||
|
||||
class Subscription:
|
||||
"""One connected SSE client: an asyncio queue bound to its event loop."""
|
||||
|
||||
__slots__ = ("loop", "queue")
|
||||
|
||||
def __init__(self, loop: asyncio.AbstractEventLoop, max_queue_size: int) -> None:
|
||||
self.loop = loop
|
||||
self.queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=max_queue_size)
|
||||
|
||||
def offer(self, payload: dict[str, Any]) -> None:
|
||||
"""Enqueue on the owning loop; drop the oldest wakeup when full."""
|
||||
|
||||
if self.queue.full():
|
||||
try:
|
||||
self.queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
pass
|
||||
try:
|
||||
self.queue.put_nowait(payload)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
|
||||
class EventBus:
|
||||
"""Small bounded fan-out bus used by the SSE endpoint.
|
||||
|
||||
Persistence is handled by SQLite; this bus only wakes currently connected
|
||||
clients. A queue per client keeps one slow browser from blocking probes or
|
||||
other subscribers.
|
||||
clients. Each client owns an ``asyncio.Queue`` so a waiting stream costs no
|
||||
thread: previously every open browser tab parked one default-executor
|
||||
thread in ``queue.get``, which starved ``asyncio.to_thread`` work such as
|
||||
the observation loop. ``publish`` is safe from worker threads and from
|
||||
the event loop; delivery is scheduled with ``call_soon_threadsafe``.
|
||||
"""
|
||||
|
||||
def __init__(self, max_queue_size: int = 100) -> None:
|
||||
self.max_queue_size = max_queue_size
|
||||
self._subscribers: set[Queue[dict[str, Any]]] = set()
|
||||
self._subscribers: set[Subscription] = set()
|
||||
self._lock = Lock()
|
||||
self._revision = 0
|
||||
|
||||
def subscribe(self) -> Queue[dict[str, Any]]:
|
||||
queue: Queue[dict[str, Any]] = Queue(maxsize=self.max_queue_size)
|
||||
with self._lock:
|
||||
self._subscribers.add(queue)
|
||||
return queue
|
||||
@property
|
||||
def revision(self) -> int:
|
||||
"""Monotonic count of published events; read caches compare it."""
|
||||
|
||||
def unsubscribe(self, queue: Queue[dict[str, Any]]) -> None:
|
||||
with self._lock:
|
||||
self._subscribers.discard(queue)
|
||||
return self._revision
|
||||
|
||||
def subscribe(self) -> Subscription:
|
||||
subscription = Subscription(asyncio.get_running_loop(), self.max_queue_size)
|
||||
with self._lock:
|
||||
self._subscribers.add(subscription)
|
||||
return subscription
|
||||
|
||||
def unsubscribe(self, subscription: Subscription) -> None:
|
||||
with self._lock:
|
||||
self._subscribers.discard(subscription)
|
||||
|
||||
@property
|
||||
def subscriber_count(self) -> int:
|
||||
with self._lock:
|
||||
return len(self._subscribers)
|
||||
|
||||
def publish(self, payload: dict[str, Any]) -> None:
|
||||
with self._lock:
|
||||
self._revision += 1
|
||||
subscribers = tuple(self._subscribers)
|
||||
for queue in subscribers:
|
||||
for subscription in subscribers:
|
||||
try:
|
||||
queue.put_nowait(payload)
|
||||
except Full:
|
||||
# Drop the oldest event for a lagging client, then retain the
|
||||
# newest state notification. The durable event journal remains
|
||||
# the source of truth if a client reconnects.
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except Empty:
|
||||
pass
|
||||
try:
|
||||
queue.put_nowait(payload)
|
||||
except Full:
|
||||
pass
|
||||
subscription.loop.call_soon_threadsafe(subscription.offer, payload)
|
||||
except RuntimeError:
|
||||
# The client's loop has closed (shutdown); it can no longer
|
||||
# receive anything, so forget it.
|
||||
self.unsubscribe(subscription)
|
||||
|
||||
|
||||
event_bus = EventBus()
|
||||
|
||||
|
||||
@@ -56,8 +94,10 @@ def sse_payload(payload: dict[str, Any]) -> str:
|
||||
return f"event: {event_type}\ndata: {json.dumps(payload, default=str)}\n\n"
|
||||
|
||||
|
||||
async def next_event(queue: Queue[dict[str, Any]], timeout: float = 15.0) -> dict[str, Any] | None:
|
||||
async def next_event(
|
||||
subscription: Subscription, timeout: float = SSE_KEEPALIVE_SECONDS
|
||||
) -> dict[str, Any] | None:
|
||||
try:
|
||||
return await asyncio.to_thread(queue.get, True, timeout)
|
||||
except Empty:
|
||||
return await asyncio.wait_for(subscription.queue.get(), timeout)
|
||||
except TimeoutError:
|
||||
return None
|
||||
@@ -98,10 +98,10 @@ def resolve_health(
|
||||
else:
|
||||
reason_code = HealthResolutionReason.NO_CURRENT_COMPONENT_EVIDENCE
|
||||
explanation = (
|
||||
"Only application-scoped evidence is current; it is shown for provenance and is not "
|
||||
"used as component health."
|
||||
"Alleen bewijs op applicatieniveau is actueel; het wordt getoond als herkomst en niet "
|
||||
"gebruikt als componentgezondheid."
|
||||
if app_claims
|
||||
else "No current component-scoped health evidence is available."
|
||||
else "Er is geen actueel gezondheidsbewijs op componentniveau."
|
||||
)
|
||||
return HealthResolution(reason_code=reason_code, explanation=explanation, evidence=annotated)
|
||||
|
||||
@@ -118,7 +118,7 @@ def resolve_health(
|
||||
)
|
||||
selected = ordered[0]
|
||||
conflict = len({item.status for item in eligible}) > 1
|
||||
scope_note = " Conflicting current component signals are retained below." if conflict else ""
|
||||
scope_note = " Tegenstrijdige actuele componentsignalen worden hieronder bewaard." if conflict else ""
|
||||
return HealthResolution(
|
||||
status=selected.status,
|
||||
selected_evidence_id=selected.id,
|
||||
@@ -126,8 +126,8 @@ def resolve_health(
|
||||
confidence=selected.confidence,
|
||||
reason_code=HealthResolutionReason.SELECTED_COMPONENT_EVIDENCE,
|
||||
explanation=(
|
||||
f"{selected.provider.value} selected from current component-scoped evidence "
|
||||
f"(confidence {selected.confidence:.2f}).{scope_note}"
|
||||
f"{selected.provider.value} gekozen uit actueel bewijs op componentniveau "
|
||||
f"(zekerheid {selected.confidence:.2f}).{scope_note}"
|
||||
),
|
||||
conflict=conflict,
|
||||
evidence=annotated,
|
||||
|
||||
+27
-38
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
from .models import HealthStatus, Service
|
||||
from .store import Store
|
||||
|
||||
@@ -14,51 +16,38 @@ SEVERITY = {
|
||||
}
|
||||
|
||||
|
||||
def _direct_status(service: Service, store: Store) -> HealthStatus:
|
||||
return store.service_health(service.id)
|
||||
class TopologyHealth(NamedTuple):
|
||||
services: list[Service]
|
||||
direct: dict[str, HealthStatus]
|
||||
effective: dict[str, HealthStatus]
|
||||
dependency: dict[str, HealthStatus]
|
||||
|
||||
|
||||
def _dependency_status(service: Service, store: Store, visiting: set[str]) -> HealthStatus:
|
||||
if not service.depends_on:
|
||||
return HealthStatus.HEALTHY
|
||||
def topology_health(store: Store, *, services: list[Service] | None = None) -> TopologyHealth:
|
||||
"""Resolve direct, dependency and effective health for the whole graph.
|
||||
|
||||
states: list[HealthStatus] = []
|
||||
for dependency_id in service.depends_on:
|
||||
dependency = store.get_service(dependency_id)
|
||||
if dependency is None:
|
||||
states.append(HealthStatus.UNKNOWN)
|
||||
continue
|
||||
states.append(_effective_status(dependency, store, visiting))
|
||||
return max(states, key=lambda status: SEVERITY[status])
|
||||
One bulk evidence read plus one in-memory traversal. The former
|
||||
per-service recursive walk re-queried SQLite for every node on every
|
||||
dependency path and dominated server CPU.
|
||||
"""
|
||||
|
||||
services = store.list_services() if services is None else services
|
||||
direct, _ = store.service_healths([service.id for service in services])
|
||||
effective, dependency = status_maps(services, direct_statuses=direct)
|
||||
return TopologyHealth(services, direct, effective, dependency)
|
||||
|
||||
|
||||
def _effective_status(service: Service, store: Store, visiting: set[str]) -> HealthStatus:
|
||||
if service.id in visiting:
|
||||
return HealthStatus.UNKNOWN
|
||||
next_visiting = set(visiting)
|
||||
next_visiting.add(service.id)
|
||||
own = _direct_status(service, store)
|
||||
if own == HealthStatus.MAINTENANCE:
|
||||
return own
|
||||
dependency = _dependency_status(service, store, next_visiting)
|
||||
if own == HealthStatus.DOWN:
|
||||
return own
|
||||
if dependency == HealthStatus.DOWN:
|
||||
return HealthStatus.DEGRADED
|
||||
if dependency in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} and own in {
|
||||
HealthStatus.HEALTHY,
|
||||
HealthStatus.RECOVERING,
|
||||
}:
|
||||
return HealthStatus.DEGRADED
|
||||
return own
|
||||
def _single(service: Service, store: Store) -> TopologyHealth:
|
||||
services = [item for item in store.list_services() if item.id != service.id]
|
||||
return topology_health(store, services=[*services, service])
|
||||
|
||||
|
||||
def dependency_status(service: Service, store: Store) -> HealthStatus:
|
||||
return _dependency_status(service, store, {service.id})
|
||||
return _single(service, store).dependency.get(service.id, HealthStatus.UNKNOWN)
|
||||
|
||||
|
||||
def effective_status(service: Service, store: Store) -> HealthStatus:
|
||||
return _effective_status(service, store, set())
|
||||
return _single(service, store).effective.get(service.id, HealthStatus.UNKNOWN)
|
||||
|
||||
|
||||
def status_maps(
|
||||
@@ -134,15 +123,15 @@ def validate_dependencies(
|
||||
service_id: str | None = None,
|
||||
) -> None:
|
||||
if len(depends_on) != len(set(depends_on)):
|
||||
raise ValueError("Dependencies must be unique")
|
||||
raise ValueError("Afhankelijkheden moeten uniek zijn")
|
||||
if service_id and service_id in depends_on:
|
||||
raise ValueError("A service cannot depend on itself")
|
||||
raise ValueError("Een service kan niet van zichzelf afhangen")
|
||||
|
||||
services = store.list_services()
|
||||
known_ids = {service.id for service in services}
|
||||
missing = [dependency_id for dependency_id in depends_on if dependency_id not in known_ids]
|
||||
if missing:
|
||||
raise ValueError(f"Unknown dependency ids: {', '.join(missing)}")
|
||||
raise ValueError(f"Onbekende afhankelijkheids-id's: {', '.join(missing)}")
|
||||
|
||||
if service_id is None:
|
||||
return
|
||||
@@ -154,7 +143,7 @@ def validate_dependencies(
|
||||
|
||||
def visit(node: str) -> None:
|
||||
if node in visiting:
|
||||
raise ValueError("Dependency graph contains a cycle")
|
||||
raise ValueError("De afhankelijkheidsgrafiek bevat een kring")
|
||||
if node in visited:
|
||||
return
|
||||
visiting.add(node)
|
||||
|
||||
+29
-15
@@ -1,9 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from .health import dependency_status, effective_status
|
||||
from .health import TopologyHealth, topology_health
|
||||
from .models import (
|
||||
Criticality,
|
||||
HealthStatus,
|
||||
@@ -37,11 +38,12 @@ def affected_service_ids(root_id: str, services: list[Service]) -> list[str]:
|
||||
return sorted(seen)
|
||||
|
||||
|
||||
def incident_overview(store: Store) -> IncidentOverview:
|
||||
services = store.list_services()
|
||||
def incident_overview(store: Store, *, health: TopologyHealth | None = None) -> IncidentOverview:
|
||||
health = health or topology_health(store)
|
||||
services = health.services
|
||||
by_id = {service.id: service for service in services}
|
||||
effective = {service.id: effective_status(service, store) for service in services}
|
||||
dependency = {service.id: dependency_status(service, store) for service in services}
|
||||
effective = health.effective
|
||||
dependency = health.dependency
|
||||
|
||||
down_ids = sorted(service_id for service_id, state in effective.items() if state == HealthStatus.DOWN)
|
||||
degraded_ids = sorted(
|
||||
@@ -62,9 +64,9 @@ def incident_overview(store: Store) -> IncidentOverview:
|
||||
continue
|
||||
affected_ids = affected_service_ids(service.id, services)
|
||||
affected_names = [by_id[item].name for item in affected_ids if item in by_id]
|
||||
reason = "Service is down and no configured dependency is also down"
|
||||
reason = "Service is uitgevallen en geen enkele ingestelde afhankelijkheid is ook uitgevallen"
|
||||
if dependency[service.id] == HealthStatus.UNKNOWN:
|
||||
reason += "; at least one dependency has unknown health"
|
||||
reason += "; minstens één afhankelijkheid heeft een onbekende gezondheid"
|
||||
candidates.append(
|
||||
IncidentCandidate(
|
||||
service_id=service.id,
|
||||
@@ -91,15 +93,27 @@ def _severity(service: Service) -> Criticality:
|
||||
return service.criticality
|
||||
|
||||
|
||||
def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recovered; incident resolved") -> list[IncidentRecord]:
|
||||
# Probes, observation and API handlers reconcile from different threads. Two
|
||||
# concurrent passes could both see "no active incident" for a root and each
|
||||
# open one (the later duplicate cleanup then left two journal entries).
|
||||
_SYNC_LOCK = threading.RLock()
|
||||
|
||||
|
||||
def sync_incidents(store: Store, *, resolution_reason: str = "Hoofdoorzaak hersteld; incident opgelost") -> list[IncidentRecord]:
|
||||
with _SYNC_LOCK:
|
||||
return _sync_incidents_locked(store, resolution_reason=resolution_reason)
|
||||
|
||||
|
||||
def _sync_incidents_locked(store: Store, *, resolution_reason: str) -> list[IncidentRecord]:
|
||||
"""Reconcile the current graph with the durable incident journal.
|
||||
|
||||
A root service has at most one active incident. Repeated probe cycles only
|
||||
update its affected set; a later independent outage gets a new id.
|
||||
"""
|
||||
|
||||
overview = incident_overview(store)
|
||||
services = {service.id: service for service in store.list_services()}
|
||||
health = topology_health(store)
|
||||
overview = incident_overview(store, health=health)
|
||||
services = {service.id: service for service in health.services}
|
||||
now = datetime.now(UTC)
|
||||
active: dict[str, IncidentRecord] = {}
|
||||
for incident in sorted(
|
||||
@@ -115,7 +129,7 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
|
||||
store.append_incident_event(
|
||||
incident.id,
|
||||
"incident.duplicate_resolved",
|
||||
"Duplicate active incident closed; the earliest incident remains active",
|
||||
"Dubbel actief incident gesloten; het oudste incident blijft actief",
|
||||
incident.root_service_id,
|
||||
)
|
||||
continue
|
||||
@@ -133,7 +147,7 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
|
||||
root_service_name=candidate.service_name,
|
||||
started_at=now,
|
||||
detected_at=now,
|
||||
title=f"{candidate.service_name} is down",
|
||||
title=f"{candidate.service_name} is uitgevallen",
|
||||
root_cause_reason=candidate.reason,
|
||||
affected_service_ids=candidate.affected_service_ids,
|
||||
affected_service_names=candidate.affected_service_names,
|
||||
@@ -144,7 +158,7 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
|
||||
store.append_incident_event(
|
||||
incident.id,
|
||||
"incident.opened",
|
||||
f"Incident opened: {candidate.service_name} is the likely root cause",
|
||||
f"Incident geopend: {candidate.service_name} is de waarschijnlijke hoofdoorzaak",
|
||||
candidate.service_id,
|
||||
)
|
||||
active[candidate.service_id] = incident
|
||||
@@ -158,14 +172,14 @@ def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recover
|
||||
}
|
||||
)
|
||||
store.upsert_incident(updated)
|
||||
store.append_incident_event(incident.id, "incident.updated", "Downstream impact recalculated", candidate.service_id)
|
||||
store.append_incident_event(incident.id, "incident.updated", "Impact op afhankelijke services herberekend", candidate.service_id)
|
||||
|
||||
for root_id, incident in list(active.items()):
|
||||
if any(candidate.service_id == root_id for candidate in overview.candidates):
|
||||
continue
|
||||
root = services.get(root_id)
|
||||
if (
|
||||
(root is None or store.service_health(root_id) not in {
|
||||
(root is None or health.direct.get(root_id, HealthStatus.UNKNOWN) not in {
|
||||
HealthStatus.DOWN,
|
||||
HealthStatus.DEGRADED,
|
||||
HealthStatus.RECOVERING,
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Background loops that never die silently, and their readiness record.
|
||||
|
||||
Before v1.6.3 the observation and discovery loops caught only ``OSError``,
|
||||
``RuntimeError`` and ``ValueError``. Any other exception, for example
|
||||
``sqlite3.OperationalError: database is locked``, ended the task without a
|
||||
log line; ServiceMap then looked healthy while it no longer observed
|
||||
anything. ``run_periodic`` logs every failure, retries with bounded
|
||||
backoff and records the last success so ``/ready`` can report a stalled
|
||||
loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from threading import Lock
|
||||
|
||||
logger = logging.getLogger("servicemap.loops")
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoopStatus:
|
||||
name: str
|
||||
interval_seconds: float
|
||||
started_at: float = field(default_factory=time.time)
|
||||
last_success_at: float | None = None
|
||||
last_failure_at: float | None = None
|
||||
last_error: str | None = None
|
||||
consecutive_failures: int = 0
|
||||
paused: bool = False
|
||||
|
||||
def stale_after_seconds(self) -> float:
|
||||
# Three missed cycles, with a floor for very short intervals.
|
||||
return max(3 * self.interval_seconds, 180.0)
|
||||
|
||||
def healthy(self, now: float) -> bool:
|
||||
if self.paused:
|
||||
return True
|
||||
reference = self.last_success_at if self.last_success_at is not None else self.started_at
|
||||
return now - reference <= self.stale_after_seconds()
|
||||
|
||||
def as_dict(self, now: float) -> dict[str, object]:
|
||||
def iso(value: float | None) -> str | None:
|
||||
return None if value is None else time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(value))
|
||||
|
||||
return {
|
||||
"name": self.name,
|
||||
"healthy": self.healthy(now),
|
||||
"interval_seconds": self.interval_seconds,
|
||||
"last_success_at": iso(self.last_success_at),
|
||||
"last_failure_at": iso(self.last_failure_at),
|
||||
"last_error": self.last_error,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"paused": self.paused,
|
||||
}
|
||||
|
||||
|
||||
class LoopRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._loops: dict[str, LoopStatus] = {}
|
||||
self._lock = Lock()
|
||||
|
||||
def register(self, name: str, interval_seconds: float) -> LoopStatus:
|
||||
with self._lock:
|
||||
status = LoopStatus(name=name, interval_seconds=interval_seconds)
|
||||
self._loops[name] = status
|
||||
return status
|
||||
|
||||
def unregister(self, name: str) -> None:
|
||||
with self._lock:
|
||||
self._loops.pop(name, None)
|
||||
|
||||
def snapshot(self) -> list[dict[str, object]]:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
return [status.as_dict(now) for status in self._loops.values()]
|
||||
|
||||
def all_healthy(self) -> bool:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
return all(status.healthy(now) for status in self._loops.values())
|
||||
|
||||
|
||||
loop_registry = LoopRegistry()
|
||||
|
||||
|
||||
def record_success(status: LoopStatus) -> None:
|
||||
status.last_success_at = time.time()
|
||||
status.consecutive_failures = 0
|
||||
status.paused = False
|
||||
|
||||
|
||||
def record_failure(status: LoopStatus, error: BaseException) -> None:
|
||||
status.last_failure_at = time.time()
|
||||
status.consecutive_failures += 1
|
||||
status.last_error = f"{type(error).__name__}: {str(error)[:300]}"
|
||||
|
||||
|
||||
async def run_periodic(
|
||||
name: str,
|
||||
work: Callable[[], Awaitable[object]],
|
||||
*,
|
||||
interval_seconds: float,
|
||||
initial_delay_seconds: float = 0.0,
|
||||
paused: Callable[[], bool] | None = None,
|
||||
registry: LoopRegistry = loop_registry,
|
||||
max_backoff_seconds: float = 300.0,
|
||||
) -> None:
|
||||
"""Run ``work`` every ``interval_seconds`` until cancelled.
|
||||
|
||||
A failure is logged with its traceback and retried after an exponential
|
||||
backoff capped at ``max_backoff_seconds``. ``paused`` lets a loop
|
||||
idle deliberately (for example while the synthetic demo is active)
|
||||
without being reported as stalled.
|
||||
"""
|
||||
|
||||
status = registry.register(name, interval_seconds)
|
||||
try:
|
||||
if initial_delay_seconds > 0:
|
||||
await asyncio.sleep(initial_delay_seconds)
|
||||
while True:
|
||||
delay = interval_seconds
|
||||
try:
|
||||
if paused is not None and paused():
|
||||
status.paused = True
|
||||
else:
|
||||
await work()
|
||||
record_success(status)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
record_failure(status, exc)
|
||||
# interval, 2x, 4x … capped: a persistent fault does not spin,
|
||||
# a long interval still retries within max_backoff_seconds.
|
||||
delay = min(
|
||||
max_backoff_seconds,
|
||||
max(5.0, interval_seconds) * 2 ** min(status.consecutive_failures - 1, 5),
|
||||
)
|
||||
logger.exception(
|
||||
"Background loop %s failed (%s consecutive); retrying in %.0fs",
|
||||
name,
|
||||
status.consecutive_failures,
|
||||
delay,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
finally:
|
||||
registry.unregister(name)
|
||||
+430
-100
@@ -1,26 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Query, Response, status
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
|
||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from .auth import AuthConfig, AuthorizationError, authorize_request
|
||||
from .alerts import AlertManager, AlertSettingsUpdate, AlertSettingsView, AlertTestResult
|
||||
from .auth import AuthConfig, AuthorizationError, admin_token, admin_token_path, authorize_request
|
||||
from .backups import BackupInfo, BackupManager
|
||||
from .config_io import export_document, import_document, parse_document, serialize_document
|
||||
from .demo import DEMO_ROOT_CAUSE_NAME, DEMO_SCENARIO_NAME, build_demo_records
|
||||
from .discovery import DiscoveryManager
|
||||
from .discovery_store import DiscoveryStore
|
||||
from .docker_discovery import discover_containers
|
||||
from .docker_provider import docker_provider_mode
|
||||
from .events import event_bus, next_event, sse_payload
|
||||
from .health import dependency_status, effective_status, status_maps, validate_dependencies
|
||||
from .events import SSE_RETRY_MS, event_bus, next_event, sse_payload
|
||||
from .health import status_maps, topology_health, validate_dependencies
|
||||
from .incidents import incident_overview, sync_incidents
|
||||
from .loops import loop_registry, run_periodic
|
||||
from .models import (
|
||||
AvailabilitySummary,
|
||||
BulkProbeResult,
|
||||
@@ -39,7 +43,10 @@ from .models import (
|
||||
EdgeMutationRequest,
|
||||
EventRecord,
|
||||
HealthResolution,
|
||||
HealthStatus,
|
||||
IncidentNoteRequest,
|
||||
IncidentOverview,
|
||||
IncidentResolveRequest,
|
||||
LogicalRelation,
|
||||
MaintenanceRequest,
|
||||
OverviewSummary,
|
||||
@@ -60,6 +67,14 @@ from .models import (
|
||||
)
|
||||
from .observation import ObservationManager
|
||||
from .probes import probe_many, probe_service
|
||||
from .response_cache import (
|
||||
GENERATED_AT_HEADER,
|
||||
CachedRepresentation,
|
||||
RepresentationCache,
|
||||
etag_matches,
|
||||
next_freshness_boundary,
|
||||
representation,
|
||||
)
|
||||
from .scheduler import scheduled_probe_loop
|
||||
from .semantic import build_semantic_topology
|
||||
from .store import Store
|
||||
@@ -83,6 +98,16 @@ def _env_float(name: str, default: float, minimum: float = 0.1) -> float:
|
||||
return default
|
||||
|
||||
|
||||
logger = logging.getLogger("servicemap")
|
||||
if not logger.handlers:
|
||||
# Uvicorn configures only its own loggers; without a handler our INFO
|
||||
# lines (maintenance results, loop recovery) were never emitted.
|
||||
_handler = logging.StreamHandler()
|
||||
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
|
||||
logger.addHandler(_handler)
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False
|
||||
|
||||
PROBE_INTERVAL = _env_int("SERVICEMAP_PROBE_INTERVAL", 0)
|
||||
PROBE_TIMEOUT = _env_float("SERVICEMAP_PROBE_TIMEOUT", 5.0)
|
||||
PROBE_RETRIES = _env_int("SERVICEMAP_PROBE_RETRIES", 1)
|
||||
@@ -90,8 +115,48 @@ PROBE_CONCURRENCY = _env_int("SERVICEMAP_PROBE_CONCURRENCY", 10, minimum=1)
|
||||
RETENTION_DAYS = _env_int("SERVICEMAP_RETENTION_DAYS", 30, minimum=1)
|
||||
EVENT_RETENTION_DAYS = _env_int("SERVICEMAP_EVENT_RETENTION_DAYS", 90, minimum=1)
|
||||
DISCOVERY_INTERVAL = _env_int("SERVICEMAP_DISCOVERY_INTERVAL_SECONDS", 0, minimum=0)
|
||||
OBSERVATION_INTERVAL = _env_int("SERVICEMAP_OBSERVATION_INTERVAL_SECONDS", 60, minimum=10)
|
||||
OBSERVATION_INTERVAL = _env_int("SERVICEMAP_OBSERVATION_INTERVAL_SECONDS", 60)
|
||||
if OBSERVATION_INTERVAL > 0:
|
||||
OBSERVATION_INTERVAL = max(10, OBSERVATION_INTERVAL)
|
||||
# Superseded health-evidence history; current claims are never pruned. 0 disables.
|
||||
EVIDENCE_RETENTION_DAYS = _env_int("SERVICEMAP_EVIDENCE_RETENTION_DAYS", 7, minimum=0)
|
||||
MAINTENANCE_INTERVAL = _env_int("SERVICEMAP_MAINTENANCE_INTERVAL_SECONDS", 3600, minimum=60)
|
||||
# Discovery runs whose per-container observations are kept.
|
||||
DISCOVERY_OBSERVATION_RUNS = _env_int("SERVICEMAP_DISCOVERY_OBSERVATION_RUNS", 10, minimum=1)
|
||||
# Verified database backups in <data>/backups; 0 hours disables the schedule.
|
||||
BACKUP_INTERVAL_HOURS = _env_int("SERVICEMAP_BACKUP_INTERVAL_HOURS", 24, minimum=0)
|
||||
BACKUP_KEEP = _env_int("SERVICEMAP_BACKUP_KEEP", 14, minimum=1)
|
||||
observation_manager = ObservationManager(store, interval_seconds=OBSERVATION_INTERVAL)
|
||||
semantic_cache = RepresentationCache(max_age_seconds=10.0)
|
||||
_alert_manager: AlertManager | None = None
|
||||
|
||||
|
||||
def backup_manager() -> BackupManager:
|
||||
return BackupManager(store, keep=BACKUP_KEEP)
|
||||
|
||||
|
||||
def alert_manager() -> AlertManager:
|
||||
"""Bound to the current store (tests swap ``store``)."""
|
||||
|
||||
global _alert_manager
|
||||
if _alert_manager is None or _alert_manager.store is not store:
|
||||
_alert_manager = AlertManager(store)
|
||||
return _alert_manager
|
||||
|
||||
|
||||
def run_maintenance() -> dict[str, int]:
|
||||
"""Bounded retention for tables that otherwise grow with uptime."""
|
||||
|
||||
if store.demo_active():
|
||||
return {}
|
||||
pruned = {
|
||||
"health_evidence": store.prune_health_evidence(EVIDENCE_RETENTION_DAYS),
|
||||
"probe_results": store.prune_probes(RETENTION_DAYS, EVENT_RETENTION_DAYS),
|
||||
"discovery_observations": discovery_store.prune_observations(DISCOVERY_OBSERVATION_RUNS),
|
||||
"topology_change_events": discovery_store.prune_change_events(EVENT_RETENTION_DAYS),
|
||||
}
|
||||
store.checkpoint()
|
||||
return pruned
|
||||
|
||||
|
||||
def _require_configured_discovery_targets(
|
||||
@@ -102,16 +167,22 @@ def _require_configured_discovery_targets(
|
||||
configured_docker = os.getenv("SERVICEMAP_DOCKER_HOST", "unix:///var/run/docker.sock")
|
||||
configured_appops = os.getenv("SERVICEMAP_APPOPS_URL", "")
|
||||
if docker_host and docker_host != configured_docker:
|
||||
raise HTTPException(status_code=403, detail="Docker discovery target is not configured")
|
||||
raise HTTPException(status_code=403, detail="Er is geen Docker-detectiedoel ingesteld")
|
||||
if appops_url and appops_url.rstrip("/") != configured_appops.rstrip("/"):
|
||||
raise HTTPException(status_code=403, detail="AppOps discovery target is not configured")
|
||||
raise HTTPException(status_code=403, detail="Er is geen AppOps-detectiedoel ingesteld")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
try:
|
||||
if AuthConfig.from_environment().mode == "token":
|
||||
# Create the token on first start so its location is logged
|
||||
# before anyone needs it.
|
||||
admin_token()
|
||||
logger.info("Editing requires the admin token in %s", admin_token_path())
|
||||
except (AuthorizationError, OSError):
|
||||
logger.exception("ServiceMap admin token is unavailable; edits are refused")
|
||||
task: asyncio.Task[None] | None = None
|
||||
discovery_task: asyncio.Task[None] | None = None
|
||||
observation_task: asyncio.Task[None] | None = None
|
||||
if PROBE_INTERVAL > 0:
|
||||
task = asyncio.create_task(
|
||||
scheduled_probe_loop(
|
||||
@@ -121,37 +192,88 @@ async def lifespan(_: FastAPI):
|
||||
retries=PROBE_RETRIES,
|
||||
concurrency=PROBE_CONCURRENCY,
|
||||
retention_days=RETENTION_DAYS,
|
||||
event_retention_days=EVENT_RETENTION_DAYS,
|
||||
)
|
||||
)
|
||||
background: list[asyncio.Task[None]] = []
|
||||
if DISCOVERY_INTERVAL > 0:
|
||||
async def discovery_loop() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(DISCOVERY_INTERVAL)
|
||||
if store.demo_active():
|
||||
continue
|
||||
try:
|
||||
request = DiscoveryRunRequest()
|
||||
result = await asyncio.to_thread(discovery_manager.run, request)
|
||||
await asyncio.to_thread(
|
||||
discovery_manager.reconcile,
|
||||
DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=False),
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
# Discovery is review-first; a failed background run is
|
||||
# persisted and must not take down the monitoring loop.
|
||||
continue
|
||||
discovery_task = asyncio.create_task(discovery_loop())
|
||||
async def observation_loop() -> None:
|
||||
while True:
|
||||
if not store.demo_active():
|
||||
try:
|
||||
await asyncio.to_thread(observation_manager.refresh)
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
# Provider availability is persisted by the observer; an
|
||||
# unavailable provider must not stop scheduled probing.
|
||||
pass
|
||||
await asyncio.sleep(OBSERVATION_INTERVAL)
|
||||
observation_task = asyncio.create_task(observation_loop())
|
||||
async def discovery_cycle() -> None:
|
||||
result = await asyncio.to_thread(discovery_manager.run, DiscoveryRunRequest())
|
||||
await asyncio.to_thread(
|
||||
discovery_manager.reconcile,
|
||||
DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=False),
|
||||
)
|
||||
|
||||
background.append(asyncio.create_task(run_periodic(
|
||||
"discovery",
|
||||
discovery_cycle,
|
||||
interval_seconds=DISCOVERY_INTERVAL,
|
||||
initial_delay_seconds=DISCOVERY_INTERVAL,
|
||||
paused=store.demo_active,
|
||||
)))
|
||||
|
||||
if OBSERVATION_INTERVAL > 0:
|
||||
async def observation_cycle() -> None:
|
||||
await asyncio.to_thread(observation_manager.refresh)
|
||||
|
||||
background.append(asyncio.create_task(run_periodic(
|
||||
"observation",
|
||||
observation_cycle,
|
||||
interval_seconds=OBSERVATION_INTERVAL,
|
||||
paused=store.demo_active,
|
||||
)))
|
||||
|
||||
async def alert_cycle() -> None:
|
||||
sent = await asyncio.to_thread(alert_manager().run_cycle)
|
||||
if sent:
|
||||
logger.info("Sent %s incident notification(s)", sent)
|
||||
|
||||
background.append(asyncio.create_task(run_periodic(
|
||||
"alerts",
|
||||
alert_cycle,
|
||||
interval_seconds=30,
|
||||
paused=store.demo_active,
|
||||
)))
|
||||
|
||||
if BACKUP_INTERVAL_HOURS > 0:
|
||||
async def backup_cycle() -> None:
|
||||
info = await asyncio.to_thread(backup_manager().create)
|
||||
logger.info("Backup %s written (%s bytes, verified)", info.name, info.size_bytes)
|
||||
|
||||
background.append(asyncio.create_task(run_periodic(
|
||||
"backups",
|
||||
backup_cycle,
|
||||
interval_seconds=BACKUP_INTERVAL_HOURS * 3600,
|
||||
# First backup shortly after start, then on the schedule.
|
||||
initial_delay_seconds=120,
|
||||
paused=store.demo_active,
|
||||
)))
|
||||
|
||||
async def maintenance_window_cycle() -> None:
|
||||
ended = await asyncio.to_thread(store.expire_maintenance_windows)
|
||||
if ended:
|
||||
logger.info("Maintenance windows ended for %s service(s)", len(ended))
|
||||
|
||||
background.append(asyncio.create_task(run_periodic(
|
||||
"maintenance-windows",
|
||||
maintenance_window_cycle,
|
||||
interval_seconds=60,
|
||||
paused=store.demo_active,
|
||||
)))
|
||||
|
||||
async def maintenance_cycle() -> None:
|
||||
pruned = await asyncio.to_thread(run_maintenance)
|
||||
if any(pruned.values()):
|
||||
logger.info("Maintenance pruned %s", pruned)
|
||||
|
||||
# Start shortly after boot so a long-running deployment sheds its backlog
|
||||
# without delaying startup, then repeat on a slow cadence.
|
||||
background.append(asyncio.create_task(run_periodic(
|
||||
"maintenance",
|
||||
maintenance_cycle,
|
||||
interval_seconds=MAINTENANCE_INTERVAL,
|
||||
initial_delay_seconds=30,
|
||||
)))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
@@ -159,38 +281,74 @@ async def lifespan(_: FastAPI):
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
if discovery_task:
|
||||
discovery_task.cancel()
|
||||
for background_task in background:
|
||||
background_task.cancel()
|
||||
for background_task in background:
|
||||
with suppress(asyncio.CancelledError):
|
||||
await discovery_task
|
||||
if observation_task:
|
||||
observation_task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await observation_task
|
||||
await background_task
|
||||
|
||||
|
||||
app = FastAPI(title="ServiceMap API", version="1.6.2", lifespan=lifespan)
|
||||
app = FastAPI(title="ServiceMap API", version="1.6.3", lifespan=lifespan)
|
||||
|
||||
|
||||
# Bounded request bodies: the largest legitimate payload is a config import.
|
||||
MAX_REQUEST_BYTES = 4 * 1024 * 1024
|
||||
|
||||
SECURITY_HEADERS = {
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"X-Frame-Options": "DENY",
|
||||
"Referrer-Policy": "no-referrer",
|
||||
"Cross-Origin-Opener-Policy": "same-origin",
|
||||
# The SPA ships no inline scripts or third-party assets; React Flow and
|
||||
# the layout use inline style attributes.
|
||||
"Content-Security-Policy": (
|
||||
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
|
||||
"img-src 'self' data: blob:; font-src 'self'; connect-src 'self'; "
|
||||
"object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def authorization_boundary(request, call_next):
|
||||
try:
|
||||
authorize_request(request)
|
||||
except HTTPException as exc:
|
||||
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
|
||||
return await call_next(request)
|
||||
declared = request.headers.get("content-length", "")
|
||||
if declared.isdigit() and int(declared) > MAX_REQUEST_BYTES:
|
||||
response = JSONResponse(status_code=413, content={"detail": "De aanvraag is te groot"})
|
||||
else:
|
||||
try:
|
||||
authorize_request(request)
|
||||
except HTTPException as exc:
|
||||
response = JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={"detail": exc.detail},
|
||||
headers=dict(exc.headers or {}),
|
||||
)
|
||||
else:
|
||||
response = await call_next(request)
|
||||
for name, value in SECURITY_HEADERS.items():
|
||||
response.headers.setdefault(name, value)
|
||||
path = request.url.path
|
||||
if path.startswith("/assets/"):
|
||||
# Vite fingerprints asset names, so they never change in place.
|
||||
response.headers.setdefault("Cache-Control", "public, max-age=31536000, immutable")
|
||||
elif not path.startswith("/api/") and path not in {"/health", "/ready"}:
|
||||
# index.html (and other unhashed files) must be revalidated, or a
|
||||
# browser keeps running the previous release after a deploy.
|
||||
response.headers.setdefault("Cache-Control", "no-cache")
|
||||
return response
|
||||
|
||||
|
||||
def _snapshot(service: Service) -> ServiceSnapshot:
|
||||
services = store.list_services()
|
||||
services = [item for item in store.list_services() if item.id != service.id]
|
||||
health = topology_health(store, services=[*services, service])
|
||||
dependents = [item.name for item in services if service.id in item.depends_on]
|
||||
return ServiceSnapshot(
|
||||
service=service,
|
||||
latest_probe=store.latest_probe(service.id),
|
||||
availability_24h=store.availability_summary(service.id, 24),
|
||||
dependency_status=dependency_status(service, store),
|
||||
effective_status=effective_status(service, store),
|
||||
direct_status=store.service_health(service.id),
|
||||
dependency_status=health.dependency.get(service.id, HealthStatus.UNKNOWN),
|
||||
effective_status=health.effective.get(service.id, HealthStatus.UNKNOWN),
|
||||
direct_status=health.direct.get(service.id, HealthStatus.UNKNOWN),
|
||||
flapping=service.flapping,
|
||||
flap_count=service.flap_count,
|
||||
next_probe_at=store.next_probe_at(service, PROBE_INTERVAL),
|
||||
@@ -199,6 +357,7 @@ def _snapshot(service: Service) -> ServiceSnapshot:
|
||||
evidence=discovery_store.evidence_for_service(service.id),
|
||||
health_evidence=store.health_evidence_for_service(service.id),
|
||||
health_resolution=store.health_resolution(service.id),
|
||||
maintenance_until=store.maintenance_windows().get(service.id),
|
||||
)
|
||||
|
||||
|
||||
@@ -209,9 +368,34 @@ def _name_conflict(name: str, *, exclude_id: str | None = None) -> bool:
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
"""Liveness: the process answers. Used by the container healthcheck."""
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/ready")
|
||||
def ready() -> JSONResponse:
|
||||
"""Readiness: the database takes writes and no background loop stalled.
|
||||
|
||||
A loop is stalled when it has not completed a cycle for three intervals
|
||||
(at least three minutes); its last error is included for diagnosis.
|
||||
"""
|
||||
|
||||
checks: dict[str, object] = {"loops": loop_registry.snapshot()}
|
||||
database_ok = True
|
||||
try:
|
||||
store.ready_check()
|
||||
except sqlite3.Error as exc:
|
||||
database_ok = False
|
||||
checks["database_error"] = f"{type(exc).__name__}: {exc}"
|
||||
checks["database"] = "ok" if database_ok else "unavailable"
|
||||
ok = database_ok and loop_registry.all_healthy()
|
||||
return JSONResponse(
|
||||
status_code=status.HTTP_200_OK if ok else status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
content={"status": "ready" if ok else "not_ready", **checks},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/meta")
|
||||
def meta() -> dict[str, str | int | float | bool]:
|
||||
try:
|
||||
@@ -220,7 +404,7 @@ def meta() -> dict[str, str | int | float | bool]:
|
||||
auth_mode = "misconfigured"
|
||||
return {
|
||||
"name": "ServiceMap",
|
||||
"version": "1.6.2",
|
||||
"version": "1.6.3",
|
||||
"stage": "v1",
|
||||
"scheduled_probing": PROBE_INTERVAL > 0,
|
||||
"probe_interval_seconds": PROBE_INTERVAL,
|
||||
@@ -237,9 +421,73 @@ def meta() -> dict[str, str | int | float | bool]:
|
||||
"docker_provider": docker_provider_mode(),
|
||||
"appops_configured": bool(os.getenv("SERVICEMAP_APPOPS_URL")),
|
||||
"auth_mode": auth_mode,
|
||||
# What a browser needs before it may change anything.
|
||||
"write_auth": {"token": "admin_token", "oidc": "oidc"}.get(auth_mode, "none"),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/settings/alerts", response_model=AlertSettingsView)
|
||||
def get_alert_settings() -> AlertSettingsView:
|
||||
return alert_manager().view()
|
||||
|
||||
|
||||
@app.put("/api/v1/settings/alerts", response_model=AlertSettingsView)
|
||||
def put_alert_settings(payload: AlertSettingsUpdate) -> AlertSettingsView:
|
||||
try:
|
||||
return alert_manager().update(payload)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.post("/api/v1/settings/alerts/test", response_model=AlertTestResult)
|
||||
def test_alert() -> AlertTestResult:
|
||||
return alert_manager().send_test()
|
||||
|
||||
|
||||
@app.get("/api/v1/backups", response_model=list[BackupInfo])
|
||||
def list_backups() -> list[BackupInfo]:
|
||||
return backup_manager().list()
|
||||
|
||||
|
||||
@app.post("/api/v1/backups", response_model=BackupInfo, status_code=status.HTTP_201_CREATED)
|
||||
def create_backup() -> BackupInfo:
|
||||
try:
|
||||
return backup_manager().create()
|
||||
except (ValueError, RuntimeError, sqlite3.Error) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/api/v1/backups/{name}")
|
||||
def download_backup(name: str) -> FileResponse:
|
||||
try:
|
||||
path = backup_manager().path_for(name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
return FileResponse(path, media_type="application/vnd.sqlite3", filename=name)
|
||||
|
||||
|
||||
@app.post("/api/v1/backups/{name}/restore", response_model=BackupInfo)
|
||||
def restore_backup(name: str) -> BackupInfo:
|
||||
"""Restore ``name`` into the live database; returns the safety copy taken first."""
|
||||
|
||||
try:
|
||||
safety = backup_manager().restore(name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except (RuntimeError, sqlite3.Error) as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
semantic_cache.clear()
|
||||
event_bus.publish({"event_type": "topology.changed", "message": f"Database teruggezet vanaf {name}"})
|
||||
return safety
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/verify")
|
||||
def verify_write_access() -> dict[str, bool]:
|
||||
"""Succeeds only for a caller allowed to mutate (checked by the middleware)."""
|
||||
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def _service_snapshots(
|
||||
*, include_hidden: bool = False, include_context: bool = True
|
||||
) -> list[ServiceSnapshot]:
|
||||
@@ -268,6 +516,10 @@ def _service_snapshots(
|
||||
service.id: [item.name for item in services if service.id in item.depends_on]
|
||||
for service in services
|
||||
}
|
||||
visible_ids = [service.id for service in services]
|
||||
windows = store.maintenance_windows()
|
||||
sources_by_service = discovery_store.sources_for_services(visible_ids) if include_context else {}
|
||||
relations_by_service = discovery_store.evidence_for_services(visible_ids) if include_context else {}
|
||||
return [
|
||||
ServiceSnapshot(
|
||||
service=service,
|
||||
@@ -284,10 +536,11 @@ def _service_snapshots(
|
||||
else None
|
||||
),
|
||||
dependents=dependent_names.get(service.id, []),
|
||||
sources=discovery_store.sources_for_service(service.id) if include_context else [],
|
||||
evidence=discovery_store.evidence_for_service(service.id) if include_context else [],
|
||||
sources=sources_by_service.get(service.id, []),
|
||||
evidence=relations_by_service.get(service.id, []),
|
||||
health_evidence=evidence_by_service.get(service.id, []),
|
||||
health_resolution=resolutions.get(service.id, HealthResolution()),
|
||||
maintenance_until=windows.get(service.id),
|
||||
)
|
||||
for service in services
|
||||
]
|
||||
@@ -301,21 +554,21 @@ def list_services(include_hidden: bool = Query(default=False)) -> list[ServiceSn
|
||||
@app.post("/api/v1/services", response_model=Service, status_code=status.HTTP_201_CREATED)
|
||||
def create_service(payload: ServiceCreate) -> Service:
|
||||
if _name_conflict(payload.name):
|
||||
raise HTTPException(status_code=409, detail="Service name already exists")
|
||||
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam")
|
||||
try:
|
||||
validate_dependencies(payload.depends_on, store)
|
||||
return store.create_service(payload)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise HTTPException(status_code=409, detail="Service name already exists") from exc
|
||||
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam") from exc
|
||||
|
||||
|
||||
@app.get("/api/v1/services/{service_id}", response_model=ServiceSnapshot)
|
||||
def get_service(service_id: str) -> ServiceSnapshot:
|
||||
service = store.get_service(service_id)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
return _snapshot(service)
|
||||
|
||||
|
||||
@@ -323,18 +576,18 @@ def get_service(service_id: str) -> ServiceSnapshot:
|
||||
def update_service(service_id: str, payload: ServiceUpdate) -> Service:
|
||||
before = store.get_service(service_id)
|
||||
if before is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
if _name_conflict(payload.name, exclude_id=service_id):
|
||||
raise HTTPException(status_code=409, detail="Service name already exists")
|
||||
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam")
|
||||
try:
|
||||
validate_dependencies(payload.depends_on, store, service_id=service_id)
|
||||
updated = store.update_service(service_id, payload)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise HTTPException(status_code=409, detail="Service name already exists") from exc
|
||||
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam") from exc
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
if before.source_type:
|
||||
discovery_manager.record_service_override(before, updated)
|
||||
return updated
|
||||
@@ -343,7 +596,7 @@ def update_service(service_id: str, payload: ServiceUpdate) -> Service:
|
||||
@app.delete("/api/v1/services/{service_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
def delete_service(service_id: str) -> None:
|
||||
if not store.delete_service(service_id):
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
|
||||
|
||||
@app.post("/api/v1/services/{service_id}/probe", response_model=ProbeResult)
|
||||
@@ -353,10 +606,10 @@ async def run_probe(
|
||||
retries: int | None = Query(default=None, ge=0, le=5),
|
||||
) -> ProbeResult:
|
||||
if store.demo_active():
|
||||
raise HTTPException(status_code=409, detail="Explore demo uses synthetic probe data; live probing is paused")
|
||||
raise HTTPException(status_code=409, detail="De demo gebruikt synthetische controlegegevens; live controles zijn gepauzeerd")
|
||||
service = store.get_service(service_id)
|
||||
if service is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
result = await probe_service(
|
||||
service,
|
||||
timeout_seconds=timeout_seconds,
|
||||
@@ -394,21 +647,21 @@ async def run_all_probes(
|
||||
@app.get("/api/v1/services/{service_id}/history", response_model=list[ProbeResult])
|
||||
def probe_history(service_id: str, limit: int = Query(default=50, ge=1, le=500)) -> list[ProbeResult]:
|
||||
if store.get_service(service_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
return store.probe_history(service_id, limit)
|
||||
|
||||
|
||||
@app.get("/api/v1/services/{service_id}/availability", response_model=AvailabilitySummary)
|
||||
def availability(service_id: str, hours: int = Query(default=24, ge=1, le=24 * 90)) -> AvailabilitySummary:
|
||||
if store.get_service(service_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
return store.availability_summary(service_id, hours)
|
||||
|
||||
|
||||
@app.get("/api/v1/services/{service_id}/evidence", response_model=HealthResolution)
|
||||
def service_evidence(service_id: str) -> HealthResolution:
|
||||
if store.get_service(service_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
return store.health_resolution(service_id)
|
||||
|
||||
|
||||
@@ -419,9 +672,12 @@ def providers() -> list[ProviderStatus]:
|
||||
|
||||
@app.post("/api/v1/services/{service_id}/maintenance", response_model=Service)
|
||||
def maintenance(service_id: str, payload: MaintenanceRequest) -> Service:
|
||||
updated = store.set_maintenance(service_id, payload.enabled, payload.reason)
|
||||
try:
|
||||
updated = store.set_maintenance(service_id, payload.enabled, payload.reason, payload.until)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="Service not found")
|
||||
raise HTTPException(status_code=404, detail="Service niet gevonden")
|
||||
return updated
|
||||
|
||||
|
||||
@@ -437,7 +693,7 @@ def incidents() -> IncidentOverview:
|
||||
def get_incident(incident_id: str) -> dict[str, object]:
|
||||
incident = store.get_incident(incident_id)
|
||||
if incident is None:
|
||||
raise HTTPException(status_code=404, detail="Incident not found")
|
||||
raise HTTPException(status_code=404, detail="Incident niet gevonden")
|
||||
return {"incident": incident.model_dump(mode="json"), "timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)]}
|
||||
|
||||
|
||||
@@ -448,30 +704,60 @@ def acknowledge_incident(incident_id: str) -> dict[str, object]:
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
if incident is None:
|
||||
raise HTTPException(status_code=404, detail="Incident not found")
|
||||
raise HTTPException(status_code=404, detail="Incident niet gevonden")
|
||||
return {"incident": incident.model_dump(mode="json"), "timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)]}
|
||||
|
||||
|
||||
def _incident_payload(incident_id: str) -> dict[str, object]:
|
||||
incident = store.get_incident(incident_id)
|
||||
if incident is None:
|
||||
raise HTTPException(status_code=404, detail="Incident niet gevonden")
|
||||
return {
|
||||
"incident": incident.model_dump(mode="json"),
|
||||
"timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)],
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/v1/incidents/{incident_id}/resolve")
|
||||
def resolve_incident(incident_id: str, payload: IncidentResolveRequest | None = None) -> dict[str, object]:
|
||||
if store.resolve_incident(incident_id, payload.note if payload else None) is None:
|
||||
raise HTTPException(status_code=404, detail="Incident niet gevonden")
|
||||
return _incident_payload(incident_id)
|
||||
|
||||
|
||||
@app.post("/api/v1/incidents/{incident_id}/notes")
|
||||
def add_incident_note(incident_id: str, payload: IncidentNoteRequest) -> dict[str, object]:
|
||||
if store.add_incident_note(incident_id, payload.text.strip()) is None:
|
||||
raise HTTPException(status_code=404, detail="Incident niet gevonden")
|
||||
return _incident_payload(incident_id)
|
||||
|
||||
|
||||
@app.get("/api/v1/events", response_model=list[EventRecord])
|
||||
def events(limit: int = Query(default=50, ge=1, le=500), service_id: str | None = None) -> list[EventRecord]:
|
||||
return store.list_events(limit=limit, service_id=service_id)
|
||||
def events(
|
||||
limit: int = Query(default=50, ge=1, le=500),
|
||||
service_id: str | None = None,
|
||||
event_type: str | None = Query(default=None, max_length=80),
|
||||
before: datetime | None = None,
|
||||
) -> list[EventRecord]:
|
||||
return store.list_events(limit=limit, service_id=service_id, event_type=event_type, before=before)
|
||||
|
||||
|
||||
@app.get("/api/v1/events/stream")
|
||||
async def event_stream() -> StreamingResponse:
|
||||
queue = event_bus.subscribe()
|
||||
subscription = event_bus.subscribe()
|
||||
|
||||
async def stream():
|
||||
try:
|
||||
yield ": connected\n\n"
|
||||
# ``retry`` tells EventSource how long to wait before reconnecting.
|
||||
yield f"retry: {SSE_RETRY_MS}\n: connected\n\n"
|
||||
while True:
|
||||
payload = await next_event(queue)
|
||||
payload = await next_event(subscription)
|
||||
if payload is None:
|
||||
yield ": keepalive\n\n"
|
||||
else:
|
||||
yield sse_payload(payload)
|
||||
finally:
|
||||
event_bus.unsubscribe(queue)
|
||||
event_bus.unsubscribe(subscription)
|
||||
|
||||
return StreamingResponse(
|
||||
stream(),
|
||||
@@ -483,7 +769,8 @@ async def event_stream() -> StreamingResponse:
|
||||
@app.get("/api/v1/overview", response_model=OverviewSummary)
|
||||
def overview() -> OverviewSummary:
|
||||
sync_incidents(store)
|
||||
snapshots = _service_snapshots(include_hidden=False)
|
||||
# Only status and coverage are counted; discovery context is not needed.
|
||||
snapshots = _service_snapshots(include_hidden=False, include_context=False)
|
||||
overview_state = incident_overview(store)
|
||||
counts: dict[str, int] = {}
|
||||
coverage: dict[str, int] = {}
|
||||
@@ -521,14 +808,20 @@ def import_config(payload: ConfigImportRequest) -> ConfigImportResult:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
|
||||
def _demo_enabled() -> bool:
|
||||
return os.getenv("SERVICEMAP_DEMO_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _demo_state() -> DemoState:
|
||||
enabled = _demo_enabled()
|
||||
if not store.demo_active():
|
||||
return DemoState(active=False)
|
||||
return DemoState(active=False, enabled=enabled)
|
||||
sync_incidents(store)
|
||||
overview_state = incident_overview(store)
|
||||
candidate = overview_state.candidates[0] if overview_state.candidates else None
|
||||
return DemoState(
|
||||
active=True,
|
||||
enabled=enabled,
|
||||
synthetic=True,
|
||||
scenario_name=DEMO_SCENARIO_NAME,
|
||||
service_count=len(store.list_services()),
|
||||
@@ -557,15 +850,10 @@ def demo_state() -> DemoState:
|
||||
|
||||
|
||||
@app.post("/api/v1/demo", response_model=DemoMutationResult)
|
||||
def load_demo() -> DemoMutationResult:
|
||||
services, probes = build_demo_records()
|
||||
result = store.activate_demo(services, probes)
|
||||
sync_incidents(store)
|
||||
return _demo_result(result)
|
||||
|
||||
|
||||
@app.post("/api/v1/demo/reset", response_model=DemoMutationResult)
|
||||
def reset_demo() -> DemoMutationResult:
|
||||
def load_demo() -> DemoMutationResult:
|
||||
if not _demo_enabled():
|
||||
raise HTTPException(status_code=409, detail="Start de demo in een aparte ServiceMap-instantie")
|
||||
services, probes = build_demo_records()
|
||||
result = store.activate_demo(services, probes)
|
||||
sync_incidents(store)
|
||||
@@ -578,7 +866,7 @@ async def exit_demo() -> DemoMutationResult:
|
||||
# Refresh live provider evidence immediately after restoring the saved
|
||||
# production topology. Test stores intentionally do not inherit the
|
||||
# process-global observer, but production always does.
|
||||
if restored and observation_manager.store is store:
|
||||
if restored and OBSERVATION_INTERVAL > 0 and observation_manager.store is store:
|
||||
await asyncio.to_thread(observation_manager.refresh)
|
||||
return _demo_result(restored=restored)
|
||||
|
||||
@@ -586,7 +874,7 @@ async def exit_demo() -> DemoMutationResult:
|
||||
@app.post("/api/v1/discovery/run", response_model=DiscoveryResult)
|
||||
def run_discovery(payload: DiscoveryRunRequest | None = None) -> DiscoveryResult:
|
||||
if store.demo_active():
|
||||
raise HTTPException(status_code=409, detail="Exit Explore demo before discovering production infrastructure")
|
||||
raise HTTPException(status_code=409, detail="Sluit de demo af voor je de productie-infrastructuur detecteert")
|
||||
request = payload or DiscoveryRunRequest()
|
||||
_require_configured_discovery_targets(request.docker_host, request.appops_url)
|
||||
try:
|
||||
@@ -613,7 +901,7 @@ def discovery_suggestions(
|
||||
try:
|
||||
suggestion_status = SuggestionStatus(status_filter) if status_filter else None
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail="Unknown suggestion status") from exc
|
||||
raise HTTPException(status_code=422, detail="Onbekende status van het voorstel") from exc
|
||||
return discovery_store.list_suggestions(status=suggestion_status)
|
||||
|
||||
|
||||
@@ -636,7 +924,7 @@ def reject_discovery_suggestion(suggestion_id: str) -> TopologySuggestion:
|
||||
@app.post("/api/v1/discovery/reconcile", response_model=ReconcileResult)
|
||||
def reconcile_discovery(payload: DiscoveryReconcileRequest | None = None) -> ReconcileResult:
|
||||
if store.demo_active():
|
||||
raise HTTPException(status_code=409, detail="Exit Explore demo before reconciling production infrastructure")
|
||||
raise HTTPException(status_code=409, detail="Sluit de demo af voor je de productie-infrastructuur verwerkt")
|
||||
try:
|
||||
return discovery_manager.reconcile(payload or DiscoveryReconcileRequest())
|
||||
except ValueError as exc:
|
||||
@@ -649,7 +937,7 @@ def repair_discovered_probes(
|
||||
run_id: str | None = Query(default=None),
|
||||
) -> ProbeRepairReport:
|
||||
if store.demo_active():
|
||||
raise HTTPException(status_code=409, detail="Exit Explore demo before repairing production probe endpoints")
|
||||
raise HTTPException(status_code=409, detail="Sluit de demo af voor je controle-endpoints in productie herstelt")
|
||||
try:
|
||||
return discovery_manager.repair_probe_reachability(run_id=run_id, dry_run=dry_run)
|
||||
except ValueError as exc:
|
||||
@@ -686,8 +974,14 @@ def topology_diff(
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/api/v1/topology/semantic", response_model=SemanticTopology)
|
||||
def semantic_topology(include_hidden: bool = Query(default=False)) -> SemanticTopology:
|
||||
def _semantic_representation(include_hidden: bool) -> CachedRepresentation:
|
||||
revision = event_bus.revision
|
||||
# The store identity keeps a swapped store (tests, demo restore paths)
|
||||
# from ever reading another database's cached topology.
|
||||
cache_key = (id(store), include_hidden)
|
||||
cached = semantic_cache.get(cache_key, revision)
|
||||
if cached is not None:
|
||||
return cached
|
||||
snapshots = _service_snapshots(include_hidden=include_hidden, include_context=False)
|
||||
incident_state = incident_overview(store)
|
||||
root_ids = {candidate.service_id for candidate in incident_state.candidates}
|
||||
@@ -696,12 +990,48 @@ def semantic_topology(include_hidden: bool = Query(default=False)) -> SemanticTo
|
||||
for candidate in incident_state.candidates
|
||||
for service_id in candidate.affected_service_ids
|
||||
}
|
||||
return build_semantic_topology(
|
||||
topology = build_semantic_topology(
|
||||
snapshots,
|
||||
discovery_store.list_logical_relations(),
|
||||
root_service_ids=root_ids,
|
||||
affected_service_ids=affected_ids,
|
||||
)
|
||||
value = representation(topology)
|
||||
boundary = next_freshness_boundary(
|
||||
[item for snapshot in snapshots for item in snapshot.health_evidence],
|
||||
topology.generated_at,
|
||||
)
|
||||
semantic_cache.put(
|
||||
cache_key,
|
||||
revision,
|
||||
value,
|
||||
valid_for_seconds=(
|
||||
(boundary - topology.generated_at).total_seconds() if boundary is not None else None
|
||||
),
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
@app.get("/api/v1/topology/semantic", response_model=SemanticTopology)
|
||||
def semantic_topology(
|
||||
request: Request, include_hidden: bool = Query(default=False)
|
||||
) -> Response:
|
||||
"""Semantic topology with a content ETag and conditional GET support.
|
||||
|
||||
Clients that send ``If-None-Match`` receive ``304`` without a body while
|
||||
the topology is unchanged; ``X-ServiceMap-Generated-At`` then states when
|
||||
the server last verified it. Browsers revalidate automatically.
|
||||
"""
|
||||
|
||||
value = _semantic_representation(include_hidden)
|
||||
headers = {
|
||||
"ETag": value.etag,
|
||||
"Cache-Control": "no-cache",
|
||||
GENERATED_AT_HEADER: value.generated_at,
|
||||
}
|
||||
if etag_matches(request.headers.get("if-none-match"), value.etag):
|
||||
return Response(status_code=status.HTTP_304_NOT_MODIFIED, headers=headers)
|
||||
return Response(content=value.body, media_type="application/json", headers=headers)
|
||||
|
||||
|
||||
@app.post("/api/v1/topology/edges", response_model=RelationEvidence, status_code=status.HTTP_201_CREATED)
|
||||
|
||||
+28
-14
@@ -190,8 +190,8 @@ class RelationType(StrEnum):
|
||||
class ServiceCreate(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=120)
|
||||
kind: ServiceKind
|
||||
url: str | None = None
|
||||
host: str | None = Field(default=None, validation_alias=AliasChoices("host", "hostname"))
|
||||
url: str | None = Field(default=None, max_length=2048)
|
||||
host: str | None = Field(default=None, max_length=253, validation_alias=AliasChoices("host", "hostname"))
|
||||
port: int | None = Field(default=None, ge=1, le=65535)
|
||||
depends_on: list[str] = Field(default_factory=list)
|
||||
description: str | None = Field(default=None, max_length=500)
|
||||
@@ -243,27 +243,27 @@ class ServiceCreate(BaseModel):
|
||||
if not self.monitoring_enabled and not self.url and not self.host and self.port is None:
|
||||
return self
|
||||
if self.kind == ServiceKind.HTTP and not self.url:
|
||||
raise ValueError("HTTP services require url")
|
||||
raise ValueError("HTTP-services vragen een url")
|
||||
if self.kind == ServiceKind.HTTP:
|
||||
parsed = urlparse(self.url or "")
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError("HTTP services require an absolute http(s) url")
|
||||
raise ValueError("HTTP-services vragen een absolute http(s)-url")
|
||||
if self.host or self.port:
|
||||
raise ValueError("HTTP services must use url instead of host/port")
|
||||
raise ValueError("HTTP-services gebruiken url in plaats van host/poort")
|
||||
if self.kind in {ServiceKind.TCP, ServiceKind.TLS} and (not self.host or self.port is None):
|
||||
raise ValueError(f"{self.kind.value.upper()} services require host and port")
|
||||
raise ValueError(f"{self.kind.value.upper()} services vragen een host en poort")
|
||||
if self.kind == ServiceKind.DNS and not self.host:
|
||||
raise ValueError("DNS services require hostname in host")
|
||||
raise ValueError("DNS-services vragen een hostnaam in host")
|
||||
if self.kind in {ServiceKind.TCP, ServiceKind.TLS, ServiceKind.DNS} and self.url:
|
||||
raise ValueError(f"{self.kind.value.upper()} services must use host/port instead of url")
|
||||
raise ValueError(f"{self.kind.value.upper()} services gebruiken host/poort in plaats van url")
|
||||
if self.kind == ServiceKind.DNS and self.port is not None:
|
||||
raise ValueError("DNS services use a hostname without a port")
|
||||
raise ValueError("DNS-services gebruiken een hostnaam zonder poort")
|
||||
if (
|
||||
self.expected_status_min is not None
|
||||
and self.expected_status_max is not None
|
||||
and self.expected_status_min > self.expected_status_max
|
||||
):
|
||||
raise ValueError("expected_status_min cannot exceed expected_status_max")
|
||||
raise ValueError("expected_status_min mag niet groter zijn dan expected_status_max")
|
||||
return self
|
||||
|
||||
|
||||
@@ -327,7 +327,7 @@ class HealthResolution(BaseModel):
|
||||
selected_provider: EvidenceProvider | None = None
|
||||
confidence: float | None = Field(default=None, ge=0, le=1)
|
||||
reason_code: HealthResolutionReason = HealthResolutionReason.NO_CURRENT_COMPONENT_EVIDENCE
|
||||
explanation: str = "No current component-scoped health evidence is available."
|
||||
explanation: str = "Er is geen actueel gezondheidsbewijs op componentniveau."
|
||||
conflict: bool = False
|
||||
evidence: list[HealthEvidence] = Field(default_factory=list)
|
||||
|
||||
@@ -337,7 +337,7 @@ class ApplicationHealthResolution(BaseModel):
|
||||
|
||||
status: HealthStatus = HealthStatus.UNKNOWN
|
||||
confidence: float | None = Field(default=None, ge=0, le=1)
|
||||
explanation: str = "No current application or component health evidence is available."
|
||||
explanation: str = "Er is geen actueel gezondheidsbewijs voor applicatie of componenten."
|
||||
reason_code: HealthResolutionReason = HealthResolutionReason.NO_APPLICATION_EVIDENCE
|
||||
selected_evidence_id: str | None = None
|
||||
selected_provider: EvidenceProvider | None = None
|
||||
@@ -372,6 +372,8 @@ class ServiceSnapshot(BaseModel):
|
||||
evidence: list[RelationEvidence] = Field(default_factory=list)
|
||||
health_evidence: list[HealthEvidence] = Field(default_factory=list)
|
||||
health_resolution: HealthResolution = Field(default_factory=HealthResolution)
|
||||
# End of a scheduled maintenance window, if one is set.
|
||||
maintenance_until: datetime | None = None
|
||||
|
||||
|
||||
class AvailabilitySummary(BaseModel):
|
||||
@@ -658,7 +660,7 @@ class EdgeMutationRequest(BaseModel):
|
||||
target_service_id: str
|
||||
relation_type: RelationType = RelationType.DEPENDS_ON
|
||||
lock: bool = True
|
||||
explanation: str = Field(default="Manual topology relationship", max_length=500)
|
||||
explanation: str = Field(default="Handmatige topologierelatie", max_length=500)
|
||||
|
||||
|
||||
class EdgeActionRequest(BaseModel):
|
||||
@@ -678,7 +680,8 @@ class ConfigDocument(BaseModel):
|
||||
|
||||
|
||||
class ConfigImportRequest(BaseModel):
|
||||
content: str
|
||||
# About 2 MB of YAML/JSON is far beyond any real topology.
|
||||
content: str = Field(max_length=2_000_000)
|
||||
format: str = Field(pattern="^(yaml|json)$")
|
||||
mode: str = Field(default="merge", pattern="^(merge|replace)$")
|
||||
|
||||
@@ -691,6 +694,7 @@ class ConfigImportResult(BaseModel):
|
||||
|
||||
class DemoState(BaseModel):
|
||||
active: bool
|
||||
enabled: bool = False
|
||||
synthetic: bool = False
|
||||
scenario_name: str | None = None
|
||||
service_count: int = 0
|
||||
@@ -860,3 +864,13 @@ class OverviewSummary(BaseModel):
|
||||
class MaintenanceRequest(BaseModel):
|
||||
enabled: bool
|
||||
reason: str | None = Field(default=None, max_length=240)
|
||||
# Optional end of the window; maintenance ends automatically afterwards.
|
||||
until: datetime | None = None
|
||||
|
||||
|
||||
class IncidentNoteRequest(BaseModel):
|
||||
text: str = Field(min_length=1, max_length=2000)
|
||||
|
||||
|
||||
class IncidentResolveRequest(BaseModel):
|
||||
note: str | None = Field(default=None, max_length=2000)
|
||||
@@ -111,14 +111,14 @@ class ObservationManager:
|
||||
and not docker_observations
|
||||
and any(ref.startswith("docker:") for ref in services_by_ref)
|
||||
):
|
||||
raise ValueError("Docker broker returned no containers for configured services")
|
||||
raise ValueError("De Docker-broker gaf geen containers terug voor de ingestelde services")
|
||||
self.store.set_provider_status(
|
||||
ProviderStatus(
|
||||
provider=EvidenceProvider.DOCKER,
|
||||
status=ProviderHealth.HEALTHY,
|
||||
observed_at=now,
|
||||
valid_until=valid_until,
|
||||
detail=f"{len(docker_observations)} containers observed",
|
||||
detail=f"{len(docker_observations)} containers waargenomen",
|
||||
metadata={"container_count": len(docker_observations)},
|
||||
)
|
||||
)
|
||||
@@ -147,7 +147,7 @@ class ObservationManager:
|
||||
observed_at=now,
|
||||
valid_until=valid_until,
|
||||
received_at=now,
|
||||
summary="Docker healthcheck" if observation.healthcheck_present else "Docker runtime state",
|
||||
summary="Docker-healthcheck" if observation.healthcheck_present else "Docker-runtimestatus",
|
||||
metadata={
|
||||
"container_id": observation.container_id,
|
||||
"state": observation.state,
|
||||
@@ -190,7 +190,7 @@ class ObservationManager:
|
||||
observed_at=now,
|
||||
valid_until=valid_until,
|
||||
received_at=now,
|
||||
summary="AppOps application health; never used as component direct health",
|
||||
summary="Applicatiegezondheid via AppOps; nooit gebruikt als directe componentgezondheid",
|
||||
metadata={
|
||||
"app_id": app_id,
|
||||
"category": info.category,
|
||||
|
||||
+41
-8
@@ -41,10 +41,36 @@ def _http_status(status_code: int, service: Service) -> HealthStatus:
|
||||
return HealthStatus.DOWN
|
||||
|
||||
|
||||
_MAX_BODY_BYTES = 64 * 1024
|
||||
_SSL_CONTEXT: ssl.SSLContext | None = None
|
||||
|
||||
|
||||
def _ssl_context() -> ssl.SSLContext:
|
||||
"""Build the CA bundle context once instead of on every HTTP probe."""
|
||||
|
||||
global _SSL_CONTEXT
|
||||
if _SSL_CONTEXT is None:
|
||||
import certifi
|
||||
|
||||
_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
|
||||
return _SSL_CONTEXT
|
||||
|
||||
|
||||
async def _probe_once_with_metadata(service: Service, timeout_seconds: float) -> tuple[HealthStatus, str, dict[str, object]]:
|
||||
if service.kind == ServiceKind.HTTP:
|
||||
async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True) as client:
|
||||
response = await client.get(service.url or "")
|
||||
# One overall deadline (httpx timeouts are per phase, so a slow-drip
|
||||
# body could run for many times the timeout) and a bounded body read:
|
||||
# the status line answers the health question.
|
||||
async with asyncio.timeout(timeout_seconds):
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout_seconds, follow_redirects=True, max_redirects=5, verify=_ssl_context()
|
||||
) as client:
|
||||
async with client.stream("GET", service.url or "") as response:
|
||||
received = 0
|
||||
async for chunk in response.aiter_raw():
|
||||
received += len(chunk)
|
||||
if received >= _MAX_BODY_BYTES:
|
||||
break
|
||||
status = _http_status(response.status_code, service)
|
||||
return status, f"HTTP {response.status_code}", {"status_code": response.status_code}
|
||||
|
||||
@@ -72,7 +98,7 @@ async def _probe_once_with_metadata(service: Service, timeout_seconds: float) ->
|
||||
del reader
|
||||
ssl_object = writer.get_extra_info("ssl_object")
|
||||
metadata: dict[str, object] = {"hostname": service.host, "port": service.port}
|
||||
detail = f"TLS {service.host}:{service.port} certificate reachable"
|
||||
detail = f"TLS-certificaat van {service.host}:{service.port} bereikbaar"
|
||||
status = HealthStatus.HEALTHY
|
||||
if ssl_object is not None:
|
||||
certificate = ssl_object.getpeercert() or {}
|
||||
@@ -84,10 +110,10 @@ async def _probe_once_with_metadata(service: Service, timeout_seconds: float) ->
|
||||
warning_days = max(1, int(os.getenv("SERVICEMAP_TLS_WARNING_DAYS", "30")))
|
||||
if days_remaining <= 0:
|
||||
status = HealthStatus.DOWN
|
||||
detail = f"TLS certificate expired on {expires_at.date().isoformat()}"
|
||||
detail = f"TLS-certificaat verlopen op {expires_at.date().isoformat()}"
|
||||
elif days_remaining <= warning_days:
|
||||
status = HealthStatus.DEGRADED
|
||||
detail = f"TLS certificate expires in {round(days_remaining, 1)} days"
|
||||
detail = f"TLS-certificaat verloopt over {round(days_remaining, 1)} dagen"
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return status, detail, metadata
|
||||
@@ -122,7 +148,7 @@ async def probe_service(
|
||||
service_id=service.id,
|
||||
status=HealthStatus.UNKNOWN,
|
||||
latency_ms=None,
|
||||
detail="Monitoring disabled for service",
|
||||
detail="Bewaking uitgeschakeld voor deze service",
|
||||
checked_at=datetime.now(UTC),
|
||||
trigger=trigger,
|
||||
)
|
||||
@@ -132,7 +158,7 @@ async def probe_service(
|
||||
retry_count = retry_count if retry_count is not None else default_retries
|
||||
attempts = max(1, retry_count + 1)
|
||||
status = HealthStatus.DOWN
|
||||
detail = "Probe failed"
|
||||
detail = "Controle mislukt"
|
||||
latency_ms: float | None = None
|
||||
metadata: dict[str, object] = {}
|
||||
|
||||
@@ -152,9 +178,16 @@ async def probe_service(
|
||||
detail = f"{type(exc).__name__}: {exc}"
|
||||
if isinstance(exc, ssl.CertificateError):
|
||||
status = HealthStatus.DOWN
|
||||
detail = f"TLS hostname mismatch: {exc}"
|
||||
detail = f"TLS-hostnaam komt niet overeen: {exc}"
|
||||
if attempt < attempts:
|
||||
await asyncio.sleep(min(0.2 * attempt, 1.0))
|
||||
except Exception as exc: # noqa: BLE001 - one bad target must not fail a whole cycle
|
||||
# e.g. httpx.InvalidURL or an IDNA UnicodeError: a permanent
|
||||
# configuration error, reported as down without retrying.
|
||||
latency_ms = round((perf_counter() - started) * 1000, 2)
|
||||
status = HealthStatus.DOWN
|
||||
detail = f"Controlefout ({type(exc).__name__}): {exc}"
|
||||
break
|
||||
|
||||
return ProbeResult(
|
||||
id=str(uuid4()),
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Validated, briefly cached JSON representations for hot read endpoints.
|
||||
|
||||
The semantic topology is read by the UI after every event burst and by
|
||||
integrations such as ITWorx Forge on a fixed poll. Its content changes far
|
||||
less often than it is read: in production only ``generated_at`` differed
|
||||
between polls. Two mechanisms make an unchanged read nearly free:
|
||||
|
||||
* A strong ``ETag`` computed over the content *excluding* ``generated_at``.
|
||||
A client that sends ``If-None-Match`` gets ``304 Not Modified`` with no
|
||||
body. Because the tag is derived from the content itself it is always
|
||||
correct, including after writes that publish no event.
|
||||
* A short-lived computation cache. It is reused only while no event has
|
||||
been published since it was built, no evidence freshness boundary
|
||||
(fresh→stale→expired) has been crossed, and it is younger than
|
||||
``max_age_seconds``. The age cap bounds staleness for the few write paths
|
||||
that publish no event (for example discovery relation evidence).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from threading import Lock
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .models import HealthEvidence
|
||||
|
||||
GENERATED_AT_HEADER = "X-ServiceMap-Generated-At"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CachedRepresentation:
|
||||
body: bytes
|
||||
etag: str
|
||||
generated_at: str
|
||||
|
||||
|
||||
def representation(model: BaseModel, *, timestamp_field: str = "generated_at") -> CachedRepresentation:
|
||||
"""Serialize once and tag the content independent of its timestamp."""
|
||||
|
||||
content = model.model_dump_json(exclude={timestamp_field})
|
||||
stamp = model.model_dump_json(include={timestamp_field})
|
||||
# Splice ``{"generated_at": ...}`` in front of the remaining fields: one
|
||||
# serialization for the body, the hash covers everything but the stamp.
|
||||
body = stamp if content == "{}" else f"{stamp[:-1]},{content[1:]}"
|
||||
digest = hashlib.sha256(content.encode()).hexdigest()[:32]
|
||||
generated_at = str(getattr(model, timestamp_field).isoformat())
|
||||
return CachedRepresentation(body=body.encode(), etag=f'"{digest}"', generated_at=generated_at)
|
||||
|
||||
|
||||
def etag_matches(if_none_match: str | None, etag: str) -> bool:
|
||||
if not if_none_match:
|
||||
return False
|
||||
for candidate in if_none_match.split(","):
|
||||
value = candidate.strip()
|
||||
if value == "*" or value.removeprefix("W/") == etag:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def next_freshness_boundary(evidence: list[HealthEvidence], now: datetime) -> datetime | None:
|
||||
"""Earliest future instant at which any claim changes freshness."""
|
||||
|
||||
boundary: datetime | None = None
|
||||
for item in evidence:
|
||||
ttl = max(item.valid_until - item.observed_at, timedelta(seconds=1))
|
||||
for instant in (item.valid_until, item.valid_until + ttl):
|
||||
if instant > now and (boundary is None or instant < boundary):
|
||||
boundary = instant
|
||||
return boundary
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Entry:
|
||||
value: CachedRepresentation
|
||||
revision: int
|
||||
expires_at: float
|
||||
|
||||
|
||||
class RepresentationCache:
|
||||
"""Thread-safe cache keyed by request variant and event revision."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_age_seconds: float = 10.0,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self.max_age_seconds = max_age_seconds
|
||||
self._clock = clock
|
||||
self._entries: dict[object, _Entry] = {}
|
||||
self._lock = Lock()
|
||||
|
||||
def get(self, key: object, revision: int) -> CachedRepresentation | None:
|
||||
with self._lock:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
if entry.revision != revision or self._clock() >= entry.expires_at:
|
||||
del self._entries[key]
|
||||
return None
|
||||
return entry.value
|
||||
|
||||
def put(
|
||||
self,
|
||||
key: object,
|
||||
revision: int,
|
||||
value: CachedRepresentation,
|
||||
*,
|
||||
valid_for_seconds: float | None = None,
|
||||
) -> None:
|
||||
ttl = self.max_age_seconds if valid_for_seconds is None else min(self.max_age_seconds, valid_for_seconds)
|
||||
if ttl <= 0:
|
||||
return
|
||||
with self._lock:
|
||||
self._entries[key] = _Entry(value=value, revision=revision, expires_at=self._clock() + ttl)
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._entries.clear()
|
||||
@@ -7,7 +7,7 @@ from time import monotonic
|
||||
from .probes import probe_many
|
||||
from .store import Store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = logging.getLogger("servicemap.scheduler")
|
||||
|
||||
|
||||
async def scheduled_probe_loop(
|
||||
@@ -18,6 +18,7 @@ async def scheduled_probe_loop(
|
||||
retries: int,
|
||||
concurrency: int,
|
||||
retention_days: int,
|
||||
event_retention_days: int | None = None,
|
||||
) -> None:
|
||||
next_due: dict[str, float] = {}
|
||||
while True:
|
||||
@@ -58,7 +59,12 @@ async def scheduled_probe_loop(
|
||||
sleep_seconds = min([interval_seconds, *next_intervals]) if next_intervals else interval_seconds
|
||||
else:
|
||||
sleep_seconds = interval_seconds
|
||||
store.prune_probes(retention_days)
|
||||
if event_retention_days is None:
|
||||
store.prune_probes(retention_days)
|
||||
else:
|
||||
# Events have their own (longer) retention; pruning them at the
|
||||
# probe retention silently shortened the journal.
|
||||
store.prune_probes(retention_days, event_retention_days)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # pragma: no cover - defensive daemon boundary
|
||||
|
||||
@@ -214,7 +214,7 @@ def _application_resolution(
|
||||
return ApplicationHealthResolution(
|
||||
status=status,
|
||||
confidence=max((item.health_resolution.confidence or 1.0 for item in direct_down), default=None),
|
||||
explanation=f"{names} has current component failure evidence; aggregate health does not override it.",
|
||||
explanation=f"{names} heeft actueel bewijs van een componentstoring; de totale gezondheid overschrijft dat niet.",
|
||||
reason_code=HealthResolutionReason.COMPONENT_FAILURE,
|
||||
**common,
|
||||
)
|
||||
@@ -223,7 +223,7 @@ def _application_resolution(
|
||||
return ApplicationHealthResolution(
|
||||
status=HealthStatus.DEGRADED,
|
||||
confidence=selected.confidence if selected else None,
|
||||
explanation=f"{names} is impacted through a dependency health signal.",
|
||||
explanation=f"{names} wordt geraakt via het gezondheidssignaal van een afhankelijkheid.",
|
||||
reason_code=HealthResolutionReason.COMPONENT_IMPACT,
|
||||
**common,
|
||||
)
|
||||
@@ -232,8 +232,8 @@ def _application_resolution(
|
||||
status=selected.status,
|
||||
confidence=selected.confidence,
|
||||
explanation=(
|
||||
f"{selected.provider.value} application observation selected "
|
||||
f"(confidence {selected.confidence:.2f}); component coverage is {coverage:.0f}% ."
|
||||
f"{selected.provider.value} applicatiewaarneming gekozen "
|
||||
f"(zekerheid {selected.confidence:.2f}); componentdekking is {coverage:.0f}%."
|
||||
),
|
||||
reason_code=HealthResolutionReason.SELECTED_APPLICATION_EVIDENCE,
|
||||
**common,
|
||||
@@ -244,7 +244,7 @@ def _application_resolution(
|
||||
status = HealthStatus.DEGRADED
|
||||
return ApplicationHealthResolution(
|
||||
status=status,
|
||||
explanation=f"{len(resolved)} of {len(members)} components have current direct health evidence.",
|
||||
explanation=f"{len(resolved)} van {len(members)} componenten hebben actueel direct gezondheidsbewijs.",
|
||||
reason_code=HealthResolutionReason.COMPONENT_COVERAGE,
|
||||
**common,
|
||||
)
|
||||
@@ -257,20 +257,20 @@ def _application_resolution(
|
||||
)
|
||||
return ApplicationHealthResolution(
|
||||
status=HealthStatus.UNKNOWN,
|
||||
explanation="Application evidence is no longer current; waiting for a provider refresh.",
|
||||
explanation="Het applicatiebewijs is niet meer actueel; wachten op een nieuwe meting van de bron.",
|
||||
reason_code=reason,
|
||||
**common,
|
||||
)
|
||||
if members and all(item.direct_status == HealthStatus.UNMONITORED for item in members):
|
||||
return ApplicationHealthResolution(
|
||||
status=HealthStatus.UNMONITORED,
|
||||
explanation="No active health signal is configured for this application.",
|
||||
explanation="Voor deze applicatie is geen actief gezondheidssignaal ingesteld.",
|
||||
reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE,
|
||||
**common,
|
||||
)
|
||||
return ApplicationHealthResolution(
|
||||
status=HealthStatus.UNKNOWN,
|
||||
explanation="No current application or component health evidence is available.",
|
||||
explanation="Er is geen actueel gezondheidsbewijs voor applicatie of componenten.",
|
||||
reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE,
|
||||
**common,
|
||||
)
|
||||
|
||||
+273
-52
@@ -52,6 +52,16 @@ def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if value is not None else None
|
||||
|
||||
|
||||
# Per-observation bookkeeping that changes on every probe. It is stored, but a
|
||||
# change in only these keys is not a material evidence change and must not
|
||||
# publish an SSE wakeup (that turned every probe into a full UI refresh).
|
||||
VOLATILE_EVIDENCE_METADATA = frozenset({"probe_id", "latency_ms", "trigger"})
|
||||
|
||||
|
||||
def _stable_metadata(metadata: dict[str, object]) -> dict[str, object]:
|
||||
return {key: value for key, value in metadata.items() if key not in VOLATILE_EVIDENCE_METADATA}
|
||||
|
||||
|
||||
def _bool(value: object, default: bool = False) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
@@ -70,14 +80,24 @@ class Store:
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys = ON")
|
||||
connection.execute("PRAGMA busy_timeout = 5000")
|
||||
# Per connection, not persistent: without this every commit ran at
|
||||
# the FULL default (an fsync per write). NORMAL is durable under WAL.
|
||||
connection.execute("PRAGMA synchronous = NORMAL")
|
||||
return connection
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._connect() as db:
|
||||
db.execute("PRAGMA journal_mode = WAL")
|
||||
db.execute("PRAGMA synchronous = NORMAL")
|
||||
# Bound the WAL file after checkpoints (64 MiB).
|
||||
db.execute("PRAGMA journal_size_limit = 67108864")
|
||||
db.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS maintenance_windows (
|
||||
service_id TEXT PRIMARY KEY,
|
||||
ends_at TEXT NOT NULL,
|
||||
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS services (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
@@ -770,7 +790,7 @@ class Store:
|
||||
with self._connect() as db:
|
||||
self._upsert_service(db, updated)
|
||||
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {updated.name} updated", {})
|
||||
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service updated"})
|
||||
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service bijgewerkt"})
|
||||
return updated
|
||||
|
||||
def delete_service(self, service_id: str) -> bool:
|
||||
@@ -787,14 +807,24 @@ class Store:
|
||||
"UPDATE services SET depends_on = ?, updated_at = ? WHERE id = ?",
|
||||
(json.dumps([value for value in dependencies if value != service_id]), _now().isoformat(), item["id"]),
|
||||
)
|
||||
db.execute("UPDATE incidents SET affected_service_ids = replace(affected_service_ids, ?, '') WHERE status != 'resolved'", (service_id,))
|
||||
# Remove the id as a JSON element. A textual replace() left an
|
||||
# empty-string entry behind (["a","b"] -> ["","b"]).
|
||||
for incident in db.execute(
|
||||
"SELECT id, affected_service_ids FROM incidents WHERE status != 'resolved'"
|
||||
).fetchall():
|
||||
affected = self._json_list(incident["affected_service_ids"])
|
||||
if service_id in affected:
|
||||
db.execute(
|
||||
"UPDATE incidents SET affected_service_ids = ? WHERE id = ?",
|
||||
(json.dumps([value for value in affected if value != service_id]), incident["id"]),
|
||||
)
|
||||
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {row['name']} deleted", {})
|
||||
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service deleted"})
|
||||
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service verwijderd"})
|
||||
return True
|
||||
|
||||
def import_config_services(self, definitions: list[ConfigService], mode: str) -> ConfigImportResult:
|
||||
if mode not in {"merge", "replace"}:
|
||||
raise ValueError("Import mode must be merge or replace")
|
||||
raise ValueError("Importmodus moet merge of replace zijn")
|
||||
|
||||
with self._connect() as db:
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
@@ -813,7 +843,7 @@ class Store:
|
||||
missing = [item for item in definition.depends_on if item.casefold() not in ids_by_name]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Service {definition.name} references unknown dependencies: {', '.join(missing)}"
|
||||
f"Service {definition.name} verwijst naar onbekende afhankelijkheden: {', '.join(missing)}"
|
||||
)
|
||||
|
||||
created = 0
|
||||
@@ -822,8 +852,16 @@ class Store:
|
||||
for definition in definitions:
|
||||
key = definition.name.casefold()
|
||||
current = existing.get(key)
|
||||
payload = definition.model_dump()
|
||||
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
|
||||
if current is None:
|
||||
payload = definition.model_dump()
|
||||
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
|
||||
else:
|
||||
# Merge: only fields the document states change. Rebuilding
|
||||
# from model_dump() reset every omitted field to its default.
|
||||
payload = current.model_dump(include=set(ServiceCreate.model_fields))
|
||||
payload.update(definition.model_dump(exclude_unset=True, exclude={"depends_on"}))
|
||||
if "depends_on" in definition.model_fields_set:
|
||||
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
|
||||
if current is None:
|
||||
service = Service(id=ids_by_name[key], created_at=now, updated_at=now, **payload)
|
||||
created += 1
|
||||
@@ -845,8 +883,8 @@ class Store:
|
||||
self._insert_service(db, service)
|
||||
else:
|
||||
self._upsert_service(db, service)
|
||||
self._insert_event_db(db, "topology.changed", None, None, None, None, "Configuration imported", {"mode": mode})
|
||||
event_bus.publish({"event_type": "topology.changed", "message": "Configuration imported"})
|
||||
self._insert_event_db(db, "topology.changed", None, None, None, None, "Configuratie geïmporteerd", {"mode": mode})
|
||||
event_bus.publish({"event_type": "topology.changed", "message": "Configuratie geïmporteerd"})
|
||||
return ConfigImportResult(created=created, updated=updated, total=len(definitions))
|
||||
|
||||
def replace_all_services(self, definitions: list[ServiceCreate]) -> list[Service]:
|
||||
@@ -916,12 +954,29 @@ class Store:
|
||||
("status", evidence.status.value),
|
||||
("confidence", evidence.confidence),
|
||||
("summary", evidence.summary),
|
||||
("metadata", json.dumps(evidence.metadata, sort_keys=True)),
|
||||
)
|
||||
)
|
||||
)
|
||||
) or _stable_metadata(self._json_object(prior["metadata"])) != _stable_metadata(evidence.metadata)
|
||||
if prior is not None and prior["status"] == evidence.status.value:
|
||||
saved = evidence.model_copy(update={"id": prior["id"], "active": True, "superseded_by": None})
|
||||
ttl = evidence.valid_until - evidence.observed_at
|
||||
prior_valid_until = datetime.fromisoformat(prior["valid_until"])
|
||||
if (
|
||||
not materially_changed
|
||||
and prior["service_id"] == evidence.service_id
|
||||
and prior_valid_until - _now() > ttl / 2
|
||||
):
|
||||
# Unchanged and comfortably fresh: skip the timestamp-only
|
||||
# rewrite. It is renewed once less than half the TTL is
|
||||
# left, so the claim never goes stale, with about half
|
||||
# the writes of renewing on every observation cycle.
|
||||
saved = self._row_to_health_evidence(prior)
|
||||
for service_id in links:
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
|
||||
(saved.id, service_id),
|
||||
)
|
||||
return saved
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE health_evidence SET service_id = ?, confidence = ?, observed_at = ?, valid_until = ?,
|
||||
@@ -980,18 +1035,24 @@ class Store:
|
||||
"service_id": evidence.service_id,
|
||||
"provider": evidence.provider.value,
|
||||
"status": evidence.status.value,
|
||||
"message": "Health evidence changed",
|
||||
"message": "Gezondheidsbewijs gewijzigd",
|
||||
}
|
||||
)
|
||||
return saved
|
||||
|
||||
def health_evidence_for_service(self, service_id: str, *, limit: int = 100) -> list[HealthEvidence]:
|
||||
# A direct claim and a linked claim are matched by two independently
|
||||
# indexed lookups. An ``OR`` across the LEFT JOIN cannot use either
|
||||
# index and forced a full ``health_evidence`` scan on every call.
|
||||
with self._connect() as db:
|
||||
rows = db.execute(
|
||||
"""
|
||||
SELECT DISTINCT e.* FROM health_evidence e
|
||||
LEFT JOIN health_evidence_service_links link ON link.evidence_id = e.id
|
||||
WHERE e.service_id = ? OR link.service_id = ?
|
||||
SELECT e.* FROM health_evidence e
|
||||
WHERE e.id IN (
|
||||
SELECT id FROM health_evidence WHERE service_id = ?
|
||||
UNION
|
||||
SELECT evidence_id FROM health_evidence_service_links WHERE service_id = ?
|
||||
)
|
||||
ORDER BY e.observed_at DESC, e.rowid DESC LIMIT ?
|
||||
""",
|
||||
(service_id, service_id, limit),
|
||||
@@ -1015,27 +1076,41 @@ class Store:
|
||||
return result
|
||||
placeholders = ", ".join("?" for _ in ids)
|
||||
with self._connect() as db:
|
||||
# Both match paths use their own index; the per-service limit is
|
||||
# applied in SQL so superseded history is not materialised.
|
||||
rows = db.execute(
|
||||
f"""
|
||||
SELECT e.*, e.service_id AS direct_service_id, link.service_id AS linked_service_id
|
||||
FROM health_evidence e
|
||||
LEFT JOIN health_evidence_service_links link ON link.evidence_id = e.id
|
||||
WHERE e.service_id IN ({placeholders}) OR link.service_id IN ({placeholders})
|
||||
ORDER BY e.observed_at DESC, e.rowid DESC
|
||||
WITH matched(evidence_id, matched_service_id) AS (
|
||||
SELECT id, service_id FROM health_evidence WHERE service_id IN ({placeholders})
|
||||
UNION
|
||||
SELECT evidence_id, service_id FROM health_evidence_service_links
|
||||
WHERE service_id IN ({placeholders})
|
||||
),
|
||||
ranked AS (
|
||||
SELECT e.*, m.matched_service_id,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY m.matched_service_id
|
||||
ORDER BY e.observed_at DESC, e.rowid DESC
|
||||
) AS rank_in_service
|
||||
FROM matched m
|
||||
JOIN health_evidence e ON e.id = m.evidence_id
|
||||
)
|
||||
SELECT * FROM ranked
|
||||
WHERE rank_in_service <= ?
|
||||
ORDER BY matched_service_id, rank_in_service
|
||||
""",
|
||||
[*ids, *ids],
|
||||
[*ids, *ids, limit_per_service],
|
||||
).fetchall()
|
||||
seen: dict[str, set[str]] = {service_id: set() for service_id in ids}
|
||||
parsed: dict[str, HealthEvidence] = {}
|
||||
for row in rows:
|
||||
evidence = self._row_to_health_evidence(row)
|
||||
for service_id in {row["direct_service_id"], row["linked_service_id"]}:
|
||||
if (
|
||||
service_id in result
|
||||
and evidence.id not in seen[service_id]
|
||||
and len(result[service_id]) < limit_per_service
|
||||
):
|
||||
result[service_id].append(evidence)
|
||||
seen[service_id].add(evidence.id)
|
||||
service_id = row["matched_service_id"]
|
||||
if service_id not in result:
|
||||
continue
|
||||
evidence = parsed.get(row["id"])
|
||||
if evidence is None:
|
||||
evidence = self._row_to_health_evidence(row)
|
||||
parsed[row["id"]] = evidence
|
||||
result[service_id].append(evidence)
|
||||
return result
|
||||
|
||||
def application_evidence_links(self, provider: EvidenceProvider) -> dict[str, set[str]]:
|
||||
@@ -1098,7 +1173,7 @@ class Store:
|
||||
"event_type": "provider.status_changed",
|
||||
"provider": value.provider.value,
|
||||
"status": value.status.value,
|
||||
"message": "Provider availability changed",
|
||||
"message": "Beschikbaarheid van de bron gewijzigd",
|
||||
}
|
||||
)
|
||||
return value
|
||||
@@ -1205,10 +1280,37 @@ class Store:
|
||||
status_by_id[service_id] = self._legacy_health_from_row(row)
|
||||
return status_by_id, resolutions
|
||||
|
||||
def set_maintenance(self, service_id: str, enabled: bool, reason: str | None = None) -> Service | None:
|
||||
def maintenance_windows(self) -> dict[str, datetime]:
|
||||
with self._connect() as db:
|
||||
rows = db.execute("SELECT service_id, ends_at FROM maintenance_windows").fetchall()
|
||||
return {row["service_id"]: datetime.fromisoformat(row["ends_at"]) for row in rows}
|
||||
|
||||
def expire_maintenance_windows(self, now: datetime | None = None) -> list[str]:
|
||||
"""End maintenance for every window whose end time has passed."""
|
||||
|
||||
now = now or _now()
|
||||
with self._connect() as db:
|
||||
rows = db.execute(
|
||||
"SELECT service_id FROM maintenance_windows WHERE ends_at <= ?", (now.isoformat(),)
|
||||
).fetchall()
|
||||
ended: list[str] = []
|
||||
for row in rows:
|
||||
if self.set_maintenance(row["service_id"], False, "Maintenance window ended") is not None:
|
||||
ended.append(row["service_id"])
|
||||
return ended
|
||||
|
||||
def set_maintenance(
|
||||
self,
|
||||
service_id: str,
|
||||
enabled: bool,
|
||||
reason: str | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> Service | None:
|
||||
service = self.get_service(service_id)
|
||||
if service is None:
|
||||
return None
|
||||
if enabled and until is not None and until <= _now():
|
||||
raise ValueError("Het einde van het onderhoud moet in de toekomst liggen")
|
||||
updated = service.model_copy(
|
||||
update={
|
||||
"maintenance_enabled": enabled,
|
||||
@@ -1221,6 +1323,14 @@ class Store:
|
||||
)
|
||||
with self._connect() as db:
|
||||
self._upsert_service(db, updated)
|
||||
if enabled and until is not None:
|
||||
db.execute(
|
||||
"INSERT INTO maintenance_windows (service_id, ends_at) VALUES (?, ?) "
|
||||
"ON CONFLICT(service_id) DO UPDATE SET ends_at = excluded.ends_at",
|
||||
(service_id, until.isoformat()),
|
||||
)
|
||||
else:
|
||||
db.execute("DELETE FROM maintenance_windows WHERE service_id = ?", (service_id,))
|
||||
self._insert_event_db(
|
||||
db,
|
||||
"service.maintenance_changed",
|
||||
@@ -1228,15 +1338,18 @@ class Store:
|
||||
None,
|
||||
service.state.value,
|
||||
HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
|
||||
f"Maintenance {'enabled' if enabled else 'disabled'} for {service.name}",
|
||||
{"reason": reason} if reason else {},
|
||||
f"Onderhoud {'gestart' if enabled else 'beëindigd'} voor {service.name}",
|
||||
{
|
||||
**({"reason": reason} if reason else {}),
|
||||
**({"until": until.isoformat()} if enabled and until else {}),
|
||||
},
|
||||
)
|
||||
event_bus.publish(
|
||||
{
|
||||
"event_type": "service.maintenance_changed",
|
||||
"service_id": service_id,
|
||||
"status": HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
|
||||
"message": f"Maintenance {'enabled' if enabled else 'disabled'}",
|
||||
"message": f"Onderhoud {'gestart' if enabled else 'beëindigd'}",
|
||||
}
|
||||
)
|
||||
from .incidents import sync_incidents
|
||||
@@ -1330,11 +1443,11 @@ class Store:
|
||||
None,
|
||||
service.state.value,
|
||||
status.value,
|
||||
f"Docker runtime observation updated for {service.name}",
|
||||
f"Docker-runtimewaarneming bijgewerkt voor {service.name}",
|
||||
{"runtime_state": runtime_state, "runtime_health": runtime_health, "monitoring_state": state_kind.value},
|
||||
)
|
||||
if service.state != status:
|
||||
event_bus.publish({"event_type": "service.runtime_observed", "service_id": service_id, "status": status.value, "message": "Docker runtime observation updated"})
|
||||
event_bus.publish({"event_type": "service.runtime_observed", "service_id": service_id, "status": status.value, "message": "Docker-runtimewaarneming bijgewerkt"})
|
||||
observed_at = _now()
|
||||
self.record_health_evidence(
|
||||
HealthEvidence(
|
||||
@@ -1350,7 +1463,7 @@ class Store:
|
||||
observed_at=observed_at,
|
||||
valid_until=observed_at + timedelta(seconds=max(60, int(service.interval_seconds or 60) * 2)),
|
||||
received_at=observed_at,
|
||||
summary="Docker runtime observation" if provider == EvidenceProvider.DOCKER else "Runtime observation",
|
||||
summary="Docker-runtimewaarneming" if provider == EvidenceProvider.DOCKER else "Runtimewaarneming",
|
||||
metadata={"runtime_state": runtime_state, "runtime_health": runtime_health},
|
||||
)
|
||||
)
|
||||
@@ -1389,7 +1502,7 @@ class Store:
|
||||
"service_id": probe.service_id,
|
||||
"from_status": previous.value,
|
||||
"to_status": next_state.value,
|
||||
"message": f"Service changed from {previous.value} to {next_state.value}",
|
||||
"message": f"Service gewijzigd van {previous.value} to {next_state.value}",
|
||||
"metadata": {"trigger": probe.trigger, "probe_id": probe.id},
|
||||
}
|
||||
self._insert_event_db(
|
||||
@@ -1420,11 +1533,11 @@ class Store:
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
f"Flapping detected: {flap_count} state changes in 30 minutes",
|
||||
f"Wisselvalligheid vastgesteld: {flap_count} statuswijzigingen in 30 minuten",
|
||||
{"state_changes": flap_count, "window_minutes": 30},
|
||||
created_at=now,
|
||||
)
|
||||
events.append({"event_type": "flapping.detected", "service_id": probe.service_id, "message": "Flapping detected"})
|
||||
events.append({"event_type": "flapping.detected", "service_id": probe.service_id, "message": "Wisselvalligheid vastgesteld"})
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE services
|
||||
@@ -1449,7 +1562,7 @@ class Store:
|
||||
"service_id": probe.service_id,
|
||||
"status": probe.status.value,
|
||||
"trigger": probe.trigger,
|
||||
"message": "Probe completed",
|
||||
"message": "Controle voltooid",
|
||||
}
|
||||
)
|
||||
for event in events:
|
||||
@@ -1618,6 +1731,52 @@ class Store:
|
||||
db.execute("DELETE FROM events WHERE created_at < ? AND event_type NOT LIKE 'incident.%'", (event_cutoff.isoformat(),))
|
||||
return cursor.rowcount
|
||||
|
||||
def checkpoint(self) -> None:
|
||||
"""Fold the WAL back into the database and refresh planner statistics.
|
||||
|
||||
Without a TRUNCATE checkpoint the WAL file keeps its high-water size.
|
||||
"""
|
||||
|
||||
with self._connect() as db:
|
||||
db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
db.execute("PRAGMA optimize")
|
||||
|
||||
def ready_check(self) -> None:
|
||||
"""Raise if the database cannot take a write lock right now."""
|
||||
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute("ROLLBACK")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def prune_health_evidence(self, retention_days: int) -> int:
|
||||
"""Delete superseded evidence history older than the retention window.
|
||||
|
||||
Only inactive rows are removed. A row becomes inactive exclusively
|
||||
when a newer claim for the same provider/subject supersedes it, so
|
||||
every subject keeps its current (active) claim and health resolution
|
||||
is unaffected. Without this, every status flip leaves a permanent row
|
||||
and evidence reads grow linearly with uptime.
|
||||
"""
|
||||
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
cutoff = _now() - timedelta(days=retention_days)
|
||||
with self._connect() as db:
|
||||
cursor = db.execute(
|
||||
"DELETE FROM health_evidence WHERE active = 0 AND superseded_by IS NOT NULL AND received_at < ?",
|
||||
(cutoff.isoformat(),),
|
||||
)
|
||||
# Links cascade through the foreign key; also remove any orphan
|
||||
# links left by databases that predate enforced foreign keys.
|
||||
db.execute(
|
||||
"DELETE FROM health_evidence_service_links "
|
||||
"WHERE evidence_id NOT IN (SELECT id FROM health_evidence)"
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
def _insert_event_db(
|
||||
self,
|
||||
db: sqlite3.Connection,
|
||||
@@ -1669,12 +1828,37 @@ class Store:
|
||||
event_bus.publish(result.model_dump(mode="json"))
|
||||
return result
|
||||
|
||||
def list_events(self, limit: int = 50, service_id: str | None = None) -> list[EventRecord]:
|
||||
with self._connect() as db:
|
||||
if service_id:
|
||||
rows = db.execute("SELECT * FROM events WHERE service_id = ? ORDER BY created_at DESC LIMIT ?", (service_id, limit)).fetchall()
|
||||
def list_events(
|
||||
self,
|
||||
limit: int = 50,
|
||||
service_id: str | None = None,
|
||||
event_type: str | None = None,
|
||||
before: datetime | None = None,
|
||||
) -> list[EventRecord]:
|
||||
"""Newest first. ``event_type`` ending in ``.`` matches a family
|
||||
(``incident.``); ``before`` pages to older events."""
|
||||
|
||||
clauses: list[str] = []
|
||||
params: list[object] = []
|
||||
if service_id:
|
||||
clauses.append("service_id = ?")
|
||||
params.append(service_id)
|
||||
if event_type:
|
||||
if event_type.endswith("."):
|
||||
# Exact prefix match without LIKE wildcards.
|
||||
clauses.append("substr(event_type, 1, ?) = ?")
|
||||
params.extend([len(event_type), event_type])
|
||||
else:
|
||||
rows = db.execute("SELECT * FROM events ORDER BY created_at DESC LIMIT ?", (limit,)).fetchall()
|
||||
clauses.append("event_type = ?")
|
||||
params.append(event_type)
|
||||
if before is not None:
|
||||
clauses.append("created_at < ?")
|
||||
params.append(before.isoformat())
|
||||
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
||||
with self._connect() as db:
|
||||
rows = db.execute(
|
||||
f"SELECT * FROM events {where} ORDER BY created_at DESC LIMIT ?", [*params, limit]
|
||||
).fetchall()
|
||||
return [self._row_to_event(row) for row in rows]
|
||||
|
||||
def demo_active(self) -> bool:
|
||||
@@ -1928,7 +2112,7 @@ class Store:
|
||||
""",
|
||||
(snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, json.dumps([service.id for service in services]), started_at),
|
||||
)
|
||||
event_bus.publish({"event_type": "topology.changed", "message": "Synthetic demo loaded"})
|
||||
event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo geladen"})
|
||||
return ConfigImportResult(created=len(services), updated=0, total=len(services))
|
||||
|
||||
def exit_demo(self) -> bool:
|
||||
@@ -1947,7 +2131,7 @@ class Store:
|
||||
session["snapshot_health_evidence"] or "{}",
|
||||
)
|
||||
db.execute("DELETE FROM demo_sessions WHERE id = 1")
|
||||
event_bus.publish({"event_type": "topology.changed", "message": "Synthetic demo exited"})
|
||||
event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo afgesloten"})
|
||||
return True
|
||||
|
||||
def list_incidents(self, active_only: bool = False, limit: int = 100) -> list[IncidentRecord]:
|
||||
@@ -1999,14 +2183,51 @@ class Store:
|
||||
event_bus.publish(payload)
|
||||
return IncidentEvent(id=event_id, incident_id=incident_id, event_type=event_type, message=message, created_at=created_at, service_id=service_id)
|
||||
|
||||
def resolve_incident(self, incident_id: str, note: str | None = None) -> IncidentRecord | None:
|
||||
"""Close an incident by hand (e.g. a stale one).
|
||||
|
||||
If its root is still down, the next incident sync opens a new one.
|
||||
"""
|
||||
|
||||
incident = self.get_incident(incident_id)
|
||||
if incident is None:
|
||||
return None
|
||||
if incident.status == IncidentStatus.RESOLVED:
|
||||
return incident
|
||||
now = _now()
|
||||
updated = incident.model_copy(
|
||||
update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now}
|
||||
)
|
||||
self.upsert_incident(updated)
|
||||
self.append_incident_event(
|
||||
incident_id,
|
||||
"incident.resolved",
|
||||
f"Incident handmatig opgelost{f': {note}' if note else ''}",
|
||||
incident.root_service_id,
|
||||
)
|
||||
return updated
|
||||
|
||||
def add_incident_note(self, incident_id: str, text: str) -> IncidentRecord | None:
|
||||
incident = self.get_incident(incident_id)
|
||||
if incident is None:
|
||||
return None
|
||||
self.append_incident_event(incident_id, "incident.note", text, incident.root_service_id)
|
||||
updated = incident.model_copy(update={"latest_update_at": _now()})
|
||||
self.upsert_incident(updated)
|
||||
return updated
|
||||
|
||||
def acknowledge_incident(self, incident_id: str) -> IncidentRecord | None:
|
||||
incident = self.get_incident(incident_id)
|
||||
if incident is None:
|
||||
return None
|
||||
if incident.status == IncidentStatus.RESOLVED:
|
||||
raise ValueError("Resolved incidents cannot be acknowledged")
|
||||
raise ValueError("Opgeloste incidenten kunnen niet bevestigd worden")
|
||||
if incident.status == IncidentStatus.ACKNOWLEDGED:
|
||||
# Idempotent: a repeated acknowledge keeps the original time and
|
||||
# adds no duplicate timeline entry.
|
||||
return incident
|
||||
now = _now()
|
||||
updated = incident.model_copy(update={"status": IncidentStatus.ACKNOWLEDGED, "acknowledged_at": now, "latest_update_at": now})
|
||||
self.upsert_incident(updated)
|
||||
self.append_incident_event(incident_id, "incident.acknowledged", "Incident acknowledged", incident.root_service_id)
|
||||
self.append_incident_event(incident_id, "incident.acknowledged", "Incident bevestigd", incident.root_service_id)
|
||||
return updated
|
||||
Reference in new issue
Block a user