Files
NuklearRabbit 41a8eee259 ServiceMap: update dependencies and backend/app (+5 areas)
Public changes compared with the previous published export:
16 files changed.

Affected areas:
- backend/app: 3 files
- dependencies: 5 files
- deployment and CI: 2 files
- documentation: 2 files
- frontend/src: 1 file
- project files: 1 file
- tests: 2 files

Changed files:
- Updated: ".env.example"
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/app/incidents.py"
- Updated: "backend/app/main.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Updated: "backend/tests/test_docker_provider.py"
- Updated: "backend/tests/test_v163_health_performance.py"
- Updated: "docker-compose.broker.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "frontend/src/App.tsx"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated ServiceMap source from 8c297f756ff84aaa917ae9c48535e835f8c09ec7
2026-11-20 16:01:36 +01:00

2239 lines
111 KiB
Python

from __future__ import annotations
import json
import os
import sqlite3
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from uuid import uuid4
from .events import event_bus
from .evidence import resolve_health, with_freshness
from .models import (
AvailabilitySummary,
ConfigImportResult,
ConfigService,
EventRecord,
EvidenceProvider,
EvidenceScope,
HealthEvidence,
HealthResolution,
HealthStatus,
IncidentEvent,
IncidentRecord,
IncidentStatus,
MonitoringState,
ProbeResult,
ProviderStatus,
Service,
ServiceCreate,
ServiceUpdate,
)
SERVICE_COLUMNS = (
"id, name, kind, url, host, port, depends_on, description, tags, \"group\", "
"environment, owner, enabled, monitoring_enabled, monitoring_state, probe_source, probe_reachability, probe_confidence, "
"runtime_state, runtime_health, application, domain, component_role, application_locked, domain_locked, component_role_locked, criticality, category, visibility, lifecycle, "
"source_type, source_ref, first_seen_at, last_seen_at, last_discovery_status, container_image, "
"compose_project, layout_x, layout_y, layout_pinned, "
"interval_seconds, timeout_seconds, retries, failure_threshold, recovery_threshold, "
"expected_status_min, expected_status_max, availability_target, maintenance_enabled, "
"maintenance_reason, state, failure_streak, recovery_streak, flapping, flap_count, "
"last_state_change_at, created_at, updated_at"
)
def _now() -> datetime:
return datetime.now(UTC)
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if value is not None else None
# Per-observation bookkeeping that changes on every probe. It is stored, but a
# change in only these keys is not a material evidence change and must not
# publish an SSE wakeup (that turned every probe into a full UI refresh).
VOLATILE_EVIDENCE_METADATA = frozenset({"probe_id", "latency_ms", "trigger"})
def _stable_metadata(metadata: dict[str, object]) -> dict[str, object]:
return {key: value for key, value in metadata.items() if key not in VOLATILE_EVIDENCE_METADATA}
def _bool(value: object, default: bool = False) -> bool:
if value is None:
return default
return bool(int(value)) if isinstance(value, (int, str)) and str(value).isdigit() else bool(value)
class Store:
def __init__(self, db_path: str | None = None) -> None:
raw_path = db_path or os.getenv("SERVICEMAP_DB", "data/servicemap.db")
self.path = Path(raw_path)
self.path.parent.mkdir(parents=True, exist_ok=True)
self._init_schema()
def _connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.path, timeout=5.0)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
connection.execute("PRAGMA busy_timeout = 5000")
# Per connection, not persistent: without this every commit ran at
# the FULL default (an fsync per write). NORMAL is durable under WAL.
connection.execute("PRAGMA synchronous = NORMAL")
return connection
def _init_schema(self) -> None:
with self._connect() as db:
db.execute("PRAGMA journal_mode = WAL")
db.execute("PRAGMA synchronous = NORMAL")
# Bound the WAL file after checkpoints (64 MiB).
db.execute("PRAGMA journal_size_limit = 67108864")
db.executescript(
"""
CREATE TABLE IF NOT EXISTS maintenance_windows (
service_id TEXT PRIMARY KEY,
ends_at TEXT NOT NULL,
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS services (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
kind TEXT NOT NULL,
url TEXT,
host TEXT,
port INTEGER,
depends_on TEXT NOT NULL DEFAULT '[]',
description TEXT,
tags TEXT NOT NULL DEFAULT '[]',
"group" TEXT,
environment TEXT,
owner TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
monitoring_enabled INTEGER NOT NULL DEFAULT 1,
monitoring_state TEXT NOT NULL DEFAULT 'active_probe',
probe_source TEXT NOT NULL DEFAULT 'manual',
probe_reachability TEXT NOT NULL DEFAULT 'verified',
probe_confidence TEXT NOT NULL DEFAULT 'confirmed',
runtime_state TEXT,
runtime_health TEXT,
application TEXT,
domain TEXT,
component_role TEXT NOT NULL DEFAULT 'unknown',
application_locked INTEGER NOT NULL DEFAULT 0,
domain_locked INTEGER NOT NULL DEFAULT 0,
component_role_locked INTEGER NOT NULL DEFAULT 0,
criticality TEXT NOT NULL DEFAULT 'medium',
category TEXT NOT NULL DEFAULT 'unknown',
visibility TEXT NOT NULL DEFAULT 'visible',
lifecycle TEXT NOT NULL DEFAULT 'active',
source_type TEXT,
source_ref TEXT,
first_seen_at TEXT,
last_seen_at TEXT,
last_discovery_status TEXT,
container_image TEXT,
compose_project TEXT,
layout_x REAL,
layout_y REAL,
layout_pinned INTEGER NOT NULL DEFAULT 0,
interval_seconds INTEGER,
timeout_seconds REAL,
retries INTEGER,
failure_threshold INTEGER NOT NULL DEFAULT 1,
recovery_threshold INTEGER NOT NULL DEFAULT 1,
expected_status_min INTEGER,
expected_status_max INTEGER,
availability_target REAL,
maintenance_enabled INTEGER NOT NULL DEFAULT 0,
maintenance_reason TEXT,
state TEXT NOT NULL DEFAULT 'unknown',
failure_streak INTEGER NOT NULL DEFAULT 0,
recovery_streak INTEGER NOT NULL DEFAULT 0,
flapping INTEGER NOT NULL DEFAULT 0,
flap_count INTEGER NOT NULL DEFAULT 0,
last_state_change_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT
);
CREATE TABLE IF NOT EXISTS probe_results (
id TEXT PRIMARY KEY,
service_id TEXT NOT NULL,
status TEXT NOT NULL,
latency_ms REAL,
detail TEXT NOT NULL,
checked_at TEXT NOT NULL,
trigger TEXT NOT NULL DEFAULT 'manual',
metadata TEXT NOT NULL DEFAULT '{}',
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_probe_service_time
ON probe_results(service_id, checked_at DESC);
CREATE INDEX IF NOT EXISTS idx_probe_time
ON probe_results(checked_at DESC);
CREATE TABLE IF NOT EXISTS events (
id TEXT PRIMARY KEY,
event_type TEXT NOT NULL,
service_id TEXT,
incident_id TEXT,
from_status TEXT,
to_status TEXT,
message TEXT NOT NULL,
metadata TEXT NOT NULL DEFAULT '{}',
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_events_time ON events(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_events_service_time ON events(service_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_events_incident_time ON events(incident_id, created_at ASC);
CREATE TABLE IF NOT EXISTS incidents (
id TEXT PRIMARY KEY,
status TEXT NOT NULL,
root_service_id TEXT NOT NULL,
started_at TEXT NOT NULL,
detected_at TEXT NOT NULL,
acknowledged_at TEXT,
resolved_at TEXT,
title TEXT NOT NULL,
root_cause_reason TEXT NOT NULL,
affected_service_ids TEXT NOT NULL DEFAULT '[]',
severity TEXT NOT NULL DEFAULT 'medium',
latest_update_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_incidents_status ON incidents(status, latest_update_at DESC);
CREATE INDEX IF NOT EXISTS idx_incidents_root ON incidents(root_service_id, started_at DESC);
CREATE TABLE IF NOT EXISTS demo_sessions (
id INTEGER PRIMARY KEY CHECK (id = 1),
snapshot_services TEXT NOT NULL,
snapshot_probes TEXT NOT NULL,
snapshot_events TEXT NOT NULL DEFAULT '[]',
snapshot_incidents TEXT NOT NULL DEFAULT '[]',
snapshot_discovery TEXT NOT NULL DEFAULT '{}',
snapshot_health_evidence TEXT NOT NULL DEFAULT '{}',
demo_service_ids TEXT NOT NULL,
started_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS health_evidence (
id TEXT PRIMARY KEY,
subject_type TEXT NOT NULL,
subject_id TEXT NOT NULL,
service_id TEXT,
provider TEXT NOT NULL,
provider_ref TEXT,
status TEXT NOT NULL,
scope TEXT NOT NULL,
confidence REAL NOT NULL,
observed_at TEXT NOT NULL,
valid_until TEXT NOT NULL,
received_at TEXT NOT NULL,
summary TEXT NOT NULL DEFAULT '',
metadata TEXT NOT NULL DEFAULT '{}',
active INTEGER NOT NULL DEFAULT 1,
superseded_by TEXT,
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_health_evidence_subject
ON health_evidence(subject_type, subject_id, provider, active, observed_at DESC);
CREATE INDEX IF NOT EXISTS idx_health_evidence_service
ON health_evidence(service_id, active, observed_at DESC);
CREATE TABLE IF NOT EXISTS health_evidence_service_links (
evidence_id TEXT NOT NULL,
service_id TEXT NOT NULL,
PRIMARY KEY(evidence_id, service_id),
FOREIGN KEY(evidence_id) REFERENCES health_evidence(id) ON DELETE CASCADE,
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_health_evidence_link_service
ON health_evidence_service_links(service_id, evidence_id);
CREATE TABLE IF NOT EXISTS provider_status (
provider TEXT PRIMARY KEY,
status TEXT NOT NULL,
observed_at TEXT NOT NULL,
valid_until TEXT NOT NULL,
detail TEXT NOT NULL DEFAULT '',
metadata TEXT NOT NULL DEFAULT '{}'
);
"""
)
self._migrate_columns(db)
self._backfill_health_evidence(db)
@staticmethod
def _migrate_columns(db: sqlite3.Connection) -> None:
columns = {row["name"] for row in db.execute("PRAGMA table_info(services)").fetchall()}
additions = {
"updated_at": "TEXT",
"description": "TEXT",
"tags": "TEXT NOT NULL DEFAULT '[]'",
"group": "TEXT",
"environment": "TEXT",
"owner": "TEXT",
"enabled": "INTEGER NOT NULL DEFAULT 1",
"monitoring_enabled": "INTEGER NOT NULL DEFAULT 1",
"monitoring_state": "TEXT NOT NULL DEFAULT 'active_probe'",
"probe_source": "TEXT NOT NULL DEFAULT 'manual'",
"probe_reachability": "TEXT NOT NULL DEFAULT 'verified'",
"probe_confidence": "TEXT NOT NULL DEFAULT 'confirmed'",
"runtime_state": "TEXT",
"runtime_health": "TEXT",
"application": "TEXT",
"domain": "TEXT",
"component_role": "TEXT NOT NULL DEFAULT 'unknown'",
"application_locked": "INTEGER NOT NULL DEFAULT 0",
"domain_locked": "INTEGER NOT NULL DEFAULT 0",
"component_role_locked": "INTEGER NOT NULL DEFAULT 0",
"criticality": "TEXT NOT NULL DEFAULT 'medium'",
"category": "TEXT NOT NULL DEFAULT 'unknown'",
"visibility": "TEXT NOT NULL DEFAULT 'visible'",
"lifecycle": "TEXT NOT NULL DEFAULT 'active'",
"source_type": "TEXT",
"source_ref": "TEXT",
"first_seen_at": "TEXT",
"last_seen_at": "TEXT",
"last_discovery_status": "TEXT",
"container_image": "TEXT",
"compose_project": "TEXT",
"layout_x": "REAL",
"layout_y": "REAL",
"layout_pinned": "INTEGER NOT NULL DEFAULT 0",
"interval_seconds": "INTEGER",
"timeout_seconds": "REAL",
"retries": "INTEGER",
"failure_threshold": "INTEGER NOT NULL DEFAULT 1",
"recovery_threshold": "INTEGER NOT NULL DEFAULT 1",
"expected_status_min": "INTEGER",
"expected_status_max": "INTEGER",
"availability_target": "REAL",
"maintenance_enabled": "INTEGER NOT NULL DEFAULT 0",
"maintenance_reason": "TEXT",
"state": "TEXT NOT NULL DEFAULT 'unknown'",
"failure_streak": "INTEGER NOT NULL DEFAULT 0",
"recovery_streak": "INTEGER NOT NULL DEFAULT 0",
"flapping": "INTEGER NOT NULL DEFAULT 0",
"flap_count": "INTEGER NOT NULL DEFAULT 0",
"last_state_change_at": "TEXT",
}
for name, definition in additions.items():
if name not in columns:
quoted = f'"{name}"' if name == "group" else name
db.execute(f"ALTER TABLE services ADD COLUMN {quoted} {definition}")
db.execute("UPDATE services SET updated_at = created_at WHERE updated_at IS NULL")
probe_columns = {row["name"] for row in db.execute("PRAGMA table_info(probe_results)").fetchall()}
for name, definition in {"trigger": "TEXT NOT NULL DEFAULT 'manual'", "metadata": "TEXT NOT NULL DEFAULT '{}'"}.items():
if name not in probe_columns:
db.execute(f"ALTER TABLE probe_results ADD COLUMN {name} {definition}")
# v1.0.1 derived health directly from the latest probe. Backfill the
# durable v1.1 state column so an in-place migration does not make an
# existing healthy/down service appear unknown until the next cycle.
db.execute(
"""
UPDATE services
SET state = COALESCE(
(SELECT latest.status FROM probe_results latest
WHERE latest.service_id = services.id
ORDER BY latest.checked_at DESC, latest.rowid DESC LIMIT 1),
state
),
last_state_change_at = COALESCE(
last_state_change_at,
(SELECT latest.checked_at FROM probe_results latest
WHERE latest.service_id = services.id
ORDER BY latest.checked_at DESC, latest.rowid DESC LIMIT 1)
)
WHERE state = 'unknown'
AND EXISTS (SELECT 1 FROM probe_results existing WHERE existing.service_id = services.id)
"""
)
demo_columns = {row["name"] for row in db.execute("PRAGMA table_info(demo_sessions)").fetchall()}
for name in ("snapshot_events", "snapshot_incidents", "snapshot_discovery", "snapshot_health_evidence"):
if name not in demo_columns:
default = "'{}'" if name in {"snapshot_discovery", "snapshot_health_evidence"} else "'[]'"
db.execute(f"ALTER TABLE demo_sessions ADD COLUMN {name} TEXT NOT NULL DEFAULT {default}")
@staticmethod
def _backfill_health_evidence(db: sqlite3.Connection) -> None:
"""Preserve legitimate recent v1.5 observations during the v1.6 migration.
This is a one-time projection of existing probe/runtime facts, not a
fabricated monitoring result. Historic observations stay expired and
visible in provenance; only their original interval makes them current.
AppOps rows are deliberately excluded because pre-v1.6 persisted an
application-scoped status as component runtime health.
"""
if db.execute("SELECT 1 FROM health_evidence LIMIT 1").fetchone():
return
latest = db.execute(
"""
SELECT p.* FROM probe_results p
WHERE p.rowid = (
SELECT candidate.rowid FROM probe_results candidate
WHERE candidate.service_id = p.service_id
ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1
)
"""
).fetchall()
intervals = {
row["id"]: max(60, int(row["interval_seconds"] or 60) * 2)
for row in db.execute("SELECT id, interval_seconds FROM services").fetchall()
}
for row in latest:
observed = datetime.fromisoformat(row["checked_at"])
ttl = timedelta(seconds=intervals.get(row["service_id"], 120))
evidence_id = f"migration-probe-{row['id']}"
db.execute(
"""
INSERT INTO health_evidence (
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
confidence, observed_at, valid_until, received_at, summary, metadata, active
) VALUES (?, 'service', ?, ?, 'active_probe', ?, ?, 'component', 1.0, ?, ?, ?, ?, ?, 1)
""",
(
evidence_id,
row["service_id"],
row["service_id"],
row["id"],
row["status"],
observed.isoformat(),
(observed + ttl).isoformat(),
observed.isoformat(),
"Migrated latest active probe result",
row["metadata"] or "{}",
),
)
db.execute(
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
(evidence_id, row["service_id"]),
)
runtime_rows = db.execute(
"""
SELECT id, runtime_state, runtime_health, updated_at, monitoring_state
FROM services
WHERE COALESCE(monitoring_state, '') != 'appops_observation'
AND (runtime_state IS NOT NULL OR runtime_health IS NOT NULL)
"""
).fetchall()
for row in runtime_rows:
observed = datetime.fromisoformat(row["updated_at"] or _now().isoformat())
runtime_state = str(row["runtime_state"] or "").casefold()
runtime_health = str(row["runtime_health"] or "").casefold()
if runtime_health in {"unhealthy", "down"} or runtime_state in {"exited", "dead", "stopped"}:
status = HealthStatus.DOWN
elif runtime_health == "degraded" or runtime_state == "restarting":
status = HealthStatus.DEGRADED
elif runtime_health == "healthy":
status = HealthStatus.HEALTHY
else:
continue
evidence_id = f"migration-runtime-{row['id']}"
db.execute(
"""
INSERT INTO health_evidence (
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
confidence, observed_at, valid_until, received_at, summary, metadata, active
) VALUES (?, 'service', ?, ?, 'docker', 'legacy-runtime', ?, 'component', 0.7, ?, ?, ?, ?, ?, 1)
""",
(
evidence_id,
row["id"],
row["id"],
status.value,
observed.isoformat(),
(observed + timedelta(minutes=10)).isoformat(),
observed.isoformat(),
"Migrated Docker runtime observation",
json.dumps({"runtime_state": row["runtime_state"], "runtime_health": row["runtime_health"]}),
),
)
db.execute(
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
(evidence_id, row["id"]),
)
@staticmethod
def _json_list(value: object, fallback: list[str] | None = None) -> list[str]:
if isinstance(value, list):
return [str(item) for item in value]
if isinstance(value, str):
try:
parsed = json.loads(value)
return [str(item) for item in parsed] if isinstance(parsed, list) else (fallback or [])
except json.JSONDecodeError:
return fallback or []
return fallback or []
@staticmethod
def _json_object(value: object) -> dict[str, object]:
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
parsed = json.loads(value)
return parsed if isinstance(parsed, dict) else {}
except json.JSONDecodeError:
return {}
return {}
@classmethod
def _row_to_service(cls, row: sqlite3.Row | dict[str, Any]) -> Service:
created_at = datetime.fromisoformat(str(row["created_at"]))
raw_updated = row.get("updated_at") if isinstance(row, dict) else row["updated_at"]
raw_state = row.get("state", "unknown") if isinstance(row, dict) else row["state"]
try:
state = HealthStatus(raw_state or "unknown")
except ValueError:
state = HealthStatus.UNKNOWN
return Service(
id=row["id"],
name=row["name"],
kind=row["kind"],
url=row["url"],
host=row["host"],
port=row["port"],
depends_on=cls._json_list(row["depends_on"]),
description=row.get("description") if isinstance(row, dict) else row["description"],
tags=cls._json_list(row.get("tags", "[]") if isinstance(row, dict) else row["tags"]),
group=row.get("group") if isinstance(row, dict) else row["group"],
environment=row.get("environment") if isinstance(row, dict) else row["environment"],
owner=row.get("owner") if isinstance(row, dict) else row["owner"],
enabled=_bool(row.get("enabled", 1) if isinstance(row, dict) else row["enabled"], True),
monitoring_enabled=_bool(
row.get("monitoring_enabled", 1) if isinstance(row, dict) else row["monitoring_enabled"],
True,
),
monitoring_state=(row.get("monitoring_state", "active_probe") if isinstance(row, dict) else row["monitoring_state"]) or "active_probe",
probe_source=(row.get("probe_source", "manual") if isinstance(row, dict) else row["probe_source"]) or "manual",
probe_reachability=(row.get("probe_reachability", "verified") if isinstance(row, dict) else row["probe_reachability"]) or "verified",
probe_confidence=(row.get("probe_confidence", "confirmed") if isinstance(row, dict) else row["probe_confidence"]) or "confirmed",
runtime_state=row.get("runtime_state") if isinstance(row, dict) else row["runtime_state"],
runtime_health=row.get("runtime_health") if isinstance(row, dict) else row["runtime_health"],
application=row.get("application") if isinstance(row, dict) else row["application"],
domain=row.get("domain") if isinstance(row, dict) else row["domain"],
component_role=(row.get("component_role", "unknown") if isinstance(row, dict) else row["component_role"]) or "unknown",
application_locked=_bool(row.get("application_locked", 0) if isinstance(row, dict) else row["application_locked"]),
domain_locked=_bool(row.get("domain_locked", 0) if isinstance(row, dict) else row["domain_locked"]),
component_role_locked=_bool(row.get("component_role_locked", 0) if isinstance(row, dict) else row["component_role_locked"]),
criticality=row.get("criticality", "medium") if isinstance(row, dict) else row["criticality"],
category=row.get("category", "unknown") if isinstance(row, dict) else row["category"],
visibility=row.get("visibility", "visible") if isinstance(row, dict) else row["visibility"],
lifecycle=row.get("lifecycle", "active") if isinstance(row, dict) else row["lifecycle"],
source_type=row.get("source_type") if isinstance(row, dict) else row["source_type"],
source_ref=row.get("source_ref") if isinstance(row, dict) else row["source_ref"],
first_seen_at=(
datetime.fromisoformat(row["first_seen_at"])
if (row.get("first_seen_at") if isinstance(row, dict) else row["first_seen_at"])
else None
),
last_seen_at=(
datetime.fromisoformat(row["last_seen_at"])
if (row.get("last_seen_at") if isinstance(row, dict) else row["last_seen_at"])
else None
),
last_discovery_status=(
row.get("last_discovery_status")
if isinstance(row, dict)
else row["last_discovery_status"]
),
container_image=row.get("container_image") if isinstance(row, dict) else row["container_image"],
compose_project=row.get("compose_project") if isinstance(row, dict) else row["compose_project"],
layout_x=row.get("layout_x") if isinstance(row, dict) else row["layout_x"],
layout_y=row.get("layout_y") if isinstance(row, dict) else row["layout_y"],
layout_pinned=_bool(
row.get("layout_pinned", 0) if isinstance(row, dict) else row["layout_pinned"]
),
interval_seconds=row.get("interval_seconds") if isinstance(row, dict) else row["interval_seconds"],
timeout_seconds=row.get("timeout_seconds") if isinstance(row, dict) else row["timeout_seconds"],
retries=row.get("retries") if isinstance(row, dict) else row["retries"],
failure_threshold=(row.get("failure_threshold", 1) if isinstance(row, dict) else row["failure_threshold"]) or 1,
recovery_threshold=(row.get("recovery_threshold", 1) if isinstance(row, dict) else row["recovery_threshold"]) or 1,
expected_status_min=row.get("expected_status_min") if isinstance(row, dict) else row["expected_status_min"],
expected_status_max=row.get("expected_status_max") if isinstance(row, dict) else row["expected_status_max"],
availability_target=row.get("availability_target") if isinstance(row, dict) else row["availability_target"],
maintenance_enabled=_bool(row.get("maintenance_enabled", 0) if isinstance(row, dict) else row["maintenance_enabled"]),
maintenance_reason=row.get("maintenance_reason") if isinstance(row, dict) else row["maintenance_reason"],
state=state,
failure_streak=(row.get("failure_streak", 0) if isinstance(row, dict) else row["failure_streak"]) or 0,
recovery_streak=(row.get("recovery_streak", 0) if isinstance(row, dict) else row["recovery_streak"]) or 0,
flapping=_bool(row.get("flapping", 0) if isinstance(row, dict) else row["flapping"]),
flap_count=(row.get("flap_count", 0) if isinstance(row, dict) else row["flap_count"]) or 0,
last_state_change_at=(
datetime.fromisoformat(row["last_state_change_at"])
if (row.get("last_state_change_at") if isinstance(row, dict) else row["last_state_change_at"])
else None
),
created_at=created_at,
updated_at=datetime.fromisoformat(str(raw_updated)) if raw_updated else created_at,
)
@classmethod
def _row_to_probe(cls, row: sqlite3.Row | dict[str, Any]) -> ProbeResult:
return ProbeResult(
id=row["id"],
service_id=row["service_id"],
status=HealthStatus(row["status"]),
latency_ms=row["latency_ms"],
detail=row["detail"],
checked_at=datetime.fromisoformat(row["checked_at"]),
trigger=(row.get("trigger", "manual") if isinstance(row, dict) else row["trigger"]) or "manual",
metadata=cls._json_object(row.get("metadata", "{}") if isinstance(row, dict) else row["metadata"]),
)
@classmethod
def _row_to_event(cls, row: sqlite3.Row | dict[str, Any]) -> EventRecord:
raw_from = row["from_status"]
raw_to = row["to_status"]
return EventRecord(
id=row["id"],
event_type=row["event_type"],
service_id=row["service_id"],
incident_id=row["incident_id"],
from_status=HealthStatus(raw_from) if raw_from else None,
to_status=HealthStatus(raw_to) if raw_to else None,
message=row["message"],
metadata=cls._json_object(row["metadata"]),
created_at=datetime.fromisoformat(row["created_at"]),
)
@classmethod
def _row_to_incident(cls, row: sqlite3.Row | dict[str, Any], services: dict[str, Service] | None = None) -> IncidentRecord:
services = services or {}
root = services.get(row["root_service_id"])
affected_ids = cls._json_list(row["affected_service_ids"])
return IncidentRecord(
id=row["id"],
status=IncidentStatus(row["status"]),
root_service_id=row["root_service_id"],
root_service_name=root.name if root else row["root_service_id"],
started_at=datetime.fromisoformat(row["started_at"]),
detected_at=datetime.fromisoformat(row["detected_at"]),
acknowledged_at=datetime.fromisoformat(row["acknowledged_at"]) if row["acknowledged_at"] else None,
resolved_at=datetime.fromisoformat(row["resolved_at"]) if row["resolved_at"] else None,
title=row["title"],
root_cause_reason=row["root_cause_reason"],
affected_service_ids=affected_ids,
affected_service_names=[services[item].name for item in affected_ids if item in services],
severity=row["severity"],
latest_update_at=datetime.fromisoformat(row["latest_update_at"]),
)
@staticmethod
def _service_params(service: Service) -> dict[str, Any]:
return {
"id": service.id,
"name": service.name,
"kind": service.kind.value,
"url": service.url,
"host": service.host,
"port": service.port,
"depends_on": json.dumps(service.depends_on),
"description": service.description,
"tags": json.dumps(service.tags),
"group": service.group,
"environment": service.environment,
"owner": service.owner,
"enabled": int(service.enabled),
"monitoring_enabled": int(service.monitoring_enabled),
"monitoring_state": service.monitoring_state.value,
"probe_source": service.probe_source.value,
"probe_reachability": service.probe_reachability.value,
"probe_confidence": service.probe_confidence.value,
"runtime_state": service.runtime_state,
"runtime_health": service.runtime_health,
"application": service.application,
"domain": service.domain,
"component_role": service.component_role.value,
"application_locked": int(service.application_locked),
"domain_locked": int(service.domain_locked),
"component_role_locked": int(service.component_role_locked),
"criticality": service.criticality.value,
"category": service.category.value,
"visibility": service.visibility.value,
"lifecycle": service.lifecycle.value,
"source_type": service.source_type,
"source_ref": service.source_ref,
"first_seen_at": _iso(service.first_seen_at),
"last_seen_at": _iso(service.last_seen_at),
"last_discovery_status": service.last_discovery_status,
"container_image": service.container_image,
"compose_project": service.compose_project,
"layout_x": service.layout_x,
"layout_y": service.layout_y,
"layout_pinned": int(service.layout_pinned),
"interval_seconds": service.interval_seconds,
"timeout_seconds": service.timeout_seconds,
"retries": service.retries,
"failure_threshold": service.failure_threshold,
"recovery_threshold": service.recovery_threshold,
"expected_status_min": service.expected_status_min,
"expected_status_max": service.expected_status_max,
"availability_target": service.availability_target,
"maintenance_enabled": int(service.maintenance_enabled),
"maintenance_reason": service.maintenance_reason,
"state": service.state.value,
"failure_streak": service.failure_streak,
"recovery_streak": service.recovery_streak,
"flapping": int(service.flapping),
"flap_count": service.flap_count,
"last_state_change_at": _iso(service.last_state_change_at),
"created_at": service.created_at.isoformat(),
"updated_at": service.updated_at.isoformat(),
}
@classmethod
def _insert_service(cls, db: sqlite3.Connection, service: Service) -> None:
params = cls._service_params(service)
db.execute(
f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES ("
":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, "
":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, "
":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, "
":compose_project, :layout_x, :layout_y, :layout_pinned, "
":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, "
":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, "
":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, "
":last_state_change_at, :created_at, :updated_at)",
params,
)
@classmethod
def _upsert_service(cls, db: sqlite3.Connection, service: Service) -> None:
params = cls._service_params(service)
db.execute(
f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES ("
":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, "
":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, "
":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, "
":compose_project, :layout_x, :layout_y, :layout_pinned, "
":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, "
":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, "
":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, "
":last_state_change_at, :created_at, :updated_at) "
"ON CONFLICT(id) DO UPDATE SET "
"name=excluded.name, kind=excluded.kind, url=excluded.url, host=excluded.host, port=excluded.port, "
"depends_on=excluded.depends_on, description=excluded.description, tags=excluded.tags, "
'"group"=excluded."group", environment=excluded.environment, owner=excluded.owner, '
"enabled=excluded.enabled, monitoring_enabled=excluded.monitoring_enabled, monitoring_state=excluded.monitoring_state, "
"probe_source=excluded.probe_source, probe_reachability=excluded.probe_reachability, probe_confidence=excluded.probe_confidence, "
"runtime_state=excluded.runtime_state, runtime_health=excluded.runtime_health, application=excluded.application, domain=excluded.domain, component_role=excluded.component_role, "
"application_locked=excluded.application_locked, domain_locked=excluded.domain_locked, component_role_locked=excluded.component_role_locked, criticality=excluded.criticality, "
"category=excluded.category, visibility=excluded.visibility, lifecycle=excluded.lifecycle, "
"source_type=excluded.source_type, source_ref=excluded.source_ref, first_seen_at=excluded.first_seen_at, "
"last_seen_at=excluded.last_seen_at, last_discovery_status=excluded.last_discovery_status, "
"container_image=excluded.container_image, compose_project=excluded.compose_project, "
"layout_x=excluded.layout_x, layout_y=excluded.layout_y, layout_pinned=excluded.layout_pinned, "
"interval_seconds=excluded.interval_seconds, "
"timeout_seconds=excluded.timeout_seconds, retries=excluded.retries, failure_threshold=excluded.failure_threshold, "
"recovery_threshold=excluded.recovery_threshold, expected_status_min=excluded.expected_status_min, "
"expected_status_max=excluded.expected_status_max, availability_target=excluded.availability_target, "
"maintenance_enabled=excluded.maintenance_enabled, maintenance_reason=excluded.maintenance_reason, "
"state=excluded.state, failure_streak=excluded.failure_streak, recovery_streak=excluded.recovery_streak, "
"flapping=excluded.flapping, flap_count=excluded.flap_count, last_state_change_at=excluded.last_state_change_at, "
"created_at=excluded.created_at, updated_at=excluded.updated_at",
params,
)
def list_services(self) -> list[Service]:
with self._connect() as db:
rows = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services ORDER BY name COLLATE NOCASE").fetchall()
return [self._row_to_service(row) for row in rows]
def get_service(self, service_id: str) -> Service | None:
with self._connect() as db:
row = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services WHERE id = ?", (service_id,)).fetchone()
return self._row_to_service(row) if row else None
def get_service_by_name(self, name: str) -> Service | None:
with self._connect() as db:
row = db.execute(
f"SELECT {SERVICE_COLUMNS} FROM services WHERE name = ? COLLATE NOCASE", (name,)
).fetchone()
return self._row_to_service(row) if row else None
def create_service(self, payload: ServiceCreate) -> Service:
now = _now()
service = Service(id=str(uuid4()), created_at=now, updated_at=now, **payload.model_dump())
with self._connect() as db:
self._insert_service(db, service)
self._insert_event_db(db, "topology.changed", None, None, None, None, f"Service {service.name} created", {})
event_bus.publish({"event_type": "topology.changed", "message": f"Service {service.name} created"})
return service
def update_service(self, service_id: str, payload: ServiceUpdate) -> Service | None:
existing = self.get_service(service_id)
if existing is None:
return None
updated = Service(
id=service_id,
created_at=existing.created_at,
updated_at=_now(),
state=existing.state,
failure_streak=existing.failure_streak,
recovery_streak=existing.recovery_streak,
flapping=existing.flapping,
flap_count=existing.flap_count,
last_state_change_at=existing.last_state_change_at,
**payload.model_dump(),
)
with self._connect() as db:
self._upsert_service(db, updated)
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {updated.name} updated", {})
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service bijgewerkt"})
return updated
def delete_service(self, service_id: str) -> bool:
with self._connect() as db:
row = db.execute("SELECT name FROM services WHERE id = ?", (service_id,)).fetchone()
if row is None:
return False
db.execute("DELETE FROM services WHERE id = ?", (service_id,))
rows = db.execute("SELECT id, depends_on FROM services").fetchall()
for item in rows:
dependencies = self._json_list(item["depends_on"])
if service_id in dependencies:
db.execute(
"UPDATE services SET depends_on = ?, updated_at = ? WHERE id = ?",
(json.dumps([value for value in dependencies if value != service_id]), _now().isoformat(), item["id"]),
)
# Remove the id as a JSON element. A textual replace() left an
# empty-string entry behind (["a","b"] -> ["","b"]).
for incident in db.execute(
"SELECT id, affected_service_ids FROM incidents WHERE status != 'resolved'"
).fetchall():
affected = self._json_list(incident["affected_service_ids"])
if service_id in affected:
db.execute(
"UPDATE incidents SET affected_service_ids = ? WHERE id = ?",
(json.dumps([value for value in affected if value != service_id]), incident["id"]),
)
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {row['name']} deleted", {})
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service verwijderd"})
return True
def import_config_services(self, definitions: list[ConfigService], mode: str) -> ConfigImportResult:
if mode not in {"merge", "replace"}:
raise ValueError("Importmodus moet merge of replace zijn")
with self._connect() as db:
db.execute("BEGIN IMMEDIATE")
rows = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services").fetchall()
existing = {row["name"].casefold(): self._row_to_service(row) for row in rows}
if mode == "replace":
db.execute("DELETE FROM incidents")
db.execute("DELETE FROM events")
db.execute("DELETE FROM services")
existing = {}
ids_by_name = {name: service.id for name, service in existing.items()}
for definition in definitions:
ids_by_name.setdefault(definition.name.casefold(), str(uuid4()))
for definition in definitions:
missing = [item for item in definition.depends_on if item.casefold() not in ids_by_name]
if missing:
raise ValueError(
f"Service {definition.name} verwijst naar onbekende afhankelijkheden: {', '.join(missing)}"
)
created = 0
updated = 0
now = _now()
for definition in definitions:
key = definition.name.casefold()
current = existing.get(key)
if current is None:
payload = definition.model_dump()
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
else:
# Merge: only fields the document states change. Rebuilding
# from model_dump() reset every omitted field to its default.
payload = current.model_dump(include=set(ServiceCreate.model_fields))
payload.update(definition.model_dump(exclude_unset=True, exclude={"depends_on"}))
if "depends_on" in definition.model_fields_set:
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
if current is None:
service = Service(id=ids_by_name[key], created_at=now, updated_at=now, **payload)
created += 1
else:
service = Service(
id=current.id,
created_at=current.created_at,
updated_at=now,
state=current.state,
failure_streak=current.failure_streak,
recovery_streak=current.recovery_streak,
flapping=current.flapping,
flap_count=current.flap_count,
last_state_change_at=current.last_state_change_at,
**payload,
)
updated += 1
if current is None:
self._insert_service(db, service)
else:
self._upsert_service(db, service)
self._insert_event_db(db, "topology.changed", None, None, None, None, "Configuratie geïmporteerd", {"mode": mode})
event_bus.publish({"event_type": "topology.changed", "message": "Configuratie geïmporteerd"})
return ConfigImportResult(created=created, updated=updated, total=len(definitions))
def replace_all_services(self, definitions: list[ServiceCreate]) -> list[Service]:
with self._connect() as db:
db.execute("DELETE FROM services")
return [self.create_service(item) for item in definitions]
@staticmethod
def _row_to_health_evidence(row: sqlite3.Row) -> HealthEvidence:
return with_freshness(
HealthEvidence(
id=row["id"],
subject_type=row["subject_type"],
subject_id=row["subject_id"],
service_id=row["service_id"],
provider=row["provider"],
provider_ref=row["provider_ref"],
status=row["status"],
scope=row["scope"],
confidence=float(row["confidence"]),
observed_at=datetime.fromisoformat(row["observed_at"]),
valid_until=datetime.fromisoformat(row["valid_until"]),
received_at=datetime.fromisoformat(row["received_at"]),
summary=row["summary"] or "",
metadata=Store._json_object(row["metadata"]),
active=_bool(row["active"], True),
superseded_by=row["superseded_by"],
)
)
def record_health_evidence(
self,
evidence: HealthEvidence,
*,
service_ids: list[str] | None = None,
) -> HealthEvidence:
"""Store a current provider claim and retain superseded provenance.
Repeated discovery cycles renew the same active claim instead of
growing unbounded identical rows. A changed claim deactivates only
the prior claim from that provider/subject; other providers remain
independently auditable and can surface a conflict.
"""
links = sorted({*(service_ids or []), *({evidence.service_id} if evidence.service_id else set())})
with self._connect() as db:
prior = db.execute(
"""
SELECT * FROM health_evidence
WHERE subject_type = ? AND subject_id = ? AND provider = ?
AND COALESCE(provider_ref, '') = COALESCE(?, '') AND scope = ? AND active = 1
ORDER BY observed_at DESC, rowid DESC LIMIT 1
""",
(
evidence.subject_type,
evidence.subject_id,
evidence.provider.value,
evidence.provider_ref,
evidence.scope.value,
),
).fetchone()
saved = evidence
materially_changed = prior is None or any(
(
prior[field] != value
for field, value in (
("status", evidence.status.value),
("confidence", evidence.confidence),
("summary", evidence.summary),
)
)
) or _stable_metadata(self._json_object(prior["metadata"])) != _stable_metadata(evidence.metadata)
if prior is not None and prior["status"] == evidence.status.value:
saved = evidence.model_copy(update={"id": prior["id"], "active": True, "superseded_by": None})
ttl = evidence.valid_until - evidence.observed_at
prior_valid_until = datetime.fromisoformat(prior["valid_until"])
if (
not materially_changed
and prior["service_id"] == evidence.service_id
and prior_valid_until - _now() > ttl / 2
):
# Unchanged and comfortably fresh: skip the timestamp-only
# rewrite. It is renewed once less than half the TTL is
# left, so the claim never goes stale, with about half
# the writes of renewing on every observation cycle.
saved = self._row_to_health_evidence(prior)
for service_id in links:
db.execute(
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
(saved.id, service_id),
)
return saved
db.execute(
"""
UPDATE health_evidence SET service_id = ?, confidence = ?, observed_at = ?, valid_until = ?,
received_at = ?, summary = ?, metadata = ?, active = 1, superseded_by = NULL
WHERE id = ?
""",
(
saved.service_id,
saved.confidence,
saved.observed_at.isoformat(),
saved.valid_until.isoformat(),
saved.received_at.isoformat(),
saved.summary,
json.dumps(saved.metadata, sort_keys=True),
saved.id,
),
)
else:
if prior is not None:
db.execute("UPDATE health_evidence SET active = 0, superseded_by = ? WHERE id = ?", (evidence.id, prior["id"]))
db.execute(
"""
INSERT INTO health_evidence (
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
confidence, observed_at, valid_until, received_at, summary, metadata, active, superseded_by
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
saved.id,
saved.subject_type,
saved.subject_id,
saved.service_id,
saved.provider.value,
saved.provider_ref,
saved.status.value,
saved.scope.value,
saved.confidence,
saved.observed_at.isoformat(),
saved.valid_until.isoformat(),
saved.received_at.isoformat(),
saved.summary,
json.dumps(saved.metadata, sort_keys=True),
int(saved.active),
saved.superseded_by,
),
)
for service_id in links:
db.execute(
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
(saved.id, service_id),
)
if materially_changed:
event_bus.publish(
{
"event_type": "health.evidence_changed",
"service_id": evidence.service_id,
"provider": evidence.provider.value,
"status": evidence.status.value,
"message": "Gezondheidsbewijs gewijzigd",
}
)
return saved
def health_evidence_for_service(self, service_id: str, *, limit: int = 100) -> list[HealthEvidence]:
# A direct claim and a linked claim are matched by two independently
# indexed lookups. An ``OR`` across the LEFT JOIN cannot use either
# index and forced a full ``health_evidence`` scan on every call.
with self._connect() as db:
rows = db.execute(
"""
SELECT e.* FROM health_evidence e
WHERE e.id IN (
SELECT id FROM health_evidence WHERE service_id = ?
UNION
SELECT evidence_id FROM health_evidence_service_links WHERE service_id = ?
)
ORDER BY e.observed_at DESC, e.rowid DESC LIMIT ?
""",
(service_id, service_id, limit),
).fetchall()
return [self._row_to_health_evidence(row) for row in rows]
def health_evidence_for_services(
self, service_ids: list[str], *, limit_per_service: int = 100
) -> dict[str, list[HealthEvidence]]:
"""Fetch linked evidence for a topology in one bounded query.
A semantic topology can contain many applications. Resolving each
application with a separate evidence query creates an avoidable N+1
SQLite scan, so this method projects the evidence links once and then
groups them in memory.
"""
ids = sorted(set(service_ids))
result = {service_id: [] for service_id in ids}
if not ids:
return result
placeholders = ", ".join("?" for _ in ids)
with self._connect() as db:
# Both match paths use their own index; the per-service limit is
# applied in SQL so superseded history is not materialised.
rows = db.execute(
f"""
WITH matched(evidence_id, matched_service_id) AS (
SELECT id, service_id FROM health_evidence WHERE service_id IN ({placeholders})
UNION
SELECT evidence_id, service_id FROM health_evidence_service_links
WHERE service_id IN ({placeholders})
),
ranked AS (
SELECT e.*, m.matched_service_id,
ROW_NUMBER() OVER (
PARTITION BY m.matched_service_id
ORDER BY e.observed_at DESC, e.rowid DESC
) AS rank_in_service
FROM matched m
JOIN health_evidence e ON e.id = m.evidence_id
)
SELECT * FROM ranked
WHERE rank_in_service <= ?
ORDER BY matched_service_id, rank_in_service
""",
[*ids, *ids, limit_per_service],
).fetchall()
parsed: dict[str, HealthEvidence] = {}
for row in rows:
service_id = row["matched_service_id"]
if service_id not in result:
continue
evidence = parsed.get(row["id"])
if evidence is None:
evidence = self._row_to_health_evidence(row)
parsed[row["id"]] = evidence
result[service_id].append(evidence)
return result
def application_evidence_links(self, provider: EvidenceProvider) -> dict[str, set[str]]:
"""Return persisted application-claim links keyed by provider reference."""
with self._connect() as db:
rows = db.execute(
"""
SELECT e.provider_ref, e.subject_id, link.service_id
FROM health_evidence e
JOIN health_evidence_service_links link ON link.evidence_id = e.id
WHERE e.provider = ? AND e.scope = ?
""",
(provider.value, EvidenceScope.APPLICATION.value),
).fetchall()
result: dict[str, set[str]] = {}
for row in rows:
key = row["provider_ref"] or row["subject_id"]
if key:
result.setdefault(key, set()).add(row["service_id"])
return result
def health_resolution(self, service_id: str) -> HealthResolution:
return resolve_health(self.health_evidence_for_service(service_id))
def set_provider_status(self, value: ProviderStatus) -> ProviderStatus:
with self._connect() as db:
previous = db.execute(
"SELECT status, detail, metadata FROM provider_status WHERE provider = ?",
(value.provider.value,),
).fetchone()
db.execute(
"""
INSERT INTO provider_status (provider, status, observed_at, valid_until, detail, metadata)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(provider) DO UPDATE SET status=excluded.status, observed_at=excluded.observed_at,
valid_until=excluded.valid_until, detail=excluded.detail, metadata=excluded.metadata
""",
(
value.provider.value,
value.status.value,
value.observed_at.isoformat(),
value.valid_until.isoformat(),
value.detail,
json.dumps(value.metadata, sort_keys=True),
),
)
if previous is None or any(
(
previous[field] != expected
for field, expected in (
("status", value.status.value),
("detail", value.detail),
("metadata", json.dumps(value.metadata, sort_keys=True)),
)
)
):
event_bus.publish(
{
"event_type": "provider.status_changed",
"provider": value.provider.value,
"status": value.status.value,
"message": "Beschikbaarheid van de bron gewijzigd",
}
)
return value
def provider_statuses(self) -> list[ProviderStatus]:
with self._connect() as db:
rows = db.execute("SELECT * FROM provider_status ORDER BY provider").fetchall()
return [
ProviderStatus(
provider=row["provider"],
status=row["status"],
observed_at=datetime.fromisoformat(row["observed_at"]),
valid_until=datetime.fromisoformat(row["valid_until"]),
detail=row["detail"] or "",
metadata=self._json_object(row["metadata"]),
)
for row in rows
]
@staticmethod
def _legacy_health_from_row(row: sqlite3.Row) -> HealthStatus:
"""Compatibility fallback for pre-v1.6 data without component evidence."""
runtime_state = str(row["runtime_state"] or "").casefold()
runtime_health = str(row["runtime_health"] or "").casefold()
if runtime_health == "unhealthy" or runtime_state in {"exited", "dead", "stopped"}:
return HealthStatus.DOWN
if runtime_state == "restarting":
return HealthStatus.DEGRADED
if not _bool(row["monitoring_enabled"], True):
if runtime_health == "healthy":
return HealthStatus.HEALTHY
if str(row["monitoring_state"] or "").casefold() in {"unmonitored", "observed"}:
return HealthStatus.UNMONITORED
return HealthStatus.UNKNOWN
try:
return HealthStatus(row["state"] or "unknown")
except ValueError:
return HealthStatus.UNKNOWN
def service_health(self, service_id: str) -> HealthStatus:
with self._connect() as db:
row = db.execute(
"SELECT state, maintenance_enabled, enabled, monitoring_enabled, monitoring_state, runtime_state, runtime_health "
"FROM services WHERE id = ?",
(service_id,),
).fetchone()
if row is None or not _bool(row["enabled"], True):
return HealthStatus.UNKNOWN
if _bool(row["maintenance_enabled"]):
return HealthStatus.MAINTENANCE
resolution = self.health_resolution(service_id)
if resolution.selected_evidence_id:
return resolution.status
# Once v1.6 evidence exists, an expired/stale claim must not silently
# keep a historical state alive. Legacy fallback is only for rows
# that predate evidence entirely. Old AppOps-only rows were always
# aggregate application observations, never component health.
if resolution.evidence:
return HealthStatus.UNKNOWN
if str(row["monitoring_state"] or "").casefold() == MonitoringState.APPOPS.value:
return HealthStatus.UNMONITORED
return self._legacy_health_from_row(row)
def service_healths(
self,
service_ids: list[str],
*,
evidence_by_service: dict[str, list[HealthEvidence]] | None = None,
) -> tuple[dict[str, HealthStatus], dict[str, HealthResolution]]:
"""Resolve direct health for a service set without per-service SQL reads."""
ids = sorted(set(service_ids))
if not ids:
return {}, {}
evidence_by_service = evidence_by_service or self.health_evidence_for_services(ids)
placeholders = ", ".join("?" for _ in ids)
with self._connect() as db:
rows = db.execute(
f"""
SELECT id, state, maintenance_enabled, enabled, monitoring_enabled,
monitoring_state, runtime_state, runtime_health
FROM services WHERE id IN ({placeholders})
""",
ids,
).fetchall()
status_by_id: dict[str, HealthStatus] = {}
resolutions: dict[str, HealthResolution] = {}
for row in rows:
service_id = row["id"]
resolution = resolve_health(evidence_by_service.get(service_id, []))
resolutions[service_id] = resolution
if not _bool(row["enabled"], True):
status_by_id[service_id] = HealthStatus.UNKNOWN
elif _bool(row["maintenance_enabled"]):
status_by_id[service_id] = HealthStatus.MAINTENANCE
elif resolution.selected_evidence_id:
status_by_id[service_id] = resolution.status
elif resolution.evidence:
status_by_id[service_id] = HealthStatus.UNKNOWN
elif str(row["monitoring_state"] or "").casefold() == MonitoringState.APPOPS.value:
status_by_id[service_id] = HealthStatus.UNMONITORED
else:
status_by_id[service_id] = self._legacy_health_from_row(row)
return status_by_id, resolutions
def maintenance_windows(self) -> dict[str, datetime]:
with self._connect() as db:
rows = db.execute("SELECT service_id, ends_at FROM maintenance_windows").fetchall()
return {row["service_id"]: datetime.fromisoformat(row["ends_at"]) for row in rows}
def expire_maintenance_windows(self, now: datetime | None = None) -> list[str]:
"""End maintenance for every window whose end time has passed."""
now = now or _now()
with self._connect() as db:
rows = db.execute(
"SELECT service_id FROM maintenance_windows WHERE ends_at <= ?", (now.isoformat(),)
).fetchall()
ended: list[str] = []
for row in rows:
if self.set_maintenance(row["service_id"], False, "Maintenance window ended") is not None:
ended.append(row["service_id"])
return ended
def set_maintenance(
self,
service_id: str,
enabled: bool,
reason: str | None = None,
until: datetime | None = None,
) -> Service | None:
service = self.get_service(service_id)
if service is None:
return None
if enabled and until is not None and until <= _now():
raise ValueError("Het einde van het onderhoud moet in de toekomst liggen")
updated = service.model_copy(
update={
"maintenance_enabled": enabled,
"maintenance_reason": reason if enabled else None,
"state": HealthStatus.MAINTENANCE if enabled else HealthStatus.UNKNOWN,
"failure_streak": 0,
"recovery_streak": 0,
"updated_at": _now(),
}
)
with self._connect() as db:
self._upsert_service(db, updated)
if enabled and until is not None:
db.execute(
"INSERT INTO maintenance_windows (service_id, ends_at) VALUES (?, ?) "
"ON CONFLICT(service_id) DO UPDATE SET ends_at = excluded.ends_at",
(service_id, until.isoformat()),
)
else:
db.execute("DELETE FROM maintenance_windows WHERE service_id = ?", (service_id,))
self._insert_event_db(
db,
"service.maintenance_changed",
service_id,
None,
service.state.value,
HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
f"Onderhoud {'gestart' if enabled else 'beëindigd'} voor {service.name}",
{
**({"reason": reason} if reason else {}),
**({"until": until.isoformat()} if enabled and until else {}),
},
)
event_bus.publish(
{
"event_type": "service.maintenance_changed",
"service_id": service_id,
"status": HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
"message": f"Onderhoud {'gestart' if enabled else 'beëindigd'}",
}
)
from .incidents import sync_incidents
sync_incidents(self)
return updated
def _stable_state(self, row: sqlite3.Row, raw_status: HealthStatus) -> tuple[HealthStatus, int, int]:
if not _bool(row["enabled"], True) or not _bool(row["monitoring_enabled"], True):
monitoring_state = str(row["monitoring_state"] or "").casefold()
return (HealthStatus.UNMONITORED if monitoring_state in {"unmonitored", "observed"} else HealthStatus.UNKNOWN), 0, 0
if _bool(row["maintenance_enabled"]):
return HealthStatus.MAINTENANCE, 0, 0
previous = HealthStatus(row["state"] or "unknown")
failure_streak = int(row["failure_streak"] or 0)
recovery_streak = int(row["recovery_streak"] or 0)
failure_threshold = max(1, int(row["failure_threshold"] or 1))
recovery_threshold = max(1, int(row["recovery_threshold"] or 1))
if raw_status in {HealthStatus.DOWN, HealthStatus.DEGRADED}:
failure_streak += 1
recovery_streak = 0
if failure_streak < failure_threshold:
if previous == HealthStatus.UNKNOWN:
return HealthStatus.UNKNOWN, failure_streak, recovery_streak
if previous == HealthStatus.RECOVERING:
return HealthStatus.RECOVERING, failure_streak, recovery_streak
return previous, failure_streak, recovery_streak
return raw_status, failure_streak, recovery_streak
if raw_status == HealthStatus.HEALTHY:
recovery_streak += 1
failure_streak = 0
if previous in {HealthStatus.DOWN, HealthStatus.DEGRADED, HealthStatus.RECOVERING} and recovery_streak < recovery_threshold:
return HealthStatus.RECOVERING, failure_streak, recovery_streak
return HealthStatus.HEALTHY, failure_streak, recovery_streak
return HealthStatus.UNMONITORED if raw_status == HealthStatus.UNMONITORED else HealthStatus.UNKNOWN, 0, 0
def set_runtime_observation(
self,
service_id: str,
*,
runtime_state: str | None,
runtime_health: str | None,
monitoring_state: MonitoringState | str,
provider: EvidenceProvider = EvidenceProvider.DOCKER,
provider_ref: str | None = None,
confidence: float = 0.9,
) -> Service | None:
"""Persist Docker runtime truth without fabricating a network probe."""
service = self.get_service(service_id)
if service is None:
return None
state_kind = MonitoringState(monitoring_state)
normalized_state = (runtime_state or "").casefold()
normalized_health = (runtime_health or "").casefold()
if service.maintenance_enabled:
status = HealthStatus.MAINTENANCE
elif normalized_health in {"unhealthy", "down"} or normalized_state in {"exited", "dead", "stopped"}:
status = HealthStatus.DOWN
elif normalized_health == "degraded" or normalized_state == "restarting":
status = HealthStatus.DEGRADED
elif normalized_health == "healthy":
status = HealthStatus.HEALTHY
elif state_kind in {
MonitoringState.UNMONITORED,
MonitoringState.OBSERVED,
MonitoringState.APPOPS,
}:
status = HealthStatus.UNMONITORED
else:
status = service.state
updated = service.model_copy(
update={
"runtime_state": runtime_state,
"runtime_health": runtime_health,
"monitoring_state": state_kind,
"state": status,
"failure_streak": 0 if status in {HealthStatus.UNMONITORED, HealthStatus.HEALTHY} else service.failure_streak,
"recovery_streak": 0 if status == HealthStatus.UNMONITORED else service.recovery_streak,
"last_state_change_at": _now() if service.state != status else service.last_state_change_at,
"updated_at": _now(),
}
)
with self._connect() as db:
self._upsert_service(db, updated)
if service.state != status:
self._insert_event_db(
db,
"service.runtime_observed",
service_id,
None,
service.state.value,
status.value,
f"Docker-runtimewaarneming bijgewerkt voor {service.name}",
{"runtime_state": runtime_state, "runtime_health": runtime_health, "monitoring_state": state_kind.value},
)
if service.state != status:
event_bus.publish({"event_type": "service.runtime_observed", "service_id": service_id, "status": status.value, "message": "Docker-runtimewaarneming bijgewerkt"})
observed_at = _now()
self.record_health_evidence(
HealthEvidence(
id=str(uuid4()),
subject_type="service",
subject_id=service_id,
service_id=service_id,
provider=provider,
provider_ref=provider_ref or service.source_ref,
status=status,
scope=EvidenceScope.COMPONENT,
confidence=confidence,
observed_at=observed_at,
valid_until=observed_at + timedelta(seconds=max(60, int(service.interval_seconds or 60) * 2)),
received_at=observed_at,
summary="Docker-runtimewaarneming" if provider == EvidenceProvider.DOCKER else "Runtimewaarneming",
metadata={"runtime_state": runtime_state, "runtime_health": runtime_health},
)
)
return updated
def save_probe(self, probe: ProbeResult) -> ProbeResult:
events: list[dict[str, Any]] = []
with self._connect() as db:
row = db.execute("SELECT * FROM services WHERE id = ?", (probe.service_id,)).fetchone()
if row is None:
return probe
db.execute(
"""
INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
probe.id,
probe.service_id,
probe.status.value,
probe.latency_ms,
probe.detail,
probe.checked_at.isoformat(),
probe.trigger,
json.dumps(probe.metadata),
),
)
previous = HealthStatus(row["state"] or "unknown")
next_state, failure_streak, recovery_streak = self._stable_state(row, probe.status)
now = probe.checked_at
state_changed = next_state != previous
last_state_change = now.isoformat() if state_changed else row["last_state_change_at"]
if state_changed:
event = {
"event_type": "service.state_changed",
"service_id": probe.service_id,
"from_status": previous.value,
"to_status": next_state.value,
"message": f"Service gewijzigd van {previous.value} to {next_state.value}",
"metadata": {"trigger": probe.trigger, "probe_id": probe.id},
}
self._insert_event_db(
db,
event["event_type"],
probe.service_id,
None,
previous.value,
next_state.value,
event["message"],
event["metadata"],
created_at=now,
)
events.append(event)
since = (now - timedelta(minutes=30)).isoformat()
flap_count = int(
db.execute(
"SELECT COUNT(*) FROM events WHERE service_id = ? AND event_type = 'service.state_changed' AND created_at >= ?",
(probe.service_id, since),
).fetchone()[0]
)
flapping = flap_count >= int(os.getenv("SERVICEMAP_FLAP_THRESHOLD", "6"))
if flapping and not _bool(row["flapping"]):
self._insert_event_db(
db,
"flapping.detected",
probe.service_id,
None,
None,
None,
f"Wisselvalligheid vastgesteld: {flap_count} statuswijzigingen in 30 minuten",
{"state_changes": flap_count, "window_minutes": 30},
created_at=now,
)
events.append({"event_type": "flapping.detected", "service_id": probe.service_id, "message": "Wisselvalligheid vastgesteld"})
db.execute(
"""
UPDATE services
SET state = ?, failure_streak = ?, recovery_streak = ?, flapping = ?, flap_count = ?,
last_state_change_at = ?, updated_at = ?
WHERE id = ?
""",
(
next_state.value,
failure_streak,
recovery_streak,
int(flapping),
flap_count,
last_state_change,
now.isoformat(),
probe.service_id,
),
)
event_bus.publish(
{
"event_type": "probe.completed",
"service_id": probe.service_id,
"status": probe.status.value,
"trigger": probe.trigger,
"message": "Controle voltooid",
}
)
for event in events:
event_bus.publish(event)
self.record_health_evidence(
HealthEvidence(
id=f"probe-{probe.id}",
subject_type="service",
subject_id=probe.service_id,
service_id=probe.service_id,
provider=EvidenceProvider.PROBE,
provider_ref="active-probe",
# The evidence mirrors the hysteresis-filtered service state,
# while the raw probe result remains intact in history.
status=next_state,
scope=EvidenceScope.COMPONENT,
confidence=1.0,
observed_at=probe.checked_at,
valid_until=probe.checked_at + timedelta(seconds=max(60, int(os.getenv("SERVICEMAP_PROBE_INTERVAL", "60")) * 2)),
received_at=_now(),
summary=probe.detail,
metadata={"probe_id": probe.id, "trigger": probe.trigger, "raw_status": probe.status.value, "latency_ms": probe.latency_ms, **probe.metadata},
)
)
from .incidents import sync_incidents
sync_incidents(self)
return probe
def latest_probe(self, service_id: str) -> ProbeResult | None:
with self._connect() as db:
row = db.execute(
"SELECT * FROM probe_results WHERE service_id = ? ORDER BY checked_at DESC, rowid DESC LIMIT 1",
(service_id,),
).fetchone()
return self._row_to_probe(row) if row else None
def latest_probes(self, service_ids: list[str] | None = None) -> dict[str, ProbeResult]:
with self._connect() as db:
if service_ids:
placeholders = ",".join("?" for _ in service_ids)
rows = db.execute(
f"""
SELECT p.* FROM probe_results p
WHERE p.service_id IN ({placeholders})
AND p.rowid = (
SELECT candidate.rowid FROM probe_results candidate
WHERE candidate.service_id = p.service_id
ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1
)
""",
service_ids,
).fetchall()
else:
rows = db.execute(
"""
SELECT p.* FROM probe_results p
WHERE p.rowid = (
SELECT candidate.rowid FROM probe_results candidate
WHERE candidate.service_id = p.service_id
ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1
)
"""
).fetchall()
return {row["service_id"]: self._row_to_probe(row) for row in rows}
def probe_history(self, service_id: str, limit: int = 50) -> list[ProbeResult]:
with self._connect() as db:
rows = db.execute(
"SELECT * FROM probe_results WHERE service_id = ? ORDER BY checked_at DESC, rowid DESC LIMIT ?",
(service_id, limit),
).fetchall()
return [self._row_to_probe(row) for row in rows]
def availability_summary(self, service_id: str, hours: int = 24) -> AvailabilitySummary:
hours = max(1, min(hours, 24 * 90))
since = _now() - timedelta(hours=hours)
with self._connect() as db:
row = db.execute(
"""
SELECT COUNT(*) AS total,
SUM(CASE WHEN status = 'healthy' THEN 1 ELSE 0 END) AS healthy,
SUM(CASE WHEN status = 'degraded' THEN 1 ELSE 0 END) AS degraded,
SUM(CASE WHEN status = 'down' THEN 1 ELSE 0 END) AS down_count,
AVG(latency_ms) AS avg_latency, MAX(checked_at) AS last_checked
FROM probe_results WHERE service_id = ? AND checked_at >= ?
""",
(service_id, since.isoformat()),
).fetchone()
service_row = db.execute("SELECT availability_target FROM services WHERE id = ?", (service_id,)).fetchone()
total = int(row["total"] or 0)
healthy = int(row["healthy"] or 0)
degraded = int(row["degraded"] or 0)
down = int(row["down_count"] or 0)
uptime = round(((healthy + degraded) / total) * 100, 2) if total else None
target = service_row["availability_target"] if service_row else None
return AvailabilitySummary(
hours=hours,
total_checks=total,
healthy_checks=healthy,
degraded_checks=degraded,
down_checks=down,
uptime_percent=uptime,
average_latency_ms=round(float(row["avg_latency"]), 2) if row["avg_latency"] is not None else None,
last_checked_at=datetime.fromisoformat(row["last_checked"]) if row["last_checked"] else None,
target_percent=target,
meets_target=(uptime >= target) if uptime is not None and target is not None else None,
)
def availability_summaries(self, service_ids: list[str], hours: int = 24) -> dict[str, AvailabilitySummary]:
"""Fetch table-ready availability in one grouped query, not N requests."""
if not service_ids:
return {}
hours = max(1, min(hours, 24 * 90))
since = _now() - timedelta(hours=hours)
placeholders = ",".join("?" for _ in service_ids)
with self._connect() as db:
rows = db.execute(
f"""
SELECT s.id, s.availability_target, COUNT(p.id) AS total,
SUM(CASE WHEN p.status = 'healthy' THEN 1 ELSE 0 END) AS healthy,
SUM(CASE WHEN p.status = 'degraded' THEN 1 ELSE 0 END) AS degraded,
SUM(CASE WHEN p.status = 'down' THEN 1 ELSE 0 END) AS down_count,
AVG(p.latency_ms) AS avg_latency, MAX(p.checked_at) AS last_checked
FROM services s
LEFT JOIN probe_results p ON p.service_id=s.id AND p.checked_at >= ?
WHERE s.id IN ({placeholders})
GROUP BY s.id, s.availability_target
""",
[since.isoformat(), *service_ids],
).fetchall()
result: dict[str, AvailabilitySummary] = {}
for row in rows:
total = int(row["total"] or 0)
healthy = int(row["healthy"] or 0)
degraded = int(row["degraded"] or 0)
target = row["availability_target"]
uptime = round(((healthy + degraded) / total) * 100, 2) if total else None
result[row["id"]] = AvailabilitySummary(
hours=hours,
total_checks=total,
healthy_checks=healthy,
degraded_checks=degraded,
down_checks=int(row["down_count"] or 0),
uptime_percent=uptime,
average_latency_ms=round(float(row["avg_latency"]), 2) if row["avg_latency"] is not None else None,
last_checked_at=datetime.fromisoformat(row["last_checked"]) if row["last_checked"] else None,
target_percent=target,
meets_target=(uptime >= target) if uptime is not None and target is not None else None,
)
return result
def next_probe_at(self, service: Service, global_interval: int) -> datetime | None:
if not service.enabled or not service.monitoring_enabled:
return None
last = self.latest_probe(service.id)
if last is None:
return _now()
return last.checked_at + timedelta(seconds=service.interval_seconds or global_interval)
def prune_probes(self, retention_days: int, event_retention_days: int | None = None) -> int:
cutoff = _now() - timedelta(days=retention_days)
event_cutoff = _now() - timedelta(days=event_retention_days or retention_days)
with self._connect() as db:
cursor = db.execute("DELETE FROM probe_results WHERE checked_at < ?", (cutoff.isoformat(),))
db.execute("DELETE FROM events WHERE created_at < ? AND event_type NOT LIKE 'incident.%'", (event_cutoff.isoformat(),))
return cursor.rowcount
def checkpoint(self) -> None:
"""Fold the WAL back into the database and refresh planner statistics.
Without a TRUNCATE checkpoint the WAL file keeps its high-water size.
"""
with self._connect() as db:
db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
db.execute("PRAGMA optimize")
def ready_check(self) -> None:
"""Raise if the database cannot take a write lock right now."""
connection = self._connect()
try:
connection.execute("BEGIN IMMEDIATE")
connection.execute("ROLLBACK")
finally:
connection.close()
def prune_health_evidence(self, retention_days: int) -> int:
"""Delete superseded evidence history older than the retention window.
Only inactive rows are removed. A row becomes inactive exclusively
when a newer claim for the same provider/subject supersedes it, so
every subject keeps its current (active) claim and health resolution
is unaffected. Without this, every status flip leaves a permanent row
and evidence reads grow linearly with uptime.
"""
if retention_days <= 0:
return 0
cutoff = _now() - timedelta(days=retention_days)
with self._connect() as db:
cursor = db.execute(
"DELETE FROM health_evidence WHERE active = 0 AND superseded_by IS NOT NULL AND received_at < ?",
(cutoff.isoformat(),),
)
# Links cascade through the foreign key; also remove any orphan
# links left by databases that predate enforced foreign keys.
db.execute(
"DELETE FROM health_evidence_service_links "
"WHERE evidence_id NOT IN (SELECT id FROM health_evidence)"
)
return cursor.rowcount
def _insert_event_db(
self,
db: sqlite3.Connection,
event_type: str,
service_id: str | None,
incident_id: str | None,
from_status: str | None,
to_status: str | None,
message: str,
metadata: dict[str, object],
created_at: datetime | None = None,
) -> str:
event_id = str(uuid4())
db.execute(
"""
INSERT INTO events (id, event_type, service_id, incident_id, from_status, to_status, message, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(event_id, event_type, service_id, incident_id, from_status, to_status, message, json.dumps(metadata), (created_at or _now()).isoformat()),
)
return event_id
def add_event(
self,
event_type: str,
message: str,
*,
service_id: str | None = None,
incident_id: str | None = None,
from_status: HealthStatus | None = None,
to_status: HealthStatus | None = None,
metadata: dict[str, object] | None = None,
) -> EventRecord:
created_at = _now()
with self._connect() as db:
event_id = self._insert_event_db(
db,
event_type,
service_id,
incident_id,
from_status.value if from_status else None,
to_status.value if to_status else None,
message,
metadata or {},
created_at,
)
row = db.execute("SELECT * FROM events WHERE id = ?", (event_id,)).fetchone()
result = self._row_to_event(row)
event_bus.publish(result.model_dump(mode="json"))
return result
def list_events(
self,
limit: int = 50,
service_id: str | None = None,
event_type: str | None = None,
before: datetime | None = None,
) -> list[EventRecord]:
"""Newest first. ``event_type`` ending in ``.`` matches a family
(``incident.``); ``before`` pages to older events."""
clauses: list[str] = []
params: list[object] = []
if service_id:
clauses.append("service_id = ?")
params.append(service_id)
if event_type:
if event_type.endswith("."):
# Exact prefix match without LIKE wildcards.
clauses.append("substr(event_type, 1, ?) = ?")
params.extend([len(event_type), event_type])
else:
clauses.append("event_type = ?")
params.append(event_type)
if before is not None:
clauses.append("created_at < ?")
params.append(before.isoformat())
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
with self._connect() as db:
rows = db.execute(
f"SELECT * FROM events {where} ORDER BY created_at DESC LIMIT ?", [*params, limit]
).fetchall()
return [self._row_to_event(row) for row in rows]
def demo_active(self) -> bool:
with self._connect() as db:
return db.execute("SELECT 1 FROM demo_sessions WHERE id = 1").fetchone() is not None
@staticmethod
def _snapshot_rows(db: sqlite3.Connection) -> tuple[str, str, str, str]:
services = [dict(row) for row in db.execute("SELECT * FROM services ORDER BY id").fetchall()]
probes = [dict(row) for row in db.execute("SELECT * FROM probe_results ORDER BY id").fetchall()]
events = [dict(row) for row in db.execute("SELECT * FROM events ORDER BY id").fetchall()]
incidents = [dict(row) for row in db.execute("SELECT * FROM incidents ORDER BY id").fetchall()]
return tuple(json.dumps(item, separators=(",", ":")) for item in (services, probes, events, incidents)) # type: ignore[return-value]
@staticmethod
def _snapshot_discovery_rows(db: sqlite3.Connection) -> str:
tables = (
"discovery_runs", "discovery_sources", "service_sources", "discovery_observations",
"relation_evidence", "topology_suggestions", "topology_overrides", "topology_change_events",
)
existing = {row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
snapshot: dict[str, list[dict[str, object]]] = {}
for table in tables:
if table in existing:
snapshot[table] = [dict(row) for row in db.execute(f"SELECT * FROM {table}").fetchall()]
return json.dumps(snapshot, separators=(",", ":"))
@staticmethod
def _snapshot_health_evidence_rows(db: sqlite3.Connection) -> str:
return json.dumps(
{
"health_evidence": [dict(row) for row in db.execute("SELECT * FROM health_evidence ORDER BY id").fetchall()],
"health_evidence_service_links": [
dict(row) for row in db.execute("SELECT * FROM health_evidence_service_links ORDER BY evidence_id, service_id").fetchall()
],
"provider_status": [dict(row) for row in db.execute("SELECT * FROM provider_status ORDER BY provider").fetchall()],
},
separators=(",", ":"),
)
@classmethod
def _raw_service_params(cls, raw: dict[str, Any]) -> dict[str, Any]:
defaults: dict[str, Any] = {
"description": None, "tags": "[]", "group": None, "environment": None, "owner": None,
"enabled": 1, "monitoring_enabled": 1, "monitoring_state": "active_probe", "probe_source": "manual",
"probe_reachability": "verified", "probe_confidence": "confirmed", "runtime_state": None, "runtime_health": None,
"application": None, "criticality": "medium", "category": "unknown",
"visibility": "visible", "lifecycle": "active", "source_type": None, "source_ref": None,
"first_seen_at": None, "last_seen_at": None, "last_discovery_status": None,
"container_image": None, "compose_project": None, "layout_x": None, "layout_y": None,
"layout_pinned": 0, "interval_seconds": None, "timeout_seconds": None, "retries": None,
"failure_threshold": 1, "recovery_threshold": 1, "expected_status_min": None,
"expected_status_max": None, "availability_target": None, "maintenance_enabled": 0,
"maintenance_reason": None, "state": "unknown", "failure_streak": 0, "recovery_streak": 0,
"flapping": 0, "flap_count": 0, "last_state_change_at": None, "updated_at": raw.get("created_at"),
"depends_on": "[]",
}
params = {**defaults, **raw}
params["depends_on"] = json.dumps(params["depends_on"]) if isinstance(params["depends_on"], list) else params["depends_on"]
params["tags"] = json.dumps(params["tags"]) if isinstance(params["tags"], list) else params["tags"]
params["updated_at"] = params["updated_at"] or params["created_at"]
params["metadata"] = params.get("metadata", "{}")
params["trigger"] = params.get("trigger", "manual")
return params
@classmethod
def _insert_raw_service(cls, db: sqlite3.Connection, raw: dict[str, Any]) -> None:
params = cls._raw_service_params(raw)
cls._insert_service_row(db, params)
@staticmethod
def _insert_service_row(db: sqlite3.Connection, params: dict[str, Any]) -> None:
db.execute(
f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES ("
":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, "
":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, "
":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, "
":compose_project, :layout_x, :layout_y, :layout_pinned, "
":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, "
":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, "
":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, "
":last_state_change_at, :created_at, :updated_at)",
params,
)
@classmethod
def _restore_rows(cls, db: sqlite3.Connection, snapshot_services: str, snapshot_probes: str, snapshot_events: str, snapshot_incidents: str, snapshot_discovery: str = "{}", snapshot_health_evidence: str = "{}") -> None:
services = json.loads(snapshot_services)
probes = json.loads(snapshot_probes)
events = json.loads(snapshot_events or "[]")
incidents = json.loads(snapshot_incidents or "[]")
db.execute("DELETE FROM incidents")
db.execute("DELETE FROM events")
db.execute("DELETE FROM services")
for raw in services:
cls._insert_raw_service(db, raw)
for raw in probes:
params = {
"id": raw["id"],
"service_id": raw["service_id"],
"status": raw["status"],
"latency_ms": raw.get("latency_ms"),
"detail": raw["detail"],
"checked_at": raw["checked_at"],
"trigger": raw.get("trigger", "manual"),
"metadata": raw.get("metadata", "{}"),
}
db.execute(
"INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata) VALUES (:id, :service_id, :status, :latency_ms, :detail, :checked_at, :trigger, :metadata)",
params,
)
for raw in incidents:
db.execute(
"INSERT INTO incidents (id, status, root_service_id, started_at, detected_at, acknowledged_at, resolved_at, title, root_cause_reason, affected_service_ids, severity, latest_update_at) VALUES (:id, :status, :root_service_id, :started_at, :detected_at, :acknowledged_at, :resolved_at, :title, :root_cause_reason, :affected_service_ids, :severity, :latest_update_at)",
raw,
)
for raw in events:
db.execute(
"INSERT INTO events (id, event_type, service_id, incident_id, from_status, to_status, message, metadata, created_at) VALUES (:id, :event_type, :service_id, :incident_id, :from_status, :to_status, :message, :metadata, :created_at)",
{**raw, "metadata": raw.get("metadata", "{}")},
)
cls._restore_discovery_rows(db, snapshot_discovery)
cls._restore_health_evidence_rows(db, snapshot_health_evidence)
@staticmethod
def _restore_health_evidence_rows(db: sqlite3.Connection, snapshot: str) -> None:
try:
items = json.loads(snapshot or "{}")
except json.JSONDecodeError:
items = {}
if not isinstance(items, dict):
return
db.execute("DELETE FROM health_evidence")
db.execute("DELETE FROM provider_status")
for table in ("health_evidence", "health_evidence_service_links", "provider_status"):
rows = items.get(table, [])
if not isinstance(rows, list):
continue
for raw in rows:
if not isinstance(raw, dict) or not raw:
continue
columns = list(raw)
quoted = ", ".join(f'"{column}"' for column in columns)
placeholders = ", ".join("?" for _ in columns)
db.execute(f"INSERT INTO {table} ({quoted}) VALUES ({placeholders})", tuple(raw[column] for column in columns))
@staticmethod
def _restore_discovery_rows(db: sqlite3.Connection, snapshot: str) -> None:
try:
tables = json.loads(snapshot or "{}")
except json.JSONDecodeError:
tables = {}
if not isinstance(tables, dict) or not tables:
return
order = (
"topology_change_events", "topology_overrides", "relation_evidence", "service_sources",
"discovery_observations", "topology_suggestions", "discovery_sources", "discovery_runs",
)
existing = {row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
for table in order:
if table in existing:
db.execute(f"DELETE FROM {table}")
for table in reversed(order):
rows = tables.get(table, [])
if table not in existing or not isinstance(rows, list):
continue
for raw in rows:
if not isinstance(raw, dict) or not raw:
continue
columns = list(raw)
quoted = ", ".join(f'"{column}"' for column in columns)
placeholders = ", ".join("?" for _ in columns)
db.execute(f"INSERT INTO {table} ({quoted}) VALUES ({placeholders})", tuple(raw[column] for column in columns))
@classmethod
def _insert_demo_rows(cls, db: sqlite3.Connection, services: list[Service], probes: list[ProbeResult]) -> None:
db.execute("DELETE FROM incidents")
db.execute("DELETE FROM events")
db.execute("DELETE FROM health_evidence")
db.execute("DELETE FROM provider_status")
db.execute("DELETE FROM services")
latest: dict[str, ProbeResult] = {}
for probe in probes:
if probe.service_id not in latest or probe.checked_at > latest[probe.service_id].checked_at:
latest[probe.service_id] = probe
for service in services:
seeded = service.model_copy(update={"state": latest.get(service.id, ProbeResult(id="", service_id=service.id, status=HealthStatus.UNKNOWN, latency_ms=None, detail="", checked_at=_now())).status})
cls._insert_service_row(db, cls._service_params(seeded))
for probe in probes:
db.execute(
"INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
(probe.id, probe.service_id, probe.status.value, probe.latency_ms, probe.detail, probe.checked_at.isoformat(), probe.trigger, json.dumps(probe.metadata)),
)
for probe in probes:
service = next((item for item in services if item.id == probe.service_id), None)
interval = service.interval_seconds if service and service.interval_seconds else 60
evidence_id = f"demo-probe-{probe.id}"
db.execute(
"""
INSERT INTO health_evidence (
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
confidence, observed_at, valid_until, received_at, summary, metadata, active
) VALUES (?, 'service', ?, ?, 'active_probe', 'active-probe', ?, 'component', 1.0, ?, ?, ?, ?, ?, 1)
ON CONFLICT(id) DO NOTHING
""",
(
evidence_id,
probe.service_id,
probe.service_id,
probe.status.value,
probe.checked_at.isoformat(),
(probe.checked_at + timedelta(seconds=max(60, interval * 2))).isoformat(),
probe.checked_at.isoformat(),
probe.detail,
json.dumps({"probe_id": probe.id, "trigger": probe.trigger, **probe.metadata}, sort_keys=True),
),
)
db.execute(
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
(evidence_id, probe.service_id),
)
def activate_demo(self, services: list[Service], probes: list[ProbeResult]) -> ConfigImportResult:
with self._connect() as db:
db.execute("BEGIN IMMEDIATE")
session = db.execute("SELECT * FROM demo_sessions WHERE id = 1").fetchone()
if session is None:
snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents = self._snapshot_rows(db)
snapshot_discovery = self._snapshot_discovery_rows(db)
snapshot_health_evidence = self._snapshot_health_evidence_rows(db)
started_at = _now().isoformat()
else:
snapshot_services = session["snapshot_services"]
snapshot_probes = session["snapshot_probes"]
snapshot_events = session["snapshot_events"] or "[]"
snapshot_incidents = session["snapshot_incidents"] or "[]"
snapshot_discovery = session["snapshot_discovery"] or "{}"
snapshot_health_evidence = session["snapshot_health_evidence"] or "{}"
started_at = session["started_at"]
self._insert_demo_rows(db, services, probes)
db.execute(
"""
INSERT INTO demo_sessions (id, snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, demo_service_ids, started_at)
VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET snapshot_services=excluded.snapshot_services, snapshot_probes=excluded.snapshot_probes,
snapshot_events=excluded.snapshot_events, snapshot_incidents=excluded.snapshot_incidents,
snapshot_discovery=excluded.snapshot_discovery,
snapshot_health_evidence=excluded.snapshot_health_evidence,
demo_service_ids=excluded.demo_service_ids, started_at=excluded.started_at
""",
(snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, json.dumps([service.id for service in services]), started_at),
)
event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo geladen"})
return ConfigImportResult(created=len(services), updated=0, total=len(services))
def exit_demo(self) -> bool:
with self._connect() as db:
db.execute("BEGIN IMMEDIATE")
session = db.execute("SELECT * FROM demo_sessions WHERE id = 1").fetchone()
if session is None:
return False
self._restore_rows(
db,
session["snapshot_services"],
session["snapshot_probes"],
session["snapshot_events"],
session["snapshot_incidents"],
session["snapshot_discovery"] or "{}",
session["snapshot_health_evidence"] or "{}",
)
db.execute("DELETE FROM demo_sessions WHERE id = 1")
event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo afgesloten"})
return True
def list_incidents(
self, active_only: bool = False, limit: int = 100, *, services: dict[str, Service] | None = None
) -> list[IncidentRecord]:
with self._connect() as db:
query = "SELECT * FROM incidents"
params: tuple[Any, ...] = ()
if active_only:
query += " WHERE status != 'resolved'"
query += " ORDER BY detected_at DESC LIMIT ?"
params += (limit,)
rows = db.execute(query, params).fetchall()
if not rows:
return []
if services is None:
services = {service.id: service for service in self.list_services()}
return [self._row_to_incident(row, services) for row in rows]
def get_incident(self, incident_id: str) -> IncidentRecord | None:
services = {service.id: service for service in self.list_services()}
with self._connect() as db:
row = db.execute("SELECT * FROM incidents WHERE id = ?", (incident_id,)).fetchone()
return self._row_to_incident(row, services) if row else None
def incident_timeline(self, incident_id: str, limit: int = 200) -> list[IncidentEvent]:
with self._connect() as db:
rows = db.execute(
"SELECT id, incident_id, event_type, message, created_at, service_id FROM events WHERE incident_id = ? ORDER BY created_at ASC LIMIT ?",
(incident_id, limit),
).fetchall()
return [IncidentEvent(**dict(row)) for row in rows]
def upsert_incident(self, incident: IncidentRecord) -> IncidentRecord:
with self._connect() as db:
db.execute(
"""
INSERT INTO incidents (id, status, root_service_id, started_at, detected_at, acknowledged_at, resolved_at,
title, root_cause_reason, affected_service_ids, severity, latest_update_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET status=excluded.status, acknowledged_at=excluded.acknowledged_at,
resolved_at=excluded.resolved_at, title=excluded.title, root_cause_reason=excluded.root_cause_reason,
affected_service_ids=excluded.affected_service_ids, severity=excluded.severity, latest_update_at=excluded.latest_update_at
""",
(incident.id, incident.status.value, incident.root_service_id, incident.started_at.isoformat(), incident.detected_at.isoformat(), _iso(incident.acknowledged_at), _iso(incident.resolved_at), incident.title, incident.root_cause_reason, json.dumps(incident.affected_service_ids), incident.severity.value, incident.latest_update_at.isoformat()),
)
return incident
def append_incident_event(self, incident_id: str, event_type: str, message: str, service_id: str | None = None) -> IncidentEvent:
created_at = _now()
with self._connect() as db:
event_id = self._insert_event_db(db, event_type, service_id, incident_id, None, None, message, {}, created_at)
payload = {"event_type": event_type, "incident_id": incident_id, "service_id": service_id, "message": message, "created_at": created_at.isoformat()}
event_bus.publish(payload)
return IncidentEvent(id=event_id, incident_id=incident_id, event_type=event_type, message=message, created_at=created_at, service_id=service_id)
def resolve_incident(self, incident_id: str, note: str | None = None) -> IncidentRecord | None:
"""Close an incident by hand (e.g. a stale one).
If its root is still down, the next incident sync opens a new one.
"""
incident = self.get_incident(incident_id)
if incident is None:
return None
if incident.status == IncidentStatus.RESOLVED:
return incident
now = _now()
updated = incident.model_copy(
update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now}
)
self.upsert_incident(updated)
self.append_incident_event(
incident_id,
"incident.resolved",
f"Incident handmatig opgelost{f': {note}' if note else ''}",
incident.root_service_id,
)
return updated
def add_incident_note(self, incident_id: str, text: str) -> IncidentRecord | None:
incident = self.get_incident(incident_id)
if incident is None:
return None
self.append_incident_event(incident_id, "incident.note", text, incident.root_service_id)
updated = incident.model_copy(update={"latest_update_at": _now()})
self.upsert_incident(updated)
return updated
def acknowledge_incident(self, incident_id: str) -> IncidentRecord | None:
incident = self.get_incident(incident_id)
if incident is None:
return None
if incident.status == IncidentStatus.RESOLVED:
raise ValueError("Opgeloste incidenten kunnen niet bevestigd worden")
if incident.status == IncidentStatus.ACKNOWLEDGED:
# Idempotent: a repeated acknowledge keeps the original time and
# adds no duplicate timeline entry.
return incident
now = _now()
updated = incident.model_copy(update={"status": IncidentStatus.ACKNOWLEDGED, "acknowledged_at": now, "latest_update_at": now})
self.upsert_incident(updated)
self.append_incident_event(incident_id, "incident.acknowledged", "Incident bevestigd", incident.root_service_id)
return updated