from __future__ import annotations import json import os import sqlite3 from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any from uuid import uuid4 from .events import event_bus from .evidence import resolve_health, with_freshness from .models import ( AvailabilitySummary, ConfigImportResult, ConfigService, EventRecord, EvidenceProvider, EvidenceScope, HealthEvidence, HealthResolution, HealthStatus, IncidentEvent, IncidentRecord, IncidentStatus, MonitoringState, ProbeResult, ProviderStatus, Service, ServiceCreate, ServiceUpdate, ) SERVICE_COLUMNS = ( "id, name, kind, url, host, port, depends_on, description, tags, \"group\", " "environment, owner, enabled, monitoring_enabled, monitoring_state, probe_source, probe_reachability, probe_confidence, " "runtime_state, runtime_health, application, domain, component_role, application_locked, domain_locked, component_role_locked, criticality, category, visibility, lifecycle, " "source_type, source_ref, first_seen_at, last_seen_at, last_discovery_status, container_image, " "compose_project, layout_x, layout_y, layout_pinned, " "interval_seconds, timeout_seconds, retries, failure_threshold, recovery_threshold, " "expected_status_min, expected_status_max, availability_target, maintenance_enabled, " "maintenance_reason, state, failure_streak, recovery_streak, flapping, flap_count, " "last_state_change_at, created_at, updated_at" ) def _now() -> datetime: return datetime.now(UTC) def _iso(value: datetime | None) -> str | None: return value.isoformat() if value is not None else None # Per-observation bookkeeping that changes on every probe. It is stored, but a # change in only these keys is not a material evidence change and must not # publish an SSE wakeup (that turned every probe into a full UI refresh). VOLATILE_EVIDENCE_METADATA = frozenset({"probe_id", "latency_ms", "trigger"}) def _stable_metadata(metadata: dict[str, object]) -> dict[str, object]: return {key: value for key, value in metadata.items() if key not in VOLATILE_EVIDENCE_METADATA} def _bool(value: object, default: bool = False) -> bool: if value is None: return default return bool(int(value)) if isinstance(value, (int, str)) and str(value).isdigit() else bool(value) class Store: def __init__(self, db_path: str | None = None) -> None: raw_path = db_path or os.getenv("SERVICEMAP_DB", "data/servicemap.db") self.path = Path(raw_path) self.path.parent.mkdir(parents=True, exist_ok=True) self._init_schema() def _connect(self) -> sqlite3.Connection: connection = sqlite3.connect(self.path, timeout=5.0) connection.row_factory = sqlite3.Row connection.execute("PRAGMA foreign_keys = ON") connection.execute("PRAGMA busy_timeout = 5000") # Per connection, not persistent: without this every commit ran at # the FULL default (an fsync per write). NORMAL is durable under WAL. connection.execute("PRAGMA synchronous = NORMAL") return connection def _init_schema(self) -> None: with self._connect() as db: db.execute("PRAGMA journal_mode = WAL") db.execute("PRAGMA synchronous = NORMAL") # Bound the WAL file after checkpoints (64 MiB). db.execute("PRAGMA journal_size_limit = 67108864") db.executescript( """ CREATE TABLE IF NOT EXISTS maintenance_windows ( service_id TEXT PRIMARY KEY, ends_at TEXT NOT NULL, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS services ( id TEXT PRIMARY KEY, name TEXT NOT NULL UNIQUE, kind TEXT NOT NULL, url TEXT, host TEXT, port INTEGER, depends_on TEXT NOT NULL DEFAULT '[]', description TEXT, tags TEXT NOT NULL DEFAULT '[]', "group" TEXT, environment TEXT, owner TEXT, enabled INTEGER NOT NULL DEFAULT 1, monitoring_enabled INTEGER NOT NULL DEFAULT 1, monitoring_state TEXT NOT NULL DEFAULT 'active_probe', probe_source TEXT NOT NULL DEFAULT 'manual', probe_reachability TEXT NOT NULL DEFAULT 'verified', probe_confidence TEXT NOT NULL DEFAULT 'confirmed', runtime_state TEXT, runtime_health TEXT, application TEXT, domain TEXT, component_role TEXT NOT NULL DEFAULT 'unknown', application_locked INTEGER NOT NULL DEFAULT 0, domain_locked INTEGER NOT NULL DEFAULT 0, component_role_locked INTEGER NOT NULL DEFAULT 0, criticality TEXT NOT NULL DEFAULT 'medium', category TEXT NOT NULL DEFAULT 'unknown', visibility TEXT NOT NULL DEFAULT 'visible', lifecycle TEXT NOT NULL DEFAULT 'active', source_type TEXT, source_ref TEXT, first_seen_at TEXT, last_seen_at TEXT, last_discovery_status TEXT, container_image TEXT, compose_project TEXT, layout_x REAL, layout_y REAL, layout_pinned INTEGER NOT NULL DEFAULT 0, interval_seconds INTEGER, timeout_seconds REAL, retries INTEGER, failure_threshold INTEGER NOT NULL DEFAULT 1, recovery_threshold INTEGER NOT NULL DEFAULT 1, expected_status_min INTEGER, expected_status_max INTEGER, availability_target REAL, maintenance_enabled INTEGER NOT NULL DEFAULT 0, maintenance_reason TEXT, state TEXT NOT NULL DEFAULT 'unknown', failure_streak INTEGER NOT NULL DEFAULT 0, recovery_streak INTEGER NOT NULL DEFAULT 0, flapping INTEGER NOT NULL DEFAULT 0, flap_count INTEGER NOT NULL DEFAULT 0, last_state_change_at TEXT, created_at TEXT NOT NULL, updated_at TEXT ); CREATE TABLE IF NOT EXISTS probe_results ( id TEXT PRIMARY KEY, service_id TEXT NOT NULL, status TEXT NOT NULL, latency_ms REAL, detail TEXT NOT NULL, checked_at TEXT NOT NULL, trigger TEXT NOT NULL DEFAULT 'manual', metadata TEXT NOT NULL DEFAULT '{}', FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_probe_service_time ON probe_results(service_id, checked_at DESC); CREATE INDEX IF NOT EXISTS idx_probe_time ON probe_results(checked_at DESC); CREATE TABLE IF NOT EXISTS events ( id TEXT PRIMARY KEY, event_type TEXT NOT NULL, service_id TEXT, incident_id TEXT, from_status TEXT, to_status TEXT, message TEXT NOT NULL, metadata TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_events_time ON events(created_at DESC); CREATE INDEX IF NOT EXISTS idx_events_service_time ON events(service_id, created_at DESC); CREATE INDEX IF NOT EXISTS idx_events_incident_time ON events(incident_id, created_at ASC); CREATE TABLE IF NOT EXISTS incidents ( id TEXT PRIMARY KEY, status TEXT NOT NULL, root_service_id TEXT NOT NULL, started_at TEXT NOT NULL, detected_at TEXT NOT NULL, acknowledged_at TEXT, resolved_at TEXT, title TEXT NOT NULL, root_cause_reason TEXT NOT NULL, affected_service_ids TEXT NOT NULL DEFAULT '[]', severity TEXT NOT NULL DEFAULT 'medium', latest_update_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_incidents_status ON incidents(status, latest_update_at DESC); CREATE INDEX IF NOT EXISTS idx_incidents_root ON incidents(root_service_id, started_at DESC); CREATE TABLE IF NOT EXISTS demo_sessions ( id INTEGER PRIMARY KEY CHECK (id = 1), snapshot_services TEXT NOT NULL, snapshot_probes TEXT NOT NULL, snapshot_events TEXT NOT NULL DEFAULT '[]', snapshot_incidents TEXT NOT NULL DEFAULT '[]', snapshot_discovery TEXT NOT NULL DEFAULT '{}', snapshot_health_evidence TEXT NOT NULL DEFAULT '{}', demo_service_ids TEXT NOT NULL, started_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS health_evidence ( id TEXT PRIMARY KEY, subject_type TEXT NOT NULL, subject_id TEXT NOT NULL, service_id TEXT, provider TEXT NOT NULL, provider_ref TEXT, status TEXT NOT NULL, scope TEXT NOT NULL, confidence REAL NOT NULL, observed_at TEXT NOT NULL, valid_until TEXT NOT NULL, received_at TEXT NOT NULL, summary TEXT NOT NULL DEFAULT '', metadata TEXT NOT NULL DEFAULT '{}', active INTEGER NOT NULL DEFAULT 1, superseded_by TEXT, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_health_evidence_subject ON health_evidence(subject_type, subject_id, provider, active, observed_at DESC); CREATE INDEX IF NOT EXISTS idx_health_evidence_service ON health_evidence(service_id, active, observed_at DESC); CREATE TABLE IF NOT EXISTS health_evidence_service_links ( evidence_id TEXT NOT NULL, service_id TEXT NOT NULL, PRIMARY KEY(evidence_id, service_id), FOREIGN KEY(evidence_id) REFERENCES health_evidence(id) ON DELETE CASCADE, FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_health_evidence_link_service ON health_evidence_service_links(service_id, evidence_id); CREATE TABLE IF NOT EXISTS provider_status ( provider TEXT PRIMARY KEY, status TEXT NOT NULL, observed_at TEXT NOT NULL, valid_until TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '', metadata TEXT NOT NULL DEFAULT '{}' ); """ ) self._migrate_columns(db) self._backfill_health_evidence(db) @staticmethod def _migrate_columns(db: sqlite3.Connection) -> None: columns = {row["name"] for row in db.execute("PRAGMA table_info(services)").fetchall()} additions = { "updated_at": "TEXT", "description": "TEXT", "tags": "TEXT NOT NULL DEFAULT '[]'", "group": "TEXT", "environment": "TEXT", "owner": "TEXT", "enabled": "INTEGER NOT NULL DEFAULT 1", "monitoring_enabled": "INTEGER NOT NULL DEFAULT 1", "monitoring_state": "TEXT NOT NULL DEFAULT 'active_probe'", "probe_source": "TEXT NOT NULL DEFAULT 'manual'", "probe_reachability": "TEXT NOT NULL DEFAULT 'verified'", "probe_confidence": "TEXT NOT NULL DEFAULT 'confirmed'", "runtime_state": "TEXT", "runtime_health": "TEXT", "application": "TEXT", "domain": "TEXT", "component_role": "TEXT NOT NULL DEFAULT 'unknown'", "application_locked": "INTEGER NOT NULL DEFAULT 0", "domain_locked": "INTEGER NOT NULL DEFAULT 0", "component_role_locked": "INTEGER NOT NULL DEFAULT 0", "criticality": "TEXT NOT NULL DEFAULT 'medium'", "category": "TEXT NOT NULL DEFAULT 'unknown'", "visibility": "TEXT NOT NULL DEFAULT 'visible'", "lifecycle": "TEXT NOT NULL DEFAULT 'active'", "source_type": "TEXT", "source_ref": "TEXT", "first_seen_at": "TEXT", "last_seen_at": "TEXT", "last_discovery_status": "TEXT", "container_image": "TEXT", "compose_project": "TEXT", "layout_x": "REAL", "layout_y": "REAL", "layout_pinned": "INTEGER NOT NULL DEFAULT 0", "interval_seconds": "INTEGER", "timeout_seconds": "REAL", "retries": "INTEGER", "failure_threshold": "INTEGER NOT NULL DEFAULT 1", "recovery_threshold": "INTEGER NOT NULL DEFAULT 1", "expected_status_min": "INTEGER", "expected_status_max": "INTEGER", "availability_target": "REAL", "maintenance_enabled": "INTEGER NOT NULL DEFAULT 0", "maintenance_reason": "TEXT", "state": "TEXT NOT NULL DEFAULT 'unknown'", "failure_streak": "INTEGER NOT NULL DEFAULT 0", "recovery_streak": "INTEGER NOT NULL DEFAULT 0", "flapping": "INTEGER NOT NULL DEFAULT 0", "flap_count": "INTEGER NOT NULL DEFAULT 0", "last_state_change_at": "TEXT", } for name, definition in additions.items(): if name not in columns: quoted = f'"{name}"' if name == "group" else name db.execute(f"ALTER TABLE services ADD COLUMN {quoted} {definition}") db.execute("UPDATE services SET updated_at = created_at WHERE updated_at IS NULL") probe_columns = {row["name"] for row in db.execute("PRAGMA table_info(probe_results)").fetchall()} for name, definition in {"trigger": "TEXT NOT NULL DEFAULT 'manual'", "metadata": "TEXT NOT NULL DEFAULT '{}'"}.items(): if name not in probe_columns: db.execute(f"ALTER TABLE probe_results ADD COLUMN {name} {definition}") # v1.0.1 derived health directly from the latest probe. Backfill the # durable v1.1 state column so an in-place migration does not make an # existing healthy/down service appear unknown until the next cycle. db.execute( """ UPDATE services SET state = COALESCE( (SELECT latest.status FROM probe_results latest WHERE latest.service_id = services.id ORDER BY latest.checked_at DESC, latest.rowid DESC LIMIT 1), state ), last_state_change_at = COALESCE( last_state_change_at, (SELECT latest.checked_at FROM probe_results latest WHERE latest.service_id = services.id ORDER BY latest.checked_at DESC, latest.rowid DESC LIMIT 1) ) WHERE state = 'unknown' AND EXISTS (SELECT 1 FROM probe_results existing WHERE existing.service_id = services.id) """ ) demo_columns = {row["name"] for row in db.execute("PRAGMA table_info(demo_sessions)").fetchall()} for name in ("snapshot_events", "snapshot_incidents", "snapshot_discovery", "snapshot_health_evidence"): if name not in demo_columns: default = "'{}'" if name in {"snapshot_discovery", "snapshot_health_evidence"} else "'[]'" db.execute(f"ALTER TABLE demo_sessions ADD COLUMN {name} TEXT NOT NULL DEFAULT {default}") @staticmethod def _backfill_health_evidence(db: sqlite3.Connection) -> None: """Preserve legitimate recent v1.5 observations during the v1.6 migration. This is a one-time projection of existing probe/runtime facts, not a fabricated monitoring result. Historic observations stay expired and visible in provenance; only their original interval makes them current. AppOps rows are deliberately excluded because pre-v1.6 persisted an application-scoped status as component runtime health. """ if db.execute("SELECT 1 FROM health_evidence LIMIT 1").fetchone(): return latest = db.execute( """ SELECT p.* FROM probe_results p WHERE p.rowid = ( SELECT candidate.rowid FROM probe_results candidate WHERE candidate.service_id = p.service_id ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1 ) """ ).fetchall() intervals = { row["id"]: max(60, int(row["interval_seconds"] or 60) * 2) for row in db.execute("SELECT id, interval_seconds FROM services").fetchall() } for row in latest: observed = datetime.fromisoformat(row["checked_at"]) ttl = timedelta(seconds=intervals.get(row["service_id"], 120)) evidence_id = f"migration-probe-{row['id']}" db.execute( """ INSERT INTO health_evidence ( id, subject_type, subject_id, service_id, provider, provider_ref, status, scope, confidence, observed_at, valid_until, received_at, summary, metadata, active ) VALUES (?, 'service', ?, ?, 'active_probe', ?, ?, 'component', 1.0, ?, ?, ?, ?, ?, 1) """, ( evidence_id, row["service_id"], row["service_id"], row["id"], row["status"], observed.isoformat(), (observed + ttl).isoformat(), observed.isoformat(), "Migrated latest active probe result", row["metadata"] or "{}", ), ) db.execute( "INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)", (evidence_id, row["service_id"]), ) runtime_rows = db.execute( """ SELECT id, runtime_state, runtime_health, updated_at, monitoring_state FROM services WHERE COALESCE(monitoring_state, '') != 'appops_observation' AND (runtime_state IS NOT NULL OR runtime_health IS NOT NULL) """ ).fetchall() for row in runtime_rows: observed = datetime.fromisoformat(row["updated_at"] or _now().isoformat()) runtime_state = str(row["runtime_state"] or "").casefold() runtime_health = str(row["runtime_health"] or "").casefold() if runtime_health in {"unhealthy", "down"} or runtime_state in {"exited", "dead", "stopped"}: status = HealthStatus.DOWN elif runtime_health == "degraded" or runtime_state == "restarting": status = HealthStatus.DEGRADED elif runtime_health == "healthy": status = HealthStatus.HEALTHY else: continue evidence_id = f"migration-runtime-{row['id']}" db.execute( """ INSERT INTO health_evidence ( id, subject_type, subject_id, service_id, provider, provider_ref, status, scope, confidence, observed_at, valid_until, received_at, summary, metadata, active ) VALUES (?, 'service', ?, ?, 'docker', 'legacy-runtime', ?, 'component', 0.7, ?, ?, ?, ?, ?, 1) """, ( evidence_id, row["id"], row["id"], status.value, observed.isoformat(), (observed + timedelta(minutes=10)).isoformat(), observed.isoformat(), "Migrated Docker runtime observation", json.dumps({"runtime_state": row["runtime_state"], "runtime_health": row["runtime_health"]}), ), ) db.execute( "INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)", (evidence_id, row["id"]), ) @staticmethod def _json_list(value: object, fallback: list[str] | None = None) -> list[str]: if isinstance(value, list): return [str(item) for item in value] if isinstance(value, str): try: parsed = json.loads(value) return [str(item) for item in parsed] if isinstance(parsed, list) else (fallback or []) except json.JSONDecodeError: return fallback or [] return fallback or [] @staticmethod def _json_object(value: object) -> dict[str, object]: if isinstance(value, dict): return value if isinstance(value, str): try: parsed = json.loads(value) return parsed if isinstance(parsed, dict) else {} except json.JSONDecodeError: return {} return {} @classmethod def _row_to_service(cls, row: sqlite3.Row | dict[str, Any]) -> Service: created_at = datetime.fromisoformat(str(row["created_at"])) raw_updated = row.get("updated_at") if isinstance(row, dict) else row["updated_at"] raw_state = row.get("state", "unknown") if isinstance(row, dict) else row["state"] try: state = HealthStatus(raw_state or "unknown") except ValueError: state = HealthStatus.UNKNOWN return Service( id=row["id"], name=row["name"], kind=row["kind"], url=row["url"], host=row["host"], port=row["port"], depends_on=cls._json_list(row["depends_on"]), description=row.get("description") if isinstance(row, dict) else row["description"], tags=cls._json_list(row.get("tags", "[]") if isinstance(row, dict) else row["tags"]), group=row.get("group") if isinstance(row, dict) else row["group"], environment=row.get("environment") if isinstance(row, dict) else row["environment"], owner=row.get("owner") if isinstance(row, dict) else row["owner"], enabled=_bool(row.get("enabled", 1) if isinstance(row, dict) else row["enabled"], True), monitoring_enabled=_bool( row.get("monitoring_enabled", 1) if isinstance(row, dict) else row["monitoring_enabled"], True, ), monitoring_state=(row.get("monitoring_state", "active_probe") if isinstance(row, dict) else row["monitoring_state"]) or "active_probe", probe_source=(row.get("probe_source", "manual") if isinstance(row, dict) else row["probe_source"]) or "manual", probe_reachability=(row.get("probe_reachability", "verified") if isinstance(row, dict) else row["probe_reachability"]) or "verified", probe_confidence=(row.get("probe_confidence", "confirmed") if isinstance(row, dict) else row["probe_confidence"]) or "confirmed", runtime_state=row.get("runtime_state") if isinstance(row, dict) else row["runtime_state"], runtime_health=row.get("runtime_health") if isinstance(row, dict) else row["runtime_health"], application=row.get("application") if isinstance(row, dict) else row["application"], domain=row.get("domain") if isinstance(row, dict) else row["domain"], component_role=(row.get("component_role", "unknown") if isinstance(row, dict) else row["component_role"]) or "unknown", application_locked=_bool(row.get("application_locked", 0) if isinstance(row, dict) else row["application_locked"]), domain_locked=_bool(row.get("domain_locked", 0) if isinstance(row, dict) else row["domain_locked"]), component_role_locked=_bool(row.get("component_role_locked", 0) if isinstance(row, dict) else row["component_role_locked"]), criticality=row.get("criticality", "medium") if isinstance(row, dict) else row["criticality"], category=row.get("category", "unknown") if isinstance(row, dict) else row["category"], visibility=row.get("visibility", "visible") if isinstance(row, dict) else row["visibility"], lifecycle=row.get("lifecycle", "active") if isinstance(row, dict) else row["lifecycle"], source_type=row.get("source_type") if isinstance(row, dict) else row["source_type"], source_ref=row.get("source_ref") if isinstance(row, dict) else row["source_ref"], first_seen_at=( datetime.fromisoformat(row["first_seen_at"]) if (row.get("first_seen_at") if isinstance(row, dict) else row["first_seen_at"]) else None ), last_seen_at=( datetime.fromisoformat(row["last_seen_at"]) if (row.get("last_seen_at") if isinstance(row, dict) else row["last_seen_at"]) else None ), last_discovery_status=( row.get("last_discovery_status") if isinstance(row, dict) else row["last_discovery_status"] ), container_image=row.get("container_image") if isinstance(row, dict) else row["container_image"], compose_project=row.get("compose_project") if isinstance(row, dict) else row["compose_project"], layout_x=row.get("layout_x") if isinstance(row, dict) else row["layout_x"], layout_y=row.get("layout_y") if isinstance(row, dict) else row["layout_y"], layout_pinned=_bool( row.get("layout_pinned", 0) if isinstance(row, dict) else row["layout_pinned"] ), interval_seconds=row.get("interval_seconds") if isinstance(row, dict) else row["interval_seconds"], timeout_seconds=row.get("timeout_seconds") if isinstance(row, dict) else row["timeout_seconds"], retries=row.get("retries") if isinstance(row, dict) else row["retries"], failure_threshold=(row.get("failure_threshold", 1) if isinstance(row, dict) else row["failure_threshold"]) or 1, recovery_threshold=(row.get("recovery_threshold", 1) if isinstance(row, dict) else row["recovery_threshold"]) or 1, expected_status_min=row.get("expected_status_min") if isinstance(row, dict) else row["expected_status_min"], expected_status_max=row.get("expected_status_max") if isinstance(row, dict) else row["expected_status_max"], availability_target=row.get("availability_target") if isinstance(row, dict) else row["availability_target"], maintenance_enabled=_bool(row.get("maintenance_enabled", 0) if isinstance(row, dict) else row["maintenance_enabled"]), maintenance_reason=row.get("maintenance_reason") if isinstance(row, dict) else row["maintenance_reason"], state=state, failure_streak=(row.get("failure_streak", 0) if isinstance(row, dict) else row["failure_streak"]) or 0, recovery_streak=(row.get("recovery_streak", 0) if isinstance(row, dict) else row["recovery_streak"]) or 0, flapping=_bool(row.get("flapping", 0) if isinstance(row, dict) else row["flapping"]), flap_count=(row.get("flap_count", 0) if isinstance(row, dict) else row["flap_count"]) or 0, last_state_change_at=( datetime.fromisoformat(row["last_state_change_at"]) if (row.get("last_state_change_at") if isinstance(row, dict) else row["last_state_change_at"]) else None ), created_at=created_at, updated_at=datetime.fromisoformat(str(raw_updated)) if raw_updated else created_at, ) @classmethod def _row_to_probe(cls, row: sqlite3.Row | dict[str, Any]) -> ProbeResult: return ProbeResult( id=row["id"], service_id=row["service_id"], status=HealthStatus(row["status"]), latency_ms=row["latency_ms"], detail=row["detail"], checked_at=datetime.fromisoformat(row["checked_at"]), trigger=(row.get("trigger", "manual") if isinstance(row, dict) else row["trigger"]) or "manual", metadata=cls._json_object(row.get("metadata", "{}") if isinstance(row, dict) else row["metadata"]), ) @classmethod def _row_to_event(cls, row: sqlite3.Row | dict[str, Any]) -> EventRecord: raw_from = row["from_status"] raw_to = row["to_status"] return EventRecord( id=row["id"], event_type=row["event_type"], service_id=row["service_id"], incident_id=row["incident_id"], from_status=HealthStatus(raw_from) if raw_from else None, to_status=HealthStatus(raw_to) if raw_to else None, message=row["message"], metadata=cls._json_object(row["metadata"]), created_at=datetime.fromisoformat(row["created_at"]), ) @classmethod def _row_to_incident(cls, row: sqlite3.Row | dict[str, Any], services: dict[str, Service] | None = None) -> IncidentRecord: services = services or {} root = services.get(row["root_service_id"]) affected_ids = cls._json_list(row["affected_service_ids"]) return IncidentRecord( id=row["id"], status=IncidentStatus(row["status"]), root_service_id=row["root_service_id"], root_service_name=root.name if root else row["root_service_id"], started_at=datetime.fromisoformat(row["started_at"]), detected_at=datetime.fromisoformat(row["detected_at"]), acknowledged_at=datetime.fromisoformat(row["acknowledged_at"]) if row["acknowledged_at"] else None, resolved_at=datetime.fromisoformat(row["resolved_at"]) if row["resolved_at"] else None, title=row["title"], root_cause_reason=row["root_cause_reason"], affected_service_ids=affected_ids, affected_service_names=[services[item].name for item in affected_ids if item in services], severity=row["severity"], latest_update_at=datetime.fromisoformat(row["latest_update_at"]), ) @staticmethod def _service_params(service: Service) -> dict[str, Any]: return { "id": service.id, "name": service.name, "kind": service.kind.value, "url": service.url, "host": service.host, "port": service.port, "depends_on": json.dumps(service.depends_on), "description": service.description, "tags": json.dumps(service.tags), "group": service.group, "environment": service.environment, "owner": service.owner, "enabled": int(service.enabled), "monitoring_enabled": int(service.monitoring_enabled), "monitoring_state": service.monitoring_state.value, "probe_source": service.probe_source.value, "probe_reachability": service.probe_reachability.value, "probe_confidence": service.probe_confidence.value, "runtime_state": service.runtime_state, "runtime_health": service.runtime_health, "application": service.application, "domain": service.domain, "component_role": service.component_role.value, "application_locked": int(service.application_locked), "domain_locked": int(service.domain_locked), "component_role_locked": int(service.component_role_locked), "criticality": service.criticality.value, "category": service.category.value, "visibility": service.visibility.value, "lifecycle": service.lifecycle.value, "source_type": service.source_type, "source_ref": service.source_ref, "first_seen_at": _iso(service.first_seen_at), "last_seen_at": _iso(service.last_seen_at), "last_discovery_status": service.last_discovery_status, "container_image": service.container_image, "compose_project": service.compose_project, "layout_x": service.layout_x, "layout_y": service.layout_y, "layout_pinned": int(service.layout_pinned), "interval_seconds": service.interval_seconds, "timeout_seconds": service.timeout_seconds, "retries": service.retries, "failure_threshold": service.failure_threshold, "recovery_threshold": service.recovery_threshold, "expected_status_min": service.expected_status_min, "expected_status_max": service.expected_status_max, "availability_target": service.availability_target, "maintenance_enabled": int(service.maintenance_enabled), "maintenance_reason": service.maintenance_reason, "state": service.state.value, "failure_streak": service.failure_streak, "recovery_streak": service.recovery_streak, "flapping": int(service.flapping), "flap_count": service.flap_count, "last_state_change_at": _iso(service.last_state_change_at), "created_at": service.created_at.isoformat(), "updated_at": service.updated_at.isoformat(), } @classmethod def _insert_service(cls, db: sqlite3.Connection, service: Service) -> None: params = cls._service_params(service) db.execute( f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES (" ":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, " ":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, " ":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, " ":compose_project, :layout_x, :layout_y, :layout_pinned, " ":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, " ":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, " ":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, " ":last_state_change_at, :created_at, :updated_at)", params, ) @classmethod def _upsert_service(cls, db: sqlite3.Connection, service: Service) -> None: params = cls._service_params(service) db.execute( f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES (" ":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, " ":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, " ":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, " ":compose_project, :layout_x, :layout_y, :layout_pinned, " ":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, " ":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, " ":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, " ":last_state_change_at, :created_at, :updated_at) " "ON CONFLICT(id) DO UPDATE SET " "name=excluded.name, kind=excluded.kind, url=excluded.url, host=excluded.host, port=excluded.port, " "depends_on=excluded.depends_on, description=excluded.description, tags=excluded.tags, " '"group"=excluded."group", environment=excluded.environment, owner=excluded.owner, ' "enabled=excluded.enabled, monitoring_enabled=excluded.monitoring_enabled, monitoring_state=excluded.monitoring_state, " "probe_source=excluded.probe_source, probe_reachability=excluded.probe_reachability, probe_confidence=excluded.probe_confidence, " "runtime_state=excluded.runtime_state, runtime_health=excluded.runtime_health, application=excluded.application, domain=excluded.domain, component_role=excluded.component_role, " "application_locked=excluded.application_locked, domain_locked=excluded.domain_locked, component_role_locked=excluded.component_role_locked, criticality=excluded.criticality, " "category=excluded.category, visibility=excluded.visibility, lifecycle=excluded.lifecycle, " "source_type=excluded.source_type, source_ref=excluded.source_ref, first_seen_at=excluded.first_seen_at, " "last_seen_at=excluded.last_seen_at, last_discovery_status=excluded.last_discovery_status, " "container_image=excluded.container_image, compose_project=excluded.compose_project, " "layout_x=excluded.layout_x, layout_y=excluded.layout_y, layout_pinned=excluded.layout_pinned, " "interval_seconds=excluded.interval_seconds, " "timeout_seconds=excluded.timeout_seconds, retries=excluded.retries, failure_threshold=excluded.failure_threshold, " "recovery_threshold=excluded.recovery_threshold, expected_status_min=excluded.expected_status_min, " "expected_status_max=excluded.expected_status_max, availability_target=excluded.availability_target, " "maintenance_enabled=excluded.maintenance_enabled, maintenance_reason=excluded.maintenance_reason, " "state=excluded.state, failure_streak=excluded.failure_streak, recovery_streak=excluded.recovery_streak, " "flapping=excluded.flapping, flap_count=excluded.flap_count, last_state_change_at=excluded.last_state_change_at, " "created_at=excluded.created_at, updated_at=excluded.updated_at", params, ) def list_services(self) -> list[Service]: with self._connect() as db: rows = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services ORDER BY name COLLATE NOCASE").fetchall() return [self._row_to_service(row) for row in rows] def get_service(self, service_id: str) -> Service | None: with self._connect() as db: row = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services WHERE id = ?", (service_id,)).fetchone() return self._row_to_service(row) if row else None def get_service_by_name(self, name: str) -> Service | None: with self._connect() as db: row = db.execute( f"SELECT {SERVICE_COLUMNS} FROM services WHERE name = ? COLLATE NOCASE", (name,) ).fetchone() return self._row_to_service(row) if row else None def create_service(self, payload: ServiceCreate) -> Service: now = _now() service = Service(id=str(uuid4()), created_at=now, updated_at=now, **payload.model_dump()) with self._connect() as db: self._insert_service(db, service) self._insert_event_db(db, "topology.changed", None, None, None, None, f"Service {service.name} created", {}) event_bus.publish({"event_type": "topology.changed", "message": f"Service {service.name} created"}) return service def update_service(self, service_id: str, payload: ServiceUpdate) -> Service | None: existing = self.get_service(service_id) if existing is None: return None updated = Service( id=service_id, created_at=existing.created_at, updated_at=_now(), state=existing.state, failure_streak=existing.failure_streak, recovery_streak=existing.recovery_streak, flapping=existing.flapping, flap_count=existing.flap_count, last_state_change_at=existing.last_state_change_at, **payload.model_dump(), ) with self._connect() as db: self._upsert_service(db, updated) self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {updated.name} updated", {}) event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service bijgewerkt"}) return updated def delete_service(self, service_id: str) -> bool: with self._connect() as db: row = db.execute("SELECT name FROM services WHERE id = ?", (service_id,)).fetchone() if row is None: return False db.execute("DELETE FROM services WHERE id = ?", (service_id,)) rows = db.execute("SELECT id, depends_on FROM services").fetchall() for item in rows: dependencies = self._json_list(item["depends_on"]) if service_id in dependencies: db.execute( "UPDATE services SET depends_on = ?, updated_at = ? WHERE id = ?", (json.dumps([value for value in dependencies if value != service_id]), _now().isoformat(), item["id"]), ) # Remove the id as a JSON element. A textual replace() left an # empty-string entry behind (["a","b"] -> ["","b"]). for incident in db.execute( "SELECT id, affected_service_ids FROM incidents WHERE status != 'resolved'" ).fetchall(): affected = self._json_list(incident["affected_service_ids"]) if service_id in affected: db.execute( "UPDATE incidents SET affected_service_ids = ? WHERE id = ?", (json.dumps([value for value in affected if value != service_id]), incident["id"]), ) self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {row['name']} deleted", {}) event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service verwijderd"}) return True def import_config_services(self, definitions: list[ConfigService], mode: str) -> ConfigImportResult: if mode not in {"merge", "replace"}: raise ValueError("Importmodus moet merge of replace zijn") with self._connect() as db: db.execute("BEGIN IMMEDIATE") rows = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services").fetchall() existing = {row["name"].casefold(): self._row_to_service(row) for row in rows} if mode == "replace": db.execute("DELETE FROM incidents") db.execute("DELETE FROM events") db.execute("DELETE FROM services") existing = {} ids_by_name = {name: service.id for name, service in existing.items()} for definition in definitions: ids_by_name.setdefault(definition.name.casefold(), str(uuid4())) for definition in definitions: missing = [item for item in definition.depends_on if item.casefold() not in ids_by_name] if missing: raise ValueError( f"Service {definition.name} verwijst naar onbekende afhankelijkheden: {', '.join(missing)}" ) created = 0 updated = 0 now = _now() for definition in definitions: key = definition.name.casefold() current = existing.get(key) if current is None: payload = definition.model_dump() payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on] else: # Merge: only fields the document states change. Rebuilding # from model_dump() reset every omitted field to its default. payload = current.model_dump(include=set(ServiceCreate.model_fields)) payload.update(definition.model_dump(exclude_unset=True, exclude={"depends_on"})) if "depends_on" in definition.model_fields_set: payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on] if current is None: service = Service(id=ids_by_name[key], created_at=now, updated_at=now, **payload) created += 1 else: service = Service( id=current.id, created_at=current.created_at, updated_at=now, state=current.state, failure_streak=current.failure_streak, recovery_streak=current.recovery_streak, flapping=current.flapping, flap_count=current.flap_count, last_state_change_at=current.last_state_change_at, **payload, ) updated += 1 if current is None: self._insert_service(db, service) else: self._upsert_service(db, service) self._insert_event_db(db, "topology.changed", None, None, None, None, "Configuratie geïmporteerd", {"mode": mode}) event_bus.publish({"event_type": "topology.changed", "message": "Configuratie geïmporteerd"}) return ConfigImportResult(created=created, updated=updated, total=len(definitions)) def replace_all_services(self, definitions: list[ServiceCreate]) -> list[Service]: with self._connect() as db: db.execute("DELETE FROM services") return [self.create_service(item) for item in definitions] @staticmethod def _row_to_health_evidence(row: sqlite3.Row) -> HealthEvidence: return with_freshness( HealthEvidence( id=row["id"], subject_type=row["subject_type"], subject_id=row["subject_id"], service_id=row["service_id"], provider=row["provider"], provider_ref=row["provider_ref"], status=row["status"], scope=row["scope"], confidence=float(row["confidence"]), observed_at=datetime.fromisoformat(row["observed_at"]), valid_until=datetime.fromisoformat(row["valid_until"]), received_at=datetime.fromisoformat(row["received_at"]), summary=row["summary"] or "", metadata=Store._json_object(row["metadata"]), active=_bool(row["active"], True), superseded_by=row["superseded_by"], ) ) def record_health_evidence( self, evidence: HealthEvidence, *, service_ids: list[str] | None = None, ) -> HealthEvidence: """Store a current provider claim and retain superseded provenance. Repeated discovery cycles renew the same active claim instead of growing unbounded identical rows. A changed claim deactivates only the prior claim from that provider/subject; other providers remain independently auditable and can surface a conflict. """ links = sorted({*(service_ids or []), *({evidence.service_id} if evidence.service_id else set())}) with self._connect() as db: prior = db.execute( """ SELECT * FROM health_evidence WHERE subject_type = ? AND subject_id = ? AND provider = ? AND COALESCE(provider_ref, '') = COALESCE(?, '') AND scope = ? AND active = 1 ORDER BY observed_at DESC, rowid DESC LIMIT 1 """, ( evidence.subject_type, evidence.subject_id, evidence.provider.value, evidence.provider_ref, evidence.scope.value, ), ).fetchone() saved = evidence materially_changed = prior is None or any( ( prior[field] != value for field, value in ( ("status", evidence.status.value), ("confidence", evidence.confidence), ("summary", evidence.summary), ) ) ) or _stable_metadata(self._json_object(prior["metadata"])) != _stable_metadata(evidence.metadata) if prior is not None and prior["status"] == evidence.status.value: saved = evidence.model_copy(update={"id": prior["id"], "active": True, "superseded_by": None}) ttl = evidence.valid_until - evidence.observed_at prior_valid_until = datetime.fromisoformat(prior["valid_until"]) if ( not materially_changed and prior["service_id"] == evidence.service_id and prior_valid_until - _now() > ttl / 2 ): # Unchanged and comfortably fresh: skip the timestamp-only # rewrite. It is renewed once less than half the TTL is # left, so the claim never goes stale, with about half # the writes of renewing on every observation cycle. saved = self._row_to_health_evidence(prior) for service_id in links: db.execute( "INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)", (saved.id, service_id), ) return saved db.execute( """ UPDATE health_evidence SET service_id = ?, confidence = ?, observed_at = ?, valid_until = ?, received_at = ?, summary = ?, metadata = ?, active = 1, superseded_by = NULL WHERE id = ? """, ( saved.service_id, saved.confidence, saved.observed_at.isoformat(), saved.valid_until.isoformat(), saved.received_at.isoformat(), saved.summary, json.dumps(saved.metadata, sort_keys=True), saved.id, ), ) else: if prior is not None: db.execute("UPDATE health_evidence SET active = 0, superseded_by = ? WHERE id = ?", (evidence.id, prior["id"])) db.execute( """ INSERT INTO health_evidence ( id, subject_type, subject_id, service_id, provider, provider_ref, status, scope, confidence, observed_at, valid_until, received_at, summary, metadata, active, superseded_by ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( saved.id, saved.subject_type, saved.subject_id, saved.service_id, saved.provider.value, saved.provider_ref, saved.status.value, saved.scope.value, saved.confidence, saved.observed_at.isoformat(), saved.valid_until.isoformat(), saved.received_at.isoformat(), saved.summary, json.dumps(saved.metadata, sort_keys=True), int(saved.active), saved.superseded_by, ), ) for service_id in links: db.execute( "INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)", (saved.id, service_id), ) if materially_changed: event_bus.publish( { "event_type": "health.evidence_changed", "service_id": evidence.service_id, "provider": evidence.provider.value, "status": evidence.status.value, "message": "Gezondheidsbewijs gewijzigd", } ) return saved def health_evidence_for_service(self, service_id: str, *, limit: int = 100) -> list[HealthEvidence]: # A direct claim and a linked claim are matched by two independently # indexed lookups. An ``OR`` across the LEFT JOIN cannot use either # index and forced a full ``health_evidence`` scan on every call. with self._connect() as db: rows = db.execute( """ SELECT e.* FROM health_evidence e WHERE e.id IN ( SELECT id FROM health_evidence WHERE service_id = ? UNION SELECT evidence_id FROM health_evidence_service_links WHERE service_id = ? ) ORDER BY e.observed_at DESC, e.rowid DESC LIMIT ? """, (service_id, service_id, limit), ).fetchall() return [self._row_to_health_evidence(row) for row in rows] def health_evidence_for_services( self, service_ids: list[str], *, limit_per_service: int = 100 ) -> dict[str, list[HealthEvidence]]: """Fetch linked evidence for a topology in one bounded query. A semantic topology can contain many applications. Resolving each application with a separate evidence query creates an avoidable N+1 SQLite scan, so this method projects the evidence links once and then groups them in memory. """ ids = sorted(set(service_ids)) result = {service_id: [] for service_id in ids} if not ids: return result placeholders = ", ".join("?" for _ in ids) with self._connect() as db: # Both match paths use their own index; the per-service limit is # applied in SQL so superseded history is not materialised. rows = db.execute( f""" WITH matched(evidence_id, matched_service_id) AS ( SELECT id, service_id FROM health_evidence WHERE service_id IN ({placeholders}) UNION SELECT evidence_id, service_id FROM health_evidence_service_links WHERE service_id IN ({placeholders}) ), ranked AS ( SELECT e.*, m.matched_service_id, ROW_NUMBER() OVER ( PARTITION BY m.matched_service_id ORDER BY e.observed_at DESC, e.rowid DESC ) AS rank_in_service FROM matched m JOIN health_evidence e ON e.id = m.evidence_id ) SELECT * FROM ranked WHERE rank_in_service <= ? ORDER BY matched_service_id, rank_in_service """, [*ids, *ids, limit_per_service], ).fetchall() parsed: dict[str, HealthEvidence] = {} for row in rows: service_id = row["matched_service_id"] if service_id not in result: continue evidence = parsed.get(row["id"]) if evidence is None: evidence = self._row_to_health_evidence(row) parsed[row["id"]] = evidence result[service_id].append(evidence) return result def application_evidence_links(self, provider: EvidenceProvider) -> dict[str, set[str]]: """Return persisted application-claim links keyed by provider reference.""" with self._connect() as db: rows = db.execute( """ SELECT e.provider_ref, e.subject_id, link.service_id FROM health_evidence e JOIN health_evidence_service_links link ON link.evidence_id = e.id WHERE e.provider = ? AND e.scope = ? """, (provider.value, EvidenceScope.APPLICATION.value), ).fetchall() result: dict[str, set[str]] = {} for row in rows: key = row["provider_ref"] or row["subject_id"] if key: result.setdefault(key, set()).add(row["service_id"]) return result def health_resolution(self, service_id: str) -> HealthResolution: return resolve_health(self.health_evidence_for_service(service_id)) def set_provider_status(self, value: ProviderStatus) -> ProviderStatus: with self._connect() as db: previous = db.execute( "SELECT status, detail, metadata FROM provider_status WHERE provider = ?", (value.provider.value,), ).fetchone() db.execute( """ INSERT INTO provider_status (provider, status, observed_at, valid_until, detail, metadata) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(provider) DO UPDATE SET status=excluded.status, observed_at=excluded.observed_at, valid_until=excluded.valid_until, detail=excluded.detail, metadata=excluded.metadata """, ( value.provider.value, value.status.value, value.observed_at.isoformat(), value.valid_until.isoformat(), value.detail, json.dumps(value.metadata, sort_keys=True), ), ) if previous is None or any( ( previous[field] != expected for field, expected in ( ("status", value.status.value), ("detail", value.detail), ("metadata", json.dumps(value.metadata, sort_keys=True)), ) ) ): event_bus.publish( { "event_type": "provider.status_changed", "provider": value.provider.value, "status": value.status.value, "message": "Beschikbaarheid van de bron gewijzigd", } ) return value def provider_statuses(self) -> list[ProviderStatus]: with self._connect() as db: rows = db.execute("SELECT * FROM provider_status ORDER BY provider").fetchall() return [ ProviderStatus( provider=row["provider"], status=row["status"], observed_at=datetime.fromisoformat(row["observed_at"]), valid_until=datetime.fromisoformat(row["valid_until"]), detail=row["detail"] or "", metadata=self._json_object(row["metadata"]), ) for row in rows ] @staticmethod def _legacy_health_from_row(row: sqlite3.Row) -> HealthStatus: """Compatibility fallback for pre-v1.6 data without component evidence.""" runtime_state = str(row["runtime_state"] or "").casefold() runtime_health = str(row["runtime_health"] or "").casefold() if runtime_health == "unhealthy" or runtime_state in {"exited", "dead", "stopped"}: return HealthStatus.DOWN if runtime_state == "restarting": return HealthStatus.DEGRADED if not _bool(row["monitoring_enabled"], True): if runtime_health == "healthy": return HealthStatus.HEALTHY if str(row["monitoring_state"] or "").casefold() in {"unmonitored", "observed"}: return HealthStatus.UNMONITORED return HealthStatus.UNKNOWN try: return HealthStatus(row["state"] or "unknown") except ValueError: return HealthStatus.UNKNOWN def service_health(self, service_id: str) -> HealthStatus: with self._connect() as db: row = db.execute( "SELECT state, maintenance_enabled, enabled, monitoring_enabled, monitoring_state, runtime_state, runtime_health " "FROM services WHERE id = ?", (service_id,), ).fetchone() if row is None or not _bool(row["enabled"], True): return HealthStatus.UNKNOWN if _bool(row["maintenance_enabled"]): return HealthStatus.MAINTENANCE resolution = self.health_resolution(service_id) if resolution.selected_evidence_id: return resolution.status # Once v1.6 evidence exists, an expired/stale claim must not silently # keep a historical state alive. Legacy fallback is only for rows # that predate evidence entirely. Old AppOps-only rows were always # aggregate application observations, never component health. if resolution.evidence: return HealthStatus.UNKNOWN if str(row["monitoring_state"] or "").casefold() == MonitoringState.APPOPS.value: return HealthStatus.UNMONITORED return self._legacy_health_from_row(row) def service_healths( self, service_ids: list[str], *, evidence_by_service: dict[str, list[HealthEvidence]] | None = None, ) -> tuple[dict[str, HealthStatus], dict[str, HealthResolution]]: """Resolve direct health for a service set without per-service SQL reads.""" ids = sorted(set(service_ids)) if not ids: return {}, {} evidence_by_service = evidence_by_service or self.health_evidence_for_services(ids) placeholders = ", ".join("?" for _ in ids) with self._connect() as db: rows = db.execute( f""" SELECT id, state, maintenance_enabled, enabled, monitoring_enabled, monitoring_state, runtime_state, runtime_health FROM services WHERE id IN ({placeholders}) """, ids, ).fetchall() status_by_id: dict[str, HealthStatus] = {} resolutions: dict[str, HealthResolution] = {} for row in rows: service_id = row["id"] resolution = resolve_health(evidence_by_service.get(service_id, [])) resolutions[service_id] = resolution if not _bool(row["enabled"], True): status_by_id[service_id] = HealthStatus.UNKNOWN elif _bool(row["maintenance_enabled"]): status_by_id[service_id] = HealthStatus.MAINTENANCE elif resolution.selected_evidence_id: status_by_id[service_id] = resolution.status elif resolution.evidence: status_by_id[service_id] = HealthStatus.UNKNOWN elif str(row["monitoring_state"] or "").casefold() == MonitoringState.APPOPS.value: status_by_id[service_id] = HealthStatus.UNMONITORED else: status_by_id[service_id] = self._legacy_health_from_row(row) return status_by_id, resolutions def maintenance_windows(self) -> dict[str, datetime]: with self._connect() as db: rows = db.execute("SELECT service_id, ends_at FROM maintenance_windows").fetchall() return {row["service_id"]: datetime.fromisoformat(row["ends_at"]) for row in rows} def expire_maintenance_windows(self, now: datetime | None = None) -> list[str]: """End maintenance for every window whose end time has passed.""" now = now or _now() with self._connect() as db: rows = db.execute( "SELECT service_id FROM maintenance_windows WHERE ends_at <= ?", (now.isoformat(),) ).fetchall() ended: list[str] = [] for row in rows: if self.set_maintenance(row["service_id"], False, "Maintenance window ended") is not None: ended.append(row["service_id"]) return ended def set_maintenance( self, service_id: str, enabled: bool, reason: str | None = None, until: datetime | None = None, ) -> Service | None: service = self.get_service(service_id) if service is None: return None if enabled and until is not None and until <= _now(): raise ValueError("Het einde van het onderhoud moet in de toekomst liggen") updated = service.model_copy( update={ "maintenance_enabled": enabled, "maintenance_reason": reason if enabled else None, "state": HealthStatus.MAINTENANCE if enabled else HealthStatus.UNKNOWN, "failure_streak": 0, "recovery_streak": 0, "updated_at": _now(), } ) with self._connect() as db: self._upsert_service(db, updated) if enabled and until is not None: db.execute( "INSERT INTO maintenance_windows (service_id, ends_at) VALUES (?, ?) " "ON CONFLICT(service_id) DO UPDATE SET ends_at = excluded.ends_at", (service_id, until.isoformat()), ) else: db.execute("DELETE FROM maintenance_windows WHERE service_id = ?", (service_id,)) self._insert_event_db( db, "service.maintenance_changed", service_id, None, service.state.value, HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value, f"Onderhoud {'gestart' if enabled else 'beëindigd'} voor {service.name}", { **({"reason": reason} if reason else {}), **({"until": until.isoformat()} if enabled and until else {}), }, ) event_bus.publish( { "event_type": "service.maintenance_changed", "service_id": service_id, "status": HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value, "message": f"Onderhoud {'gestart' if enabled else 'beëindigd'}", } ) from .incidents import sync_incidents sync_incidents(self) return updated def _stable_state(self, row: sqlite3.Row, raw_status: HealthStatus) -> tuple[HealthStatus, int, int]: if not _bool(row["enabled"], True) or not _bool(row["monitoring_enabled"], True): monitoring_state = str(row["monitoring_state"] or "").casefold() return (HealthStatus.UNMONITORED if monitoring_state in {"unmonitored", "observed"} else HealthStatus.UNKNOWN), 0, 0 if _bool(row["maintenance_enabled"]): return HealthStatus.MAINTENANCE, 0, 0 previous = HealthStatus(row["state"] or "unknown") failure_streak = int(row["failure_streak"] or 0) recovery_streak = int(row["recovery_streak"] or 0) failure_threshold = max(1, int(row["failure_threshold"] or 1)) recovery_threshold = max(1, int(row["recovery_threshold"] or 1)) if raw_status in {HealthStatus.DOWN, HealthStatus.DEGRADED}: failure_streak += 1 recovery_streak = 0 if failure_streak < failure_threshold: if previous == HealthStatus.UNKNOWN: return HealthStatus.UNKNOWN, failure_streak, recovery_streak if previous == HealthStatus.RECOVERING: return HealthStatus.RECOVERING, failure_streak, recovery_streak return previous, failure_streak, recovery_streak return raw_status, failure_streak, recovery_streak if raw_status == HealthStatus.HEALTHY: recovery_streak += 1 failure_streak = 0 if previous in {HealthStatus.DOWN, HealthStatus.DEGRADED, HealthStatus.RECOVERING} and recovery_streak < recovery_threshold: return HealthStatus.RECOVERING, failure_streak, recovery_streak return HealthStatus.HEALTHY, failure_streak, recovery_streak return HealthStatus.UNMONITORED if raw_status == HealthStatus.UNMONITORED else HealthStatus.UNKNOWN, 0, 0 def set_runtime_observation( self, service_id: str, *, runtime_state: str | None, runtime_health: str | None, monitoring_state: MonitoringState | str, provider: EvidenceProvider = EvidenceProvider.DOCKER, provider_ref: str | None = None, confidence: float = 0.9, ) -> Service | None: """Persist Docker runtime truth without fabricating a network probe.""" service = self.get_service(service_id) if service is None: return None state_kind = MonitoringState(monitoring_state) normalized_state = (runtime_state or "").casefold() normalized_health = (runtime_health or "").casefold() if service.maintenance_enabled: status = HealthStatus.MAINTENANCE elif normalized_health in {"unhealthy", "down"} or normalized_state in {"exited", "dead", "stopped"}: status = HealthStatus.DOWN elif normalized_health == "degraded" or normalized_state == "restarting": status = HealthStatus.DEGRADED elif normalized_health == "healthy": status = HealthStatus.HEALTHY elif state_kind in { MonitoringState.UNMONITORED, MonitoringState.OBSERVED, MonitoringState.APPOPS, }: status = HealthStatus.UNMONITORED else: status = service.state updated = service.model_copy( update={ "runtime_state": runtime_state, "runtime_health": runtime_health, "monitoring_state": state_kind, "state": status, "failure_streak": 0 if status in {HealthStatus.UNMONITORED, HealthStatus.HEALTHY} else service.failure_streak, "recovery_streak": 0 if status == HealthStatus.UNMONITORED else service.recovery_streak, "last_state_change_at": _now() if service.state != status else service.last_state_change_at, "updated_at": _now(), } ) with self._connect() as db: self._upsert_service(db, updated) if service.state != status: self._insert_event_db( db, "service.runtime_observed", service_id, None, service.state.value, status.value, f"Docker-runtimewaarneming bijgewerkt voor {service.name}", {"runtime_state": runtime_state, "runtime_health": runtime_health, "monitoring_state": state_kind.value}, ) if service.state != status: event_bus.publish({"event_type": "service.runtime_observed", "service_id": service_id, "status": status.value, "message": "Docker-runtimewaarneming bijgewerkt"}) observed_at = _now() self.record_health_evidence( HealthEvidence( id=str(uuid4()), subject_type="service", subject_id=service_id, service_id=service_id, provider=provider, provider_ref=provider_ref or service.source_ref, status=status, scope=EvidenceScope.COMPONENT, confidence=confidence, observed_at=observed_at, valid_until=observed_at + timedelta(seconds=max(60, int(service.interval_seconds or 60) * 2)), received_at=observed_at, summary="Docker-runtimewaarneming" if provider == EvidenceProvider.DOCKER else "Runtimewaarneming", metadata={"runtime_state": runtime_state, "runtime_health": runtime_health}, ) ) return updated def save_probe(self, probe: ProbeResult) -> ProbeResult: events: list[dict[str, Any]] = [] with self._connect() as db: row = db.execute("SELECT * FROM services WHERE id = ?", (probe.service_id,)).fetchone() if row is None: return probe db.execute( """ INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, ( probe.id, probe.service_id, probe.status.value, probe.latency_ms, probe.detail, probe.checked_at.isoformat(), probe.trigger, json.dumps(probe.metadata), ), ) previous = HealthStatus(row["state"] or "unknown") next_state, failure_streak, recovery_streak = self._stable_state(row, probe.status) now = probe.checked_at state_changed = next_state != previous last_state_change = now.isoformat() if state_changed else row["last_state_change_at"] if state_changed: event = { "event_type": "service.state_changed", "service_id": probe.service_id, "from_status": previous.value, "to_status": next_state.value, "message": f"Service gewijzigd van {previous.value} to {next_state.value}", "metadata": {"trigger": probe.trigger, "probe_id": probe.id}, } self._insert_event_db( db, event["event_type"], probe.service_id, None, previous.value, next_state.value, event["message"], event["metadata"], created_at=now, ) events.append(event) since = (now - timedelta(minutes=30)).isoformat() flap_count = int( db.execute( "SELECT COUNT(*) FROM events WHERE service_id = ? AND event_type = 'service.state_changed' AND created_at >= ?", (probe.service_id, since), ).fetchone()[0] ) flapping = flap_count >= int(os.getenv("SERVICEMAP_FLAP_THRESHOLD", "6")) if flapping and not _bool(row["flapping"]): self._insert_event_db( db, "flapping.detected", probe.service_id, None, None, None, f"Wisselvalligheid vastgesteld: {flap_count} statuswijzigingen in 30 minuten", {"state_changes": flap_count, "window_minutes": 30}, created_at=now, ) events.append({"event_type": "flapping.detected", "service_id": probe.service_id, "message": "Wisselvalligheid vastgesteld"}) db.execute( """ UPDATE services SET state = ?, failure_streak = ?, recovery_streak = ?, flapping = ?, flap_count = ?, last_state_change_at = ?, updated_at = ? WHERE id = ? """, ( next_state.value, failure_streak, recovery_streak, int(flapping), flap_count, last_state_change, now.isoformat(), probe.service_id, ), ) event_bus.publish( { "event_type": "probe.completed", "service_id": probe.service_id, "status": probe.status.value, "trigger": probe.trigger, "message": "Controle voltooid", } ) for event in events: event_bus.publish(event) self.record_health_evidence( HealthEvidence( id=f"probe-{probe.id}", subject_type="service", subject_id=probe.service_id, service_id=probe.service_id, provider=EvidenceProvider.PROBE, provider_ref="active-probe", # The evidence mirrors the hysteresis-filtered service state, # while the raw probe result remains intact in history. status=next_state, scope=EvidenceScope.COMPONENT, confidence=1.0, observed_at=probe.checked_at, valid_until=probe.checked_at + timedelta(seconds=max(60, int(os.getenv("SERVICEMAP_PROBE_INTERVAL", "60")) * 2)), received_at=_now(), summary=probe.detail, metadata={"probe_id": probe.id, "trigger": probe.trigger, "raw_status": probe.status.value, "latency_ms": probe.latency_ms, **probe.metadata}, ) ) from .incidents import sync_incidents sync_incidents(self) return probe def latest_probe(self, service_id: str) -> ProbeResult | None: with self._connect() as db: row = db.execute( "SELECT * FROM probe_results WHERE service_id = ? ORDER BY checked_at DESC, rowid DESC LIMIT 1", (service_id,), ).fetchone() return self._row_to_probe(row) if row else None def latest_probes(self, service_ids: list[str] | None = None) -> dict[str, ProbeResult]: with self._connect() as db: if service_ids: placeholders = ",".join("?" for _ in service_ids) rows = db.execute( f""" SELECT p.* FROM probe_results p WHERE p.service_id IN ({placeholders}) AND p.rowid = ( SELECT candidate.rowid FROM probe_results candidate WHERE candidate.service_id = p.service_id ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1 ) """, service_ids, ).fetchall() else: rows = db.execute( """ SELECT p.* FROM probe_results p WHERE p.rowid = ( SELECT candidate.rowid FROM probe_results candidate WHERE candidate.service_id = p.service_id ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1 ) """ ).fetchall() return {row["service_id"]: self._row_to_probe(row) for row in rows} def probe_history(self, service_id: str, limit: int = 50) -> list[ProbeResult]: with self._connect() as db: rows = db.execute( "SELECT * FROM probe_results WHERE service_id = ? ORDER BY checked_at DESC, rowid DESC LIMIT ?", (service_id, limit), ).fetchall() return [self._row_to_probe(row) for row in rows] def availability_summary(self, service_id: str, hours: int = 24) -> AvailabilitySummary: hours = max(1, min(hours, 24 * 90)) since = _now() - timedelta(hours=hours) with self._connect() as db: row = db.execute( """ SELECT COUNT(*) AS total, SUM(CASE WHEN status = 'healthy' THEN 1 ELSE 0 END) AS healthy, SUM(CASE WHEN status = 'degraded' THEN 1 ELSE 0 END) AS degraded, SUM(CASE WHEN status = 'down' THEN 1 ELSE 0 END) AS down_count, AVG(latency_ms) AS avg_latency, MAX(checked_at) AS last_checked FROM probe_results WHERE service_id = ? AND checked_at >= ? """, (service_id, since.isoformat()), ).fetchone() service_row = db.execute("SELECT availability_target FROM services WHERE id = ?", (service_id,)).fetchone() total = int(row["total"] or 0) healthy = int(row["healthy"] or 0) degraded = int(row["degraded"] or 0) down = int(row["down_count"] or 0) uptime = round(((healthy + degraded) / total) * 100, 2) if total else None target = service_row["availability_target"] if service_row else None return AvailabilitySummary( hours=hours, total_checks=total, healthy_checks=healthy, degraded_checks=degraded, down_checks=down, uptime_percent=uptime, average_latency_ms=round(float(row["avg_latency"]), 2) if row["avg_latency"] is not None else None, last_checked_at=datetime.fromisoformat(row["last_checked"]) if row["last_checked"] else None, target_percent=target, meets_target=(uptime >= target) if uptime is not None and target is not None else None, ) def availability_summaries(self, service_ids: list[str], hours: int = 24) -> dict[str, AvailabilitySummary]: """Fetch table-ready availability in one grouped query, not N requests.""" if not service_ids: return {} hours = max(1, min(hours, 24 * 90)) since = _now() - timedelta(hours=hours) placeholders = ",".join("?" for _ in service_ids) with self._connect() as db: rows = db.execute( f""" SELECT s.id, s.availability_target, COUNT(p.id) AS total, SUM(CASE WHEN p.status = 'healthy' THEN 1 ELSE 0 END) AS healthy, SUM(CASE WHEN p.status = 'degraded' THEN 1 ELSE 0 END) AS degraded, SUM(CASE WHEN p.status = 'down' THEN 1 ELSE 0 END) AS down_count, AVG(p.latency_ms) AS avg_latency, MAX(p.checked_at) AS last_checked FROM services s LEFT JOIN probe_results p ON p.service_id=s.id AND p.checked_at >= ? WHERE s.id IN ({placeholders}) GROUP BY s.id, s.availability_target """, [since.isoformat(), *service_ids], ).fetchall() result: dict[str, AvailabilitySummary] = {} for row in rows: total = int(row["total"] or 0) healthy = int(row["healthy"] or 0) degraded = int(row["degraded"] or 0) target = row["availability_target"] uptime = round(((healthy + degraded) / total) * 100, 2) if total else None result[row["id"]] = AvailabilitySummary( hours=hours, total_checks=total, healthy_checks=healthy, degraded_checks=degraded, down_checks=int(row["down_count"] or 0), uptime_percent=uptime, average_latency_ms=round(float(row["avg_latency"]), 2) if row["avg_latency"] is not None else None, last_checked_at=datetime.fromisoformat(row["last_checked"]) if row["last_checked"] else None, target_percent=target, meets_target=(uptime >= target) if uptime is not None and target is not None else None, ) return result def next_probe_at(self, service: Service, global_interval: int) -> datetime | None: if not service.enabled or not service.monitoring_enabled: return None last = self.latest_probe(service.id) if last is None: return _now() return last.checked_at + timedelta(seconds=service.interval_seconds or global_interval) def prune_probes(self, retention_days: int, event_retention_days: int | None = None) -> int: cutoff = _now() - timedelta(days=retention_days) event_cutoff = _now() - timedelta(days=event_retention_days or retention_days) with self._connect() as db: cursor = db.execute("DELETE FROM probe_results WHERE checked_at < ?", (cutoff.isoformat(),)) db.execute("DELETE FROM events WHERE created_at < ? AND event_type NOT LIKE 'incident.%'", (event_cutoff.isoformat(),)) return cursor.rowcount def checkpoint(self) -> None: """Fold the WAL back into the database and refresh planner statistics. Without a TRUNCATE checkpoint the WAL file keeps its high-water size. """ with self._connect() as db: db.execute("PRAGMA wal_checkpoint(TRUNCATE)") db.execute("PRAGMA optimize") def ready_check(self) -> None: """Raise if the database cannot take a write lock right now.""" connection = self._connect() try: connection.execute("BEGIN IMMEDIATE") connection.execute("ROLLBACK") finally: connection.close() def prune_health_evidence(self, retention_days: int) -> int: """Delete superseded evidence history older than the retention window. Only inactive rows are removed. A row becomes inactive exclusively when a newer claim for the same provider/subject supersedes it, so every subject keeps its current (active) claim and health resolution is unaffected. Without this, every status flip leaves a permanent row and evidence reads grow linearly with uptime. """ if retention_days <= 0: return 0 cutoff = _now() - timedelta(days=retention_days) with self._connect() as db: cursor = db.execute( "DELETE FROM health_evidence WHERE active = 0 AND superseded_by IS NOT NULL AND received_at < ?", (cutoff.isoformat(),), ) # Links cascade through the foreign key; also remove any orphan # links left by databases that predate enforced foreign keys. db.execute( "DELETE FROM health_evidence_service_links " "WHERE evidence_id NOT IN (SELECT id FROM health_evidence)" ) return cursor.rowcount def _insert_event_db( self, db: sqlite3.Connection, event_type: str, service_id: str | None, incident_id: str | None, from_status: str | None, to_status: str | None, message: str, metadata: dict[str, object], created_at: datetime | None = None, ) -> str: event_id = str(uuid4()) db.execute( """ INSERT INTO events (id, event_type, service_id, incident_id, from_status, to_status, message, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, (event_id, event_type, service_id, incident_id, from_status, to_status, message, json.dumps(metadata), (created_at or _now()).isoformat()), ) return event_id def add_event( self, event_type: str, message: str, *, service_id: str | None = None, incident_id: str | None = None, from_status: HealthStatus | None = None, to_status: HealthStatus | None = None, metadata: dict[str, object] | None = None, ) -> EventRecord: created_at = _now() with self._connect() as db: event_id = self._insert_event_db( db, event_type, service_id, incident_id, from_status.value if from_status else None, to_status.value if to_status else None, message, metadata or {}, created_at, ) row = db.execute("SELECT * FROM events WHERE id = ?", (event_id,)).fetchone() result = self._row_to_event(row) event_bus.publish(result.model_dump(mode="json")) return result def list_events( self, limit: int = 50, service_id: str | None = None, event_type: str | None = None, before: datetime | None = None, ) -> list[EventRecord]: """Newest first. ``event_type`` ending in ``.`` matches a family (``incident.``); ``before`` pages to older events.""" clauses: list[str] = [] params: list[object] = [] if service_id: clauses.append("service_id = ?") params.append(service_id) if event_type: if event_type.endswith("."): # Exact prefix match without LIKE wildcards. clauses.append("substr(event_type, 1, ?) = ?") params.extend([len(event_type), event_type]) else: clauses.append("event_type = ?") params.append(event_type) if before is not None: clauses.append("created_at < ?") params.append(before.isoformat()) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with self._connect() as db: rows = db.execute( f"SELECT * FROM events {where} ORDER BY created_at DESC LIMIT ?", [*params, limit] ).fetchall() return [self._row_to_event(row) for row in rows] def demo_active(self) -> bool: with self._connect() as db: return db.execute("SELECT 1 FROM demo_sessions WHERE id = 1").fetchone() is not None @staticmethod def _snapshot_rows(db: sqlite3.Connection) -> tuple[str, str, str, str]: services = [dict(row) for row in db.execute("SELECT * FROM services ORDER BY id").fetchall()] probes = [dict(row) for row in db.execute("SELECT * FROM probe_results ORDER BY id").fetchall()] events = [dict(row) for row in db.execute("SELECT * FROM events ORDER BY id").fetchall()] incidents = [dict(row) for row in db.execute("SELECT * FROM incidents ORDER BY id").fetchall()] return tuple(json.dumps(item, separators=(",", ":")) for item in (services, probes, events, incidents)) # type: ignore[return-value] @staticmethod def _snapshot_discovery_rows(db: sqlite3.Connection) -> str: tables = ( "discovery_runs", "discovery_sources", "service_sources", "discovery_observations", "relation_evidence", "topology_suggestions", "topology_overrides", "topology_change_events", ) existing = {row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} snapshot: dict[str, list[dict[str, object]]] = {} for table in tables: if table in existing: snapshot[table] = [dict(row) for row in db.execute(f"SELECT * FROM {table}").fetchall()] return json.dumps(snapshot, separators=(",", ":")) @staticmethod def _snapshot_health_evidence_rows(db: sqlite3.Connection) -> str: return json.dumps( { "health_evidence": [dict(row) for row in db.execute("SELECT * FROM health_evidence ORDER BY id").fetchall()], "health_evidence_service_links": [ dict(row) for row in db.execute("SELECT * FROM health_evidence_service_links ORDER BY evidence_id, service_id").fetchall() ], "provider_status": [dict(row) for row in db.execute("SELECT * FROM provider_status ORDER BY provider").fetchall()], }, separators=(",", ":"), ) @classmethod def _raw_service_params(cls, raw: dict[str, Any]) -> dict[str, Any]: defaults: dict[str, Any] = { "description": None, "tags": "[]", "group": None, "environment": None, "owner": None, "enabled": 1, "monitoring_enabled": 1, "monitoring_state": "active_probe", "probe_source": "manual", "probe_reachability": "verified", "probe_confidence": "confirmed", "runtime_state": None, "runtime_health": None, "application": None, "criticality": "medium", "category": "unknown", "visibility": "visible", "lifecycle": "active", "source_type": None, "source_ref": None, "first_seen_at": None, "last_seen_at": None, "last_discovery_status": None, "container_image": None, "compose_project": None, "layout_x": None, "layout_y": None, "layout_pinned": 0, "interval_seconds": None, "timeout_seconds": None, "retries": None, "failure_threshold": 1, "recovery_threshold": 1, "expected_status_min": None, "expected_status_max": None, "availability_target": None, "maintenance_enabled": 0, "maintenance_reason": None, "state": "unknown", "failure_streak": 0, "recovery_streak": 0, "flapping": 0, "flap_count": 0, "last_state_change_at": None, "updated_at": raw.get("created_at"), "depends_on": "[]", } params = {**defaults, **raw} params["depends_on"] = json.dumps(params["depends_on"]) if isinstance(params["depends_on"], list) else params["depends_on"] params["tags"] = json.dumps(params["tags"]) if isinstance(params["tags"], list) else params["tags"] params["updated_at"] = params["updated_at"] or params["created_at"] params["metadata"] = params.get("metadata", "{}") params["trigger"] = params.get("trigger", "manual") return params @classmethod def _insert_raw_service(cls, db: sqlite3.Connection, raw: dict[str, Any]) -> None: params = cls._raw_service_params(raw) cls._insert_service_row(db, params) @staticmethod def _insert_service_row(db: sqlite3.Connection, params: dict[str, Any]) -> None: db.execute( f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES (" ":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, " ":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, " ":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, " ":compose_project, :layout_x, :layout_y, :layout_pinned, " ":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, " ":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, " ":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, " ":last_state_change_at, :created_at, :updated_at)", params, ) @classmethod def _restore_rows(cls, db: sqlite3.Connection, snapshot_services: str, snapshot_probes: str, snapshot_events: str, snapshot_incidents: str, snapshot_discovery: str = "{}", snapshot_health_evidence: str = "{}") -> None: services = json.loads(snapshot_services) probes = json.loads(snapshot_probes) events = json.loads(snapshot_events or "[]") incidents = json.loads(snapshot_incidents or "[]") db.execute("DELETE FROM incidents") db.execute("DELETE FROM events") db.execute("DELETE FROM services") for raw in services: cls._insert_raw_service(db, raw) for raw in probes: params = { "id": raw["id"], "service_id": raw["service_id"], "status": raw["status"], "latency_ms": raw.get("latency_ms"), "detail": raw["detail"], "checked_at": raw["checked_at"], "trigger": raw.get("trigger", "manual"), "metadata": raw.get("metadata", "{}"), } db.execute( "INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata) VALUES (:id, :service_id, :status, :latency_ms, :detail, :checked_at, :trigger, :metadata)", params, ) for raw in incidents: db.execute( "INSERT INTO incidents (id, status, root_service_id, started_at, detected_at, acknowledged_at, resolved_at, title, root_cause_reason, affected_service_ids, severity, latest_update_at) VALUES (:id, :status, :root_service_id, :started_at, :detected_at, :acknowledged_at, :resolved_at, :title, :root_cause_reason, :affected_service_ids, :severity, :latest_update_at)", raw, ) for raw in events: db.execute( "INSERT INTO events (id, event_type, service_id, incident_id, from_status, to_status, message, metadata, created_at) VALUES (:id, :event_type, :service_id, :incident_id, :from_status, :to_status, :message, :metadata, :created_at)", {**raw, "metadata": raw.get("metadata", "{}")}, ) cls._restore_discovery_rows(db, snapshot_discovery) cls._restore_health_evidence_rows(db, snapshot_health_evidence) @staticmethod def _restore_health_evidence_rows(db: sqlite3.Connection, snapshot: str) -> None: try: items = json.loads(snapshot or "{}") except json.JSONDecodeError: items = {} if not isinstance(items, dict): return db.execute("DELETE FROM health_evidence") db.execute("DELETE FROM provider_status") for table in ("health_evidence", "health_evidence_service_links", "provider_status"): rows = items.get(table, []) if not isinstance(rows, list): continue for raw in rows: if not isinstance(raw, dict) or not raw: continue columns = list(raw) quoted = ", ".join(f'"{column}"' for column in columns) placeholders = ", ".join("?" for _ in columns) db.execute(f"INSERT INTO {table} ({quoted}) VALUES ({placeholders})", tuple(raw[column] for column in columns)) @staticmethod def _restore_discovery_rows(db: sqlite3.Connection, snapshot: str) -> None: try: tables = json.loads(snapshot or "{}") except json.JSONDecodeError: tables = {} if not isinstance(tables, dict) or not tables: return order = ( "topology_change_events", "topology_overrides", "relation_evidence", "service_sources", "discovery_observations", "topology_suggestions", "discovery_sources", "discovery_runs", ) existing = {row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} for table in order: if table in existing: db.execute(f"DELETE FROM {table}") for table in reversed(order): rows = tables.get(table, []) if table not in existing or not isinstance(rows, list): continue for raw in rows: if not isinstance(raw, dict) or not raw: continue columns = list(raw) quoted = ", ".join(f'"{column}"' for column in columns) placeholders = ", ".join("?" for _ in columns) db.execute(f"INSERT INTO {table} ({quoted}) VALUES ({placeholders})", tuple(raw[column] for column in columns)) @classmethod def _insert_demo_rows(cls, db: sqlite3.Connection, services: list[Service], probes: list[ProbeResult]) -> None: db.execute("DELETE FROM incidents") db.execute("DELETE FROM events") db.execute("DELETE FROM health_evidence") db.execute("DELETE FROM provider_status") db.execute("DELETE FROM services") latest: dict[str, ProbeResult] = {} for probe in probes: if probe.service_id not in latest or probe.checked_at > latest[probe.service_id].checked_at: latest[probe.service_id] = probe for service in services: seeded = service.model_copy(update={"state": latest.get(service.id, ProbeResult(id="", service_id=service.id, status=HealthStatus.UNKNOWN, latency_ms=None, detail="", checked_at=_now())).status}) cls._insert_service_row(db, cls._service_params(seeded)) for probe in probes: db.execute( "INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (probe.id, probe.service_id, probe.status.value, probe.latency_ms, probe.detail, probe.checked_at.isoformat(), probe.trigger, json.dumps(probe.metadata)), ) for probe in probes: service = next((item for item in services if item.id == probe.service_id), None) interval = service.interval_seconds if service and service.interval_seconds else 60 evidence_id = f"demo-probe-{probe.id}" db.execute( """ INSERT INTO health_evidence ( id, subject_type, subject_id, service_id, provider, provider_ref, status, scope, confidence, observed_at, valid_until, received_at, summary, metadata, active ) VALUES (?, 'service', ?, ?, 'active_probe', 'active-probe', ?, 'component', 1.0, ?, ?, ?, ?, ?, 1) ON CONFLICT(id) DO NOTHING """, ( evidence_id, probe.service_id, probe.service_id, probe.status.value, probe.checked_at.isoformat(), (probe.checked_at + timedelta(seconds=max(60, interval * 2))).isoformat(), probe.checked_at.isoformat(), probe.detail, json.dumps({"probe_id": probe.id, "trigger": probe.trigger, **probe.metadata}, sort_keys=True), ), ) db.execute( "INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)", (evidence_id, probe.service_id), ) def activate_demo(self, services: list[Service], probes: list[ProbeResult]) -> ConfigImportResult: with self._connect() as db: db.execute("BEGIN IMMEDIATE") session = db.execute("SELECT * FROM demo_sessions WHERE id = 1").fetchone() if session is None: snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents = self._snapshot_rows(db) snapshot_discovery = self._snapshot_discovery_rows(db) snapshot_health_evidence = self._snapshot_health_evidence_rows(db) started_at = _now().isoformat() else: snapshot_services = session["snapshot_services"] snapshot_probes = session["snapshot_probes"] snapshot_events = session["snapshot_events"] or "[]" snapshot_incidents = session["snapshot_incidents"] or "[]" snapshot_discovery = session["snapshot_discovery"] or "{}" snapshot_health_evidence = session["snapshot_health_evidence"] or "{}" started_at = session["started_at"] self._insert_demo_rows(db, services, probes) db.execute( """ INSERT INTO demo_sessions (id, snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, demo_service_ids, started_at) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET snapshot_services=excluded.snapshot_services, snapshot_probes=excluded.snapshot_probes, snapshot_events=excluded.snapshot_events, snapshot_incidents=excluded.snapshot_incidents, snapshot_discovery=excluded.snapshot_discovery, snapshot_health_evidence=excluded.snapshot_health_evidence, demo_service_ids=excluded.demo_service_ids, started_at=excluded.started_at """, (snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, json.dumps([service.id for service in services]), started_at), ) event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo geladen"}) return ConfigImportResult(created=len(services), updated=0, total=len(services)) def exit_demo(self) -> bool: with self._connect() as db: db.execute("BEGIN IMMEDIATE") session = db.execute("SELECT * FROM demo_sessions WHERE id = 1").fetchone() if session is None: return False self._restore_rows( db, session["snapshot_services"], session["snapshot_probes"], session["snapshot_events"], session["snapshot_incidents"], session["snapshot_discovery"] or "{}", session["snapshot_health_evidence"] or "{}", ) db.execute("DELETE FROM demo_sessions WHERE id = 1") event_bus.publish({"event_type": "topology.changed", "message": "Synthetische demo afgesloten"}) return True def list_incidents(self, active_only: bool = False, limit: int = 100) -> list[IncidentRecord]: services = {service.id: service for service in self.list_services()} with self._connect() as db: query = "SELECT * FROM incidents" params: tuple[Any, ...] = () if active_only: query += " WHERE status != 'resolved'" query += " ORDER BY detected_at DESC LIMIT ?" params += (limit,) rows = db.execute(query, params).fetchall() return [self._row_to_incident(row, services) for row in rows] def get_incident(self, incident_id: str) -> IncidentRecord | None: services = {service.id: service for service in self.list_services()} with self._connect() as db: row = db.execute("SELECT * FROM incidents WHERE id = ?", (incident_id,)).fetchone() return self._row_to_incident(row, services) if row else None def incident_timeline(self, incident_id: str, limit: int = 200) -> list[IncidentEvent]: with self._connect() as db: rows = db.execute( "SELECT id, incident_id, event_type, message, created_at, service_id FROM events WHERE incident_id = ? ORDER BY created_at ASC LIMIT ?", (incident_id, limit), ).fetchall() return [IncidentEvent(**dict(row)) for row in rows] def upsert_incident(self, incident: IncidentRecord) -> IncidentRecord: with self._connect() as db: db.execute( """ INSERT INTO incidents (id, status, root_service_id, started_at, detected_at, acknowledged_at, resolved_at, title, root_cause_reason, affected_service_ids, severity, latest_update_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET status=excluded.status, acknowledged_at=excluded.acknowledged_at, resolved_at=excluded.resolved_at, title=excluded.title, root_cause_reason=excluded.root_cause_reason, affected_service_ids=excluded.affected_service_ids, severity=excluded.severity, latest_update_at=excluded.latest_update_at """, (incident.id, incident.status.value, incident.root_service_id, incident.started_at.isoformat(), incident.detected_at.isoformat(), _iso(incident.acknowledged_at), _iso(incident.resolved_at), incident.title, incident.root_cause_reason, json.dumps(incident.affected_service_ids), incident.severity.value, incident.latest_update_at.isoformat()), ) return incident def append_incident_event(self, incident_id: str, event_type: str, message: str, service_id: str | None = None) -> IncidentEvent: created_at = _now() with self._connect() as db: event_id = self._insert_event_db(db, event_type, service_id, incident_id, None, None, message, {}, created_at) payload = {"event_type": event_type, "incident_id": incident_id, "service_id": service_id, "message": message, "created_at": created_at.isoformat()} event_bus.publish(payload) return IncidentEvent(id=event_id, incident_id=incident_id, event_type=event_type, message=message, created_at=created_at, service_id=service_id) def resolve_incident(self, incident_id: str, note: str | None = None) -> IncidentRecord | None: """Close an incident by hand (e.g. a stale one). If its root is still down, the next incident sync opens a new one. """ incident = self.get_incident(incident_id) if incident is None: return None if incident.status == IncidentStatus.RESOLVED: return incident now = _now() updated = incident.model_copy( update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now} ) self.upsert_incident(updated) self.append_incident_event( incident_id, "incident.resolved", f"Incident handmatig opgelost{f': {note}' if note else ''}", incident.root_service_id, ) return updated def add_incident_note(self, incident_id: str, text: str) -> IncidentRecord | None: incident = self.get_incident(incident_id) if incident is None: return None self.append_incident_event(incident_id, "incident.note", text, incident.root_service_id) updated = incident.model_copy(update={"latest_update_at": _now()}) self.upsert_incident(updated) return updated def acknowledge_incident(self, incident_id: str) -> IncidentRecord | None: incident = self.get_incident(incident_id) if incident is None: return None if incident.status == IncidentStatus.RESOLVED: raise ValueError("Opgeloste incidenten kunnen niet bevestigd worden") if incident.status == IncidentStatus.ACKNOWLEDGED: # Idempotent: a repeated acknowledge keeps the original time and # adds no duplicate timeline entry. return incident now = _now() updated = incident.model_copy(update={"status": IncidentStatus.ACKNOWLEDGED, "acknowledged_at": now, "latest_update_at": now}) self.upsert_incident(updated) self.append_incident_event(incident_id, "incident.acknowledged", "Incident bevestigd", incident.root_service_id) return updated