Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
195 lines
7.5 KiB
Python
195 lines
7.5 KiB
Python
from __future__ import annotations
|
|
|
|
import threading
|
|
from datetime import UTC, datetime
|
|
from uuid import uuid4
|
|
|
|
from .health import TopologyHealth, topology_health
|
|
from .models import (
|
|
Criticality,
|
|
HealthStatus,
|
|
IncidentCandidate,
|
|
IncidentOverview,
|
|
IncidentRecord,
|
|
IncidentStatus,
|
|
Service,
|
|
)
|
|
from .store import Store
|
|
|
|
|
|
def _reverse_dependencies(services: list[Service]) -> dict[str, list[str]]:
|
|
reverse: dict[str, list[str]] = {service.id: [] for service in services}
|
|
for service in services:
|
|
for dependency_id in service.depends_on:
|
|
reverse.setdefault(dependency_id, []).append(service.id)
|
|
return reverse
|
|
|
|
|
|
def affected_service_ids(root_id: str, services: list[Service]) -> list[str]:
|
|
reverse = _reverse_dependencies(services)
|
|
seen: set[str] = set()
|
|
queue = list(reverse.get(root_id, []))
|
|
while queue:
|
|
current = queue.pop(0)
|
|
if current in seen or current == root_id:
|
|
continue
|
|
seen.add(current)
|
|
queue.extend(reverse.get(current, []))
|
|
return sorted(seen)
|
|
|
|
|
|
def incident_overview(store: Store, *, health: TopologyHealth | None = None) -> IncidentOverview:
|
|
health = health or topology_health(store)
|
|
services = health.services
|
|
by_id = {service.id: service for service in services}
|
|
effective = health.effective
|
|
dependency = health.dependency
|
|
|
|
down_ids = sorted(service_id for service_id, state in effective.items() if state == HealthStatus.DOWN)
|
|
degraded_ids = sorted(
|
|
service_id for service_id, state in effective.items() if state in {HealthStatus.DEGRADED, HealthStatus.RECOVERING}
|
|
)
|
|
|
|
candidates: list[IncidentCandidate] = []
|
|
for service in services:
|
|
state = effective[service.id]
|
|
if state != HealthStatus.DOWN or service.maintenance_enabled:
|
|
continue
|
|
down_dependencies = [
|
|
dependency_id
|
|
for dependency_id in service.depends_on
|
|
if effective.get(dependency_id) == HealthStatus.DOWN
|
|
]
|
|
if down_dependencies:
|
|
continue
|
|
affected_ids = affected_service_ids(service.id, services)
|
|
affected_names = [by_id[item].name for item in affected_ids if item in by_id]
|
|
reason = "Service is uitgevallen en geen enkele ingestelde afhankelijkheid is ook uitgevallen"
|
|
if dependency[service.id] == HealthStatus.UNKNOWN:
|
|
reason += "; minstens één afhankelijkheid heeft een onbekende gezondheid"
|
|
candidates.append(
|
|
IncidentCandidate(
|
|
service_id=service.id,
|
|
service_name=service.name,
|
|
status=state,
|
|
reason=reason,
|
|
affected_service_ids=affected_ids,
|
|
affected_service_names=affected_names,
|
|
)
|
|
)
|
|
|
|
candidates.sort(key=lambda item: (-len(item.affected_service_ids), item.service_name.lower()))
|
|
return IncidentOverview(
|
|
generated_at=datetime.now(UTC),
|
|
active=bool(down_ids or degraded_ids),
|
|
candidates=candidates,
|
|
degraded_service_ids=degraded_ids,
|
|
down_service_ids=down_ids,
|
|
incidents=[],
|
|
)
|
|
|
|
|
|
def _severity(service: Service) -> Criticality:
|
|
return service.criticality
|
|
|
|
|
|
# Probes, observation and API handlers reconcile from different threads. Two
|
|
# concurrent passes could both see "no active incident" for a root and each
|
|
# open one (the later duplicate cleanup then left two journal entries).
|
|
_SYNC_LOCK = threading.RLock()
|
|
|
|
|
|
def sync_incidents(store: Store, *, resolution_reason: str = "Hoofdoorzaak hersteld; incident opgelost") -> list[IncidentRecord]:
|
|
with _SYNC_LOCK:
|
|
return _sync_incidents_locked(store, resolution_reason=resolution_reason)
|
|
|
|
|
|
def _sync_incidents_locked(store: Store, *, resolution_reason: str) -> list[IncidentRecord]:
|
|
"""Reconcile the current graph with the durable incident journal.
|
|
|
|
A root service has at most one active incident. Repeated probe cycles only
|
|
update its affected set; a later independent outage gets a new id.
|
|
"""
|
|
|
|
health = topology_health(store)
|
|
overview = incident_overview(store, health=health)
|
|
services = {service.id: service for service in health.services}
|
|
now = datetime.now(UTC)
|
|
active: dict[str, IncidentRecord] = {}
|
|
for incident in sorted(
|
|
store.list_incidents(active_only=True),
|
|
key=lambda item: (item.detected_at, item.id),
|
|
):
|
|
if incident.root_service_id in active:
|
|
store.upsert_incident(incident.model_copy(update={
|
|
"status": IncidentStatus.RESOLVED,
|
|
"resolved_at": now,
|
|
"latest_update_at": now,
|
|
}))
|
|
store.append_incident_event(
|
|
incident.id,
|
|
"incident.duplicate_resolved",
|
|
"Dubbel actief incident gesloten; het oudste incident blijft actief",
|
|
incident.root_service_id,
|
|
)
|
|
continue
|
|
active[incident.root_service_id] = incident
|
|
for candidate in overview.candidates:
|
|
service = services.get(candidate.service_id)
|
|
if service is None:
|
|
continue
|
|
incident = active.get(candidate.service_id)
|
|
if incident is None:
|
|
incident = IncidentRecord(
|
|
id=str(uuid4()),
|
|
status=IncidentStatus.OPEN,
|
|
root_service_id=candidate.service_id,
|
|
root_service_name=candidate.service_name,
|
|
started_at=now,
|
|
detected_at=now,
|
|
title=f"{candidate.service_name} is uitgevallen",
|
|
root_cause_reason=candidate.reason,
|
|
affected_service_ids=candidate.affected_service_ids,
|
|
affected_service_names=candidate.affected_service_names,
|
|
severity=_severity(service),
|
|
latest_update_at=now,
|
|
)
|
|
store.upsert_incident(incident)
|
|
store.append_incident_event(
|
|
incident.id,
|
|
"incident.opened",
|
|
f"Incident geopend: {candidate.service_name} is de waarschijnlijke hoofdoorzaak",
|
|
candidate.service_id,
|
|
)
|
|
active[candidate.service_id] = incident
|
|
elif incident.affected_service_ids != candidate.affected_service_ids:
|
|
updated = incident.model_copy(
|
|
update={
|
|
"affected_service_ids": candidate.affected_service_ids,
|
|
"affected_service_names": candidate.affected_service_names,
|
|
"latest_update_at": now,
|
|
"root_cause_reason": candidate.reason,
|
|
}
|
|
)
|
|
store.upsert_incident(updated)
|
|
store.append_incident_event(incident.id, "incident.updated", "Impact op afhankelijke services herberekend", candidate.service_id)
|
|
|
|
for root_id, incident in list(active.items()):
|
|
if any(candidate.service_id == root_id for candidate in overview.candidates):
|
|
continue
|
|
root = services.get(root_id)
|
|
if (
|
|
(root is None or health.direct.get(root_id, HealthStatus.UNKNOWN) not in {
|
|
HealthStatus.DOWN,
|
|
HealthStatus.DEGRADED,
|
|
HealthStatus.RECOVERING,
|
|
})
|
|
and incident.status != IncidentStatus.RESOLVED
|
|
):
|
|
resolved = incident.model_copy(
|
|
update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now}
|
|
)
|
|
store.upsert_incident(resolved)
|
|
store.append_incident_event(incident.id, "incident.resolved", resolution_reason, root_id)
|
|
return store.list_incidents(active_only=False)
|