ServiceMap: update tests and backend/app (+7 areas)

Public changes compared with the previous published export:
89 files changed.

Affected areas:
- backend/app: 21 files
- dependencies: 5 files
- deployment and CI: 6 files
- documentation: 2 files
- frontend: 3 files
- frontend/src: 20 files
- project files: 1 file
- tests: 30 files
- unraid: 1 file

Changed files:
- Updated: ".env.example"
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Added: "backend/app/alerts.py"
- Updated: "backend/app/auth.py"
- Added: "backend/app/backups.py"
- Updated: "backend/app/config_io.py"
- Updated: "backend/app/demo.py"
- Updated: "backend/app/discovery.py"
- Updated: "backend/app/discovery_store.py"
- Updated: "backend/app/docker_discovery.py"
- Updated: "backend/app/events.py"
- Updated: "backend/app/evidence.py"
- Updated: "backend/app/health.py"
- Updated: "backend/app/incidents.py"
- Added: "backend/app/loops.py"
- Updated: "backend/app/main.py"
- Updated: "backend/app/models.py"
- Updated: "backend/app/observation.py"
- Updated: "backend/app/probes.py"
- Added: "backend/app/response_cache.py"
- Updated: "backend/app/scheduler.py"
- Updated: "backend/app/semantic.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Updated: "backend/tests/test_api.py"
- Updated: "backend/tests/test_config_io.py"
- Updated: "backend/tests/test_demo.py"
- Updated: "backend/tests/test_discovery_v120.py"
- Updated: "backend/tests/test_health.py"
- Updated: "backend/tests/test_probes.py"
- Updated: "backend/tests/test_scheduler.py"
- Updated: "backend/tests/test_v130_probe_reachability.py"
- Updated: "backend/tests/test_v160_evidence_changes.py"
- Added: "backend/tests/test_v163_alerts.py"
- Added: "backend/tests/test_v163_backups.py"
- Added: "backend/tests/test_v163_data_integrity.py"
- Added: "backend/tests/test_v163_efficiency.py"
- Added: "backend/tests/test_v163_health_performance.py"
- Added: "backend/tests/test_v163_loops.py"
- Added: "backend/tests/test_v163_management.py"
- Added: "backend/tests/test_v163_realtime.py"
- Added: "backend/tests/test_v163_security.py"
- Added: "backend/tests/test_v163_semantic_etag.py"
- Added: "compose.demo.yaml"
- Updated: "compose.yaml"
- Added: "docker-compose.broker.yml"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Added: "frontend/e2e/a11y.spec.ts"
- Updated: "frontend/e2e/demo.spec.ts"
- Updated: "frontend/index.html"
- Updated: "frontend/nginx.conf"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "frontend/playwright.config.ts"
- Updated: "frontend/src/App.tsx"
- Updated: "frontend/src/LivingEdge.tsx"
- Added: "frontend/src/activityView.test.ts"
- Added: "frontend/src/activityView.ts"
- Updated: "frontend/src/api.ts"
- Added: "frontend/src/apiAuth.test.ts"
- Added: "frontend/src/components/ActionsMenu.tsx"
- Added: "frontend/src/components/ConfirmDialog.test.tsx"
- Added: "frontend/src/components/ConfirmDialog.tsx"
- Added: "frontend/src/components/StatusIcon.tsx"
- Updated: "frontend/src/demo/DemoWalkthrough.test.ts"
- Updated: "frontend/src/demo/DemoWalkthrough.tsx"
- Added: "frontend/src/elk-worker-url.d.ts"
- Added: "frontend/src/labels.ts"
- Updated: "frontend/src/main.tsx"
- Added: "frontend/src/realtime/refreshPlan.test.ts"
- Added: "frontend/src/realtime/refreshPlan.ts"
- Updated: "frontend/src/realtime/useRealtimeEvents.ts"
- Added: "frontend/src/routing/urlState.test.ts"
- Added: "frontend/src/routing/urlState.ts"
- Updated: "frontend/src/styles.css"
- Added: "frontend/src/topology/GraphListView.test.tsx"
- Added: "frontend/src/topology/GraphListView.tsx"
- Updated: "frontend/src/topology/ServiceNode.tsx"
- Updated: "frontend/src/topology/TopologyCommandBar.tsx"
- Added: "frontend/src/topology/semanticGraph.test.ts"
- Updated: "frontend/src/topology/semanticGraph.ts"
- Added: "frontend/src/topologyLayout.test.ts"
- Updated: "frontend/src/topologyLayout.ts"
- Updated: "package-lock.json"
- Updated: "package.json"
- Updated: "unraid/my-servicemap.xml"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
This commit is contained in:
NuklearRabbit committed 2026-11-20 13:51:46 +01:00
1 parent d7319250e8
commit adc2c70c21
91 files changed
+9205 -1691

No files matched your search

+430 -100
View File
@@ -1,26 +1,30 @@
from __future__ import annotations
import asyncio
import logging
import os
import sqlite3
from contextlib import asynccontextmanager, suppress
from datetime import timedelta
from datetime import datetime, timedelta
from pathlib import Path
from fastapi import FastAPI, HTTPException, Query, Response, status
from fastapi.responses import JSONResponse, StreamingResponse
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from .auth import AuthConfig, AuthorizationError, authorize_request
from .alerts import AlertManager, AlertSettingsUpdate, AlertSettingsView, AlertTestResult
from .auth import AuthConfig, AuthorizationError, admin_token, admin_token_path, authorize_request
from .backups import BackupInfo, BackupManager
from .config_io import export_document, import_document, parse_document, serialize_document
from .demo import DEMO_ROOT_CAUSE_NAME, DEMO_SCENARIO_NAME, build_demo_records
from .discovery import DiscoveryManager
from .discovery_store import DiscoveryStore
from .docker_discovery import discover_containers
from .docker_provider import docker_provider_mode
from .events import event_bus, next_event, sse_payload
from .health import dependency_status, effective_status, status_maps, validate_dependencies
from .events import SSE_RETRY_MS, event_bus, next_event, sse_payload
from .health import status_maps, topology_health, validate_dependencies
from .incidents import incident_overview, sync_incidents
from .loops import loop_registry, run_periodic
from .models import (
AvailabilitySummary,
BulkProbeResult,
@@ -39,7 +43,10 @@ from .models import (
EdgeMutationRequest,
EventRecord,
HealthResolution,
HealthStatus,
IncidentNoteRequest,
IncidentOverview,
IncidentResolveRequest,
LogicalRelation,
MaintenanceRequest,
OverviewSummary,
@@ -60,6 +67,14 @@ from .models import (
)
from .observation import ObservationManager
from .probes import probe_many, probe_service
from .response_cache import (
GENERATED_AT_HEADER,
CachedRepresentation,
RepresentationCache,
etag_matches,
next_freshness_boundary,
representation,
)
from .scheduler import scheduled_probe_loop
from .semantic import build_semantic_topology
from .store import Store
@@ -83,6 +98,16 @@ def _env_float(name: str, default: float, minimum: float = 0.1) -> float:
return default
logger = logging.getLogger("servicemap")
if not logger.handlers:
# Uvicorn configures only its own loggers; without a handler our INFO
# lines (maintenance results, loop recovery) were never emitted.
_handler = logging.StreamHandler()
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
logger.addHandler(_handler)
logger.setLevel(logging.INFO)
logger.propagate = False
PROBE_INTERVAL = _env_int("SERVICEMAP_PROBE_INTERVAL", 0)
PROBE_TIMEOUT = _env_float("SERVICEMAP_PROBE_TIMEOUT", 5.0)
PROBE_RETRIES = _env_int("SERVICEMAP_PROBE_RETRIES", 1)
@@ -90,8 +115,48 @@ PROBE_CONCURRENCY = _env_int("SERVICEMAP_PROBE_CONCURRENCY", 10, minimum=1)
RETENTION_DAYS = _env_int("SERVICEMAP_RETENTION_DAYS", 30, minimum=1)
EVENT_RETENTION_DAYS = _env_int("SERVICEMAP_EVENT_RETENTION_DAYS", 90, minimum=1)
DISCOVERY_INTERVAL = _env_int("SERVICEMAP_DISCOVERY_INTERVAL_SECONDS", 0, minimum=0)
OBSERVATION_INTERVAL = _env_int("SERVICEMAP_OBSERVATION_INTERVAL_SECONDS", 60, minimum=10)
OBSERVATION_INTERVAL = _env_int("SERVICEMAP_OBSERVATION_INTERVAL_SECONDS", 60)
if OBSERVATION_INTERVAL > 0:
OBSERVATION_INTERVAL = max(10, OBSERVATION_INTERVAL)
# Superseded health-evidence history; current claims are never pruned. 0 disables.
EVIDENCE_RETENTION_DAYS = _env_int("SERVICEMAP_EVIDENCE_RETENTION_DAYS", 7, minimum=0)
MAINTENANCE_INTERVAL = _env_int("SERVICEMAP_MAINTENANCE_INTERVAL_SECONDS", 3600, minimum=60)
# Discovery runs whose per-container observations are kept.
DISCOVERY_OBSERVATION_RUNS = _env_int("SERVICEMAP_DISCOVERY_OBSERVATION_RUNS", 10, minimum=1)
# Verified database backups in <data>/backups; 0 hours disables the schedule.
BACKUP_INTERVAL_HOURS = _env_int("SERVICEMAP_BACKUP_INTERVAL_HOURS", 24, minimum=0)
BACKUP_KEEP = _env_int("SERVICEMAP_BACKUP_KEEP", 14, minimum=1)
observation_manager = ObservationManager(store, interval_seconds=OBSERVATION_INTERVAL)
semantic_cache = RepresentationCache(max_age_seconds=10.0)
_alert_manager: AlertManager | None = None
def backup_manager() -> BackupManager:
return BackupManager(store, keep=BACKUP_KEEP)
def alert_manager() -> AlertManager:
"""Bound to the current store (tests swap ``store``)."""
global _alert_manager
if _alert_manager is None or _alert_manager.store is not store:
_alert_manager = AlertManager(store)
return _alert_manager
def run_maintenance() -> dict[str, int]:
"""Bounded retention for tables that otherwise grow with uptime."""
if store.demo_active():
return {}
pruned = {
"health_evidence": store.prune_health_evidence(EVIDENCE_RETENTION_DAYS),
"probe_results": store.prune_probes(RETENTION_DAYS, EVENT_RETENTION_DAYS),
"discovery_observations": discovery_store.prune_observations(DISCOVERY_OBSERVATION_RUNS),
"topology_change_events": discovery_store.prune_change_events(EVENT_RETENTION_DAYS),
}
store.checkpoint()
return pruned
def _require_configured_discovery_targets(
@@ -102,16 +167,22 @@ def _require_configured_discovery_targets(
configured_docker = os.getenv("SERVICEMAP_DOCKER_HOST", "unix:///var/run/docker.sock")
configured_appops = os.getenv("SERVICEMAP_APPOPS_URL", "")
if docker_host and docker_host != configured_docker:
raise HTTPException(status_code=403, detail="Docker discovery target is not configured")
raise HTTPException(status_code=403, detail="Er is geen Docker-detectiedoel ingesteld")
if appops_url and appops_url.rstrip("/") != configured_appops.rstrip("/"):
raise HTTPException(status_code=403, detail="AppOps discovery target is not configured")
raise HTTPException(status_code=403, detail="Er is geen AppOps-detectiedoel ingesteld")
@asynccontextmanager
async def lifespan(_: FastAPI):
try:
if AuthConfig.from_environment().mode == "token":
# Create the token on first start so its location is logged
# before anyone needs it.
admin_token()
logger.info("Editing requires the admin token in %s", admin_token_path())
except (AuthorizationError, OSError):
logger.exception("ServiceMap admin token is unavailable; edits are refused")
task: asyncio.Task[None] | None = None
discovery_task: asyncio.Task[None] | None = None
observation_task: asyncio.Task[None] | None = None
if PROBE_INTERVAL > 0:
task = asyncio.create_task(
scheduled_probe_loop(
@@ -121,37 +192,88 @@ async def lifespan(_: FastAPI):
retries=PROBE_RETRIES,
concurrency=PROBE_CONCURRENCY,
retention_days=RETENTION_DAYS,
event_retention_days=EVENT_RETENTION_DAYS,
)
)
background: list[asyncio.Task[None]] = []
if DISCOVERY_INTERVAL > 0:
async def discovery_loop() -> None:
while True:
await asyncio.sleep(DISCOVERY_INTERVAL)
if store.demo_active():
continue
try:
request = DiscoveryRunRequest()
result = await asyncio.to_thread(discovery_manager.run, request)
await asyncio.to_thread(
discovery_manager.reconcile,
DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=False),
)
except (OSError, RuntimeError, ValueError):
# Discovery is review-first; a failed background run is
# persisted and must not take down the monitoring loop.
continue
discovery_task = asyncio.create_task(discovery_loop())
async def observation_loop() -> None:
while True:
if not store.demo_active():
try:
await asyncio.to_thread(observation_manager.refresh)
except (OSError, RuntimeError, ValueError):
# Provider availability is persisted by the observer; an
# unavailable provider must not stop scheduled probing.
pass
await asyncio.sleep(OBSERVATION_INTERVAL)
observation_task = asyncio.create_task(observation_loop())
async def discovery_cycle() -> None:
result = await asyncio.to_thread(discovery_manager.run, DiscoveryRunRequest())
await asyncio.to_thread(
discovery_manager.reconcile,
DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=False),
)
background.append(asyncio.create_task(run_periodic(
"discovery",
discovery_cycle,
interval_seconds=DISCOVERY_INTERVAL,
initial_delay_seconds=DISCOVERY_INTERVAL,
paused=store.demo_active,
)))
if OBSERVATION_INTERVAL > 0:
async def observation_cycle() -> None:
await asyncio.to_thread(observation_manager.refresh)
background.append(asyncio.create_task(run_periodic(
"observation",
observation_cycle,
interval_seconds=OBSERVATION_INTERVAL,
paused=store.demo_active,
)))
async def alert_cycle() -> None:
sent = await asyncio.to_thread(alert_manager().run_cycle)
if sent:
logger.info("Sent %s incident notification(s)", sent)
background.append(asyncio.create_task(run_periodic(
"alerts",
alert_cycle,
interval_seconds=30,
paused=store.demo_active,
)))
if BACKUP_INTERVAL_HOURS > 0:
async def backup_cycle() -> None:
info = await asyncio.to_thread(backup_manager().create)
logger.info("Backup %s written (%s bytes, verified)", info.name, info.size_bytes)
background.append(asyncio.create_task(run_periodic(
"backups",
backup_cycle,
interval_seconds=BACKUP_INTERVAL_HOURS * 3600,
# First backup shortly after start, then on the schedule.
initial_delay_seconds=120,
paused=store.demo_active,
)))
async def maintenance_window_cycle() -> None:
ended = await asyncio.to_thread(store.expire_maintenance_windows)
if ended:
logger.info("Maintenance windows ended for %s service(s)", len(ended))
background.append(asyncio.create_task(run_periodic(
"maintenance-windows",
maintenance_window_cycle,
interval_seconds=60,
paused=store.demo_active,
)))
async def maintenance_cycle() -> None:
pruned = await asyncio.to_thread(run_maintenance)
if any(pruned.values()):
logger.info("Maintenance pruned %s", pruned)
# Start shortly after boot so a long-running deployment sheds its backlog
# without delaying startup, then repeat on a slow cadence.
background.append(asyncio.create_task(run_periodic(
"maintenance",
maintenance_cycle,
interval_seconds=MAINTENANCE_INTERVAL,
initial_delay_seconds=30,
)))
try:
yield
finally:
@@ -159,38 +281,74 @@ async def lifespan(_: FastAPI):
task.cancel()
with suppress(asyncio.CancelledError):
await task
if discovery_task:
discovery_task.cancel()
for background_task in background:
background_task.cancel()
for background_task in background:
with suppress(asyncio.CancelledError):
await discovery_task
if observation_task:
observation_task.cancel()
with suppress(asyncio.CancelledError):
await observation_task
await background_task
app = FastAPI(title="ServiceMap API", version="1.6.2", lifespan=lifespan)
app = FastAPI(title="ServiceMap API", version="1.6.3", lifespan=lifespan)
# Bounded request bodies: the largest legitimate payload is a config import.
MAX_REQUEST_BYTES = 4 * 1024 * 1024
SECURITY_HEADERS = {
"X-Content-Type-Options": "nosniff",
"X-Frame-Options": "DENY",
"Referrer-Policy": "no-referrer",
"Cross-Origin-Opener-Policy": "same-origin",
# The SPA ships no inline scripts or third-party assets; React Flow and
# the layout use inline style attributes.
"Content-Security-Policy": (
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; "
"img-src 'self' data: blob:; font-src 'self'; connect-src 'self'; "
"object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'"
),
}
@app.middleware("http")
async def authorization_boundary(request, call_next):
try:
authorize_request(request)
except HTTPException as exc:
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
return await call_next(request)
declared = request.headers.get("content-length", "")
if declared.isdigit() and int(declared) > MAX_REQUEST_BYTES:
response = JSONResponse(status_code=413, content={"detail": "De aanvraag is te groot"})
else:
try:
authorize_request(request)
except HTTPException as exc:
response = JSONResponse(
status_code=exc.status_code,
content={"detail": exc.detail},
headers=dict(exc.headers or {}),
)
else:
response = await call_next(request)
for name, value in SECURITY_HEADERS.items():
response.headers.setdefault(name, value)
path = request.url.path
if path.startswith("/assets/"):
# Vite fingerprints asset names, so they never change in place.
response.headers.setdefault("Cache-Control", "public, max-age=31536000, immutable")
elif not path.startswith("/api/") and path not in {"/health", "/ready"}:
# index.html (and other unhashed files) must be revalidated, or a
# browser keeps running the previous release after a deploy.
response.headers.setdefault("Cache-Control", "no-cache")
return response
def _snapshot(service: Service) -> ServiceSnapshot:
services = store.list_services()
services = [item for item in store.list_services() if item.id != service.id]
health = topology_health(store, services=[*services, service])
dependents = [item.name for item in services if service.id in item.depends_on]
return ServiceSnapshot(
service=service,
latest_probe=store.latest_probe(service.id),
availability_24h=store.availability_summary(service.id, 24),
dependency_status=dependency_status(service, store),
effective_status=effective_status(service, store),
direct_status=store.service_health(service.id),
dependency_status=health.dependency.get(service.id, HealthStatus.UNKNOWN),
effective_status=health.effective.get(service.id, HealthStatus.UNKNOWN),
direct_status=health.direct.get(service.id, HealthStatus.UNKNOWN),
flapping=service.flapping,
flap_count=service.flap_count,
next_probe_at=store.next_probe_at(service, PROBE_INTERVAL),
@@ -199,6 +357,7 @@ def _snapshot(service: Service) -> ServiceSnapshot:
evidence=discovery_store.evidence_for_service(service.id),
health_evidence=store.health_evidence_for_service(service.id),
health_resolution=store.health_resolution(service.id),
maintenance_until=store.maintenance_windows().get(service.id),
)
@@ -209,9 +368,34 @@ def _name_conflict(name: str, *, exclude_id: str | None = None) -> bool:
@app.get("/health")
def health() -> dict[str, str]:
"""Liveness: the process answers. Used by the container healthcheck."""
return {"status": "ok"}
@app.get("/ready")
def ready() -> JSONResponse:
"""Readiness: the database takes writes and no background loop stalled.
A loop is stalled when it has not completed a cycle for three intervals
(at least three minutes); its last error is included for diagnosis.
"""
checks: dict[str, object] = {"loops": loop_registry.snapshot()}
database_ok = True
try:
store.ready_check()
except sqlite3.Error as exc:
database_ok = False
checks["database_error"] = f"{type(exc).__name__}: {exc}"
checks["database"] = "ok" if database_ok else "unavailable"
ok = database_ok and loop_registry.all_healthy()
return JSONResponse(
status_code=status.HTTP_200_OK if ok else status.HTTP_503_SERVICE_UNAVAILABLE,
content={"status": "ready" if ok else "not_ready", **checks},
)
@app.get("/api/v1/meta")
def meta() -> dict[str, str | int | float | bool]:
try:
@@ -220,7 +404,7 @@ def meta() -> dict[str, str | int | float | bool]:
auth_mode = "misconfigured"
return {
"name": "ServiceMap",
"version": "1.6.2",
"version": "1.6.3",
"stage": "v1",
"scheduled_probing": PROBE_INTERVAL > 0,
"probe_interval_seconds": PROBE_INTERVAL,
@@ -237,9 +421,73 @@ def meta() -> dict[str, str | int | float | bool]:
"docker_provider": docker_provider_mode(),
"appops_configured": bool(os.getenv("SERVICEMAP_APPOPS_URL")),
"auth_mode": auth_mode,
# What a browser needs before it may change anything.
"write_auth": {"token": "admin_token", "oidc": "oidc"}.get(auth_mode, "none"),
}
@app.get("/api/v1/settings/alerts", response_model=AlertSettingsView)
def get_alert_settings() -> AlertSettingsView:
return alert_manager().view()
@app.put("/api/v1/settings/alerts", response_model=AlertSettingsView)
def put_alert_settings(payload: AlertSettingsUpdate) -> AlertSettingsView:
try:
return alert_manager().update(payload)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@app.post("/api/v1/settings/alerts/test", response_model=AlertTestResult)
def test_alert() -> AlertTestResult:
return alert_manager().send_test()
@app.get("/api/v1/backups", response_model=list[BackupInfo])
def list_backups() -> list[BackupInfo]:
return backup_manager().list()
@app.post("/api/v1/backups", response_model=BackupInfo, status_code=status.HTTP_201_CREATED)
def create_backup() -> BackupInfo:
try:
return backup_manager().create()
except (ValueError, RuntimeError, sqlite3.Error) as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@app.get("/api/v1/backups/{name}")
def download_backup(name: str) -> FileResponse:
try:
path = backup_manager().path_for(name)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
return FileResponse(path, media_type="application/vnd.sqlite3", filename=name)
@app.post("/api/v1/backups/{name}/restore", response_model=BackupInfo)
def restore_backup(name: str) -> BackupInfo:
"""Restore ``name`` into the live database; returns the safety copy taken first."""
try:
safety = backup_manager().restore(name)
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except (RuntimeError, sqlite3.Error) as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
semantic_cache.clear()
event_bus.publish({"event_type": "topology.changed", "message": f"Database teruggezet vanaf {name}"})
return safety
@app.post("/api/v1/auth/verify")
def verify_write_access() -> dict[str, bool]:
"""Succeeds only for a caller allowed to mutate (checked by the middleware)."""
return {"ok": True}
def _service_snapshots(
*, include_hidden: bool = False, include_context: bool = True
) -> list[ServiceSnapshot]:
@@ -268,6 +516,10 @@ def _service_snapshots(
service.id: [item.name for item in services if service.id in item.depends_on]
for service in services
}
visible_ids = [service.id for service in services]
windows = store.maintenance_windows()
sources_by_service = discovery_store.sources_for_services(visible_ids) if include_context else {}
relations_by_service = discovery_store.evidence_for_services(visible_ids) if include_context else {}
return [
ServiceSnapshot(
service=service,
@@ -284,10 +536,11 @@ def _service_snapshots(
else None
),
dependents=dependent_names.get(service.id, []),
sources=discovery_store.sources_for_service(service.id) if include_context else [],
evidence=discovery_store.evidence_for_service(service.id) if include_context else [],
sources=sources_by_service.get(service.id, []),
evidence=relations_by_service.get(service.id, []),
health_evidence=evidence_by_service.get(service.id, []),
health_resolution=resolutions.get(service.id, HealthResolution()),
maintenance_until=windows.get(service.id),
)
for service in services
]
@@ -301,21 +554,21 @@ def list_services(include_hidden: bool = Query(default=False)) -> list[ServiceSn
@app.post("/api/v1/services", response_model=Service, status_code=status.HTTP_201_CREATED)
def create_service(payload: ServiceCreate) -> Service:
if _name_conflict(payload.name):
raise HTTPException(status_code=409, detail="Service name already exists")
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam")
try:
validate_dependencies(payload.depends_on, store)
return store.create_service(payload)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except sqlite3.IntegrityError as exc:
raise HTTPException(status_code=409, detail="Service name already exists") from exc
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam") from exc
@app.get("/api/v1/services/{service_id}", response_model=ServiceSnapshot)
def get_service(service_id: str) -> ServiceSnapshot:
service = store.get_service(service_id)
if service is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return _snapshot(service)
@@ -323,18 +576,18 @@ def get_service(service_id: str) -> ServiceSnapshot:
def update_service(service_id: str, payload: ServiceUpdate) -> Service:
before = store.get_service(service_id)
if before is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
if _name_conflict(payload.name, exclude_id=service_id):
raise HTTPException(status_code=409, detail="Service name already exists")
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam")
try:
validate_dependencies(payload.depends_on, store, service_id=service_id)
updated = store.update_service(service_id, payload)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except sqlite3.IntegrityError as exc:
raise HTTPException(status_code=409, detail="Service name already exists") from exc
raise HTTPException(status_code=409, detail="Er bestaat al een service met deze naam") from exc
if updated is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
if before.source_type:
discovery_manager.record_service_override(before, updated)
return updated
@@ -343,7 +596,7 @@ def update_service(service_id: str, payload: ServiceUpdate) -> Service:
@app.delete("/api/v1/services/{service_id}", status_code=status.HTTP_204_NO_CONTENT)
def delete_service(service_id: str) -> None:
if not store.delete_service(service_id):
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
@app.post("/api/v1/services/{service_id}/probe", response_model=ProbeResult)
@@ -353,10 +606,10 @@ async def run_probe(
retries: int | None = Query(default=None, ge=0, le=5),
) -> ProbeResult:
if store.demo_active():
raise HTTPException(status_code=409, detail="Explore demo uses synthetic probe data; live probing is paused")
raise HTTPException(status_code=409, detail="De demo gebruikt synthetische controlegegevens; live controles zijn gepauzeerd")
service = store.get_service(service_id)
if service is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
result = await probe_service(
service,
timeout_seconds=timeout_seconds,
@@ -394,21 +647,21 @@ async def run_all_probes(
@app.get("/api/v1/services/{service_id}/history", response_model=list[ProbeResult])
def probe_history(service_id: str, limit: int = Query(default=50, ge=1, le=500)) -> list[ProbeResult]:
if store.get_service(service_id) is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return store.probe_history(service_id, limit)
@app.get("/api/v1/services/{service_id}/availability", response_model=AvailabilitySummary)
def availability(service_id: str, hours: int = Query(default=24, ge=1, le=24 * 90)) -> AvailabilitySummary:
if store.get_service(service_id) is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return store.availability_summary(service_id, hours)
@app.get("/api/v1/services/{service_id}/evidence", response_model=HealthResolution)
def service_evidence(service_id: str) -> HealthResolution:
if store.get_service(service_id) is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return store.health_resolution(service_id)
@@ -419,9 +672,12 @@ def providers() -> list[ProviderStatus]:
@app.post("/api/v1/services/{service_id}/maintenance", response_model=Service)
def maintenance(service_id: str, payload: MaintenanceRequest) -> Service:
updated = store.set_maintenance(service_id, payload.enabled, payload.reason)
try:
updated = store.set_maintenance(service_id, payload.enabled, payload.reason, payload.until)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
if updated is None:
raise HTTPException(status_code=404, detail="Service not found")
raise HTTPException(status_code=404, detail="Service niet gevonden")
return updated
@@ -437,7 +693,7 @@ def incidents() -> IncidentOverview:
def get_incident(incident_id: str) -> dict[str, object]:
incident = store.get_incident(incident_id)
if incident is None:
raise HTTPException(status_code=404, detail="Incident not found")
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return {"incident": incident.model_dump(mode="json"), "timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)]}
@@ -448,30 +704,60 @@ def acknowledge_incident(incident_id: str) -> dict[str, object]:
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
if incident is None:
raise HTTPException(status_code=404, detail="Incident not found")
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return {"incident": incident.model_dump(mode="json"), "timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)]}
def _incident_payload(incident_id: str) -> dict[str, object]:
incident = store.get_incident(incident_id)
if incident is None:
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return {
"incident": incident.model_dump(mode="json"),
"timeline": [item.model_dump(mode="json") for item in store.incident_timeline(incident_id)],
}
@app.post("/api/v1/incidents/{incident_id}/resolve")
def resolve_incident(incident_id: str, payload: IncidentResolveRequest | None = None) -> dict[str, object]:
if store.resolve_incident(incident_id, payload.note if payload else None) is None:
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return _incident_payload(incident_id)
@app.post("/api/v1/incidents/{incident_id}/notes")
def add_incident_note(incident_id: str, payload: IncidentNoteRequest) -> dict[str, object]:
if store.add_incident_note(incident_id, payload.text.strip()) is None:
raise HTTPException(status_code=404, detail="Incident niet gevonden")
return _incident_payload(incident_id)
@app.get("/api/v1/events", response_model=list[EventRecord])
def events(limit: int = Query(default=50, ge=1, le=500), service_id: str | None = None) -> list[EventRecord]:
return store.list_events(limit=limit, service_id=service_id)
def events(
limit: int = Query(default=50, ge=1, le=500),
service_id: str | None = None,
event_type: str | None = Query(default=None, max_length=80),
before: datetime | None = None,
) -> list[EventRecord]:
return store.list_events(limit=limit, service_id=service_id, event_type=event_type, before=before)
@app.get("/api/v1/events/stream")
async def event_stream() -> StreamingResponse:
queue = event_bus.subscribe()
subscription = event_bus.subscribe()
async def stream():
try:
yield ": connected\n\n"
# ``retry`` tells EventSource how long to wait before reconnecting.
yield f"retry: {SSE_RETRY_MS}\n: connected\n\n"
while True:
payload = await next_event(queue)
payload = await next_event(subscription)
if payload is None:
yield ": keepalive\n\n"
else:
yield sse_payload(payload)
finally:
event_bus.unsubscribe(queue)
event_bus.unsubscribe(subscription)
return StreamingResponse(
stream(),
@@ -483,7 +769,8 @@ async def event_stream() -> StreamingResponse:
@app.get("/api/v1/overview", response_model=OverviewSummary)
def overview() -> OverviewSummary:
sync_incidents(store)
snapshots = _service_snapshots(include_hidden=False)
# Only status and coverage are counted; discovery context is not needed.
snapshots = _service_snapshots(include_hidden=False, include_context=False)
overview_state = incident_overview(store)
counts: dict[str, int] = {}
coverage: dict[str, int] = {}
@@ -521,14 +808,20 @@ def import_config(payload: ConfigImportRequest) -> ConfigImportResult:
raise HTTPException(status_code=422, detail=str(exc)) from exc
def _demo_enabled() -> bool:
return os.getenv("SERVICEMAP_DEMO_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"}
def _demo_state() -> DemoState:
enabled = _demo_enabled()
if not store.demo_active():
return DemoState(active=False)
return DemoState(active=False, enabled=enabled)
sync_incidents(store)
overview_state = incident_overview(store)
candidate = overview_state.candidates[0] if overview_state.candidates else None
return DemoState(
active=True,
enabled=enabled,
synthetic=True,
scenario_name=DEMO_SCENARIO_NAME,
service_count=len(store.list_services()),
@@ -557,15 +850,10 @@ def demo_state() -> DemoState:
@app.post("/api/v1/demo", response_model=DemoMutationResult)
def load_demo() -> DemoMutationResult:
services, probes = build_demo_records()
result = store.activate_demo(services, probes)
sync_incidents(store)
return _demo_result(result)
@app.post("/api/v1/demo/reset", response_model=DemoMutationResult)
def reset_demo() -> DemoMutationResult:
def load_demo() -> DemoMutationResult:
if not _demo_enabled():
raise HTTPException(status_code=409, detail="Start de demo in een aparte ServiceMap-instantie")
services, probes = build_demo_records()
result = store.activate_demo(services, probes)
sync_incidents(store)
@@ -578,7 +866,7 @@ async def exit_demo() -> DemoMutationResult:
# Refresh live provider evidence immediately after restoring the saved
# production topology. Test stores intentionally do not inherit the
# process-global observer, but production always does.
if restored and observation_manager.store is store:
if restored and OBSERVATION_INTERVAL > 0 and observation_manager.store is store:
await asyncio.to_thread(observation_manager.refresh)
return _demo_result(restored=restored)
@@ -586,7 +874,7 @@ async def exit_demo() -> DemoMutationResult:
@app.post("/api/v1/discovery/run", response_model=DiscoveryResult)
def run_discovery(payload: DiscoveryRunRequest | None = None) -> DiscoveryResult:
if store.demo_active():
raise HTTPException(status_code=409, detail="Exit Explore demo before discovering production infrastructure")
raise HTTPException(status_code=409, detail="Sluit de demo af voor je de productie-infrastructuur detecteert")
request = payload or DiscoveryRunRequest()
_require_configured_discovery_targets(request.docker_host, request.appops_url)
try:
@@ -613,7 +901,7 @@ def discovery_suggestions(
try:
suggestion_status = SuggestionStatus(status_filter) if status_filter else None
except ValueError as exc:
raise HTTPException(status_code=422, detail="Unknown suggestion status") from exc
raise HTTPException(status_code=422, detail="Onbekende status van het voorstel") from exc
return discovery_store.list_suggestions(status=suggestion_status)
@@ -636,7 +924,7 @@ def reject_discovery_suggestion(suggestion_id: str) -> TopologySuggestion:
@app.post("/api/v1/discovery/reconcile", response_model=ReconcileResult)
def reconcile_discovery(payload: DiscoveryReconcileRequest | None = None) -> ReconcileResult:
if store.demo_active():
raise HTTPException(status_code=409, detail="Exit Explore demo before reconciling production infrastructure")
raise HTTPException(status_code=409, detail="Sluit de demo af voor je de productie-infrastructuur verwerkt")
try:
return discovery_manager.reconcile(payload or DiscoveryReconcileRequest())
except ValueError as exc:
@@ -649,7 +937,7 @@ def repair_discovered_probes(
run_id: str | None = Query(default=None),
) -> ProbeRepairReport:
if store.demo_active():
raise HTTPException(status_code=409, detail="Exit Explore demo before repairing production probe endpoints")
raise HTTPException(status_code=409, detail="Sluit de demo af voor je controle-endpoints in productie herstelt")
try:
return discovery_manager.repair_probe_reachability(run_id=run_id, dry_run=dry_run)
except ValueError as exc:
@@ -686,8 +974,14 @@ def topology_diff(
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/api/v1/topology/semantic", response_model=SemanticTopology)
def semantic_topology(include_hidden: bool = Query(default=False)) -> SemanticTopology:
def _semantic_representation(include_hidden: bool) -> CachedRepresentation:
revision = event_bus.revision
# The store identity keeps a swapped store (tests, demo restore paths)
# from ever reading another database's cached topology.
cache_key = (id(store), include_hidden)
cached = semantic_cache.get(cache_key, revision)
if cached is not None:
return cached
snapshots = _service_snapshots(include_hidden=include_hidden, include_context=False)
incident_state = incident_overview(store)
root_ids = {candidate.service_id for candidate in incident_state.candidates}
@@ -696,12 +990,48 @@ def semantic_topology(include_hidden: bool = Query(default=False)) -> SemanticTo
for candidate in incident_state.candidates
for service_id in candidate.affected_service_ids
}
return build_semantic_topology(
topology = build_semantic_topology(
snapshots,
discovery_store.list_logical_relations(),
root_service_ids=root_ids,
affected_service_ids=affected_ids,
)
value = representation(topology)
boundary = next_freshness_boundary(
[item for snapshot in snapshots for item in snapshot.health_evidence],
topology.generated_at,
)
semantic_cache.put(
cache_key,
revision,
value,
valid_for_seconds=(
(boundary - topology.generated_at).total_seconds() if boundary is not None else None
),
)
return value
@app.get("/api/v1/topology/semantic", response_model=SemanticTopology)
def semantic_topology(
request: Request, include_hidden: bool = Query(default=False)
) -> Response:
"""Semantic topology with a content ETag and conditional GET support.
Clients that send ``If-None-Match`` receive ``304`` without a body while
the topology is unchanged; ``X-ServiceMap-Generated-At`` then states when
the server last verified it. Browsers revalidate automatically.
"""
value = _semantic_representation(include_hidden)
headers = {
"ETag": value.etag,
"Cache-Control": "no-cache",
GENERATED_AT_HEADER: value.generated_at,
}
if etag_matches(request.headers.get("if-none-match"), value.etag):
return Response(status_code=status.HTTP_304_NOT_MODIFIED, headers=headers)
return Response(content=value.body, media_type="application/json", headers=headers)
@app.post("/api/v1/topology/edges", response_model=RelationEvidence, status_code=status.HTTP_201_CREATED)