ServiceMap: update tests and backend/app (+7 areas)

Public changes compared with the previous published export:
89 files changed.

Affected areas:
- backend/app: 21 files
- dependencies: 5 files
- deployment and CI: 6 files
- documentation: 2 files
- frontend: 3 files
- frontend/src: 20 files
- project files: 1 file
- tests: 30 files
- unraid: 1 file

Changed files:
- Updated: ".env.example"
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Added: "backend/app/alerts.py"
- Updated: "backend/app/auth.py"
- Added: "backend/app/backups.py"
- Updated: "backend/app/config_io.py"
- Updated: "backend/app/demo.py"
- Updated: "backend/app/discovery.py"
- Updated: "backend/app/discovery_store.py"
- Updated: "backend/app/docker_discovery.py"
- Updated: "backend/app/events.py"
- Updated: "backend/app/evidence.py"
- Updated: "backend/app/health.py"
- Updated: "backend/app/incidents.py"
- Added: "backend/app/loops.py"
- Updated: "backend/app/main.py"
- Updated: "backend/app/models.py"
- Updated: "backend/app/observation.py"
- Updated: "backend/app/probes.py"
- Added: "backend/app/response_cache.py"
- Updated: "backend/app/scheduler.py"
- Updated: "backend/app/semantic.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Updated: "backend/tests/test_api.py"
- Updated: "backend/tests/test_config_io.py"
- Updated: "backend/tests/test_demo.py"
- Updated: "backend/tests/test_discovery_v120.py"
- Updated: "backend/tests/test_health.py"
- Updated: "backend/tests/test_probes.py"
- Updated: "backend/tests/test_scheduler.py"
- Updated: "backend/tests/test_v130_probe_reachability.py"
- Updated: "backend/tests/test_v160_evidence_changes.py"
- Added: "backend/tests/test_v163_alerts.py"
- Added: "backend/tests/test_v163_backups.py"
- Added: "backend/tests/test_v163_data_integrity.py"
- Added: "backend/tests/test_v163_efficiency.py"
- Added: "backend/tests/test_v163_health_performance.py"
- Added: "backend/tests/test_v163_loops.py"
- Added: "backend/tests/test_v163_management.py"
- Added: "backend/tests/test_v163_realtime.py"
- Added: "backend/tests/test_v163_security.py"
- Added: "backend/tests/test_v163_semantic_etag.py"
- Added: "compose.demo.yaml"
- Updated: "compose.yaml"
- Added: "docker-compose.broker.yml"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Added: "frontend/e2e/a11y.spec.ts"
- Updated: "frontend/e2e/demo.spec.ts"
- Updated: "frontend/index.html"
- Updated: "frontend/nginx.conf"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "frontend/playwright.config.ts"
- Updated: "frontend/src/App.tsx"
- Updated: "frontend/src/LivingEdge.tsx"
- Added: "frontend/src/activityView.test.ts"
- Added: "frontend/src/activityView.ts"
- Updated: "frontend/src/api.ts"
- Added: "frontend/src/apiAuth.test.ts"
- Added: "frontend/src/components/ActionsMenu.tsx"
- Added: "frontend/src/components/ConfirmDialog.test.tsx"
- Added: "frontend/src/components/ConfirmDialog.tsx"
- Added: "frontend/src/components/StatusIcon.tsx"
- Updated: "frontend/src/demo/DemoWalkthrough.test.ts"
- Updated: "frontend/src/demo/DemoWalkthrough.tsx"
- Added: "frontend/src/elk-worker-url.d.ts"
- Added: "frontend/src/labels.ts"
- Updated: "frontend/src/main.tsx"
- Added: "frontend/src/realtime/refreshPlan.test.ts"
- Added: "frontend/src/realtime/refreshPlan.ts"
- Updated: "frontend/src/realtime/useRealtimeEvents.ts"
- Added: "frontend/src/routing/urlState.test.ts"
- Added: "frontend/src/routing/urlState.ts"
- Updated: "frontend/src/styles.css"
- Added: "frontend/src/topology/GraphListView.test.tsx"
- Added: "frontend/src/topology/GraphListView.tsx"
- Updated: "frontend/src/topology/ServiceNode.tsx"
- Updated: "frontend/src/topology/TopologyCommandBar.tsx"
- Added: "frontend/src/topology/semanticGraph.test.ts"
- Updated: "frontend/src/topology/semanticGraph.ts"
- Added: "frontend/src/topologyLayout.test.ts"
- Updated: "frontend/src/topologyLayout.ts"
- Updated: "package-lock.json"
- Updated: "package.json"
- Updated: "unraid/my-servicemap.xml"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
This commit is contained in:
NuklearRabbit committed 2026-11-20 13:51:46 +01:00
1 parent d7319250e8
commit adc2c70c21
91 files changed
+9205 -1691

No files matched your search

+89 -20
View File
@@ -4,6 +4,7 @@ import os
import re
import socket
import time
from contextlib import suppress
from dataclasses import dataclass
from urllib.parse import urlparse
@@ -233,7 +234,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
hostname = parsed.hostname if parsed else candidate.host
port = parsed.port if parsed else candidate.port
if not hostname or not port:
return candidate.model_copy(update={"preflight_detail": "No host-published endpoint to preflight"})
return candidate.model_copy(update={"preflight_detail": "Geen gepubliceerd host-endpoint om vooraf te testen"})
started = time.perf_counter()
try:
with socket.create_connection((hostname, port), timeout=0.75):
@@ -243,7 +244,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
return candidate.model_copy(update={
"reachability": ProbeReachability.VERIFIED,
"primary": True,
"preflight_detail": "TCP connect succeeded",
"preflight_detail": "TCP-verbinding gelukt",
"preflight_latency_ms": elapsed,
})
with httpx.Client(timeout=1.25, follow_redirects=False) as client:
@@ -260,7 +261,7 @@ def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCand
return candidate.model_copy(update={
"reachability": ProbeReachability.UNVERIFIED,
"primary": False,
"preflight_detail": f"Preflight failed: {type(exc).__name__}",
"preflight_detail": f"Voorafgaande test mislukt: {type(exc).__name__}",
})
@@ -292,7 +293,7 @@ def _probe_candidates(
kind=suggestion.kind if suggestion else ServiceKind.TCP,
reachability=ProbeReachability.VERIFIED,
confidence=DiscoveryConfidence.HIGH,
evidence="Docker healthcheck present (observed without a network probe)",
evidence="Docker-healthcheck aanwezig (waargenomen zonder netwerkcontrole)",
)
)
return candidates
@@ -327,7 +328,7 @@ def safe_environment_evidence(env: list[str]) -> list[str]:
def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
if not appops_url:
return AppOpsInventory({}, False, "AppOps URL is not configured")
return AppOpsInventory({}, False, "Er is geen AppOps-URL ingesteld")
base = appops_url.rstrip("/")
try:
mappings_response = httpx.get(f"{base}/api/inventory/mappings", timeout=5)
@@ -337,7 +338,7 @@ def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
mappings = mappings_response.json()
apps = apps_response.json()
except (httpx.HTTPError, ValueError) as exc:
return AppOpsInventory({}, False, f"AppOps inventory unavailable: {type(exc).__name__}")
return AppOpsInventory({}, False, f"AppOps-inventaris niet beschikbaar: {type(exc).__name__}")
app_by_id = {str(item.get("id")): item for item in apps if isinstance(item, dict)}
result: dict[str, AppOpsInfo] = {}
for mapping in mappings if isinstance(mappings, list) else []:
@@ -362,7 +363,7 @@ def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
app_type=str(app.get("app_type")) if app.get("app_type") else None,
reasons=tuple(str(item) for item in (app.get("discovery_reasons") or [])[:3]),
)
return AppOpsInventory(result, True, f"{len(result)} container mappings loaded")
return AppOpsInventory(result, True, f"{len(result)} containerkoppelingen geladen")
def appops_inventory(appops_url: str | None) -> AppOpsInventory:
@@ -371,6 +372,38 @@ def appops_inventory(appops_url: str | None) -> AppOpsInventory:
return _appops_inventory(appops_url)
_HEALTH_IN_STATUS = re.compile(r"\((healthy|unhealthy|health: starting)\)")
_observer_clients: dict[str, object] = {}
def _observer_client(docker_host: str):
"""One long-lived socket client per host (no /version + /_ping per cycle)."""
client = _observer_clients.get(docker_host)
if client is None:
client = docker.DockerClient(base_url=docker_host, timeout=5, version="auto")
_observer_clients[docker_host] = client
return client
def _drop_observer_client(docker_host: str) -> None:
client = _observer_clients.pop(docker_host, None)
close = getattr(client, "close", None)
if callable(close):
with suppress(Exception): # best effort on an already failed client
close()
def health_from_status(status_text: str) -> str | None:
"""Docker's list summary embeds health: "Up 3 hours (healthy)"."""
match = _HEALTH_IN_STATUS.search(status_text or "")
if not match:
return None
value = match.group(1)
return "starting" if value == "health: starting" else value
def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
"""Read bounded Docker runtime/health facts without discovery side effects.
@@ -378,18 +411,48 @@ def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
suggestion persistence, reconciliation and snapshot capture. It is safe
to run on the observation cadence because it cannot rewrite topology or
operator-owned configuration.
Socket mode uses a single low-level ``containers(all=True)`` call on a
long-lived client: the list summary already carries id, name, state,
labels and health. The high-level ``containers.list()`` inspected every
container individually (1 + N API calls per minute).
"""
parsed = urlparse(docker_host)
if parsed.scheme not in ALLOWED_SCHEMES:
raise ValueError("docker_host must use unix://, tcp://, http:// or https://")
if docker_provider_mode() == "socket":
try:
summaries = _observer_client(docker_host).api.containers(all=True)
except (DockerException, OSError) as exc:
_drop_observer_client(docker_host)
raise RuntimeError(f"Docker-waarneming niet beschikbaar: {type(exc).__name__}") from exc
observations: list[DockerHealthObservation] = []
for item in summaries:
names = item.get("Names") or []
name = str(names[0] if names else item.get("Id", "unknown")[:12]).lstrip("/")
labels = {str(key): str(value) for key, value in (item.get("Labels") or {}).items()}
health = health_from_status(str(item.get("Status", "")))
observations.append(
DockerHealthObservation(
container_id=str(item.get("Id", ""))[:12],
name=name,
stable_source_ref=stable_source_ref(name, labels),
state=str(item.get("State", "unknown")),
health=health,
healthcheck_present=health is not None,
)
)
return observations
return _observe_container_health_full(docker_host)
def _observe_container_health_full(docker_host: str) -> list[DockerHealthObservation]:
"""Broker mode: the broker client exposes only the high-level listing."""
client = None
try:
client = (
docker.DockerClient(base_url=docker_host, timeout=5)
if docker_provider_mode() == "socket"
else docker_client_for_inventory(docker_host)
)
client = docker_client_for_inventory(docker_host)
client.ping()
observations: list[DockerHealthObservation] = []
for container in client.containers.list(all=True):
@@ -414,7 +477,7 @@ def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
)
return observations
except (DockerException, DockerProviderError) as exc:
raise RuntimeError(f"Docker observation unavailable: {type(exc).__name__}") from exc
raise RuntimeError(f"Docker-waarneming niet beschikbaar: {type(exc).__name__}") from exc
finally:
close = getattr(client, "close", None)
if callable(close):
@@ -509,7 +572,11 @@ def discover_containers(
client = docker.DockerClient(base_url=docker_host, timeout=5) if docker_provider_mode() == "socket" else docker_client_for_inventory(docker_host)
try:
client.ping()
containers = client.containers.list(all=include_stopped)
# Always list stopped containers: they still exist. Excluding them here
# made every stopped service look "missing" and, after three runs,
# retired with monitoring disabled. ``include_stopped`` only decides
# whether a stopped container is offered as a candidate below.
containers = client.containers.list(all=True)
existing_services = existing_services or []
appops_inventory = _appops_inventory(appops_url)
appops = appops_inventory.by_container
@@ -521,6 +588,13 @@ def discover_containers(
attrs = container.attrs or {}
state = attrs.get("State", {}) or {}
config = attrs.get("Config", {}) or {}
name = str(getattr(container, "name", "unknown")).lstrip("/")
state_name = str(state.get("Status", getattr(container, "status", "unknown")))
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
stable_ref = stable_source_ref(name, raw_labels)
discovered_refs.add(stable_ref)
if not include_stopped and state_name != "running":
continue
network_settings = attrs.get("NetworkSettings", {}) or {}
raw_ports = network_settings.get("Ports", {}) or {}
ports: list[DockerPort] = []
@@ -541,15 +615,10 @@ def discover_containers(
image_tags = getattr(container.image, "tags", []) or []
image_name = image_tags[0] if image_tags else str(config.get("Image", "unknown"))
name = str(getattr(container, "name", "unknown")).lstrip("/")
health = state.get("Health", {}).get("Status") if isinstance(state.get("Health"), dict) else None
state_name = str(state.get("Status", getattr(container, "status", "unknown")))
networks = sorted((network_settings.get("Networks", {}) or {}).keys())
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
labels = _safe_labels(raw_labels)
stable_ref = stable_source_ref(name, raw_labels)
legacy_ref = _legacy_source_ref(container.id, image_name)
discovered_refs.add(stable_ref)
matching_source = next(
(
service
@@ -677,6 +746,6 @@ def discover_containers(
appops_detail=appops_inventory.detail,
)
except (DockerException, DockerProviderError) as exc:
raise ValueError(f"Docker discovery failed: {exc}") from exc
raise ValueError(f"Docker-detectie mislukt: {exc}") from exc
finally:
client.close()