Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
238 lines
9.4 KiB
Python
238 lines
9.4 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextvars
|
|
import os
|
|
import socket
|
|
import ssl
|
|
from datetime import UTC, datetime
|
|
from time import perf_counter
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
|
|
from .models import HealthStatus, ProbeResult, Service, ServiceKind
|
|
|
|
_probe_metadata: contextvars.ContextVar[dict[str, object] | None] = contextvars.ContextVar(
|
|
"servicemap_probe_metadata", default=None
|
|
)
|
|
_service_locks: dict[str, asyncio.Lock] = {}
|
|
|
|
|
|
def _lock_for(service_id: str) -> asyncio.Lock:
|
|
lock = _service_locks.get(service_id)
|
|
if lock is None:
|
|
lock = asyncio.Lock()
|
|
_service_locks[service_id] = lock
|
|
return lock
|
|
|
|
|
|
def _http_status(status_code: int, service: Service) -> HealthStatus:
|
|
minimum = service.expected_status_min
|
|
maximum = service.expected_status_max
|
|
if minimum is not None or maximum is not None:
|
|
if (minimum is not None and status_code < minimum) or (maximum is not None and status_code > maximum):
|
|
return HealthStatus.DOWN if status_code >= 500 else HealthStatus.DEGRADED
|
|
return HealthStatus.HEALTHY
|
|
if status_code < 400:
|
|
return HealthStatus.HEALTHY
|
|
if status_code < 500:
|
|
return HealthStatus.DEGRADED
|
|
return HealthStatus.DOWN
|
|
|
|
|
|
_MAX_BODY_BYTES = 64 * 1024
|
|
_SSL_CONTEXT: ssl.SSLContext | None = None
|
|
|
|
|
|
def _ssl_context() -> ssl.SSLContext:
|
|
"""Build the CA bundle context once instead of on every HTTP probe."""
|
|
|
|
global _SSL_CONTEXT
|
|
if _SSL_CONTEXT is None:
|
|
import certifi
|
|
|
|
_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
|
|
return _SSL_CONTEXT
|
|
|
|
|
|
async def _probe_once_with_metadata(service: Service, timeout_seconds: float) -> tuple[HealthStatus, str, dict[str, object]]:
|
|
if service.kind == ServiceKind.HTTP:
|
|
# One overall deadline (httpx timeouts are per phase, so a slow-drip
|
|
# body could run for many times the timeout) and a bounded body read:
|
|
# the status line answers the health question.
|
|
async with asyncio.timeout(timeout_seconds):
|
|
async with httpx.AsyncClient(
|
|
timeout=timeout_seconds, follow_redirects=True, max_redirects=5, verify=_ssl_context()
|
|
) as client:
|
|
async with client.stream("GET", service.url or "") as response:
|
|
received = 0
|
|
async for chunk in response.aiter_raw():
|
|
received += len(chunk)
|
|
if received >= _MAX_BODY_BYTES:
|
|
break
|
|
status = _http_status(response.status_code, service)
|
|
return status, f"HTTP {response.status_code}", {"status_code": response.status_code}
|
|
|
|
if service.kind == ServiceKind.DNS:
|
|
started = perf_counter()
|
|
infos = await asyncio.wait_for(
|
|
asyncio.get_running_loop().getaddrinfo(service.host, None, type=socket.SOCK_STREAM),
|
|
timeout=timeout_seconds,
|
|
)
|
|
addresses = sorted({str(info[4][0]) for info in infos if info[4]})
|
|
if not addresses:
|
|
return HealthStatus.DOWN, f"DNS {service.host} returned no addresses", {"addresses": []}
|
|
return (
|
|
HealthStatus.HEALTHY,
|
|
f"DNS {service.host} resolved ({round((perf_counter() - started) * 1000, 2)} ms)",
|
|
{"addresses": addresses[:20]},
|
|
)
|
|
|
|
if service.kind == ServiceKind.TLS:
|
|
context = ssl.create_default_context()
|
|
reader, writer = await asyncio.wait_for(
|
|
asyncio.open_connection(service.host, service.port, ssl=context, server_hostname=service.host),
|
|
timeout=timeout_seconds,
|
|
)
|
|
del reader
|
|
ssl_object = writer.get_extra_info("ssl_object")
|
|
metadata: dict[str, object] = {"hostname": service.host, "port": service.port}
|
|
detail = f"TLS-certificaat van {service.host}:{service.port} bereikbaar"
|
|
status = HealthStatus.HEALTHY
|
|
if ssl_object is not None:
|
|
certificate = ssl_object.getpeercert() or {}
|
|
not_after = certificate.get("notAfter") if isinstance(certificate, dict) else None
|
|
if not_after:
|
|
expires_at = datetime.fromtimestamp(ssl.cert_time_to_seconds(not_after), UTC)
|
|
days_remaining = (expires_at - datetime.now(UTC)).total_seconds() / 86400
|
|
metadata.update({"expires_at": expires_at.isoformat(), "days_remaining": round(days_remaining, 2)})
|
|
warning_days = max(1, int(os.getenv("SERVICEMAP_TLS_WARNING_DAYS", "30")))
|
|
if days_remaining <= 0:
|
|
status = HealthStatus.DOWN
|
|
detail = f"TLS-certificaat verlopen op {expires_at.date().isoformat()}"
|
|
elif days_remaining <= warning_days:
|
|
status = HealthStatus.DEGRADED
|
|
detail = f"TLS-certificaat verloopt over {round(days_remaining, 1)} dagen"
|
|
writer.close()
|
|
await writer.wait_closed()
|
|
return status, detail, metadata
|
|
|
|
_, writer = await asyncio.wait_for(
|
|
asyncio.open_connection(service.host, service.port),
|
|
timeout=timeout_seconds,
|
|
)
|
|
writer.close()
|
|
await writer.wait_closed()
|
|
return HealthStatus.HEALTHY, f"TCP {service.host}:{service.port} reachable", {}
|
|
|
|
|
|
async def _probe_once(service: Service, timeout_seconds: float) -> tuple[HealthStatus, str]:
|
|
status, detail, metadata = await _probe_once_with_metadata(service, timeout_seconds)
|
|
_probe_metadata.set(metadata)
|
|
return status, detail
|
|
|
|
|
|
async def probe_service(
|
|
service: Service,
|
|
timeout_seconds: float | None = 5.0,
|
|
retries: int | None = 1,
|
|
*,
|
|
trigger: str = "manual",
|
|
default_timeout_seconds: float = 5.0,
|
|
default_retries: int = 1,
|
|
) -> ProbeResult:
|
|
if not service.enabled or not service.monitoring_enabled:
|
|
return ProbeResult(
|
|
id=str(uuid4()),
|
|
service_id=service.id,
|
|
status=HealthStatus.UNKNOWN,
|
|
latency_ms=None,
|
|
detail="Bewaking uitgeschakeld voor deze service",
|
|
checked_at=datetime.now(UTC),
|
|
trigger=trigger,
|
|
)
|
|
timeout = service.timeout_seconds if timeout_seconds is None else timeout_seconds
|
|
timeout = timeout if timeout is not None else default_timeout_seconds
|
|
retry_count = service.retries if retries is None else retries
|
|
retry_count = retry_count if retry_count is not None else default_retries
|
|
attempts = max(1, retry_count + 1)
|
|
status = HealthStatus.DOWN
|
|
detail = "Controle mislukt"
|
|
latency_ms: float | None = None
|
|
metadata: dict[str, object] = {}
|
|
|
|
async with _lock_for(service.id):
|
|
for attempt in range(1, attempts + 1):
|
|
_probe_metadata.set({})
|
|
started = perf_counter()
|
|
try:
|
|
status, detail = await _probe_once(service, timeout)
|
|
metadata = _probe_metadata.get() or {}
|
|
latency_ms = round((perf_counter() - started) * 1000, 2)
|
|
if attempt > 1:
|
|
detail = f"{detail} after {attempt} attempts"
|
|
break
|
|
except (TimeoutError, httpx.HTTPError, OSError, ssl.SSLError, socket.gaierror) as exc:
|
|
latency_ms = round((perf_counter() - started) * 1000, 2)
|
|
detail = f"{type(exc).__name__}: {exc}"
|
|
if isinstance(exc, ssl.CertificateError):
|
|
status = HealthStatus.DOWN
|
|
detail = f"TLS-hostnaam komt niet overeen: {exc}"
|
|
if attempt < attempts:
|
|
await asyncio.sleep(min(0.2 * attempt, 1.0))
|
|
except Exception as exc: # noqa: BLE001 - one bad target must not fail a whole cycle
|
|
# e.g. httpx.InvalidURL or an IDNA UnicodeError: a permanent
|
|
# configuration error, reported as down without retrying.
|
|
latency_ms = round((perf_counter() - started) * 1000, 2)
|
|
status = HealthStatus.DOWN
|
|
detail = f"Controlefout ({type(exc).__name__}): {exc}"
|
|
break
|
|
|
|
return ProbeResult(
|
|
id=str(uuid4()),
|
|
service_id=service.id,
|
|
status=status,
|
|
latency_ms=latency_ms,
|
|
detail=detail[:500],
|
|
checked_at=datetime.now(UTC),
|
|
trigger=trigger,
|
|
metadata=metadata,
|
|
)
|
|
|
|
|
|
async def probe_many(
|
|
services: list[Service],
|
|
*,
|
|
timeout_seconds: float | None = 5.0,
|
|
retries: int | None = 1,
|
|
concurrency: int = 10,
|
|
trigger: str = "manual",
|
|
default_timeout_seconds: float = 5.0,
|
|
default_retries: int = 1,
|
|
) -> list[ProbeResult]:
|
|
semaphore = asyncio.Semaphore(max(1, concurrency))
|
|
|
|
async def run(service: Service) -> ProbeResult:
|
|
async with semaphore:
|
|
if trigger == "manual" and default_timeout_seconds == 5.0 and default_retries == 1:
|
|
return await probe_service(service, timeout_seconds=timeout_seconds, retries=retries)
|
|
if trigger == "manual":
|
|
return await probe_service(
|
|
service,
|
|
timeout_seconds=timeout_seconds,
|
|
retries=retries,
|
|
default_timeout_seconds=default_timeout_seconds,
|
|
default_retries=default_retries,
|
|
)
|
|
return await probe_service(
|
|
service,
|
|
timeout_seconds=timeout_seconds,
|
|
retries=retries,
|
|
trigger=trigger,
|
|
default_timeout_seconds=default_timeout_seconds,
|
|
default_retries=default_retries,
|
|
)
|
|
|
|
return list(await asyncio.gather(*(run(service) for service in services)))
|