Files
NuklearRabbit adc2c70c21 ServiceMap: update tests and backend/app (+7 areas)
Public changes compared with the previous published export:
89 files changed.

Affected areas:
- backend/app: 21 files
- dependencies: 5 files
- deployment and CI: 6 files
- documentation: 2 files
- frontend: 3 files
- frontend/src: 20 files
- project files: 1 file
- tests: 30 files
- unraid: 1 file

Changed files:
- Updated: ".env.example"
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/Dockerfile"
- Added: "backend/app/alerts.py"
- Updated: "backend/app/auth.py"
- Added: "backend/app/backups.py"
- Updated: "backend/app/config_io.py"
- Updated: "backend/app/demo.py"
- Updated: "backend/app/discovery.py"
- Updated: "backend/app/discovery_store.py"
- Updated: "backend/app/docker_discovery.py"
- Updated: "backend/app/events.py"
- Updated: "backend/app/evidence.py"
- Updated: "backend/app/health.py"
- Updated: "backend/app/incidents.py"
- Added: "backend/app/loops.py"
- Updated: "backend/app/main.py"
- Updated: "backend/app/models.py"
- Updated: "backend/app/observation.py"
- Updated: "backend/app/probes.py"
- Added: "backend/app/response_cache.py"
- Updated: "backend/app/scheduler.py"
- Updated: "backend/app/semantic.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Updated: "backend/tests/test_api.py"
- Updated: "backend/tests/test_config_io.py"
- Updated: "backend/tests/test_demo.py"
- Updated: "backend/tests/test_discovery_v120.py"
- Updated: "backend/tests/test_health.py"
- Updated: "backend/tests/test_probes.py"
- Updated: "backend/tests/test_scheduler.py"
- Updated: "backend/tests/test_v130_probe_reachability.py"
- Updated: "backend/tests/test_v160_evidence_changes.py"
- Added: "backend/tests/test_v163_alerts.py"
- Added: "backend/tests/test_v163_backups.py"
- Added: "backend/tests/test_v163_data_integrity.py"
- Added: "backend/tests/test_v163_efficiency.py"
- Added: "backend/tests/test_v163_health_performance.py"
- Added: "backend/tests/test_v163_loops.py"
- Added: "backend/tests/test_v163_management.py"
- Added: "backend/tests/test_v163_realtime.py"
- Added: "backend/tests/test_v163_security.py"
- Added: "backend/tests/test_v163_semantic_etag.py"
- Added: "compose.demo.yaml"
- Updated: "compose.yaml"
- Added: "docker-compose.broker.yml"
- Updated: "docker-compose.yml"
- Updated: "docs/ARCHITECTURE.md"
- Added: "frontend/e2e/a11y.spec.ts"
- Updated: "frontend/e2e/demo.spec.ts"
- Updated: "frontend/index.html"
- Updated: "frontend/nginx.conf"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "frontend/playwright.config.ts"
- Updated: "frontend/src/App.tsx"
- Updated: "frontend/src/LivingEdge.tsx"
- Added: "frontend/src/activityView.test.ts"
- Added: "frontend/src/activityView.ts"
- Updated: "frontend/src/api.ts"
- Added: "frontend/src/apiAuth.test.ts"
- Added: "frontend/src/components/ActionsMenu.tsx"
- Added: "frontend/src/components/ConfirmDialog.test.tsx"
- Added: "frontend/src/components/ConfirmDialog.tsx"
- Added: "frontend/src/components/StatusIcon.tsx"
- Updated: "frontend/src/demo/DemoWalkthrough.test.ts"
- Updated: "frontend/src/demo/DemoWalkthrough.tsx"
- Added: "frontend/src/elk-worker-url.d.ts"
- Added: "frontend/src/labels.ts"
- Updated: "frontend/src/main.tsx"
- Added: "frontend/src/realtime/refreshPlan.test.ts"
- Added: "frontend/src/realtime/refreshPlan.ts"
- Updated: "frontend/src/realtime/useRealtimeEvents.ts"
- Added: "frontend/src/routing/urlState.test.ts"
- Added: "frontend/src/routing/urlState.ts"
- Updated: "frontend/src/styles.css"
- Added: "frontend/src/topology/GraphListView.test.tsx"
- Added: "frontend/src/topology/GraphListView.tsx"
- Updated: "frontend/src/topology/ServiceNode.tsx"
- Updated: "frontend/src/topology/TopologyCommandBar.tsx"
- Added: "frontend/src/topology/semanticGraph.test.ts"
- Updated: "frontend/src/topology/semanticGraph.ts"
- Added: "frontend/src/topologyLayout.test.ts"
- Updated: "frontend/src/topologyLayout.ts"
- Updated: "package-lock.json"
- Updated: "package.json"
- Updated: "unraid/my-servicemap.xml"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
2026-11-20 13:51:46 +01:00

151 lines
5.1 KiB
Python

"""Background loops that never die silently, and their readiness record.
Before v1.6.3 the observation and discovery loops caught only ``OSError``,
``RuntimeError`` and ``ValueError``. Any other exception, for example
``sqlite3.OperationalError: database is locked``, ended the task without a
log line; ServiceMap then looked healthy while it no longer observed
anything. ``run_periodic`` logs every failure, retries with bounded
backoff and records the last success so ``/ready`` can report a stalled
loop.
"""
from __future__ import annotations
import asyncio
import logging
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from threading import Lock
logger = logging.getLogger("servicemap.loops")
@dataclass
class LoopStatus:
name: str
interval_seconds: float
started_at: float = field(default_factory=time.time)
last_success_at: float | None = None
last_failure_at: float | None = None
last_error: str | None = None
consecutive_failures: int = 0
paused: bool = False
def stale_after_seconds(self) -> float:
# Three missed cycles, with a floor for very short intervals.
return max(3 * self.interval_seconds, 180.0)
def healthy(self, now: float) -> bool:
if self.paused:
return True
reference = self.last_success_at if self.last_success_at is not None else self.started_at
return now - reference <= self.stale_after_seconds()
def as_dict(self, now: float) -> dict[str, object]:
def iso(value: float | None) -> str | None:
return None if value is None else time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(value))
return {
"name": self.name,
"healthy": self.healthy(now),
"interval_seconds": self.interval_seconds,
"last_success_at": iso(self.last_success_at),
"last_failure_at": iso(self.last_failure_at),
"last_error": self.last_error,
"consecutive_failures": self.consecutive_failures,
"paused": self.paused,
}
class LoopRegistry:
def __init__(self) -> None:
self._loops: dict[str, LoopStatus] = {}
self._lock = Lock()
def register(self, name: str, interval_seconds: float) -> LoopStatus:
with self._lock:
status = LoopStatus(name=name, interval_seconds=interval_seconds)
self._loops[name] = status
return status
def unregister(self, name: str) -> None:
with self._lock:
self._loops.pop(name, None)
def snapshot(self) -> list[dict[str, object]]:
now = time.time()
with self._lock:
return [status.as_dict(now) for status in self._loops.values()]
def all_healthy(self) -> bool:
now = time.time()
with self._lock:
return all(status.healthy(now) for status in self._loops.values())
loop_registry = LoopRegistry()
def record_success(status: LoopStatus) -> None:
status.last_success_at = time.time()
status.consecutive_failures = 0
status.paused = False
def record_failure(status: LoopStatus, error: BaseException) -> None:
status.last_failure_at = time.time()
status.consecutive_failures += 1
status.last_error = f"{type(error).__name__}: {str(error)[:300]}"
async def run_periodic(
name: str,
work: Callable[[], Awaitable[object]],
*,
interval_seconds: float,
initial_delay_seconds: float = 0.0,
paused: Callable[[], bool] | None = None,
registry: LoopRegistry = loop_registry,
max_backoff_seconds: float = 300.0,
) -> None:
"""Run ``work`` every ``interval_seconds`` until cancelled.
A failure is logged with its traceback and retried after an exponential
backoff capped at ``max_backoff_seconds``. ``paused`` lets a loop
idle deliberately (for example while the synthetic demo is active)
without being reported as stalled.
"""
status = registry.register(name, interval_seconds)
try:
if initial_delay_seconds > 0:
await asyncio.sleep(initial_delay_seconds)
while True:
delay = interval_seconds
try:
if paused is not None and paused():
status.paused = True
else:
await work()
record_success(status)
except asyncio.CancelledError:
raise
except Exception as exc:
record_failure(status, exc)
# interval, 2x, 4x … capped: a persistent fault does not spin,
# a long interval still retries within max_backoff_seconds.
delay = min(
max_backoff_seconds,
max(5.0, interval_seconds) * 2 ** min(status.consecutive_failures - 1, 5),
)
logger.exception(
"Background loop %s failed (%s consecutive); retrying in %.0fs",
name,
status.consecutive_failures,
delay,
)
await asyncio.sleep(delay)
finally:
registry.unregister(name)