Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
151 lines
5.1 KiB
Python
151 lines
5.1 KiB
Python
"""Background loops that never die silently, and their readiness record.
|
|
|
|
Before v1.6.3 the observation and discovery loops caught only ``OSError``,
|
|
``RuntimeError`` and ``ValueError``. Any other exception, for example
|
|
``sqlite3.OperationalError: database is locked``, ended the task without a
|
|
log line; ServiceMap then looked healthy while it no longer observed
|
|
anything. ``run_periodic`` logs every failure, retries with bounded
|
|
backoff and records the last success so ``/ready`` can report a stalled
|
|
loop.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from collections.abc import Awaitable, Callable
|
|
from dataclasses import dataclass, field
|
|
from threading import Lock
|
|
|
|
logger = logging.getLogger("servicemap.loops")
|
|
|
|
|
|
@dataclass
|
|
class LoopStatus:
|
|
name: str
|
|
interval_seconds: float
|
|
started_at: float = field(default_factory=time.time)
|
|
last_success_at: float | None = None
|
|
last_failure_at: float | None = None
|
|
last_error: str | None = None
|
|
consecutive_failures: int = 0
|
|
paused: bool = False
|
|
|
|
def stale_after_seconds(self) -> float:
|
|
# Three missed cycles, with a floor for very short intervals.
|
|
return max(3 * self.interval_seconds, 180.0)
|
|
|
|
def healthy(self, now: float) -> bool:
|
|
if self.paused:
|
|
return True
|
|
reference = self.last_success_at if self.last_success_at is not None else self.started_at
|
|
return now - reference <= self.stale_after_seconds()
|
|
|
|
def as_dict(self, now: float) -> dict[str, object]:
|
|
def iso(value: float | None) -> str | None:
|
|
return None if value is None else time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(value))
|
|
|
|
return {
|
|
"name": self.name,
|
|
"healthy": self.healthy(now),
|
|
"interval_seconds": self.interval_seconds,
|
|
"last_success_at": iso(self.last_success_at),
|
|
"last_failure_at": iso(self.last_failure_at),
|
|
"last_error": self.last_error,
|
|
"consecutive_failures": self.consecutive_failures,
|
|
"paused": self.paused,
|
|
}
|
|
|
|
|
|
class LoopRegistry:
|
|
def __init__(self) -> None:
|
|
self._loops: dict[str, LoopStatus] = {}
|
|
self._lock = Lock()
|
|
|
|
def register(self, name: str, interval_seconds: float) -> LoopStatus:
|
|
with self._lock:
|
|
status = LoopStatus(name=name, interval_seconds=interval_seconds)
|
|
self._loops[name] = status
|
|
return status
|
|
|
|
def unregister(self, name: str) -> None:
|
|
with self._lock:
|
|
self._loops.pop(name, None)
|
|
|
|
def snapshot(self) -> list[dict[str, object]]:
|
|
now = time.time()
|
|
with self._lock:
|
|
return [status.as_dict(now) for status in self._loops.values()]
|
|
|
|
def all_healthy(self) -> bool:
|
|
now = time.time()
|
|
with self._lock:
|
|
return all(status.healthy(now) for status in self._loops.values())
|
|
|
|
|
|
loop_registry = LoopRegistry()
|
|
|
|
|
|
def record_success(status: LoopStatus) -> None:
|
|
status.last_success_at = time.time()
|
|
status.consecutive_failures = 0
|
|
status.paused = False
|
|
|
|
|
|
def record_failure(status: LoopStatus, error: BaseException) -> None:
|
|
status.last_failure_at = time.time()
|
|
status.consecutive_failures += 1
|
|
status.last_error = f"{type(error).__name__}: {str(error)[:300]}"
|
|
|
|
|
|
async def run_periodic(
|
|
name: str,
|
|
work: Callable[[], Awaitable[object]],
|
|
*,
|
|
interval_seconds: float,
|
|
initial_delay_seconds: float = 0.0,
|
|
paused: Callable[[], bool] | None = None,
|
|
registry: LoopRegistry = loop_registry,
|
|
max_backoff_seconds: float = 300.0,
|
|
) -> None:
|
|
"""Run ``work`` every ``interval_seconds`` until cancelled.
|
|
|
|
A failure is logged with its traceback and retried after an exponential
|
|
backoff capped at ``max_backoff_seconds``. ``paused`` lets a loop
|
|
idle deliberately (for example while the synthetic demo is active)
|
|
without being reported as stalled.
|
|
"""
|
|
|
|
status = registry.register(name, interval_seconds)
|
|
try:
|
|
if initial_delay_seconds > 0:
|
|
await asyncio.sleep(initial_delay_seconds)
|
|
while True:
|
|
delay = interval_seconds
|
|
try:
|
|
if paused is not None and paused():
|
|
status.paused = True
|
|
else:
|
|
await work()
|
|
record_success(status)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
record_failure(status, exc)
|
|
# interval, 2x, 4x … capped: a persistent fault does not spin,
|
|
# a long interval still retries within max_backoff_seconds.
|
|
delay = min(
|
|
max_backoff_seconds,
|
|
max(5.0, interval_seconds) * 2 ** min(status.consecutive_failures - 1, 5),
|
|
)
|
|
logger.exception(
|
|
"Background loop %s failed (%s consecutive); retrying in %.0fs",
|
|
name,
|
|
status.consecutive_failures,
|
|
delay,
|
|
)
|
|
await asyncio.sleep(delay)
|
|
finally:
|
|
registry.unregister(name)
|