"""Background loops that never die silently, and their readiness record. Before v1.6.3 the observation and discovery loops caught only ``OSError``, ``RuntimeError`` and ``ValueError``. Any other exception, for example ``sqlite3.OperationalError: database is locked``, ended the task without a log line; ServiceMap then looked healthy while it no longer observed anything. ``run_periodic`` logs every failure, retries with bounded backoff and records the last success so ``/ready`` can report a stalled loop. """ from __future__ import annotations import asyncio import logging import time from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from threading import Lock logger = logging.getLogger("servicemap.loops") @dataclass class LoopStatus: name: str interval_seconds: float started_at: float = field(default_factory=time.time) last_success_at: float | None = None last_failure_at: float | None = None last_error: str | None = None consecutive_failures: int = 0 paused: bool = False def stale_after_seconds(self) -> float: # Three missed cycles, with a floor for very short intervals. return max(3 * self.interval_seconds, 180.0) def healthy(self, now: float) -> bool: if self.paused: return True reference = self.last_success_at if self.last_success_at is not None else self.started_at return now - reference <= self.stale_after_seconds() def as_dict(self, now: float) -> dict[str, object]: def iso(value: float | None) -> str | None: return None if value is None else time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(value)) return { "name": self.name, "healthy": self.healthy(now), "interval_seconds": self.interval_seconds, "last_success_at": iso(self.last_success_at), "last_failure_at": iso(self.last_failure_at), "last_error": self.last_error, "consecutive_failures": self.consecutive_failures, "paused": self.paused, } class LoopRegistry: def __init__(self) -> None: self._loops: dict[str, LoopStatus] = {} self._lock = Lock() def register(self, name: str, interval_seconds: float) -> LoopStatus: with self._lock: status = LoopStatus(name=name, interval_seconds=interval_seconds) self._loops[name] = status return status def unregister(self, name: str) -> None: with self._lock: self._loops.pop(name, None) def snapshot(self) -> list[dict[str, object]]: now = time.time() with self._lock: return [status.as_dict(now) for status in self._loops.values()] def all_healthy(self) -> bool: now = time.time() with self._lock: return all(status.healthy(now) for status in self._loops.values()) loop_registry = LoopRegistry() def record_success(status: LoopStatus) -> None: status.last_success_at = time.time() status.consecutive_failures = 0 status.paused = False def record_failure(status: LoopStatus, error: BaseException) -> None: status.last_failure_at = time.time() status.consecutive_failures += 1 status.last_error = f"{type(error).__name__}: {str(error)[:300]}" async def run_periodic( name: str, work: Callable[[], Awaitable[object]], *, interval_seconds: float, initial_delay_seconds: float = 0.0, paused: Callable[[], bool] | None = None, registry: LoopRegistry = loop_registry, max_backoff_seconds: float = 300.0, ) -> None: """Run ``work`` every ``interval_seconds`` until cancelled. A failure is logged with its traceback and retried after an exponential backoff capped at ``max_backoff_seconds``. ``paused`` lets a loop idle deliberately (for example while the synthetic demo is active) without being reported as stalled. """ status = registry.register(name, interval_seconds) try: if initial_delay_seconds > 0: await asyncio.sleep(initial_delay_seconds) while True: delay = interval_seconds try: if paused is not None and paused(): status.paused = True else: await work() record_success(status) except asyncio.CancelledError: raise except Exception as exc: record_failure(status, exc) # interval, 2x, 4x … capped: a persistent fault does not spin, # a long interval still retries within max_backoff_seconds. delay = min( max_backoff_seconds, max(5.0, interval_seconds) * 2 ** min(status.consecutive_failures - 1, 5), ) logger.exception( "Background loop %s failed (%s consecutive); retrying in %.0fs", name, status.consecutive_failures, delay, ) await asyncio.sleep(delay) finally: registry.unregister(name)