ServiceMap: update tests and backend/app (+7 areas)
Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
This commit is contained in:
1 parent
d7319250e8
commit
adc2c70c21
91 files changed
+9205
-1691
No files matched your search
@@ -0,0 +1,150 @@
|
||||
"""Background loops that never die silently, and their readiness record.
|
||||
|
||||
Before v1.6.3 the observation and discovery loops caught only ``OSError``,
|
||||
``RuntimeError`` and ``ValueError``. Any other exception, for example
|
||||
``sqlite3.OperationalError: database is locked``, ended the task without a
|
||||
log line; ServiceMap then looked healthy while it no longer observed
|
||||
anything. ``run_periodic`` logs every failure, retries with bounded
|
||||
backoff and records the last success so ``/ready`` can report a stalled
|
||||
loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from threading import Lock
|
||||
|
||||
logger = logging.getLogger("servicemap.loops")
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoopStatus:
|
||||
name: str
|
||||
interval_seconds: float
|
||||
started_at: float = field(default_factory=time.time)
|
||||
last_success_at: float | None = None
|
||||
last_failure_at: float | None = None
|
||||
last_error: str | None = None
|
||||
consecutive_failures: int = 0
|
||||
paused: bool = False
|
||||
|
||||
def stale_after_seconds(self) -> float:
|
||||
# Three missed cycles, with a floor for very short intervals.
|
||||
return max(3 * self.interval_seconds, 180.0)
|
||||
|
||||
def healthy(self, now: float) -> bool:
|
||||
if self.paused:
|
||||
return True
|
||||
reference = self.last_success_at if self.last_success_at is not None else self.started_at
|
||||
return now - reference <= self.stale_after_seconds()
|
||||
|
||||
def as_dict(self, now: float) -> dict[str, object]:
|
||||
def iso(value: float | None) -> str | None:
|
||||
return None if value is None else time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(value))
|
||||
|
||||
return {
|
||||
"name": self.name,
|
||||
"healthy": self.healthy(now),
|
||||
"interval_seconds": self.interval_seconds,
|
||||
"last_success_at": iso(self.last_success_at),
|
||||
"last_failure_at": iso(self.last_failure_at),
|
||||
"last_error": self.last_error,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"paused": self.paused,
|
||||
}
|
||||
|
||||
|
||||
class LoopRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._loops: dict[str, LoopStatus] = {}
|
||||
self._lock = Lock()
|
||||
|
||||
def register(self, name: str, interval_seconds: float) -> LoopStatus:
|
||||
with self._lock:
|
||||
status = LoopStatus(name=name, interval_seconds=interval_seconds)
|
||||
self._loops[name] = status
|
||||
return status
|
||||
|
||||
def unregister(self, name: str) -> None:
|
||||
with self._lock:
|
||||
self._loops.pop(name, None)
|
||||
|
||||
def snapshot(self) -> list[dict[str, object]]:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
return [status.as_dict(now) for status in self._loops.values()]
|
||||
|
||||
def all_healthy(self) -> bool:
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
return all(status.healthy(now) for status in self._loops.values())
|
||||
|
||||
|
||||
loop_registry = LoopRegistry()
|
||||
|
||||
|
||||
def record_success(status: LoopStatus) -> None:
|
||||
status.last_success_at = time.time()
|
||||
status.consecutive_failures = 0
|
||||
status.paused = False
|
||||
|
||||
|
||||
def record_failure(status: LoopStatus, error: BaseException) -> None:
|
||||
status.last_failure_at = time.time()
|
||||
status.consecutive_failures += 1
|
||||
status.last_error = f"{type(error).__name__}: {str(error)[:300]}"
|
||||
|
||||
|
||||
async def run_periodic(
|
||||
name: str,
|
||||
work: Callable[[], Awaitable[object]],
|
||||
*,
|
||||
interval_seconds: float,
|
||||
initial_delay_seconds: float = 0.0,
|
||||
paused: Callable[[], bool] | None = None,
|
||||
registry: LoopRegistry = loop_registry,
|
||||
max_backoff_seconds: float = 300.0,
|
||||
) -> None:
|
||||
"""Run ``work`` every ``interval_seconds`` until cancelled.
|
||||
|
||||
A failure is logged with its traceback and retried after an exponential
|
||||
backoff capped at ``max_backoff_seconds``. ``paused`` lets a loop
|
||||
idle deliberately (for example while the synthetic demo is active)
|
||||
without being reported as stalled.
|
||||
"""
|
||||
|
||||
status = registry.register(name, interval_seconds)
|
||||
try:
|
||||
if initial_delay_seconds > 0:
|
||||
await asyncio.sleep(initial_delay_seconds)
|
||||
while True:
|
||||
delay = interval_seconds
|
||||
try:
|
||||
if paused is not None and paused():
|
||||
status.paused = True
|
||||
else:
|
||||
await work()
|
||||
record_success(status)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
record_failure(status, exc)
|
||||
# interval, 2x, 4x … capped: a persistent fault does not spin,
|
||||
# a long interval still retries within max_backoff_seconds.
|
||||
delay = min(
|
||||
max_backoff_seconds,
|
||||
max(5.0, interval_seconds) * 2 ** min(status.consecutive_failures - 1, 5),
|
||||
)
|
||||
logger.exception(
|
||||
"Background loop %s failed (%s consecutive); retrying in %.0fs",
|
||||
name,
|
||||
status.consecutive_failures,
|
||||
delay,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
finally:
|
||||
registry.unregister(name)
|
||||
Reference in new issue
Block a user