ServiceMap: update tests and backend/app (+7 areas)
Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
This commit is contained in:
1 parent
d7319250e8
commit
adc2c70c21
91 files changed
+9205
-1691
No files matched your search
@@ -0,0 +1,243 @@
|
||||
"""Incident notifications via ntfy.
|
||||
|
||||
The notifier is deliberately small and idempotent:
|
||||
|
||||
* settings live in the ``settings`` table (key ``alerts``); the optional ntfy
|
||||
access token is write-only and never returned by the API;
|
||||
* ``alert_log`` records which notification (``opened``, ``resolved``,
|
||||
``reminder:<n>``) was sent for which incident, so restarts and repeated
|
||||
cycles never send duplicates;
|
||||
* an incident is announced only after it has stayed open for
|
||||
``delay_seconds`` (short flaps stay quiet) and when its severity reaches
|
||||
``min_severity``. Maintenance never opens incidents, so it is quiet too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from .models import Criticality, IncidentRecord, IncidentStatus
|
||||
from .store import Store
|
||||
|
||||
logger = logging.getLogger("servicemap.alerts")
|
||||
|
||||
_SEVERITY_RANK = {Criticality.LOW: 0, Criticality.MEDIUM: 1, Criticality.HIGH: 2, Criticality.CRITICAL: 3}
|
||||
_SETTINGS_KEY = "alerts"
|
||||
|
||||
|
||||
class AlertSettings(BaseModel):
|
||||
enabled: bool = False
|
||||
ntfy_server: str = Field(default="https://ntfy.sh", max_length=300)
|
||||
ntfy_topic: str = Field(default="", max_length=120)
|
||||
min_severity: Criticality = Criticality.MEDIUM
|
||||
delay_seconds: int = Field(default=60, ge=0, le=3600)
|
||||
notify_resolved: bool = True
|
||||
reminder_minutes: int = Field(default=0, ge=0, le=1440)
|
||||
public_url: str = Field(default="", max_length=300)
|
||||
|
||||
@field_validator("ntfy_server", "public_url")
|
||||
@classmethod
|
||||
def _http_url(cls, value: str) -> str:
|
||||
value = value.strip().rstrip("/")
|
||||
if value and urlparse(value).scheme not in {"http", "https"}:
|
||||
raise ValueError("moet een http(s)-URL zijn")
|
||||
return value
|
||||
|
||||
@field_validator("ntfy_topic")
|
||||
@classmethod
|
||||
def _topic(cls, value: str) -> str:
|
||||
value = value.strip()
|
||||
if value and not all(char.isalnum() or char in "-_" for char in value):
|
||||
raise ValueError("mag alleen letters, cijfers, - en _ bevatten")
|
||||
return value
|
||||
|
||||
|
||||
class AlertSettingsView(AlertSettings):
|
||||
"""What the API returns: the access token is never echoed."""
|
||||
|
||||
ntfy_token_set: bool = False
|
||||
|
||||
|
||||
class AlertSettingsUpdate(AlertSettings):
|
||||
# None keeps the stored token, "" removes it.
|
||||
ntfy_token: str | None = Field(default=None, max_length=200)
|
||||
|
||||
|
||||
class AlertTestResult(BaseModel):
|
||||
ok: bool
|
||||
detail: str
|
||||
|
||||
|
||||
class AlertManager:
|
||||
def __init__(self, store: Store) -> None:
|
||||
self.store = store
|
||||
self._init_schema()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
return self.store._connect()
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._connect() as db:
|
||||
db.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS alert_log (
|
||||
incident_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
sent_at TEXT NOT NULL,
|
||||
PRIMARY KEY (incident_id, kind)
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
# -- settings -----------------------------------------------------------
|
||||
def _raw(self) -> dict[str, object]:
|
||||
with self._connect() as db:
|
||||
row = db.execute("SELECT value FROM settings WHERE key = ?", (_SETTINGS_KEY,)).fetchone()
|
||||
if row is None:
|
||||
return {}
|
||||
try:
|
||||
value = json.loads(row["value"])
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
def settings(self) -> AlertSettings:
|
||||
raw = self._raw()
|
||||
return AlertSettings.model_validate({key: value for key, value in raw.items() if key != "ntfy_token"})
|
||||
|
||||
def view(self) -> AlertSettingsView:
|
||||
raw = self._raw()
|
||||
return AlertSettingsView(**self.settings().model_dump(), ntfy_token_set=bool(raw.get("ntfy_token")))
|
||||
|
||||
def update(self, update: AlertSettingsUpdate) -> AlertSettingsView:
|
||||
raw = self._raw()
|
||||
stored = update.model_dump(exclude={"ntfy_token"})
|
||||
token = raw.get("ntfy_token") if update.ntfy_token is None else update.ntfy_token.strip()
|
||||
if token:
|
||||
stored["ntfy_token"] = token
|
||||
if stored["enabled"] and not stored["ntfy_topic"]:
|
||||
raise ValueError("Stel een ntfy-topic in voor je meldingen aanzet")
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at",
|
||||
(_SETTINGS_KEY, json.dumps(stored), datetime.now(UTC).isoformat()),
|
||||
)
|
||||
return self.view()
|
||||
|
||||
# -- delivery -------------------------------------------------------------
|
||||
def _send(self, title: str, message: str, *, priority: str, tags: str, click: str | None) -> None:
|
||||
settings = self.settings()
|
||||
token = self._raw().get("ntfy_token")
|
||||
headers = {"Title": title, "Priority": priority, "Tags": tags}
|
||||
if click:
|
||||
headers["Click"] = click
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
url = f"{settings.ntfy_server}/{quote(settings.ntfy_topic)}"
|
||||
with httpx.Client(timeout=10.0, trust_env=False, follow_redirects=False) as client:
|
||||
response = client.post(url, content=message.encode("utf-8"), headers=headers)
|
||||
if response.status_code >= 300:
|
||||
raise RuntimeError(f"ntfy antwoordde met HTTP {response.status_code}")
|
||||
|
||||
def send_test(self) -> AlertTestResult:
|
||||
settings = self.settings()
|
||||
if not settings.ntfy_topic:
|
||||
return AlertTestResult(ok=False, detail="Er is geen ntfy-topic ingesteld")
|
||||
try:
|
||||
self._send(
|
||||
"Testmelding van ServiceMap",
|
||||
"Meldingen van ServiceMap komen aan op dit topic.",
|
||||
priority="default",
|
||||
tags="white_check_mark",
|
||||
click=settings.public_url or None,
|
||||
)
|
||||
except (httpx.HTTPError, RuntimeError) as exc:
|
||||
return AlertTestResult(ok=False, detail=str(exc))
|
||||
return AlertTestResult(ok=True, detail=f"Verstuurd naar {settings.ntfy_server}/{settings.ntfy_topic}")
|
||||
|
||||
def _sent(self, incident_id: str) -> set[str]:
|
||||
with self._connect() as db:
|
||||
rows = db.execute("SELECT kind FROM alert_log WHERE incident_id = ?", (incident_id,)).fetchall()
|
||||
return {row["kind"] for row in rows}
|
||||
|
||||
def _mark(self, incident_id: str, kind: str, at: datetime) -> None:
|
||||
with self._connect() as db:
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO alert_log (incident_id, kind, sent_at) VALUES (?, ?, ?)",
|
||||
(incident_id, kind, at.isoformat()),
|
||||
)
|
||||
|
||||
def _link(self, settings: AlertSettings, incident: IncidentRecord) -> str | None:
|
||||
if not settings.public_url:
|
||||
return None
|
||||
return f"{settings.public_url}/#/incidents?incident={quote(incident.id)}"
|
||||
|
||||
def plan(self, incidents: list[IncidentRecord], now: datetime) -> list[tuple[IncidentRecord, Literal["opened", "resolved"] | str]]:
|
||||
"""Notifications that are due now (pure apart from reading the log)."""
|
||||
|
||||
settings = self.settings()
|
||||
due: list[tuple[IncidentRecord, str]] = []
|
||||
for incident in incidents:
|
||||
if _SEVERITY_RANK[incident.severity] < _SEVERITY_RANK[settings.min_severity]:
|
||||
continue
|
||||
sent = self._sent(incident.id)
|
||||
if incident.status == IncidentStatus.RESOLVED:
|
||||
if settings.notify_resolved and "opened" in sent and "resolved" not in sent:
|
||||
due.append((incident, "resolved"))
|
||||
continue
|
||||
open_for = now - incident.detected_at
|
||||
if "opened" not in sent:
|
||||
if open_for >= timedelta(seconds=settings.delay_seconds):
|
||||
due.append((incident, "opened"))
|
||||
continue
|
||||
if settings.reminder_minutes and incident.status == IncidentStatus.OPEN:
|
||||
count = int(open_for.total_seconds() // (settings.reminder_minutes * 60))
|
||||
if count >= 1 and f"reminder:{count}" not in sent:
|
||||
due.append((incident, f"reminder:{count}"))
|
||||
return due
|
||||
|
||||
def run_cycle(self, now: datetime | None = None) -> int:
|
||||
settings = self.settings()
|
||||
if not settings.enabled or not settings.ntfy_topic:
|
||||
return 0
|
||||
now = now or datetime.now(UTC)
|
||||
# Recently resolved incidents still need their "resolved" message.
|
||||
incidents = self.store.list_incidents(active_only=False, limit=200)
|
||||
sent = 0
|
||||
for incident, kind in self.plan(incidents, now):
|
||||
critical = incident.severity in {Criticality.HIGH, Criticality.CRITICAL}
|
||||
if kind == "resolved":
|
||||
title = f"Opgelost: {incident.title}"
|
||||
message = f"{incident.root_service_name} is hersteld."
|
||||
priority, tags = "default", "white_check_mark"
|
||||
else:
|
||||
reminder = kind.startswith("reminder")
|
||||
title = f"{'Nog steeds uitgevallen' if reminder else 'Uitgevallen'}: {incident.title}"
|
||||
affected = len(incident.affected_service_ids)
|
||||
message = (
|
||||
f"{incident.root_cause_reason}"
|
||||
+ (f"\n{affected} afhankelijke service(s) geraakt." if affected else "")
|
||||
)
|
||||
priority, tags = ("urgent" if critical else "high"), "rotating_light"
|
||||
try:
|
||||
self._send(title, message, priority=priority, tags=tags, click=self._link(settings, incident))
|
||||
except (httpx.HTTPError, RuntimeError):
|
||||
logger.exception("Sending the %s notification for incident %s failed", kind, incident.id)
|
||||
continue
|
||||
self._mark(incident.id, kind, now)
|
||||
sent += 1
|
||||
return sent
|
||||
Reference in new issue
Block a user