Public changes compared with the previous published export: 89 files changed. Affected areas: - backend/app: 21 files - dependencies: 5 files - deployment and CI: 6 files - documentation: 2 files - frontend: 3 files - frontend/src: 20 files - project files: 1 file - tests: 30 files - unraid: 1 file Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/Dockerfile" - Added: "backend/app/alerts.py" - Updated: "backend/app/auth.py" - Added: "backend/app/backups.py" - Updated: "backend/app/config_io.py" - Updated: "backend/app/demo.py" - Updated: "backend/app/discovery.py" - Updated: "backend/app/discovery_store.py" - Updated: "backend/app/docker_discovery.py" - Updated: "backend/app/events.py" - Updated: "backend/app/evidence.py" - Updated: "backend/app/health.py" - Updated: "backend/app/incidents.py" - Added: "backend/app/loops.py" - Updated: "backend/app/main.py" - Updated: "backend/app/models.py" - Updated: "backend/app/observation.py" - Updated: "backend/app/probes.py" - Added: "backend/app/response_cache.py" - Updated: "backend/app/scheduler.py" - Updated: "backend/app/semantic.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_api.py" - Updated: "backend/tests/test_config_io.py" - Updated: "backend/tests/test_demo.py" - Updated: "backend/tests/test_discovery_v120.py" - Updated: "backend/tests/test_health.py" - Updated: "backend/tests/test_probes.py" - Updated: "backend/tests/test_scheduler.py" - Updated: "backend/tests/test_v130_probe_reachability.py" - Updated: "backend/tests/test_v160_evidence_changes.py" - Added: "backend/tests/test_v163_alerts.py" - Added: "backend/tests/test_v163_backups.py" - Added: "backend/tests/test_v163_data_integrity.py" - Added: "backend/tests/test_v163_efficiency.py" - Added: "backend/tests/test_v163_health_performance.py" - Added: "backend/tests/test_v163_loops.py" - Added: "backend/tests/test_v163_management.py" - Added: "backend/tests/test_v163_realtime.py" - Added: "backend/tests/test_v163_security.py" - Added: "backend/tests/test_v163_semantic_etag.py" - Added: "compose.demo.yaml" - Updated: "compose.yaml" - Added: "docker-compose.broker.yml" - Updated: "docker-compose.yml" - Updated: "docs/ARCHITECTURE.md" - Added: "frontend/e2e/a11y.spec.ts" - Updated: "frontend/e2e/demo.spec.ts" - Updated: "frontend/index.html" - Updated: "frontend/nginx.conf" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/playwright.config.ts" - Updated: "frontend/src/App.tsx" - Updated: "frontend/src/LivingEdge.tsx" - Added: "frontend/src/activityView.test.ts" - Added: "frontend/src/activityView.ts" - Updated: "frontend/src/api.ts" - Added: "frontend/src/apiAuth.test.ts" - Added: "frontend/src/components/ActionsMenu.tsx" - Added: "frontend/src/components/ConfirmDialog.test.tsx" - Added: "frontend/src/components/ConfirmDialog.tsx" - Added: "frontend/src/components/StatusIcon.tsx" - Updated: "frontend/src/demo/DemoWalkthrough.test.ts" - Updated: "frontend/src/demo/DemoWalkthrough.tsx" - Added: "frontend/src/elk-worker-url.d.ts" - Added: "frontend/src/labels.ts" - Updated: "frontend/src/main.tsx" - Added: "frontend/src/realtime/refreshPlan.test.ts" - Added: "frontend/src/realtime/refreshPlan.ts" - Updated: "frontend/src/realtime/useRealtimeEvents.ts" - Added: "frontend/src/routing/urlState.test.ts" - Added: "frontend/src/routing/urlState.ts" - Updated: "frontend/src/styles.css" - Added: "frontend/src/topology/GraphListView.test.tsx" - Added: "frontend/src/topology/GraphListView.tsx" - Updated: "frontend/src/topology/ServiceNode.tsx" - Updated: "frontend/src/topology/TopologyCommandBar.tsx" - Added: "frontend/src/topology/semanticGraph.test.ts" - Updated: "frontend/src/topology/semanticGraph.ts" - Added: "frontend/src/topologyLayout.test.ts" - Updated: "frontend/src/topologyLayout.ts" - Updated: "package-lock.json" - Updated: "package.json" - Updated: "unraid/my-servicemap.xml" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from af46f684e468dffd3a5190e28d3b572d4c26bfd7
244 lines
10 KiB
Python
244 lines
10 KiB
Python
"""Incident notifications via ntfy.
|
|
|
|
The notifier is deliberately small and idempotent:
|
|
|
|
* settings live in the ``settings`` table (key ``alerts``); the optional ntfy
|
|
access token is write-only and never returned by the API;
|
|
* ``alert_log`` records which notification (``opened``, ``resolved``,
|
|
``reminder:<n>``) was sent for which incident, so restarts and repeated
|
|
cycles never send duplicates;
|
|
* an incident is announced only after it has stayed open for
|
|
``delay_seconds`` (short flaps stay quiet) and when its severity reaches
|
|
``min_severity``. Maintenance never opens incidents, so it is quiet too.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Literal
|
|
from urllib.parse import quote, urlparse
|
|
|
|
import httpx
|
|
from pydantic import BaseModel, Field, field_validator
|
|
|
|
from .models import Criticality, IncidentRecord, IncidentStatus
|
|
from .store import Store
|
|
|
|
logger = logging.getLogger("servicemap.alerts")
|
|
|
|
_SEVERITY_RANK = {Criticality.LOW: 0, Criticality.MEDIUM: 1, Criticality.HIGH: 2, Criticality.CRITICAL: 3}
|
|
_SETTINGS_KEY = "alerts"
|
|
|
|
|
|
class AlertSettings(BaseModel):
|
|
enabled: bool = False
|
|
ntfy_server: str = Field(default="https://ntfy.sh", max_length=300)
|
|
ntfy_topic: str = Field(default="", max_length=120)
|
|
min_severity: Criticality = Criticality.MEDIUM
|
|
delay_seconds: int = Field(default=60, ge=0, le=3600)
|
|
notify_resolved: bool = True
|
|
reminder_minutes: int = Field(default=0, ge=0, le=1440)
|
|
public_url: str = Field(default="", max_length=300)
|
|
|
|
@field_validator("ntfy_server", "public_url")
|
|
@classmethod
|
|
def _http_url(cls, value: str) -> str:
|
|
value = value.strip().rstrip("/")
|
|
if value and urlparse(value).scheme not in {"http", "https"}:
|
|
raise ValueError("moet een http(s)-URL zijn")
|
|
return value
|
|
|
|
@field_validator("ntfy_topic")
|
|
@classmethod
|
|
def _topic(cls, value: str) -> str:
|
|
value = value.strip()
|
|
if value and not all(char.isalnum() or char in "-_" for char in value):
|
|
raise ValueError("mag alleen letters, cijfers, - en _ bevatten")
|
|
return value
|
|
|
|
|
|
class AlertSettingsView(AlertSettings):
|
|
"""What the API returns: the access token is never echoed."""
|
|
|
|
ntfy_token_set: bool = False
|
|
|
|
|
|
class AlertSettingsUpdate(AlertSettings):
|
|
# None keeps the stored token, "" removes it.
|
|
ntfy_token: str | None = Field(default=None, max_length=200)
|
|
|
|
|
|
class AlertTestResult(BaseModel):
|
|
ok: bool
|
|
detail: str
|
|
|
|
|
|
class AlertManager:
|
|
def __init__(self, store: Store) -> None:
|
|
self.store = store
|
|
self._init_schema()
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
return self.store._connect()
|
|
|
|
def _init_schema(self) -> None:
|
|
with self._connect() as db:
|
|
db.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS settings (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS alert_log (
|
|
incident_id TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
sent_at TEXT NOT NULL,
|
|
PRIMARY KEY (incident_id, kind)
|
|
);
|
|
"""
|
|
)
|
|
|
|
# -- settings -----------------------------------------------------------
|
|
def _raw(self) -> dict[str, object]:
|
|
with self._connect() as db:
|
|
row = db.execute("SELECT value FROM settings WHERE key = ?", (_SETTINGS_KEY,)).fetchone()
|
|
if row is None:
|
|
return {}
|
|
try:
|
|
value = json.loads(row["value"])
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
def settings(self) -> AlertSettings:
|
|
raw = self._raw()
|
|
return AlertSettings.model_validate({key: value for key, value in raw.items() if key != "ntfy_token"})
|
|
|
|
def view(self) -> AlertSettingsView:
|
|
raw = self._raw()
|
|
return AlertSettingsView(**self.settings().model_dump(), ntfy_token_set=bool(raw.get("ntfy_token")))
|
|
|
|
def update(self, update: AlertSettingsUpdate) -> AlertSettingsView:
|
|
raw = self._raw()
|
|
stored = update.model_dump(exclude={"ntfy_token"})
|
|
token = raw.get("ntfy_token") if update.ntfy_token is None else update.ntfy_token.strip()
|
|
if token:
|
|
stored["ntfy_token"] = token
|
|
if stored["enabled"] and not stored["ntfy_topic"]:
|
|
raise ValueError("Stel een ntfy-topic in voor je meldingen aanzet")
|
|
with self._connect() as db:
|
|
db.execute(
|
|
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, ?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at",
|
|
(_SETTINGS_KEY, json.dumps(stored), datetime.now(UTC).isoformat()),
|
|
)
|
|
return self.view()
|
|
|
|
# -- delivery -------------------------------------------------------------
|
|
def _send(self, title: str, message: str, *, priority: str, tags: str, click: str | None) -> None:
|
|
settings = self.settings()
|
|
token = self._raw().get("ntfy_token")
|
|
headers = {"Title": title, "Priority": priority, "Tags": tags}
|
|
if click:
|
|
headers["Click"] = click
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
url = f"{settings.ntfy_server}/{quote(settings.ntfy_topic)}"
|
|
with httpx.Client(timeout=10.0, trust_env=False, follow_redirects=False) as client:
|
|
response = client.post(url, content=message.encode("utf-8"), headers=headers)
|
|
if response.status_code >= 300:
|
|
raise RuntimeError(f"ntfy antwoordde met HTTP {response.status_code}")
|
|
|
|
def send_test(self) -> AlertTestResult:
|
|
settings = self.settings()
|
|
if not settings.ntfy_topic:
|
|
return AlertTestResult(ok=False, detail="Er is geen ntfy-topic ingesteld")
|
|
try:
|
|
self._send(
|
|
"Testmelding van ServiceMap",
|
|
"Meldingen van ServiceMap komen aan op dit topic.",
|
|
priority="default",
|
|
tags="white_check_mark",
|
|
click=settings.public_url or None,
|
|
)
|
|
except (httpx.HTTPError, RuntimeError) as exc:
|
|
return AlertTestResult(ok=False, detail=str(exc))
|
|
return AlertTestResult(ok=True, detail=f"Verstuurd naar {settings.ntfy_server}/{settings.ntfy_topic}")
|
|
|
|
def _sent(self, incident_id: str) -> set[str]:
|
|
with self._connect() as db:
|
|
rows = db.execute("SELECT kind FROM alert_log WHERE incident_id = ?", (incident_id,)).fetchall()
|
|
return {row["kind"] for row in rows}
|
|
|
|
def _mark(self, incident_id: str, kind: str, at: datetime) -> None:
|
|
with self._connect() as db:
|
|
db.execute(
|
|
"INSERT OR IGNORE INTO alert_log (incident_id, kind, sent_at) VALUES (?, ?, ?)",
|
|
(incident_id, kind, at.isoformat()),
|
|
)
|
|
|
|
def _link(self, settings: AlertSettings, incident: IncidentRecord) -> str | None:
|
|
if not settings.public_url:
|
|
return None
|
|
return f"{settings.public_url}/#/incidents?incident={quote(incident.id)}"
|
|
|
|
def plan(self, incidents: list[IncidentRecord], now: datetime) -> list[tuple[IncidentRecord, Literal["opened", "resolved"] | str]]:
|
|
"""Notifications that are due now (pure apart from reading the log)."""
|
|
|
|
settings = self.settings()
|
|
due: list[tuple[IncidentRecord, str]] = []
|
|
for incident in incidents:
|
|
if _SEVERITY_RANK[incident.severity] < _SEVERITY_RANK[settings.min_severity]:
|
|
continue
|
|
sent = self._sent(incident.id)
|
|
if incident.status == IncidentStatus.RESOLVED:
|
|
if settings.notify_resolved and "opened" in sent and "resolved" not in sent:
|
|
due.append((incident, "resolved"))
|
|
continue
|
|
open_for = now - incident.detected_at
|
|
if "opened" not in sent:
|
|
if open_for >= timedelta(seconds=settings.delay_seconds):
|
|
due.append((incident, "opened"))
|
|
continue
|
|
if settings.reminder_minutes and incident.status == IncidentStatus.OPEN:
|
|
count = int(open_for.total_seconds() // (settings.reminder_minutes * 60))
|
|
if count >= 1 and f"reminder:{count}" not in sent:
|
|
due.append((incident, f"reminder:{count}"))
|
|
return due
|
|
|
|
def run_cycle(self, now: datetime | None = None) -> int:
|
|
settings = self.settings()
|
|
if not settings.enabled or not settings.ntfy_topic:
|
|
return 0
|
|
now = now or datetime.now(UTC)
|
|
# Recently resolved incidents still need their "resolved" message.
|
|
incidents = self.store.list_incidents(active_only=False, limit=200)
|
|
sent = 0
|
|
for incident, kind in self.plan(incidents, now):
|
|
critical = incident.severity in {Criticality.HIGH, Criticality.CRITICAL}
|
|
if kind == "resolved":
|
|
title = f"Opgelost: {incident.title}"
|
|
message = f"{incident.root_service_name} is hersteld."
|
|
priority, tags = "default", "white_check_mark"
|
|
else:
|
|
reminder = kind.startswith("reminder")
|
|
title = f"{'Nog steeds uitgevallen' if reminder else 'Uitgevallen'}: {incident.title}"
|
|
affected = len(incident.affected_service_ids)
|
|
message = (
|
|
f"{incident.root_cause_reason}"
|
|
+ (f"\n{affected} afhankelijke service(s) geraakt." if affected else "")
|
|
)
|
|
priority, tags = ("urgent" if critical else "high"), "rotating_light"
|
|
try:
|
|
self._send(title, message, priority=priority, tags=tags, click=self._link(settings, incident))
|
|
except (httpx.HTTPError, RuntimeError):
|
|
logger.exception("Sending the %s notification for incident %s failed", kind, incident.id)
|
|
continue
|
|
self._mark(incident.id, kind, now)
|
|
sent += 1
|
|
return sent
|