M22: implement operational observability

2026-08-10 15:42:26 +02:00
parent c3f1cfc699
commit 689e499634
17 changed files with 428 additions and 9 deletions
@@ -0,0 +1,39 @@
+groups:
+  - name: mobilityops
+    rules:
+      - alert: MobilityOpsApiDown
+        expr: up{job="mobilityops-api"} == 0
+        for: 2m
+        labels: {severity: critical}
+        annotations:
+          summary: MobilityOps API metrics target is down
+      - alert: MobilityOpsDatabaseNotReady
+        expr: mobilityops_database_ready == 0
+        for: 2m
+        labels: {severity: critical}
+        annotations:
+          summary: MobilityOps database readiness is failing
+      - alert: MobilityOpsHighErrorRate
+        expr: sum(rate(mobilityops_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(mobilityops_http_requests_total[5m])), 0.001) > 0.05
+        for: 10m
+        labels: {severity: warning}
+        annotations:
+          summary: More than 5% of MobilityOps requests return 5xx
+      - alert: MobilityOpsHighLatency
+        expr: histogram_quantile(0.95, sum by (le) (rate(mobilityops_http_request_duration_seconds_bucket[5m]))) > 2
+        for: 10m
+        labels: {severity: warning}
+        annotations:
+          summary: MobilityOps p95 API latency exceeds two seconds
+      - alert: MobilityOpsOutboxBacklog
+        expr: sum(mobilityops_outbox_events{status=~"pending|delivering",scenario="operational"}) > 10
+        for: 10m
+        labels: {severity: warning}
+        annotations:
+          summary: MobilityOps operational outbox backlog exceeds ten events
+      - alert: MobilityOpsOutboxFailure
+        expr: sum(mobilityops_outbox_events{status="failed",scenario="operational"}) > 0
+        for: 2m
+        labels: {severity: critical}
+        annotations:
+          summary: MobilityOps has a real failed outbox event
@@ -0,0 +1,19 @@
+{
+  "annotations": {"list": []},
+  "editable": false,
+  "panels": [
+    {"type":"stat","title":"API target","datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"up{job=\"mobilityops-api\"}"}],"gridPos":{"h":6,"w":6,"x":0,"y":0}},
+    {"type":"stat","title":"Database ready","datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"mobilityops_database_ready"}],"gridPos":{"h":6,"w":6,"x":6,"y":0}},
+    {"type":"stat","title":"Operational outbox backlog","datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(mobilityops_outbox_events{status=~\"pending|delivering\",scenario=\"operational\"})"}],"gridPos":{"h":6,"w":6,"x":12,"y":0}},
+    {"type":"stat","title":"Operational outbox failures","datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(mobilityops_outbox_events{status=\"failed\",scenario=\"operational\"})"}],"gridPos":{"h":6,"w":6,"x":18,"y":0}},
+    {"type":"timeseries","title":"Requests per second","datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum by (status) (rate(mobilityops_http_requests_total[5m]))","legendFormat":"{{status}}"}],"gridPos":{"h":9,"w":12,"x":0,"y":6}},
+    {"type":"timeseries","title":"API latency percentiles","datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum by (le) (rate(mobilityops_http_request_duration_seconds_bucket[5m])))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum by (le) (rate(mobilityops_http_request_duration_seconds_bucket[5m])))","legendFormat":"p95"}],"gridPos":{"h":9,"w":12,"x":12,"y":6}}
+  ],
+  "schemaVersion": 41,
+  "tags": ["mobilityops"],
+  "templating": {"list": []},
+  "time": {"from":"now-6h","to":"now"},
+  "title": "MobilityOps operational overview",
+  "uid": "mobilityops-overview",
+  "version": 1
+}
@@ -0,0 +1,9 @@
+apiVersion: 1
+providers:
+  - name: MobilityOps
+    folder: MobilityOps
+    type: file
+    disableDeletion: true
+    editable: false
+    options:
+      path: /var/lib/grafana/dashboards
@@ -0,0 +1,9 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
@@ -0,0 +1,12 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+scrape_configs:
+  - job_name: mobilityops-api
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["api:8000"]