M22: implement operational observability

2026-08-10 15:42:26 +02:00
parent c3f1cfc699
commit 689e499634
17 changed files with 428 additions and 9 deletions
@@ -0,0 +1,39 @@
+groups:
+  - name: mobilityops
+    rules:
+      - alert: MobilityOpsApiDown
+        expr: up{job="mobilityops-api"} == 0
+        for: 2m
+        labels: {severity: critical}
+        annotations:
+          summary: MobilityOps API metrics target is down
+      - alert: MobilityOpsDatabaseNotReady
+        expr: mobilityops_database_ready == 0
+        for: 2m
+        labels: {severity: critical}
+        annotations:
+          summary: MobilityOps database readiness is failing
+      - alert: MobilityOpsHighErrorRate
+        expr: sum(rate(mobilityops_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(mobilityops_http_requests_total[5m])), 0.001) > 0.05
+        for: 10m
+        labels: {severity: warning}
+        annotations:
+          summary: More than 5% of MobilityOps requests return 5xx
+      - alert: MobilityOpsHighLatency
+        expr: histogram_quantile(0.95, sum by (le) (rate(mobilityops_http_request_duration_seconds_bucket[5m]))) > 2
+        for: 10m
+        labels: {severity: warning}
+        annotations:
+          summary: MobilityOps p95 API latency exceeds two seconds
+      - alert: MobilityOpsOutboxBacklog
+        expr: sum(mobilityops_outbox_events{status=~"pending|delivering",scenario="operational"}) > 10
+        for: 10m
+        labels: {severity: warning}
+        annotations:
+          summary: MobilityOps operational outbox backlog exceeds ten events
+      - alert: MobilityOpsOutboxFailure
+        expr: sum(mobilityops_outbox_events{status="failed",scenario="operational"}) > 0
+        for: 2m
+        labels: {severity: critical}
+        annotations:
+          summary: MobilityOps has a real failed outbox event