From b00d33af112d5bd1b74389ab1c1e9fdfcc99126e Mon Sep 17 00:00:00 2001
From: NuklearRabbit <145918611+NuklearRabbit@users.noreply.github.com>
Date: Mon, 10 Aug 2026 16:15:23 +0200
Subject: [PATCH] M26: harden clean observability acceptance

---
 PROJECT_STATE.md                         | 17 +++++++++++++++++
 backend/app/api/routers/observability.py | 10 ++++++++--
 backend/app/core/observability.py        |  2 +-
 backend/app/main.py                      |  8 ++++----
 4 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md
index f1119f7..6070657 100644
--- a/PROJECT_STATE.md
+++ b/PROJECT_STATE.md
@@ -2594,3 +2594,20 @@ evidence yet."
   prior Starlette/httpx warning is confirmed absent in the rebuilt environment.
 - Exact next action: run complete clean acceptance, push all five milestone commits,
   create a verified live backup, redeploy and execute live browser acceptance.
+
+## M26 — clean acceptance hardening (2026-08-10)
+
+- The clean full-suite run exposed and fixed two observability regressions that targeted
+  runs against a stale image had missed: readiness now returns immediately with 503 only
+  on database failure and reaches the healthy 200 branch correctly; Prometheus outbox
+  series now use the documented label order.
+- All synthetic/operational and pending/delivering/succeeded/failed outbox combinations
+  are initialized to zero before persisted counts are applied. Fresh installs, restores
+  and demo resets therefore produce stable zero-valued metrics instead of misleading
+  `no data` panels and absent alert inputs.
+- Evidence from the final rebuilt isolated stack: complete backend **239 passed with zero
+  warnings**, ruff passed and mypy passed across 58 source files. Frontend TypeScript
+  lint and production build pass; full and production-only npm audits both report zero
+  vulnerabilities.
+- Exact next action: commit/push, take a verified pre-deployment backup, redeploy to
+  Unraid, then run migration, health, browser and full Playwright acceptance.
diff --git a/backend/app/api/routers/observability.py b/backend/app/api/routers/observability.py
index 195207f..35be438 100644
--- a/backend/app/api/routers/observability.py
+++ b/backend/app/api/routers/observability.py
@@ -10,7 +10,7 @@ from sqlalchemy import func, select
 from app.core.config import get_settings
 from app.core.db import SessionLocal
 from app.core.observability import OUTBOX_EVENTS
-from app.models.outbox import DEMO_SCENARIO_ERROR_CODE, OutboxEvent
+from app.models.outbox import DELIVERY_STATUSES, DEMO_SCENARIO_ERROR_CODE, OutboxEvent
 
 router = APIRouter(tags=["observability"])
 settings = get_settings()
@@ -26,8 +26,14 @@ def _refresh_database_metrics() -> None:
             ).group_by(OutboxEvent.delivery_status, "demo")
         ).all()
     OUTBOX_EVENTS.clear()
+    # Keep every time series present even when a state currently contains no rows.
+    # Stable zero-valued series make dashboards and alerts deterministic after resets,
+    # restores and fresh installations instead of turning "zero" into "no data".
+    for scenario in ("synthetic", "operational"):
+        for status in DELIVERY_STATUSES:
+            OUTBOX_EVENTS.labels(scenario, status).set(0)
     for status, is_demo, count in rows:
-        OUTBOX_EVENTS.labels(str(status), "synthetic" if is_demo else "operational").set(count)
+        OUTBOX_EVENTS.labels("synthetic" if is_demo else "operational", str(status)).set(count)
 
 
 @router.get("/metrics", include_in_schema=False)
diff --git a/backend/app/core/observability.py b/backend/app/core/observability.py
index 8409615..0229cab 100644
--- a/backend/app/core/observability.py
+++ b/backend/app/core/observability.py
@@ -30,7 +30,7 @@ HTTP_IN_PROGRESS = Gauge(
 OUTBOX_EVENTS = Gauge(
     "mobilityops_outbox_events",
     "Persisted outbox events by state and scenario type.",
-    ("status", "scenario"),
+    ("scenario", "status"),
 )
 DATABASE_READY = Gauge(
     "mobilityops_database_ready",
diff --git a/backend/app/main.py b/backend/app/main.py
index e880b35..5f6484a 100644
--- a/backend/app/main.py
+++ b/backend/app/main.py
@@ -148,11 +148,11 @@ def readiness() -> JSONResponse:
             db.execute(text("SELECT 1"))
     except Exception:  # noqa: BLE001 -- readiness must convert infrastructure errors to 503
         DATABASE_READY.set(0)
+        return JSONResponse(
+            status_code=503,
+            content={"status": "not_ready", "service": "mobilityops-api", "database": "down"},
+        )
     DATABASE_READY.set(1)
-    return JSONResponse(
-        status_code=503,
-        content={"status": "not_ready", "service": "mobilityops-api", "database": "down"},
-    )
     return JSONResponse(content={"status": "ready", "service": "mobilityops-api", "database": "up"})