M54: harden operations and demo resilience

backend/app/services/integration_status.py

@@ -7,7 +7,7 @@ from datetime import UTC, datetime, timedelta
 from typing import Any, Literal
 
 import httpx
-from sqlalchemy import Row, func, select
+from sqlalchemy import Row, func, or_, select
 from sqlalchemy.orm import Session
 
 from app.core.config import get_settings
@@ -93,7 +93,10 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
     latest_failure_at = db.scalar(
         select(func.max(OutboxEvent.updated_at)).where(
             OutboxEvent.delivery_status == "failed",
-            OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
+            or_(
+                OutboxEvent.last_error_code.is_(None),
+                OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
+            ),
         )
     )
     latest_demo_scenario_at = db.scalar(
@@ -106,11 +109,17 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
     state: Literal["disabled", "unavailable", "degraded", "operational", "no_evidence"]
     if not settings.n8n_dispatch_enabled:
         state = "disabled"
+    elif not settings.n8n_webhook_url:
+        # Persisted history does not make a currently unconfigured dispatcher green.
+        state = "unavailable"
     elif unexpected_failed > 0 and succeeded == 0:
         state = "unavailable"
     elif unexpected_failed > 0:
         state = "degraded"
-    elif succeeded > 0 or pending > 0 or delivering > 0:
+    # Queued/in-flight work proves only that MobilityOps has work for the dispatcher;
+    # it does not prove that n8n has ever accepted a delivery.  A green state requires
+    # at least one persisted successful round trip.
+    elif succeeded > 0:
         state = "operational"
     else:
         state = "no_evidence"
@@ -128,8 +137,14 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
     # it finishes uploading procedures to RAGcore (app/api/routers/integrations.py::
     # procedures_sync_result), the same "the workflow's own callback is the evidence"
     # pattern the scheduled scan and error handler already use below.
-    latest_procedure_sync_at = db.scalar(
-        select(func.max(AuditEvent.occurred_at)).where(AuditEvent.action == "n8n_procedures_synced")
+    latest_procedure_sync_row = db.execute(
+        select(AuditEvent.occurred_at, AuditEvent.after_json, AuditEvent.metadata_json)
+        .where(AuditEvent.action == "n8n_procedures_synced")
+        .order_by(AuditEvent.occurred_at.desc())
+        .limit(1)
+    ).first()
+    latest_procedure_sync_at = (
+        latest_procedure_sync_row[0] if latest_procedure_sync_row is not None else None
     )
 
     # Error handler evidence: registrations posted by the "Fleet Ops — Workflow Error
@@ -198,6 +213,24 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
             seen_at = failure_signal[0]
             last_status = "failed"
             execution_id = failure_signal[1]
+        if name == "Fleet Ops — RAGcore Procedure Sync" and latest_procedure_sync_row:
+            sync_at, sync_result, sync_metadata = latest_procedure_sync_row
+            synced = (sync_result or {}).get("synced", 0)
+            failed_syncs = (sync_result or {}).get("failed", 0)
+            # The result callback is authoritative for corpus delivery. A generic
+            # succeeded heartbeat cannot turn a zero/partial upload green.
+            sync_result_failed = (
+                not isinstance(synced, int)
+                or not isinstance(failed_syncs, int)
+                or synced <= 0
+                or failed_syncs > 0
+            )
+            if sync_result_failed:
+                last_status = "failed"
+            if seen_at is None or sync_at > seen_at:
+                seen_at = sync_at
+            if sync_result_failed:
+                execution_id = (sync_metadata or {}).get("execution_id")
         workflow_state: Literal["no_evidence", "healthy", "stale", "failed"]
         if seen_at is None:
             workflow_state = "no_evidence"
@@ -265,16 +298,20 @@ def derive_mcp_hub_status(db: Session) -> McpHubIntegrationStatus:
     last_client = latest_call_row[1] if latest_call_row else None
     last_tool = (latest_call_row[2] or {}).get("tool") if latest_call_row else None
 
+    hub_reachable = _check_hub_reachable()
+
     state: Literal["not_configured", "no_evidence", "operational"]
     if not settings.mcp_hub_registration_enabled:
         state = "not_configured"
-    elif total_calls > 0:
+    elif total_calls > 0 and hub_reachable is not False:
         state = "operational"
     else:
+        # Historical tool calls remain useful telemetry, but cannot support a current
+        # operational claim when the configured Hub health endpoint is unreachable.
+        # ``hub_reachable`` stays available separately so consumers can distinguish
+        # this from a provider that simply has no call evidence yet.
         state = "no_evidence"
 
-    hub_reachable = _check_hub_reachable()
-
     return McpHubIntegrationStatus(
         registration_enabled=settings.mcp_hub_registration_enabled,
         state=state,