M14: add n8n execution health telemetry

backend/app/services/integration_status.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from datetime import UTC, datetime, timedelta
 from typing import Literal
 
 import httpx
@@ -26,6 +27,10 @@ _CANONICAL_WORKFLOWS = (
     "Fleet Ops — RAGcore Procedure Sync",
     "Fleet Ops — Workflow Error Handler",
 )
+_STALE_AFTER = {
+    "Fleet Ops — Scheduled Data Quality Scan": timedelta(hours=2, minutes=30),
+    "Fleet Ops — RAGcore Procedure Sync": timedelta(hours=30),
+}
 
 
 def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
@@ -129,20 +134,82 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
         (latest_failure_row[1] or {}).get("workflow_name") if latest_failure_row else None
     )
 
-    evidence_by_workflow = {
+    legacy_evidence_by_workflow = {
         "Fleet Ops — Vehicle Return Orchestration": latest_success_at,
         "Fleet Ops — Scheduled Data Quality Scan": latest_scan_at,
         "Fleet Ops — RAGcore Procedure Sync": latest_procedure_sync_at,
         "Fleet Ops — Workflow Error Handler": latest_handler_failure_at,
     }
-    workflows = [
-        N8nWorkflowEvidence(
-            name=name,
-            built=True,
-            last_seen_at=evidence_by_workflow[name],
+    heartbeat_by_workflow: dict[str, tuple[datetime, str, str | None]] = {}
+    heartbeat_rows = db.execute(
+        select(AuditEvent.occurred_at, AuditEvent.after_json, AuditEvent.metadata_json)
+        .where(AuditEvent.action == "n8n_workflow_heartbeat")
+        .order_by(AuditEvent.occurred_at.desc())
+    ).all()
+    for occurred_at, after, metadata in heartbeat_rows:
+        workflow_name = (after or {}).get("workflow_name")
+        if workflow_name in _CANONICAL_WORKFLOWS and workflow_name not in heartbeat_by_workflow:
+            heartbeat_by_workflow[workflow_name] = (
+                occurred_at,
+                (after or {}).get("status", "succeeded"),
+                (metadata or {}).get("execution_id"),
+            )
+
+    failure_by_workflow: dict[str, tuple[datetime, str | None]] = {}
+    failure_rows = db.execute(
+        select(AuditEvent.occurred_at, AuditEvent.after_json, AuditEvent.metadata_json)
+        .where(AuditEvent.action == "n8n_workflow_failure_registered")
+        .order_by(AuditEvent.occurred_at.desc())
+    ).all()
+    for occurred_at, after, metadata in failure_rows:
+        workflow_name = (after or {}).get("workflow_name")
+        if workflow_name in _CANONICAL_WORKFLOWS and workflow_name not in failure_by_workflow:
+            failure_by_workflow[workflow_name] = (
+                occurred_at,
+                (metadata or {}).get("execution_id"),
+            )
+
+    now = datetime.now(UTC)
+    workflows: list[N8nWorkflowEvidence] = []
+    for name in _CANONICAL_WORKFLOWS:
+        legacy_seen = legacy_evidence_by_workflow[name]
+        heartbeat = heartbeat_by_workflow.get(name)
+        failure_signal = failure_by_workflow.get(name)
+        seen_at = heartbeat[0] if heartbeat else legacy_seen
+        last_status: Literal["succeeded", "failed"] | None = (
+            "succeeded" if seen_at is not None else None
         )
-        for name in _CANONICAL_WORKFLOWS
-    ]
+        execution_id = heartbeat[2] if heartbeat else None
+        if heartbeat and heartbeat[1] == "failed":
+            last_status = "failed"
+        if failure_signal and (seen_at is None or failure_signal[0] > seen_at):
+            seen_at = failure_signal[0]
+            last_status = "failed"
+            execution_id = failure_signal[1]
+        workflow_state: Literal["no_evidence", "healthy", "stale", "failed"]
+        if seen_at is None:
+            workflow_state = "no_evidence"
+        elif last_status == "failed":
+            workflow_state = "failed"
+        elif name in _STALE_AFTER and now - seen_at > _STALE_AFTER[name]:
+            workflow_state = "stale"
+        else:
+            workflow_state = "healthy"
+        workflows.append(
+            N8nWorkflowEvidence(
+                name=name,
+                built=True,
+                last_seen_at=seen_at,
+                state=workflow_state,
+                last_status=last_status,
+                last_execution_id=execution_id,
+            )
+        )
+
+    if state in {"operational", "no_evidence"} and any(
+        workflow.state in {"failed", "stale"} for workflow in workflows
+    ):
+        state = "degraded"
 
     return N8nIntegrationStatus(
         configured=bool(settings.n8n_webhook_url),