M54: harden operations and demo resilience

backend/app/api/routers/integrations.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
+import hashlib
 import hmac
 import uuid
 from datetime import UTC, datetime
 from pathlib import Path
 
 from fastapi import APIRouter, Depends, Header
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
 from app.api.deps import get_db
@@ -51,6 +52,13 @@ def _require_service_token(service_token: str) -> None:
         raise AppError("UNAUTHORIZED_SERVICE", "Invalid service token.", status_code=401)
 
 
+def _lock_idempotency_key(db: Session, namespace: str, key: str) -> None:
+    """Serialize callback check+insert by a stable, transaction-scoped key."""
+    digest = hashlib.sha256(f"{namespace}:{key}".encode()).digest()
+    lock_id = int.from_bytes(digest[:8], byteorder="big", signed=True)
+    db.scalar(select(func.pg_advisory_xact_lock(lock_id)))
+
+
 @router.post("/heartbeat", response_model=N8nHeartbeatResult)
 def workflow_heartbeat(
     body: N8nHeartbeatIn,
@@ -61,6 +69,7 @@ def workflow_heartbeat(
     _require_service_token(service_token)
     if body.workflow_name not in _CANONICAL_WORKFLOW_NAMES:
         raise AppError("UNKNOWN_WORKFLOW", "Unknown Fleet Ops workflow.", status_code=422)
+    _lock_idempotency_key(db, "n8n_workflow_heartbeat", f"{body.execution_id}:{body.status}")
     already_recorded = (
         db.scalar(
             select(AuditEvent.id).where(
@@ -109,9 +118,29 @@ def return_callback(
             "INVALID_IDEMPOTENCY_KEY", "Idempotency-Key must be the event's UUID.", status_code=422
         ) from exc
 
-    event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
+    event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update())
     if event is None:
         raise AppError("EVENT_NOT_FOUND", "No outbox event matches this event ID.", status_code=404)
+    if body.event_id != event_id:
+        raise AppError(
+            "CALLBACK_EVENT_MISMATCH",
+            "Callback event_id does not match Idempotency-Key.",
+            status_code=409,
+        )
+    try:
+        expected_correlation_id = uuid.UUID(str(event.payload_json["correlation_id"]))
+    except (KeyError, TypeError, ValueError) as exc:
+        raise AppError(
+            "INVALID_EVENT_CORRELATION",
+            "The stored outbox event has no valid correlation ID.",
+            status_code=409,
+        ) from exc
+    if body.correlation_id != expected_correlation_id:
+        raise AppError(
+            "CALLBACK_CORRELATION_MISMATCH",
+            "Callback correlation_id does not match the outbox event.",
+            status_code=409,
+        )
 
     # Idempotent by event ID: n8n or our own dispatcher may redeliver the same event
     # (e.g. a lost response after a timeout), so this callback must not double-record.
@@ -131,7 +160,7 @@ def return_callback(
             actor_label="n8n",
             action="n8n_return_followup_recorded",
             entity_type="booking",
-            correlation_id=body.correlation_id,
+            correlation_id=expected_correlation_id,
             after={"follow_up": body.follow_up, "summary": body.summary},
             metadata={"event_id": str(event_id)},
         )
@@ -170,6 +199,7 @@ def workflow_error(
     other Fleet Ops n8n workflow. Idempotent on execution_id: n8n may redeliver the same
     error report (e.g. after a timed-out response), so this must not double-record."""
     _require_service_token(service_token)
+    _lock_idempotency_key(db, "n8n_workflow_failure", body.execution_id)
 
     already_recorded = (
         db.scalar(
@@ -181,19 +211,13 @@ def workflow_error(
         is not None
     )
     if not already_recorded:
-        correlation_id: uuid.UUID | None = None
-        if body.correlation_id:
-            try:
-                correlation_id = uuid.UUID(body.correlation_id)
-            except ValueError:
-                correlation_id = None
         record_audit_event(
             db,
             actor_type="service",
             actor_label="n8n error handler",
             action="n8n_workflow_failure_registered",
             entity_type="automation",
-            correlation_id=correlation_id,
+            correlation_id=body.correlation_id,
             after={
                 "workflow_id": body.workflow_id,
                 "workflow_name": body.workflow_name,
@@ -248,6 +272,7 @@ def procedures_sync_result(
     RAGcore Procedure Sync" workflow once it finishes uploading procedures to RAGcore.
     Idempotent on execution_id, matching the workflow-error and return-callback pattern."""
     _require_service_token(service_token)
+    _lock_idempotency_key(db, "n8n_procedure_sync", body.execution_id)
 
     already_recorded = (
         db.scalar(