M54: harden operations and demo resilience

backend/app/services/dispatcher.py

@@ -77,6 +77,14 @@ def _claim_due_events(batch_size: int = 5) -> list[uuid.UUID]:
         for row in rows:
             row.delivery_status = "delivering"
             row.next_attempt_at = lease_deadline
+            # The token is stored inside the internal payload (the wire envelope below
+            # explicitly selects only contract fields). It lets the outcome transaction
+            # prove that this is still the same lease after network I/O. A stale worker
+            # must never overwrite a later reclaim/retry or an idempotent callback.
+            row.payload_json = {
+                **row.payload_json,
+                "_delivery_claim_token": str(uuid.uuid4()),
+            }
         db.commit()
         return claimed_ids
     finally:
@@ -109,6 +117,9 @@ def _deliver_one(event_id: uuid.UUID) -> None:
             wire_event = None
             payload_error = f"Malformed outbox payload, missing key {exc}"
         attempts = event.attempts
+        claim_token = event.payload_json.get("_delivery_claim_token")
+        if event.delivery_status != "delivering" or not isinstance(claim_token, str):
+            return
     finally:
         db.close()
 
@@ -130,9 +141,33 @@ def _deliver_one(event_id: uuid.UUID) -> None:
             except ValueError:
                 body = None
             if isinstance(body, dict):
-                success = bool(body.get("ok", True))
-                error = None if success else f"n8n reported failure: {body}"
-                error_code = None if success else "remoteReportedFailure"
+                acknowledged = body.get("ok") is True
+                response_event_id = body.get("event_id")
+                event_id_matches = response_event_id == str(event_id)
+                result = body.get("result")
+                execution_id = result.get("execution_id") if isinstance(result, dict) else None
+                execution_id_valid = isinstance(execution_id, str) and bool(execution_id.strip())
+                success = acknowledged and event_id_matches and execution_id_valid
+                if success:
+                    error = None
+                    error_code = None
+                elif not acknowledged:
+                    error = (
+                        "n8n response did not explicitly acknowledge the event with ok=true: "
+                        f"{body}"
+                    )
+                    error_code = (
+                        "remoteReportedFailure" if body.get("ok") is False else "malformedResponse"
+                    )
+                elif not event_id_matches:
+                    error = (
+                        "n8n acknowledged a different event ID "
+                        f"(expected {event_id}, received {response_event_id!r})"
+                    )
+                    error_code = "mismatchedEventId"
+                else:
+                    error = "n8n response omitted a valid result.execution_id"
+                    error_code = "malformedResponse"
             else:
                 # A 2xx status with a non-object (or unparsable) body means the workflow
                 # itself errored before its "Respond to Webhook" node ran -- n8n's default
@@ -152,16 +187,35 @@ def _deliver_one(event_id: uuid.UUID) -> None:
 
     db = SessionLocal()
     try:
-        event = db.get(OutboxEvent, event_id)
+        event = db.scalar(
+            select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update()
+        )
         if event is None:
             return
+        if (
+            event.delivery_status != "delivering"
+            or event.attempts != attempts
+            or event.payload_json.get("_delivery_claim_token") != claim_token
+        ):
+            logger.info(
+                "Ignoring stale delivery outcome for event %s because lease ownership changed",
+                event_id,
+            )
+            return
         event.attempts = attempts + 1
+        event.payload_json = {
+            key: value
+            for key, value in event.payload_json.items()
+            if key != "_delivery_claim_token"
+        }
         if success:
             event.delivery_status = "succeeded"
             event.last_error = None
             event.last_error_code = None
             event.next_attempt_at = None
-            event.external_run_id = str((body or {}).get("event_id", event_id))
+            result = (body or {}).get("result")
+            execution_id = result.get("execution_id") if isinstance(result, dict) else None
+            event.external_run_id = execution_id if isinstance(execution_id, str) else None
         else:
             event.last_error = (error or "delivery failed")[:2000]
             event.last_error_code = error_code or "unknownError"