M54: harden operations and demo resilience

backend/tests/test_dispatcher.py

@@ -4,6 +4,7 @@ import uuid
 from datetime import UTC, datetime, timedelta
 from types import SimpleNamespace
 
+import pytest
 from sqlalchemy import select
 
 from app.core.config import get_settings
@@ -65,12 +66,17 @@ def test_claim_marks_events_delivering():
 
 def test_deliver_one_success(monkeypatch):
     event_id = _make_pending_event("MO-002")
+    execution_id = "n8n-execution-123"
     dispatcher._claim_due_events()
 
     def fake_post(url, json, headers, timeout):
         return SimpleNamespace(
             raise_for_status=lambda: None,
-            json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
+            json=lambda: {
+                "ok": True,
+                "event_id": str(event_id),
+                "result": {"execution_id": execution_id},
+            },
         )
 
     monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
@@ -79,7 +85,7 @@ def test_deliver_one_success(monkeypatch):
     event = _get_event(event_id)
     assert event.delivery_status == "succeeded"
     assert event.attempts == 1
-    assert event.external_run_id == str(event_id)
+    assert event.external_run_id == execution_id
     assert event.last_error is None
     assert event.last_error_code is None
 
@@ -129,6 +135,106 @@ def test_deliver_one_treats_empty_2xx_body_as_failure(monkeypatch):
     assert event.last_error_code == "malformedResponse"
 
 
+def test_deliver_one_rejects_json_object_without_explicit_success_ack(monkeypatch):
+    event_id = _make_pending_event("MO-010")
+    dispatcher._claim_due_events()
+
+    def fake_post(url, json, headers, timeout):
+        return SimpleNamespace(
+            raise_for_status=lambda: None,
+            json=lambda: {},
+            status_code=200,
+        )
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+    dispatcher._deliver_one(event_id)
+
+    event = _get_event(event_id)
+    assert event.delivery_status == "pending"
+    assert event.attempts == 1
+    assert event.last_error_code == "malformedResponse"
+    assert event.external_run_id is None
+
+
+@pytest.mark.parametrize("result", [{}, {"execution_id": 123}, {"execution_id": " "}])
+def test_deliver_one_rejects_ack_without_valid_execution_id(monkeypatch, result):
+    event_id = _make_pending_event("MO-EXEC-ID")
+    dispatcher._claim_due_events()
+
+    def fake_post(url, json, headers, timeout):
+        return SimpleNamespace(
+            raise_for_status=lambda: None,
+            json=lambda: {"ok": True, "event_id": str(event_id), "result": result},
+            status_code=200,
+        )
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+    dispatcher._deliver_one(event_id)
+    event = _get_event(event_id)
+    assert event.delivery_status == "pending"
+    assert event.last_error_code == "malformedResponse"
+    assert event.external_run_id is None
+
+
+def test_deliver_one_rejects_success_ack_for_a_different_event(monkeypatch):
+    event_id = _make_pending_event("MO-011")
+    dispatcher._claim_due_events()
+    other_event_id = uuid.uuid4()
+
+    def fake_post(url, json, headers, timeout):
+        return SimpleNamespace(
+            raise_for_status=lambda: None,
+            json=lambda: {"ok": True, "event_id": str(other_event_id)},
+            status_code=200,
+        )
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+    dispatcher._deliver_one(event_id)
+
+    event = _get_event(event_id)
+    assert event.delivery_status == "pending"
+    assert event.attempts == 1
+    assert event.last_error_code == "mismatchedEventId"
+    assert event.external_run_id is None
+
+
+def test_late_delivery_outcome_cannot_overwrite_a_newer_lease(monkeypatch):
+    event_id = _make_pending_event("MO-LEASE-RACE")
+    dispatcher._claim_due_events()
+
+    def fake_post(url, json, headers, timeout):
+        # Simulate lease expiry/reclaim while the original worker is still in network
+        # I/O. The later claimant owns a different token and its state must win.
+        with SessionLocal() as db:
+            event = db.scalar(
+                select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update()
+            )
+            event.payload_json = {
+                **event.payload_json,
+                "_delivery_claim_token": "newer-worker-token",
+            }
+            event.next_attempt_at = datetime.now(UTC) + timedelta(minutes=1)
+            db.commit()
+        return SimpleNamespace(
+            raise_for_status=lambda: None,
+            json=lambda: {
+                "ok": True,
+                "event_id": str(event_id),
+                "result": {"execution_id": "stale-worker-execution"},
+            },
+            status_code=200,
+        )
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+    dispatcher._deliver_one(event_id)
+
+    event = _get_event(event_id)
+    assert event.delivery_status == "delivering"
+    assert event.attempts == 0
+    assert event.external_run_id is None
+    assert event.payload_json["_delivery_claim_token"] == "newer-worker-token"
+
+
 def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
     event_id = _make_pending_event("MO-004")
     settings = get_settings()
@@ -139,7 +245,6 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
     monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
 
     for _ in range(settings.n8n_max_attempts):
-        dispatcher._claim_due_events()
         db = SessionLocal()
         try:
             event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
@@ -147,6 +252,8 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
             db.commit()
         finally:
             db.close()
+        claimed = dispatcher._claim_due_events(batch_size=1000)
+        assert event_id in claimed
         dispatcher._deliver_one(event_id)
 
     event = _get_event(event_id)
@@ -259,7 +366,11 @@ def test_run_dispatch_cycle_recovers_a_stale_lease_before_claiming(monkeypatch):
     def fake_post(url, json, headers, timeout):
         return SimpleNamespace(
             raise_for_status=lambda: None,
-            json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
+            json=lambda: {
+                "ok": True,
+                "event_id": str(event_id),
+                "result": {"execution_id": "n8n-recovered-execution"},
+            },
         )
 
     monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
@@ -275,7 +386,11 @@ def test_run_dispatch_cycle_end_to_end(monkeypatch):
     def fake_post(url, json, headers, timeout):
         return SimpleNamespace(
             raise_for_status=lambda: None,
-            json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
+            json=lambda: {
+                "ok": True,
+                "event_id": str(event_id),
+                "result": {"execution_id": "n8n-cycle-execution"},
+            },
         )
 
     monkeypatch.setattr(dispatcher.httpx, "post", fake_post)