fix(automation): recover stale outbox delivering leases

_claim_due_events flipped rows to 'delivering' and committed before the
HTTP call; if the process died between that commit and the outcome-
recording transaction, the row stayed 'delivering' forever with no reclaim
path -- a real gap, not previously documented as an accepted limitation.

Give each claim a lease deadline (reusing next_attempt_at, since it's only
otherwise meaningful for pending-status backoff scheduling) and sweep
expired leases back to pending at the start of every dispatch cycle, before
claiming new work. attempts is preserved so the count still reflects true
history. Only leases past their deadline are touched, so a still-alive
worker mid-delivery is never disturbed or double-processed.

backend/app/services/dispatcher.py

@@ -22,8 +22,42 @@ def _backoff_seconds(attempts: int) -> int:
     return min(2**attempts, 60)
 
 
+def _reclaim_stale_deliveries(batch_size: int = 10) -> int:
+    """Recover events stuck in 'delivering' because the process that claimed them died
+    before recording an outcome. Only leases whose deadline has passed are touched, so an
+    in-flight delivery from a still-alive worker is never disturbed or double-processed;
+    `attempts` is preserved so the count reflects true history."""
+    db = SessionLocal()
+    try:
+        now = datetime.now(UTC)
+        rows = db.scalars(
+            select(OutboxEvent)
+            .where(
+                OutboxEvent.delivery_status == "delivering",
+                OutboxEvent.next_attempt_at.is_not(None),
+                OutboxEvent.next_attempt_at <= now,
+            )
+            .limit(batch_size)
+            .with_for_update(skip_locked=True)
+        ).all()
+        for row in rows:
+            row.delivery_status = "pending"
+            row.next_attempt_at = None
+            row.last_error = (
+                "Recovered from a stale 'delivering' lease "
+                f"(no outcome recorded within {settings.n8n_delivery_lease_seconds:.0f}s; "
+                f"the process likely crashed mid-delivery). attempts preserved at {row.attempts}."
+            )[:2000]
+        db.commit()
+        return len(rows)
+    finally:
+        db.close()
+
+
 def _claim_due_events(batch_size: int = 5) -> list[uuid.UUID]:
-    """Claim a batch of due events with a short-lived transaction (no network I/O held open)."""
+    """Claim a batch of due events with a short-lived transaction (no network I/O held open).
+    Each claimed row gets a lease deadline (next_attempt_at) so a crash between this claim
+    and the outcome being recorded is recoverable by _reclaim_stale_deliveries."""
     db = SessionLocal()
     try:
         now = datetime.now(UTC)
@@ -38,8 +72,10 @@ def _claim_due_events(batch_size: int = 5) -> list[uuid.UUID]:
             .with_for_update(skip_locked=True)
         ).all()
         claimed_ids = [row.event_id for row in rows]
+        lease_deadline = now + timedelta(seconds=settings.n8n_delivery_lease_seconds)
         for row in rows:
             row.delivery_status = "delivering"
+            row.next_attempt_at = lease_deadline
         db.commit()
         return claimed_ids
     finally:
@@ -120,7 +156,8 @@ def _deliver_one(event_id: uuid.UUID) -> None:
 
 
 def run_dispatch_cycle() -> int:
-    """Run one claim+deliver cycle. Returns the number of events processed."""
+    """Run one reclaim+claim+deliver cycle. Returns the number of events processed."""
+    _reclaim_stale_deliveries()
     claimed = _claim_due_events()
     for event_id in claimed:
         _deliver_one(event_id)