M4: implement n8n automation

Outbox dispatcher (background thread, FOR UPDATE SKIP LOCKED claim, exponential backoff, no transaction held during HTTP I/O). n8n callback endpoint with shared-secret auth and idempotency by event ID. Automation nav + UI with manual retry. 49 backend tests passing, ruff clean. Fixed a crash-on-redelivery bug in seeded outbox payloads and made the dispatcher defensive against malformed payloads. Verified the full live round trip against a real n8n instance: return -> outbox -> dispatcher -> n8n workflow -> callback -> succeeded, including the S5 failed-retry demo scenario.
2026-08-01 22:39:25 +02:00
parent a7cbeaae3b
commit 59d663a43a
17 changed files with 814 additions and 7 deletions
@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from types import SimpleNamespace
+
+from sqlalchemy import select
+
+from app.core.config import get_settings
+from app.core.db import SessionLocal
+from app.models.booking import Booking
+from app.models.outbox import OutboxEvent
+from app.services import dispatcher
+
+
+def _make_pending_event(vehicle_ref: str) -> uuid.UUID:
+    db = SessionLocal()
+    try:
+        booking = db.scalar(select(Booking).where(Booking.status == "returned").limit(1))
+        event = OutboxEvent(
+            event_id=uuid.uuid4(),
+            event_type="vehicle.returned.v1",
+            aggregate_type="booking",
+            aggregate_id=booking.id,
+            payload_json={
+                "correlation_id": str(uuid.uuid4()),
+                "aggregate": {
+                    "type": "booking",
+                    "id": str(booking.id),
+                    "public_ref": booking.public_ref,
+                },
+                "data": {
+                    "vehicle_ref": vehicle_ref,
+                    "inspection_ref": "INSP-TEST",
+                    "resulting_vehicle_status": "cleaning",
+                    "attention_reasons": [],
+                },
+                "aggregate_ref": booking.public_ref,
+            },
+            occurred_at=datetime.now(UTC),
+            delivery_status="pending",
+            attempts=0,
+        )
+        db.add(event)
+        db.commit()
+        return event.event_id
+    finally:
+        db.close()
+
+
+def _get_event(event_id: uuid.UUID) -> OutboxEvent:
+    db = SessionLocal()
+    try:
+        return db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
+    finally:
+        db.close()
+
+
+def test_claim_marks_events_delivering():
+    event_id = _make_pending_event("MO-001")
+    claimed = dispatcher._claim_due_events()
+    assert event_id in claimed
+    assert _get_event(event_id).delivery_status == "delivering"
+
+
+def test_deliver_one_success(monkeypatch):
+    event_id = _make_pending_event("MO-002")
+    dispatcher._claim_due_events()
+
+    def fake_post(url, json, timeout):
+        return SimpleNamespace(
+            raise_for_status=lambda: None,
+            json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
+        )
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+    dispatcher._deliver_one(event_id)
+
+    event = _get_event(event_id)
+    assert event.delivery_status == "succeeded"
+    assert event.attempts == 1
+    assert event.external_run_id == str(event_id)
+    assert event.last_error is None
+
+
+def test_deliver_one_failure_schedules_retry(monkeypatch):
+    event_id = _make_pending_event("MO-003")
+    dispatcher._claim_due_events()
+
+    def fake_post(url, json, timeout):
+        raise dispatcher.httpx.ConnectError("simulated connection failure")
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+    dispatcher._deliver_one(event_id)
+
+    event = _get_event(event_id)
+    assert event.delivery_status == "pending"
+    assert event.attempts == 1
+    assert event.next_attempt_at is not None
+    assert "simulated connection failure" in event.last_error
+
+
+def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
+    event_id = _make_pending_event("MO-004")
+    settings = get_settings()
+
+    def fake_post(url, json, timeout):
+        raise dispatcher.httpx.ConnectError("still down")
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+
+    for _ in range(settings.n8n_max_attempts):
+        dispatcher._claim_due_events()
+        db = SessionLocal()
+        try:
+            event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
+            event.next_attempt_at = None
+            db.commit()
+        finally:
+            db.close()
+        dispatcher._deliver_one(event_id)
+
+    event = _get_event(event_id)
+    assert event.delivery_status == "failed"
+    assert event.attempts == settings.n8n_max_attempts
+
+
+def test_deliver_one_handles_malformed_payload_without_getting_stuck(monkeypatch):
+    # Regression test: seeded/legacy outbox rows may lack the full event envelope. Delivery
+    # must resolve the claimed "delivering" row to pending/failed, never leave it stuck.
+    db = SessionLocal()
+    try:
+        booking = db.scalar(select(Booking).limit(1))
+        event = OutboxEvent(
+            event_id=uuid.uuid4(),
+            event_type="vehicle.returned.v1",
+            aggregate_type="booking",
+            aggregate_id=booking.id,
+            payload_json={"aggregate_ref": booking.public_ref},  # missing correlation_id/etc.
+            occurred_at=datetime.now(UTC),
+            delivery_status="pending",
+            attempts=0,
+        )
+        db.add(event)
+        db.commit()
+        event_id = event.event_id
+    finally:
+        db.close()
+
+    def fake_post(url, json, timeout):
+        raise AssertionError("must not attempt delivery with a malformed payload")
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+
+    dispatcher._claim_due_events()
+    dispatcher._deliver_one(event_id)
+
+    event = _get_event(event_id)
+    assert event.delivery_status in ("pending", "failed")
+    assert event.attempts == 1
+    assert "Malformed outbox payload" in event.last_error
+
+
+def test_run_dispatch_cycle_end_to_end(monkeypatch):
+    event_id = _make_pending_event("MO-005")
+
+    def fake_post(url, json, timeout):
+        return SimpleNamespace(
+            raise_for_status=lambda: None,
+            json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
+        )
+
+    monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+    processed = dispatcher.run_dispatch_cycle()
+
+    assert processed >= 1
+    assert _get_event(event_id).delivery_status == "succeeded"
@@ -0,0 +1,84 @@
+import uuid
+
+from app.core.config import get_settings
+
+
+def _callback_headers(event_id: str, token: str | None = None):
+    settings = get_settings()
+    return {
+        "Idempotency-Key": event_id,
+        "X-Service-Token": token if token is not None else settings.n8n_callback_token,
+    }
+
+
+def test_callback_rejects_wrong_service_token(client):
+    response = client.post(
+        "/api/v1/integrations/n8n/return-callback",
+        json={"follow_up": "cleaning"},
+        headers=_callback_headers(str(uuid.uuid4()), token="wrong-token"),
+    )
+    assert response.status_code == 401
+
+
+def test_callback_unknown_event_returns_404(client):
+    response = client.post(
+        "/api/v1/integrations/n8n/return-callback",
+        json={"follow_up": "cleaning"},
+        headers=_callback_headers(str(uuid.uuid4())),
+    )
+    assert response.status_code == 404
+
+
+def test_callback_is_idempotent_by_event_id(client, ops_client):
+    from sqlalchemy import select
+
+    from app.core.db import SessionLocal
+    from app.models.booking import Booking
+    from app.models.outbox import OutboxEvent
+
+    db = SessionLocal()
+    try:
+        booking = db.scalar(select(Booking).limit(1))
+        event = OutboxEvent(
+            event_id=uuid.uuid4(),
+            event_type="vehicle.returned.v1",
+            aggregate_type="booking",
+            aggregate_id=booking.id,
+            payload_json={
+                "correlation_id": str(uuid.uuid4()),
+                "aggregate": {
+                    "type": "booking",
+                    "id": str(booking.id),
+                    "public_ref": booking.public_ref,
+                },
+                "data": {},
+                "aggregate_ref": booking.public_ref,
+            },
+            occurred_at=db.execute(select(Booking.starts_at).limit(1)).scalar(),
+            delivery_status="delivering",
+            attempts=1,
+        )
+        db.add(event)
+        db.commit()
+        event_id = str(event.event_id)
+    finally:
+        db.close()
+
+    first = client.post(
+        "/api/v1/integrations/n8n/return-callback",
+        json={"follow_up": "cleaning", "summary": "test"},
+        headers=_callback_headers(event_id),
+    )
+    second = client.post(
+        "/api/v1/integrations/n8n/return-callback",
+        json={"follow_up": "cleaning", "summary": "test"},
+        headers=_callback_headers(event_id),
+    )
+    assert first.status_code == 200
+    assert second.status_code == 200
+
+    audit_events = ops_client.get(
+        "/api/v1/audit", params={"action": "n8n_return_followup_recorded"}
+    ).json()
+    matching = [e for e in audit_events if e["metadata"]["event_id"] == event_id]
+    assert len(matching) == 1
@@ -0,0 +1,38 @@
+def test_list_workflows_requires_operations_manager(employee_client):
+    response = employee_client.get("/api/v1/workflows")
+    assert response.status_code == 403
+
+
+def test_list_workflows_includes_seeded_failed_run(ops_client):
+    response = ops_client.get("/api/v1/workflows", params={"status": "failed"})
+    assert response.status_code == 200
+    runs = response.json()
+    assert len(runs) >= 1
+    assert all(r["status"] == "failed" for r in runs)
+
+
+def test_retry_requires_failed_status(ops_client):
+    succeeded = ops_client.get("/api/v1/workflows", params={"status": "succeeded"}).json()
+    target = succeeded[0]["event_id"]
+    response = ops_client.post(f"/api/v1/workflows/{target}/retry")
+    assert response.status_code == 409
+    assert response.json()["error"]["code"] == "NOT_RETRYABLE"
+
+
+def test_retry_failed_run_moves_to_pending_and_audits(ops_client):
+    failed = ops_client.get("/api/v1/workflows", params={"status": "failed"}).json()
+    target = failed[0]["event_id"]
+
+    response = ops_client.post(f"/api/v1/workflows/{target}/retry")
+    assert response.status_code == 200
+    assert response.json()["status"] == "pending"
+
+    audit_events = ops_client.get("/api/v1/audit", params={"action": "workflow_retry"}).json()
+    assert len(audit_events) >= 1
+
+
+def test_retry_requires_operations_manager(employee_client):
+    response = employee_client.post(
+        "/api/v1/workflows/00000000-0000-4000-8000-000000000020/retry"
+    )
+    assert response.status_code == 403