M54: harden operations and demo resilience

backend/tests/test_integrations.py

@@ -1,4 +1,5 @@
 import uuid
+from concurrent.futures import ThreadPoolExecutor
 
 from app.core.config import get_settings
 
@@ -12,19 +13,29 @@ def _callback_headers(event_id: str, token: str | None = None):
 
 
 def test_callback_rejects_wrong_service_token(client):
+    event_id = str(uuid.uuid4())
     response = client.post(
         "/api/v1/integrations/n8n/return-callback",
-        json={"follow_up": "cleaning"},
-        headers=_callback_headers(str(uuid.uuid4()), token="wrong-token"),
+        json={
+            "event_id": event_id,
+            "correlation_id": str(uuid.uuid4()),
+            "follow_up": "cleaning",
+        },
+        headers=_callback_headers(event_id, token="wrong-token"),
     )
     assert response.status_code == 401
 
 
 def test_callback_unknown_event_returns_404(client):
+    event_id = str(uuid.uuid4())
     response = client.post(
         "/api/v1/integrations/n8n/return-callback",
-        json={"follow_up": "cleaning"},
-        headers=_callback_headers(str(uuid.uuid4())),
+        json={
+            "event_id": event_id,
+            "correlation_id": str(uuid.uuid4()),
+            "follow_up": "cleaning",
+        },
+        headers=_callback_headers(event_id),
     )
     assert response.status_code == 404
 
@@ -39,13 +50,14 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
     db = SessionLocal()
     try:
         booking = db.scalar(select(Booking).limit(1))
+        correlation_id = uuid.uuid4()
         event = OutboxEvent(
             event_id=uuid.uuid4(),
             event_type="vehicle.returned.v1",
             aggregate_type="booking",
             aggregate_id=booking.id,
             payload_json={
-                "correlation_id": str(uuid.uuid4()),
+                "correlation_id": str(correlation_id),
                 "aggregate": {
                     "type": "booking",
                     "id": str(booking.id),
@@ -66,12 +78,22 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
 
     first = client.post(
         "/api/v1/integrations/n8n/return-callback",
-        json={"follow_up": "cleaning", "summary": "test"},
+        json={
+            "event_id": event_id,
+            "correlation_id": str(correlation_id),
+            "follow_up": "cleaning",
+            "summary": "test",
+        },
         headers=_callback_headers(event_id),
     )
     second = client.post(
         "/api/v1/integrations/n8n/return-callback",
-        json={"follow_up": "cleaning", "summary": "test"},
+        json={
+            "event_id": event_id,
+            "correlation_id": str(correlation_id),
+            "follow_up": "cleaning",
+            "summary": "test",
+        },
         headers=_callback_headers(event_id),
     )
     assert first.status_code == 200
@@ -82,6 +104,79 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
     ).json()
     matching = [e for e in audit_events if e["metadata"]["event_id"] == event_id]
     assert len(matching) == 1
+    assert matching[0]["correlation_id"] == str(correlation_id)
+
+
+def test_callback_rejects_crossed_event_or_correlation(client):
+    from sqlalchemy import select
+
+    from app.core.db import SessionLocal
+    from app.models.outbox import OutboxEvent
+
+    with SessionLocal() as db:
+        event = db.scalar(select(OutboxEvent).limit(1))
+        event_id = str(event.event_id)
+        correlation_id = event.payload_json["correlation_id"]
+
+    crossed_event = client.post(
+        "/api/v1/integrations/n8n/return-callback",
+        json={
+            "event_id": str(uuid.uuid4()),
+            "correlation_id": correlation_id,
+            "follow_up": "cleaning",
+        },
+        headers=_callback_headers(event_id),
+    )
+    assert crossed_event.status_code == 409
+    assert crossed_event.json()["error"]["code"] == "CALLBACK_EVENT_MISMATCH"
+
+    crossed_correlation = client.post(
+        "/api/v1/integrations/n8n/return-callback",
+        json={
+            "event_id": event_id,
+            "correlation_id": str(uuid.uuid4()),
+            "follow_up": "cleaning",
+        },
+        headers=_callback_headers(event_id),
+    )
+    assert crossed_correlation.status_code == 409
+    assert crossed_correlation.json()["error"]["code"] == "CALLBACK_CORRELATION_MISMATCH"
+
+
+def test_callback_concurrent_retries_record_one_audit(client, ops_client):
+    from sqlalchemy import select
+
+    from app.core.db import SessionLocal
+    from app.models.outbox import OutboxEvent
+
+    with SessionLocal() as db:
+        event = db.scalar(select(OutboxEvent).limit(1))
+        event_id = str(event.event_id)
+        correlation_id = event.payload_json["correlation_id"]
+    body = {
+        "event_id": event_id,
+        "correlation_id": correlation_id,
+        "follow_up": "cleaning",
+    }
+
+    def post_callback(_index: int):
+        return client.post(
+            "/api/v1/integrations/n8n/return-callback",
+            json=body,
+            headers=_callback_headers(event_id),
+        )
+
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        responses = list(pool.map(post_callback, range(2)))
+    assert [response.status_code for response in responses] == [200, 200]
+    matching = [
+        event
+        for event in ops_client.get(
+            "/api/v1/audit", params={"action": "n8n_return_followup_recorded"}
+        ).json()
+        if event["metadata"]["event_id"] == event_id
+    ]
+    assert len(matching) == 1
 
 
 def test_scheduled_scan_rejects_wrong_service_token(client):
@@ -183,6 +278,42 @@ def test_workflow_error_registers_and_is_idempotent_by_execution_id(client, ops_
     assert matching[0]["after"]["retry_action"] == "n8n will retry automatically"
 
 
+def test_workflow_error_preserves_correlation_and_rejects_malformed(client, ops_client):
+    settings = get_settings()
+    headers = {"X-Service-Token": settings.n8n_callback_token}
+    execution_id = str(uuid.uuid4())
+    correlation_id = str(uuid.uuid4())
+    accepted = client.post(
+        "/api/v1/integrations/n8n/workflow-error",
+        json=_workflow_error_body(execution_id, correlation_id=correlation_id),
+        headers=headers,
+    )
+    assert accepted.status_code == 200
+    matching = [
+        event
+        for event in ops_client.get(
+            "/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
+        ).json()
+        if event["metadata"]["execution_id"] == execution_id
+    ]
+    assert len(matching) == 1
+    assert matching[0]["correlation_id"] == correlation_id
+
+    malformed_execution = str(uuid.uuid4())
+    malformed = client.post(
+        "/api/v1/integrations/n8n/workflow-error",
+        json=_workflow_error_body(malformed_execution, correlation_id="not-a-uuid"),
+        headers=headers,
+    )
+    assert malformed.status_code == 422
+    assert all(
+        event["metadata"]["execution_id"] != malformed_execution
+        for event in ops_client.get(
+            "/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
+        ).json()
+    )
+
+
 def test_workflow_error_bounds_summary_length(client):
     settings = get_settings()
     response = client.post(
@@ -266,6 +397,26 @@ def test_procedures_sync_result_registers_and_is_idempotent(client, ops_client):
     assert ragcore_sync["last_seen_at"] is not None
 
 
+def test_failed_or_partial_procedure_sync_is_never_reported_healthy(client, ops_client):
+    settings = get_settings()
+    headers = {"X-Service-Token": settings.n8n_callback_token}
+    for synced, failed in ((0, 33), (32, 1)):
+        execution_id = str(uuid.uuid4())
+        response = client.post(
+            "/api/v1/integrations/n8n/procedures-sync-result",
+            json={"execution_id": execution_id, "synced": synced, "failed": failed},
+            headers=headers,
+        )
+        assert response.status_code == 200
+
+        status = ops_client.get("/api/v1/integrations/status").json()["n8n"]
+        workflow = next(w for w in status["workflows"] if "RAGcore" in w["name"])
+        assert workflow["state"] == "failed"
+        assert workflow["last_status"] == "failed"
+        assert workflow["last_execution_id"] == execution_id
+        assert status["state"] == "degraded"
+
+
 def test_workflow_heartbeat_is_idempotent_and_drives_live_status(client, ops_client):
     settings = get_settings()
     execution_id = str(uuid.uuid4())