fix: stop the prepared demo failure from degrading n8n integration health

The demo seed plants exactly one failed delivery (BK-H-0020) to demonstrate
retry and audit. Because derive_n8n_status() counted any failure, every fresh
reset pinned the n8n integration to "degraded" -- the demo showed a warning
about a prop, which tells a viewer something untrue about the automation.

The seeded failure now carries its own error code, demoScenarioTimeout, rather
than the generic connectionError a real timeout produces. No column and no
migration: last_error_code already existed, is already surfaced to the UI and is
already localizable.

- integration status splits failed into unexpected_failed and
  demo_scenario_failed; only unexpected failures may move the state. A staged
  failure alone leaves n8n operational.
- latest_failure_at is a health signal and now ignores the staged failure;
  latest_demo_scenario_at reports it separately.
- /api/v1/workflows exposes is_demo_scenario. The Automation page labels the run
  as a prepared demo scenario, explains that it is a simulated temporary failure
  that does not affect automation health, and offers a distinct "retry demo
  scenario" action. Translated in nl-BE, en-GB and fr-BE.
- the carve-out stays narrow: a real failure still degrades n8n, and a genuine
  later failure of the same event overwrites the demo code with the real one.
- the retry itself is unchanged and real: the event goes back on the outbox and
  the dispatcher delivers it to n8n like any other, so 19+1 becomes 20+0 only on
  an actual round trip. The audit records which kind of failure was retried.

Tests that assert on the seeded scenario now reseed first, since earlier test
files legitimately mutate the outbox and the suite shares one database.

Verified locally against a real PostgreSQL 16: 181 passed, ruff clean, mypy
clean (50 files), tsc clean, frontend build clean. Not deployed and not
browser-verified.

backend/app/api/routers/workflows.py

@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session
 
 from app.api.deps import get_db, require_operations_manager
 from app.core.errors import AppError
-from app.models.outbox import OutboxEvent
+from app.models.outbox import OutboxEvent, is_demo_scenario_failure
 from app.schemas import AutomationRunOut, CurrentUser
 from app.services.audit import record_audit_event
 
@@ -24,6 +24,7 @@ def _to_out(event: OutboxEvent) -> AutomationRunOut:
         attempts=event.attempts,
         last_error=event.last_error,
         last_error_code=event.last_error_code,
+        is_demo_scenario=is_demo_scenario_failure(event),
         occurred_at=event.occurred_at,
     )
 
@@ -63,15 +64,27 @@ def retry_workflow(
             status_code=409,
         )
 
+    # Captured before the status flips, so the audit records what was actually retried.
+    was_demo_scenario = is_demo_scenario_failure(event)
+
     event.delivery_status = "pending"
     event.next_attempt_at = None
+    # The retry itself is real either way: the event goes back on the outbox and the
+    # dispatcher delivers it to the configured n8n webhook like any other. The only
+    # difference recorded here is *what* was retried -- a staged demo failure or a real
+    # one -- so the audit trail never implies a production incident was resolved when a
+    # prop was.
     record_audit_event(
         db,
         actor_type="user",
         actor_label=user.display_name,
         action="workflow_retry",
         entity_type="outbox_event",
-        metadata={"event_id": event_id, "previous_attempts": event.attempts},
+        metadata={
+            "event_id": event_id,
+            "previous_attempts": event.attempts,
+            "demo_scenario": was_demo_scenario,
+        },
     )
     db.commit()
     return _to_out(event)

backend/app/models/outbox.py

@@ -10,6 +10,25 @@ from app.models.mixins import TimestampMixin
 
 DELIVERY_STATUSES = ("pending", "delivering", "succeeded", "failed")
 
+# The one delivery failure the demo seed deliberately plants (BK-H-0020, see
+# seed/workflow_runs.csv). It exists to show retry and audit working, so it must never
+# be read as an integration-health problem: it is a scripted prop, not evidence that
+# n8n is unhealthy. A dedicated error code -- rather than the generic
+# "connectionError" a real timeout produces -- is what lets every reader tell the two
+# apart without guessing from the message text.
+#
+# It is deliberately a `last_error_code` value and not a new column: the code is
+# already persisted, already surfaced to the UI, and already localizable, so no schema
+# change or migration is needed. A genuine later failure of this same event overwrites
+# the code with the real one, which is exactly right -- from that moment it *is* a real
+# failure.
+DEMO_SCENARIO_ERROR_CODE = "demoScenarioTimeout"
+
+
+def is_demo_scenario_failure(event: "OutboxEvent") -> bool:
+    """True for the prepared demo failure, false for every real one."""
+    return event.delivery_status == "failed" and event.last_error_code == DEMO_SCENARIO_ERROR_CODE
+
 
 class OutboxEvent(TimestampMixin, Base):
     __tablename__ = "outbox_events"

backend/app/schemas.py

@@ -266,11 +266,21 @@ class N8nIntegrationStatus(BaseModel):
     dispatch_enabled: bool
     state: Literal["disabled", "unavailable", "degraded", "operational", "no_evidence"]
     pending: int
-    delivering: int
+    #: Every failed delivery, staged and real together -- the number a viewer sees in
+    #: the run list.
     failed: int
+    #: Failures that were not planted by the demo seed. This is the only failure count
+    #: that may influence `state`.
+    unexpected_failed: int = 0
+    #: Prepared demo failures (see `app.models.outbox.DEMO_SCENARIO_ERROR_CODE`).
+    #: Present so the UI can label them instead of implying the automation is broken.
+    demo_scenario_failed: int = 0
+    delivering: int
     succeeded: int
     latest_success_at: datetime | None
+    #: Most recent *real* failure; a staged one never sets this.
     latest_failure_at: datetime | None
+    latest_demo_scenario_at: datetime | None = None
     expected_workflow_count: int
     known_workflow_count: int
     workflows: list[N8nWorkflowEvidence]
@@ -365,6 +375,10 @@ class AutomationRunOut(BaseModel):
     attempts: int
     last_error: str | None
     last_error_code: str | None
+    #: True for the deliberately seeded demo failure. The UI uses this to label the run
+    #: as a prepared scenario and to offer the demo retry, instead of presenting it as
+    #: an unexplained production error.
+    is_demo_scenario: bool = False
     occurred_at: datetime
 
 

backend/app/seed_loader.py

@@ -18,7 +18,7 @@ from app.models.data_quality import DataQualityIssue
 from app.models.idempotency import IdempotencyRecord
 from app.models.inspection import Inspection
 from app.models.maintenance import MaintenanceRecord
-from app.models.outbox import OutboxEvent
+from app.models.outbox import DEMO_SCENARIO_ERROR_CODE, OutboxEvent
 from app.models.user import User
 from app.models.vehicle import Vehicle
 from app.services.audit import record_audit_event
@@ -323,7 +323,10 @@ def load_seed(db: Session) -> SeedResult:
                 "last_error": row["last_error"] or None,
                 # The seed dataset's one synthetic failure (BK-H-0020) models a
                 # connection-timeout-style delivery failure -- see workflow_runs.csv.
-                "last_error_code": "connectionError" if row["last_error"] else None,
+                # It is coded as a *prepared demo scenario*, not as a real
+                # connectionError, so integration health never degrades because of a
+                # prop and a viewer is told plainly that this failure is staged.
+                "last_error_code": DEMO_SCENARIO_ERROR_CODE if row["last_error"] else None,
                 "external_run_id": None,
             }
         )

backend/app/services/integration_status.py

@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session
 
 from app.core.config import get_settings
 from app.models.audit import AuditEvent
-from app.models.outbox import OutboxEvent
+from app.models.outbox import DEMO_SCENARIO_ERROR_CODE, OutboxEvent
 from app.schemas import (
     McpHubIntegrationStatus,
     N8nErrorHandlerStatus,
@@ -40,19 +40,49 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
     failed = counts.get("failed", 0)
     succeeded = counts.get("succeeded", 0)
 
+    # Prepared demo failures are props, not health signals. They stay visible and
+    # counted -- hiding them would be its own kind of lie -- but they are counted
+    # *separately*, and only genuinely unexpected failures are allowed to move n8n off
+    # "operational". Without this split the demo seed's single staged failure pins the
+    # integration to "degraded" forever, which tells a viewer something untrue about
+    # the automation.
+    demo_scenario_failed = (
+        db.scalar(
+            select(func.count())
+            .select_from(OutboxEvent)
+            .where(
+                OutboxEvent.delivery_status == "failed",
+                OutboxEvent.last_error_code == DEMO_SCENARIO_ERROR_CODE,
+            )
+        )
+        or 0
+    )
+    unexpected_failed = max(failed - demo_scenario_failed, 0)
+
     latest_success_at = db.scalar(
         select(func.max(OutboxEvent.updated_at)).where(OutboxEvent.delivery_status == "succeeded")
     )
+    # Health talks about real failures only, so the "latest failure" a health reader
+    # sees must exclude the staged one too.
     latest_failure_at = db.scalar(
-        select(func.max(OutboxEvent.updated_at)).where(OutboxEvent.delivery_status == "failed")
+        select(func.max(OutboxEvent.updated_at)).where(
+            OutboxEvent.delivery_status == "failed",
+            OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
+        )
+    )
+    latest_demo_scenario_at = db.scalar(
+        select(func.max(OutboxEvent.updated_at)).where(
+            OutboxEvent.delivery_status == "failed",
+            OutboxEvent.last_error_code == DEMO_SCENARIO_ERROR_CODE,
+        )
     )
 
     state: Literal["disabled", "unavailable", "degraded", "operational", "no_evidence"]
     if not settings.n8n_dispatch_enabled:
         state = "disabled"
-    elif failed > 0 and succeeded == 0:
+    elif unexpected_failed > 0 and succeeded == 0:
         state = "unavailable"
-    elif failed > 0:
+    elif unexpected_failed > 0:
         state = "degraded"
     elif succeeded > 0 or pending > 0 or delivering > 0:
         state = "operational"
@@ -112,9 +142,12 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
         pending=pending,
         delivering=delivering,
         failed=failed,
+        unexpected_failed=unexpected_failed,
+        demo_scenario_failed=demo_scenario_failed,
         succeeded=succeeded,
         latest_success_at=latest_success_at,
         latest_failure_at=latest_failure_at,
+        latest_demo_scenario_at=latest_demo_scenario_at,
         expected_workflow_count=len(_CANONICAL_WORKFLOWS),
         known_workflow_count=sum(1 for w in workflows if w.last_seen_at is not None),
         workflows=workflows,

backend/tests/test_integration_status.py

@@ -1,3 +1,24 @@
+from sqlalchemy import select
+
+from app.core.db import SessionLocal
+from app.models.outbox import OutboxEvent
+from app.seed_loader import reset_and_seed
+
+
+def _reseed() -> None:
+    """Restore the canonical demo dataset (19 succeeded + 1 prepared failure).
+
+    The suite shares one session-scoped database and earlier files legitimately mutate
+    the outbox, so any test that asserts on the *seeded* scenario has to re-establish it
+    rather than depend on file ordering.
+    """
+    db = SessionLocal()
+    try:
+        reset_and_seed(db)
+    finally:
+        db.close()
+
+
 def test_integration_status_requires_operations_manager(employee_client):
     response = employee_client.get("/api/v1/integrations/status")
     assert response.status_code == 403
@@ -9,6 +30,7 @@ def test_integration_status_requires_authentication(client):
 
 
 def test_integration_status_reflects_seeded_mixed_outcomes(ops_client):
+    _reseed()
     response = ops_client.get("/api/v1/integrations/status")
     assert response.status_code == 200
     body = response.json()
@@ -16,20 +38,65 @@ def test_integration_status_reflects_seeded_mixed_outcomes(ops_client):
     n8n = body["n8n"]
     assert n8n["dispatch_enabled"] is True
     assert n8n["succeeded"] >= 1
+    # The seeded failure stays visible and counted...
     assert n8n["failed"] >= 1
-    # The seed deliberately carries both failed and succeeded events, so a single most-
-    # recent-event read would misreport health -- the aggregate must call this "degraded",
-    # not "operational" or "unavailable".
-    assert n8n["state"] == "degraded"
+    assert n8n["demo_scenario_failed"] == 1
+    # ...but it is a prepared prop, so it is not an unexpected failure and must not
+    # move the integration off "operational". A staged failure that degrades the
+    # health badge tells a viewer something untrue about the automation.
+    assert n8n["unexpected_failed"] == 0
+    assert n8n["state"] == "operational"
     assert n8n["latest_success_at"] is not None
-    assert n8n["latest_failure_at"] is not None
+    # "latest failure" is a health signal, so the staged one never sets it; it is
+    # reported separately instead.
+    assert n8n["latest_failure_at"] is None
+    assert n8n["latest_demo_scenario_at"] is not None
 
     mcp_hub = body["mcp_hub"]
     assert mcp_hub["registration_enabled"] is False
     assert mcp_hub["state"] == "not_configured"
 
 
+def test_a_real_failure_still_degrades_the_integration(ops_client):
+    """The demo carve-out must be narrow: a failure that is not the prepared scenario
+    still degrades n8n, otherwise this change would hide real breakage."""
+    _reseed()
+    db = SessionLocal()
+    try:
+        real_failure = db.scalar(
+            select(OutboxEvent).where(OutboxEvent.delivery_status == "succeeded").limit(1)
+        )
+        assert real_failure is not None
+        restore = (real_failure.delivery_status, real_failure.last_error_code)
+        real_failure.delivery_status = "failed"
+        real_failure.last_error_code = "connectionError"
+        db.commit()
+
+        n8n = ops_client.get("/api/v1/integrations/status").json()["n8n"]
+        assert n8n["unexpected_failed"] == 1
+        assert n8n["state"] == "degraded"
+        assert n8n["latest_failure_at"] is not None
+    finally:
+        real_failure.delivery_status, real_failure.last_error_code = restore
+        db.commit()
+        db.close()
+
+
+def test_prepared_demo_failure_is_reset_back_by_a_demo_reset(ops_client):
+    """A demo reset must recreate the intended 19 succeeded + 1 prepared failure, so the
+    scenario can be shown again after it has been retried away."""
+    _reseed()
+
+    n8n = ops_client.get("/api/v1/integrations/status").json()["n8n"]
+    assert n8n["succeeded"] == 19
+    assert n8n["failed"] == 1
+    assert n8n["demo_scenario_failed"] == 1
+    assert n8n["unexpected_failed"] == 0
+    assert n8n["state"] == "operational"
+
+
 def test_integration_status_is_operational_once_all_failed_events_resolved(ops_client):
+    _reseed()
     failed = ops_client.get("/api/v1/workflows", params={"status": "failed"}).json()
     for run in failed:
         retried = ops_client.post(f"/api/v1/workflows/{run['event_id']}/retry")

backend/tests/test_seed.py

@@ -7,7 +7,7 @@ from app.models.audit import AuditEvent
 from app.models.booking import Booking
 from app.models.customer import Customer
 from app.models.data_quality import DataQualityIssue
-from app.models.outbox import OutboxEvent
+from app.models.outbox import DEMO_SCENARIO_ERROR_CODE, OutboxEvent
 from app.models.user import User
 from app.models.vehicle import Vehicle
 from app.seed_loader import SEED_AUTHORED_ANCHOR, reset_and_seed
@@ -142,7 +142,10 @@ def test_seed_scenario_s5_failed_workflow_run():
         assert failed.delivery_status == "failed"
         assert failed.attempts >= 1
         assert failed.last_error
-        assert failed.last_error_code == "connectionError"
+        # Coded as a prepared demo scenario, not as a real connectionError: the whole
+        # point of this row is to demonstrate retry and audit, so nothing downstream
+        # may read it as evidence that the n8n integration is unhealthy.
+        assert failed.last_error_code == DEMO_SCENARIO_ERROR_CODE
     finally:
         db.close()
 

backend/tests/test_workflows.py

@@ -1,9 +1,28 @@
+from app.core.db import SessionLocal
+from app.seed_loader import reset_and_seed
+
+
+def _reseed() -> None:
+    """Restore the canonical demo dataset (19 succeeded + 1 prepared failure).
+
+    The suite shares one session-scoped database and earlier files legitimately mutate
+    the outbox, so any test that asserts on the *seeded* scenario has to re-establish it
+    rather than depend on file ordering.
+    """
+    db = SessionLocal()
+    try:
+        reset_and_seed(db)
+    finally:
+        db.close()
+
+
 def test_list_workflows_requires_operations_manager(employee_client):
     response = employee_client.get("/api/v1/workflows")
     assert response.status_code == 403
 
 
 def test_list_workflows_includes_seeded_failed_run(ops_client):
+    _reseed()
     response = ops_client.get("/api/v1/workflows", params={"status": "failed"})
     assert response.status_code == 200
     runs = response.json()
@@ -20,6 +39,7 @@ def test_retry_requires_failed_status(ops_client):
 
 
 def test_retry_failed_run_moves_to_pending_and_audits(ops_client):
+    _reseed()
     failed = ops_client.get("/api/v1/workflows", params={"status": "failed"}).json()
     target = failed[0]["event_id"]
 
@@ -36,3 +56,42 @@ def test_retry_requires_operations_manager(employee_client):
         "/api/v1/workflows/00000000-0000-4000-8000-000000000020/retry"
     )
     assert response.status_code == 403
+
+
+def test_seeded_failure_is_labelled_as_a_prepared_demo_scenario(ops_client):
+    """The one seeded failure must announce itself as staged. An unexplained red row in
+    a demo reads as a broken product; a labelled one reads as the retry story it is."""
+    _reseed()
+    runs = ops_client.get("/api/v1/workflows", params={"status": "failed"}).json()
+    demo_runs = [run for run in runs if run["is_demo_scenario"]]
+    assert len(demo_runs) == 1
+    assert demo_runs[0]["last_error_code"] == "demoScenarioTimeout"
+    assert demo_runs[0]["aggregate_ref"] == "BK-H-0020"
+
+
+def test_succeeded_runs_are_never_marked_as_a_demo_scenario(ops_client):
+    runs = ops_client.get("/api/v1/workflows", params={"status": "succeeded"}).json()
+    assert runs
+    assert all(run["is_demo_scenario"] is False for run in runs)
+
+
+def test_retry_of_the_demo_scenario_is_audited_as_a_demo_scenario(ops_client):
+    """The retry is a real redelivery either way; the audit records which kind of
+    failure it resolved so a staged retry is never mistaken for a production fix."""
+    _reseed()
+    failed = ops_client.get("/api/v1/workflows", params={"status": "failed"}).json()
+    demo_run = next(run for run in failed if run["is_demo_scenario"])
+
+    response = ops_client.post(f"/api/v1/workflows/{demo_run['event_id']}/retry")
+    assert response.status_code == 200
+    assert response.json()["status"] == "pending"
+
+    audit = ops_client.get("/api/v1/audit", params={"action": "workflow_retry"}).json()
+    entry = next(
+        event
+        for event in audit
+        if (event.get("metadata") or event.get("metadata_json") or {}).get("event_id")
+        == demo_run["event_id"]
+    )
+    metadata = entry.get("metadata") or entry.get("metadata_json") or {}
+    assert metadata["demo_scenario"] is True