fix: stop the prepared demo failure from degrading n8n integration health
The demo seed plants exactly one failed delivery (BK-H-0020) to demonstrate retry and audit. Because derive_n8n_status() counted any failure, every fresh reset pinned the n8n integration to "degraded" -- the demo showed a warning about a prop, which tells a viewer something untrue about the automation. The seeded failure now carries its own error code, demoScenarioTimeout, rather than the generic connectionError a real timeout produces. No column and no migration: last_error_code already existed, is already surfaced to the UI and is already localizable. - integration status splits failed into unexpected_failed and demo_scenario_failed; only unexpected failures may move the state. A staged failure alone leaves n8n operational. - latest_failure_at is a health signal and now ignores the staged failure; latest_demo_scenario_at reports it separately. - /api/v1/workflows exposes is_demo_scenario. The Automation page labels the run as a prepared demo scenario, explains that it is a simulated temporary failure that does not affect automation health, and offers a distinct "retry demo scenario" action. Translated in nl-BE, en-GB and fr-BE. - the carve-out stays narrow: a real failure still degrades n8n, and a genuine later failure of the same event overwrites the demo code with the real one. - the retry itself is unchanged and real: the event goes back on the outbox and the dispatcher delivers it to n8n like any other, so 19+1 becomes 20+0 only on an actual round trip. The audit records which kind of failure was retried. Tests that assert on the seeded scenario now reseed first, since earlier test files legitimately mutate the outbox and the suite shares one database. Verified locally against a real PostgreSQL 16: 181 passed, ruff clean, mypy clean (50 files), tsc clean, frontend build clean. Not deployed and not browser-verified.
This commit is contained in:
@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import get_db, require_operations_manager
|
||||
from app.core.errors import AppError
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.outbox import OutboxEvent, is_demo_scenario_failure
|
||||
from app.schemas import AutomationRunOut, CurrentUser
|
||||
from app.services.audit import record_audit_event
|
||||
|
||||
@@ -24,6 +24,7 @@ def _to_out(event: OutboxEvent) -> AutomationRunOut:
|
||||
attempts=event.attempts,
|
||||
last_error=event.last_error,
|
||||
last_error_code=event.last_error_code,
|
||||
is_demo_scenario=is_demo_scenario_failure(event),
|
||||
occurred_at=event.occurred_at,
|
||||
)
|
||||
|
||||
@@ -63,15 +64,27 @@ def retry_workflow(
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
# Captured before the status flips, so the audit records what was actually retried.
|
||||
was_demo_scenario = is_demo_scenario_failure(event)
|
||||
|
||||
event.delivery_status = "pending"
|
||||
event.next_attempt_at = None
|
||||
# The retry itself is real either way: the event goes back on the outbox and the
|
||||
# dispatcher delivers it to the configured n8n webhook like any other. The only
|
||||
# difference recorded here is *what* was retried -- a staged demo failure or a real
|
||||
# one -- so the audit trail never implies a production incident was resolved when a
|
||||
# prop was.
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=user.display_name,
|
||||
action="workflow_retry",
|
||||
entity_type="outbox_event",
|
||||
metadata={"event_id": event_id, "previous_attempts": event.attempts},
|
||||
metadata={
|
||||
"event_id": event_id,
|
||||
"previous_attempts": event.attempts,
|
||||
"demo_scenario": was_demo_scenario,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
return _to_out(event)
|
||||
|
||||
@@ -10,6 +10,25 @@ from app.models.mixins import TimestampMixin
|
||||
|
||||
DELIVERY_STATUSES = ("pending", "delivering", "succeeded", "failed")
|
||||
|
||||
# The one delivery failure the demo seed deliberately plants (BK-H-0020, see
|
||||
# seed/workflow_runs.csv). It exists to show retry and audit working, so it must never
|
||||
# be read as an integration-health problem: it is a scripted prop, not evidence that
|
||||
# n8n is unhealthy. A dedicated error code -- rather than the generic
|
||||
# "connectionError" a real timeout produces -- is what lets every reader tell the two
|
||||
# apart without guessing from the message text.
|
||||
#
|
||||
# It is deliberately a `last_error_code` value and not a new column: the code is
|
||||
# already persisted, already surfaced to the UI, and already localizable, so no schema
|
||||
# change or migration is needed. A genuine later failure of this same event overwrites
|
||||
# the code with the real one, which is exactly right -- from that moment it *is* a real
|
||||
# failure.
|
||||
DEMO_SCENARIO_ERROR_CODE = "demoScenarioTimeout"
|
||||
|
||||
|
||||
def is_demo_scenario_failure(event: "OutboxEvent") -> bool:
|
||||
"""True for the prepared demo failure, false for every real one."""
|
||||
return event.delivery_status == "failed" and event.last_error_code == DEMO_SCENARIO_ERROR_CODE
|
||||
|
||||
|
||||
class OutboxEvent(TimestampMixin, Base):
|
||||
__tablename__ = "outbox_events"
|
||||
|
||||
+15
-1
@@ -266,11 +266,21 @@ class N8nIntegrationStatus(BaseModel):
|
||||
dispatch_enabled: bool
|
||||
state: Literal["disabled", "unavailable", "degraded", "operational", "no_evidence"]
|
||||
pending: int
|
||||
delivering: int
|
||||
#: Every failed delivery, staged and real together -- the number a viewer sees in
|
||||
#: the run list.
|
||||
failed: int
|
||||
#: Failures that were not planted by the demo seed. This is the only failure count
|
||||
#: that may influence `state`.
|
||||
unexpected_failed: int = 0
|
||||
#: Prepared demo failures (see `app.models.outbox.DEMO_SCENARIO_ERROR_CODE`).
|
||||
#: Present so the UI can label them instead of implying the automation is broken.
|
||||
demo_scenario_failed: int = 0
|
||||
delivering: int
|
||||
succeeded: int
|
||||
latest_success_at: datetime | None
|
||||
#: Most recent *real* failure; a staged one never sets this.
|
||||
latest_failure_at: datetime | None
|
||||
latest_demo_scenario_at: datetime | None = None
|
||||
expected_workflow_count: int
|
||||
known_workflow_count: int
|
||||
workflows: list[N8nWorkflowEvidence]
|
||||
@@ -365,6 +375,10 @@ class AutomationRunOut(BaseModel):
|
||||
attempts: int
|
||||
last_error: str | None
|
||||
last_error_code: str | None
|
||||
#: True for the deliberately seeded demo failure. The UI uses this to label the run
|
||||
#: as a prepared scenario and to offer the demo retry, instead of presenting it as
|
||||
#: an unexplained production error.
|
||||
is_demo_scenario: bool = False
|
||||
occurred_at: datetime
|
||||
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ from app.models.data_quality import DataQualityIssue
|
||||
from app.models.idempotency import IdempotencyRecord
|
||||
from app.models.inspection import Inspection
|
||||
from app.models.maintenance import MaintenanceRecord
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.outbox import DEMO_SCENARIO_ERROR_CODE, OutboxEvent
|
||||
from app.models.user import User
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.services.audit import record_audit_event
|
||||
@@ -323,7 +323,10 @@ def load_seed(db: Session) -> SeedResult:
|
||||
"last_error": row["last_error"] or None,
|
||||
# The seed dataset's one synthetic failure (BK-H-0020) models a
|
||||
# connection-timeout-style delivery failure -- see workflow_runs.csv.
|
||||
"last_error_code": "connectionError" if row["last_error"] else None,
|
||||
# It is coded as a *prepared demo scenario*, not as a real
|
||||
# connectionError, so integration health never degrades because of a
|
||||
# prop and a viewer is told plainly that this failure is staged.
|
||||
"last_error_code": DEMO_SCENARIO_ERROR_CODE if row["last_error"] else None,
|
||||
"external_run_id": None,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.models.audit import AuditEvent
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.outbox import DEMO_SCENARIO_ERROR_CODE, OutboxEvent
|
||||
from app.schemas import (
|
||||
McpHubIntegrationStatus,
|
||||
N8nErrorHandlerStatus,
|
||||
@@ -40,19 +40,49 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
|
||||
failed = counts.get("failed", 0)
|
||||
succeeded = counts.get("succeeded", 0)
|
||||
|
||||
# Prepared demo failures are props, not health signals. They stay visible and
|
||||
# counted -- hiding them would be its own kind of lie -- but they are counted
|
||||
# *separately*, and only genuinely unexpected failures are allowed to move n8n off
|
||||
# "operational". Without this split the demo seed's single staged failure pins the
|
||||
# integration to "degraded" forever, which tells a viewer something untrue about
|
||||
# the automation.
|
||||
demo_scenario_failed = (
|
||||
db.scalar(
|
||||
select(func.count())
|
||||
.select_from(OutboxEvent)
|
||||
.where(
|
||||
OutboxEvent.delivery_status == "failed",
|
||||
OutboxEvent.last_error_code == DEMO_SCENARIO_ERROR_CODE,
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
unexpected_failed = max(failed - demo_scenario_failed, 0)
|
||||
|
||||
latest_success_at = db.scalar(
|
||||
select(func.max(OutboxEvent.updated_at)).where(OutboxEvent.delivery_status == "succeeded")
|
||||
)
|
||||
# Health talks about real failures only, so the "latest failure" a health reader
|
||||
# sees must exclude the staged one too.
|
||||
latest_failure_at = db.scalar(
|
||||
select(func.max(OutboxEvent.updated_at)).where(OutboxEvent.delivery_status == "failed")
|
||||
select(func.max(OutboxEvent.updated_at)).where(
|
||||
OutboxEvent.delivery_status == "failed",
|
||||
OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
|
||||
)
|
||||
)
|
||||
latest_demo_scenario_at = db.scalar(
|
||||
select(func.max(OutboxEvent.updated_at)).where(
|
||||
OutboxEvent.delivery_status == "failed",
|
||||
OutboxEvent.last_error_code == DEMO_SCENARIO_ERROR_CODE,
|
||||
)
|
||||
)
|
||||
|
||||
state: Literal["disabled", "unavailable", "degraded", "operational", "no_evidence"]
|
||||
if not settings.n8n_dispatch_enabled:
|
||||
state = "disabled"
|
||||
elif failed > 0 and succeeded == 0:
|
||||
elif unexpected_failed > 0 and succeeded == 0:
|
||||
state = "unavailable"
|
||||
elif failed > 0:
|
||||
elif unexpected_failed > 0:
|
||||
state = "degraded"
|
||||
elif succeeded > 0 or pending > 0 or delivering > 0:
|
||||
state = "operational"
|
||||
@@ -112,9 +142,12 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
|
||||
pending=pending,
|
||||
delivering=delivering,
|
||||
failed=failed,
|
||||
unexpected_failed=unexpected_failed,
|
||||
demo_scenario_failed=demo_scenario_failed,
|
||||
succeeded=succeeded,
|
||||
latest_success_at=latest_success_at,
|
||||
latest_failure_at=latest_failure_at,
|
||||
latest_demo_scenario_at=latest_demo_scenario_at,
|
||||
expected_workflow_count=len(_CANONICAL_WORKFLOWS),
|
||||
known_workflow_count=sum(1 for w in workflows if w.last_seen_at is not None),
|
||||
workflows=workflows,
|
||||
|
||||
Reference in New Issue
Block a user