M54: harden operations and demo resilience

backend/tests/test_integration_status.py

@@ -1,8 +1,14 @@
-from sqlalchemy import select
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import delete, select
 
 from app.core.db import SessionLocal
+from app.models.audit import AuditEvent
 from app.models.outbox import OutboxEvent
 from app.seed_loader import reset_and_seed
+from app.services import integration_status
 
 
 def _reseed() -> None:
@@ -108,6 +114,101 @@ def test_integration_status_is_operational_once_all_failed_events_resolved(ops_c
     assert body["state"] == "operational"
 
 
+@pytest.mark.parametrize("delivery_status", ["pending", "delivering"])
+def test_queued_work_without_any_success_is_not_reported_operational(delivery_status):
+    _reseed()
+    db = SessionLocal()
+    try:
+        for event in db.scalars(select(OutboxEvent)).all():
+            event.delivery_status = delivery_status
+            event.last_error = None
+            event.last_error_code = None
+        db.execute(
+            delete(AuditEvent).where(
+                AuditEvent.action.in_(
+                    {
+                        "data_quality_scan_run",
+                        "n8n_procedures_synced",
+                        "n8n_workflow_failure_registered",
+                        "n8n_workflow_heartbeat",
+                    }
+                )
+            )
+        )
+        db.flush()
+
+        status = integration_status.derive_n8n_status(db)
+
+        assert status.succeeded == 0
+        assert getattr(status, delivery_status) > 0
+        assert status.state == "no_evidence"
+    finally:
+        db.rollback()
+        db.close()
+
+
+def test_historical_success_is_not_green_when_webhook_is_unconfigured(monkeypatch):
+    _reseed()
+    with SessionLocal() as db:
+        monkeypatch.setattr(integration_status.settings, "n8n_dispatch_enabled", True)
+        monkeypatch.setattr(integration_status.settings, "n8n_webhook_url", "")
+        status = integration_status.derive_n8n_status(db)
+        assert status.configured is False
+        assert status.succeeded > 0
+        assert status.state == "unavailable"
+
+
+def test_null_coded_real_failure_sets_latest_failure_timestamp():
+    _reseed()
+    with SessionLocal() as db:
+        event = db.scalar(
+            select(OutboxEvent).where(OutboxEvent.delivery_status == "succeeded").limit(1)
+        )
+        event.delivery_status = "failed"
+        event.last_error_code = None
+        db.flush()
+        status = integration_status.derive_n8n_status(db)
+        assert status.unexpected_failed == 1
+        assert status.latest_failure_at is not None
+        db.rollback()
+
+
+def test_unreachable_hub_invalidates_historical_operational_claim(monkeypatch):
+    db = SessionLocal()
+    try:
+        db.add(
+            AuditEvent(
+                actor_type="service",
+                actor_id=None,
+                actor_label="itworx-mcp-hub",
+                action="mcp_tool_request",
+                entity_type="integration",
+                entity_id=None,
+                correlation_id=uuid.uuid4(),
+                before_json=None,
+                after_json=None,
+                metadata_json={"tool": "operations_summary"},
+                occurred_at=datetime.now(UTC),
+            )
+        )
+        db.flush()
+        monkeypatch.setattr(integration_status.settings, "mcp_hub_registration_enabled", True)
+        monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: False)
+
+        unreachable = integration_status.derive_mcp_hub_status(db)
+
+        assert unreachable.total_calls > 0
+        assert unreachable.hub_reachable is False
+        assert unreachable.state == "no_evidence"
+
+        monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: True)
+        reachable = integration_status.derive_mcp_hub_status(db)
+        assert reachable.state == "operational"
+    finally:
+        db.rollback()
+        db.close()
+
+
 def test_integration_status_lists_all_four_canonical_workflows(ops_client):
     body = ops_client.get("/api/v1/integrations/status").json()["n8n"]
     assert body["expected_workflow_count"] == 4