M54: harden operations and demo resilience

backend/tests/test_auth.py

@@ -1,3 +1,15 @@
+import threading
+
+import pytest
+from sqlalchemy import delete, func, select
+
+from app.core.db import SessionLocal, engine
+from app.models.audit import AuditEvent
+from app.models.vehicle import Vehicle
+from app.seed_loader import reset_and_seed
+from app.services.audit import record_audit_event
+
+
 def test_unauthenticated_dashboard_is_rejected(client):
     response = client.get("/api/v1/dashboard")
     assert response.status_code == 401
@@ -101,15 +113,106 @@ def test_demo_reset_rejects_concurrent_rebuild(ops_client):
     assert response.status_code == 409
 
 
+def test_demo_reset_database_lock_blocks_another_api_replica(ops_client):
+    import app.api.routers.demo as demo_router
+
+    # Use a raw connection: an ordinary Session intentionally participates in the
+    # shared mutation barrier and would make an exclusive reset wait before it can test
+    # this separate non-blocking replica lock.
+    with engine.connect() as lock_connection:
+        lock_connection.scalar(select(func.pg_advisory_lock(demo_router._RESET_ADVISORY_LOCK_ID)))
+        try:
+            response = ops_client.post("/api/v1/demo/reset")
+        finally:
+            lock_connection.scalar(
+                select(func.pg_advisory_unlock(demo_router._RESET_ADVISORY_LOCK_ID))
+            )
+    assert response.status_code == 409
+
+
+def test_demo_reset_waits_for_active_mutation_and_then_replaces_it_atomically():
+    started = threading.Event()
+    completed = threading.Event()
+    errors: list[BaseException] = []
+
+    def reset_in_other_session() -> None:
+        started.set()
+        try:
+            with SessionLocal() as reset_db:
+                reset_and_seed(reset_db)
+        except BaseException as exc:  # pragma: no cover - assertion reports thread failures
+            errors.append(exc)
+        finally:
+            completed.set()
+
+    with SessionLocal() as mutation_db:
+        vehicle = mutation_db.scalar(
+            select(Vehicle).where(Vehicle.public_ref == "MO-001").with_for_update()
+        )
+        vehicle.location = "Concurrent mutation marker"
+        mutation_db.flush()
+        worker = threading.Thread(target=reset_in_other_session, daemon=True)
+        worker.start()
+        assert started.wait(timeout=1)
+        assert not completed.wait(timeout=0.2)
+        mutation_db.commit()
+
+    worker.join(timeout=10)
+    assert not worker.is_alive(), "reset deadlocked behind the active mutation"
+    assert errors == []
+    with SessionLocal() as db:
+        restored = db.scalar(select(Vehicle).where(Vehicle.public_ref == "MO-001"))
+        assert restored.location != "Concurrent mutation marker"
+
+
 def test_demo_reset_cooldown_returns_retry_after(ops_client, monkeypatch):
     import app.api.routers.demo as demo_router
 
     monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 60)
-    monkeypatch.setattr(demo_router, "_last_reset_monotonic", demo_router.time.monotonic())
+    with SessionLocal() as db:
+        cooldown_event = record_audit_event(
+            db,
+            actor_type="user",
+            actor_label="Cooldown test",
+            action="demo_reset",
+            entity_type="system",
+        )
+        db.commit()
+        cooldown_event_id = cooldown_event.id
     response = ops_client.post("/api/v1/demo/reset")
     assert response.status_code == 429
     assert int(response.headers["retry-after"]) >= 1
-    monkeypatch.setattr(demo_router, "_last_reset_monotonic", 0.0)
+    with SessionLocal() as db:
+        db.execute(delete(AuditEvent).where(AuditEvent.id == cooldown_event_id))
+        db.commit()
+
+
+def test_demo_reset_rolls_back_every_change_when_integrity_check_fails(ops_client, monkeypatch):
+    import app.api.routers.demo as demo_router
+
+    monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 0)
+    with SessionLocal() as db:
+        probe = record_audit_event(
+            db,
+            actor_type="system",
+            actor_label="Atomic reset test",
+            action="reset_atomicity_probe",
+            entity_type="system",
+        )
+        db.commit()
+        probe_id = probe.id
+
+    def fail_integrity_check(_db):
+        raise RuntimeError("Injected integrity-check failure")
+
+    monkeypatch.setattr(demo_router, "scenario_integrity_report", fail_integrity_check)
+    with pytest.raises(RuntimeError, match="Injected integrity-check failure"):
+        ops_client.post("/api/v1/demo/reset")
+
+    with SessionLocal() as db:
+        assert db.scalar(select(AuditEvent).where(AuditEvent.id == probe_id)) is not None
+        db.execute(delete(AuditEvent).where(AuditEvent.id == probe_id))
+        db.commit()
 
 
 def test_reset_is_rejected_when_demo_allow_reset_is_disabled(ops_client, monkeypatch):