M54: harden operations and demo resilience

backend/app/api/routers/demo.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import threading
 import time
 import uuid
+from datetime import UTC, datetime
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
 from sqlalchemy import func, select
@@ -10,7 +11,9 @@ from sqlalchemy.orm import Session
 
 from app.api.deps import get_current_user, get_db, require_operations_manager
 from app.core.config import get_settings
+from app.core.db import begin_exclusive_demo_reset, end_exclusive_demo_reset, engine
 from app.core.security import SessionPayload, create_session_token, read_session_token
+from app.models.audit import AuditEvent
 from app.models.user import User
 from app.schemas import CurrentUser, DemoLoginRequest, DemoManifestOut
 from app.seed_loader import reset_and_seed
@@ -21,7 +24,6 @@ from app.services.sessions import revoke_session
 router = APIRouter(prefix="/api/v1/demo", tags=["demo"])
 settings = get_settings()
 _reset_guard = threading.Lock()
-_last_reset_monotonic = 0.0
 _RESET_ADVISORY_LOCK_ID = 706_533_149
 
 
@@ -110,7 +112,6 @@ def demo_reset(
     db: Session = Depends(get_db),
     user: CurrentUser = Depends(require_operations_manager),
 ) -> dict:
-    global _last_reset_monotonic
     if not settings.mobilityops_demo_mode:
         # Outside demo mode the reset endpoint must not exist at all: it wipes
         # operational data and replaces it with synthetic records.
@@ -122,19 +123,35 @@ def demo_reset(
         )
     if not _reset_guard.acquire(blocking=False):
         raise HTTPException(status_code=409, detail="A demo reset is already running.")
+    replica_lock_connection = None
     try:
-        elapsed = time.monotonic() - _last_reset_monotonic
-        if _last_reset_monotonic and elapsed < settings.demo_reset_cooldown_seconds:
+        # A session-level lock on its own connection rejects another replica immediately;
+        # the main DB session can then safely end its auth read transaction and wait on
+        # the normal shared/exclusive data barrier without releasing this replica guard.
+        replica_lock_connection = engine.connect()
+        locked = replica_lock_connection.scalar(
+            select(func.pg_try_advisory_lock(_RESET_ADVISORY_LOCK_ID))
+        )
+        if not locked:
+            raise HTTPException(status_code=409, detail="A demo reset is already running.")
+        begin_exclusive_demo_reset(db)
+        # The audit timestamp is shared by every replica. A process-local monotonic
+        # timestamp cannot protect a multi-replica deployment.
+        last_reset_at = db.scalar(
+            select(AuditEvent.occurred_at)
+            .where(AuditEvent.action == "demo_reset")
+            .order_by(AuditEvent.occurred_at.desc())
+            .limit(1)
+        )
+        elapsed = (datetime.now(UTC) - last_reset_at).total_seconds() if last_reset_at else None
+        if elapsed is not None and elapsed < settings.demo_reset_cooldown_seconds:
             retry_after = max(1, int(settings.demo_reset_cooldown_seconds - elapsed + 0.999))
             raise HTTPException(
                 status_code=429,
                 detail=f"Demo reset is cooling down. Retry in {retry_after} seconds.",
                 headers={"Retry-After": str(retry_after)},
             )
-        locked = db.scalar(select(func.pg_try_advisory_xact_lock(_RESET_ADVISORY_LOCK_ID)))
-        if not locked:
-            raise HTTPException(status_code=409, detail="A demo reset is already running.")
-        result = reset_and_seed(db, preserve_integration_telemetry=True)
+        result = reset_and_seed(db, preserve_integration_telemetry=True, commit=False)
         integrity = scenario_integrity_report(db)
         record_audit_event(
             db,
@@ -149,8 +166,15 @@ def demo_reset(
             },
         )
         db.commit()
-        _last_reset_monotonic = time.monotonic()
+        end_exclusive_demo_reset(db)
     finally:
+        if replica_lock_connection is not None:
+            try:
+                replica_lock_connection.scalar(
+                    select(func.pg_advisory_unlock(_RESET_ADVISORY_LOCK_ID))
+                )
+            finally:
+                replica_lock_connection.close()
         _reset_guard.release()
     response.delete_cookie(settings.session_cookie_name)
     return {