M54: harden operations and demo resilience

backend/app/api/routers/bookings.py

@@ -30,6 +30,7 @@ from app.schemas import (
     ReturnPreviewResult,
 )
 from app.services.audit import record_audit_event
+from app.services.data_quality import open_odometer_regression_issue
 from app.services.returns import preview_vehicle_return, register_vehicle_return
 
 router = APIRouter(prefix="/api/v1/bookings", tags=["bookings"])
@@ -146,8 +147,14 @@ def create_booking(
 ) -> BookingOut:
     if body.ends_at <= body.starts_at:
         raise HTTPException(status_code=422, detail="Booking end must be after its start")
-    customer = db.scalar(select(Customer).where(Customer.public_ref == body.customer_ref))
-    if customer is None or customer.merged_into_customer_id is not None:
+    customer = db.scalar(
+        select(Customer).where(Customer.public_ref == body.customer_ref).with_for_update()
+    )
+    if (
+        customer is None
+        or customer.merged_into_customer_id is not None
+        or customer.anonymized_at is not None
+    ):
         raise HTTPException(status_code=422, detail="Customer is unavailable for booking")
     # Serialise booking creation per vehicle. The overlap check must run after
     # acquiring this lock, otherwise two concurrent requests can both pass it.
@@ -179,7 +186,9 @@ def create_booking(
         status="reserved",
         start_odometer_km=None,
         end_odometer_km=None,
-        requirements_complete=body.requirements_complete,
+        # Requirements are intentionally confirmed in a separate, audited action.
+        # Never allow booking creation to bypass that operational checkpoint.
+        requirements_complete=False,
     )
     db.add(booking)
     db.flush()
@@ -225,6 +234,15 @@ def checkout_booking(
     if active_conflict is not None:
         raise HTTPException(status_code=409, detail="Vehicle already has an active booking")
 
+    correlation_id = uuid.uuid4()
+    before_booking = {
+        "status": booking.status,
+        "start_odometer_km": booking.start_odometer_km,
+    }
+    before_vehicle = {
+        "operational_status": vehicle.operational_status,
+        "odometer_km": vehicle.odometer_km,
+    }
     attention_reasons: list[str] = []
     if body.start_odometer_km < vehicle.odometer_km:
         attention_reasons.append("odometer_regression")
@@ -250,6 +268,18 @@ def checkout_booking(
         completed_by=user.display_name,
     )
     db.add(inspection)
+    if "odometer_regression" in attention_reasons:
+        open_odometer_regression_issue(
+            db,
+            vehicle=vehicle,
+            reading_ref=inspection.public_ref,
+            reading_km=body.start_odometer_km,
+            canonical_km=vehicle.odometer_km,
+            source_type="checkout",
+            related_refs=[booking.public_ref, inspection.public_ref],
+            actor_label=user.display_name,
+            correlation_id=correlation_id,
+        )
     if attention_reasons:
         booking.status = "blocked"
         vehicle.operational_status = (
@@ -269,13 +299,31 @@ def checkout_booking(
         action="booking_checkout_recorded",
         entity_type="booking",
         entity_id=booking.id,
+        correlation_id=correlation_id,
+        before=before_booking,
         after={
             "inspection_ref": inspection.public_ref,
             "booking_status": booking.status,
+            "start_odometer_km": booking.start_odometer_km,
             "vehicle_status": vehicle.operational_status,
             "attention_reasons": attention_reasons,
         },
     )
+    record_audit_event(
+        db,
+        actor_type="user",
+        actor_label=user.display_name,
+        action="vehicle_status_changed",
+        entity_type="vehicle",
+        entity_id=vehicle.id,
+        correlation_id=correlation_id,
+        before=before_vehicle,
+        after={
+            "operational_status": vehicle.operational_status,
+            "odometer_km": vehicle.odometer_km,
+        },
+        metadata={"booking_ref": booking.public_ref, "inspection_ref": inspection.public_ref},
+    )
     db.commit()
     return CheckoutBookingResult(
         booking_ref=booking.public_ref,

backend/app/api/routers/customers.py

@@ -21,6 +21,7 @@ def search_customers(
         select(Customer)
         .where(
             Customer.merged_into_customer_id.is_(None),
+            Customer.anonymized_at.is_(None),
             or_(
                 Customer.public_ref.ilike(term),
                 Customer.first_name.ilike(term),

backend/app/api/routers/data_quality.py

@@ -11,6 +11,7 @@ from app.models.booking import Booking
 from app.models.customer import Customer
 from app.models.data_quality import DataQualityIssue
 from app.models.inspection import Inspection
+from app.models.maintenance import MaintenanceRecord
 from app.models.user import User
 from app.models.vehicle import Vehicle
 from app.schemas import (
@@ -224,6 +225,8 @@ _PREFIX_TO_TYPE = {
     "MO-": "vehicle",
     "BK-": "booking",
     "INSP-": "inspection",
+    "MAINT-": "maintenance",
+    "MNT-": "maintenance",
 }
 
 
@@ -292,6 +295,17 @@ def _snapshot(entity_type: str, ref: str, db: Session) -> dict | None:
             "completed_at": inspection.completed_at.isoformat(),
             "booking_ref": booking.public_ref if booking else None,
         }
+    if entity_type == "maintenance":
+        record = db.scalar(select(MaintenanceRecord).where(MaintenanceRecord.public_ref == ref))
+        if record is None:
+            return None
+        return {
+            "entity_type": "maintenance",
+            "public_ref": record.public_ref,
+            "odometer_km": record.odometer_km,
+            "occurred_at": record.occurred_at.isoformat(),
+            "category": record.category,
+        }
     return None
 
 

backend/app/api/routers/demo.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import threading
 import time
 import uuid
+from datetime import UTC, datetime
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
 from sqlalchemy import func, select
@@ -10,7 +11,9 @@ from sqlalchemy.orm import Session
 
 from app.api.deps import get_current_user, get_db, require_operations_manager
 from app.core.config import get_settings
+from app.core.db import begin_exclusive_demo_reset, end_exclusive_demo_reset, engine
 from app.core.security import SessionPayload, create_session_token, read_session_token
+from app.models.audit import AuditEvent
 from app.models.user import User
 from app.schemas import CurrentUser, DemoLoginRequest, DemoManifestOut
 from app.seed_loader import reset_and_seed
@@ -21,7 +24,6 @@ from app.services.sessions import revoke_session
 router = APIRouter(prefix="/api/v1/demo", tags=["demo"])
 settings = get_settings()
 _reset_guard = threading.Lock()
-_last_reset_monotonic = 0.0
 _RESET_ADVISORY_LOCK_ID = 706_533_149
 
 
@@ -110,7 +112,6 @@ def demo_reset(
     db: Session = Depends(get_db),
     user: CurrentUser = Depends(require_operations_manager),
 ) -> dict:
-    global _last_reset_monotonic
     if not settings.mobilityops_demo_mode:
         # Outside demo mode the reset endpoint must not exist at all: it wipes
         # operational data and replaces it with synthetic records.
@@ -122,19 +123,35 @@ def demo_reset(
         )
     if not _reset_guard.acquire(blocking=False):
         raise HTTPException(status_code=409, detail="A demo reset is already running.")
+    replica_lock_connection = None
     try:
-        elapsed = time.monotonic() - _last_reset_monotonic
-        if _last_reset_monotonic and elapsed < settings.demo_reset_cooldown_seconds:
+        # A session-level lock on its own connection rejects another replica immediately;
+        # the main DB session can then safely end its auth read transaction and wait on
+        # the normal shared/exclusive data barrier without releasing this replica guard.
+        replica_lock_connection = engine.connect()
+        locked = replica_lock_connection.scalar(
+            select(func.pg_try_advisory_lock(_RESET_ADVISORY_LOCK_ID))
+        )
+        if not locked:
+            raise HTTPException(status_code=409, detail="A demo reset is already running.")
+        begin_exclusive_demo_reset(db)
+        # The audit timestamp is shared by every replica. A process-local monotonic
+        # timestamp cannot protect a multi-replica deployment.
+        last_reset_at = db.scalar(
+            select(AuditEvent.occurred_at)
+            .where(AuditEvent.action == "demo_reset")
+            .order_by(AuditEvent.occurred_at.desc())
+            .limit(1)
+        )
+        elapsed = (datetime.now(UTC) - last_reset_at).total_seconds() if last_reset_at else None
+        if elapsed is not None and elapsed < settings.demo_reset_cooldown_seconds:
             retry_after = max(1, int(settings.demo_reset_cooldown_seconds - elapsed + 0.999))
             raise HTTPException(
                 status_code=429,
                 detail=f"Demo reset is cooling down. Retry in {retry_after} seconds.",
                 headers={"Retry-After": str(retry_after)},
             )
-        locked = db.scalar(select(func.pg_try_advisory_xact_lock(_RESET_ADVISORY_LOCK_ID)))
-        if not locked:
-            raise HTTPException(status_code=409, detail="A demo reset is already running.")
-        result = reset_and_seed(db, preserve_integration_telemetry=True)
+        result = reset_and_seed(db, preserve_integration_telemetry=True, commit=False)
         integrity = scenario_integrity_report(db)
         record_audit_event(
             db,
@@ -149,8 +166,15 @@ def demo_reset(
             },
         )
         db.commit()
-        _last_reset_monotonic = time.monotonic()
+        end_exclusive_demo_reset(db)
     finally:
+        if replica_lock_connection is not None:
+            try:
+                replica_lock_connection.scalar(
+                    select(func.pg_advisory_unlock(_RESET_ADVISORY_LOCK_ID))
+                )
+            finally:
+                replica_lock_connection.close()
         _reset_guard.release()
     response.delete_cookie(settings.session_cookie_name)
     return {

backend/app/api/routers/integrations.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
+import hashlib
 import hmac
 import uuid
 from datetime import UTC, datetime
 from pathlib import Path
 
 from fastapi import APIRouter, Depends, Header
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
 from app.api.deps import get_db
@@ -51,6 +52,13 @@ def _require_service_token(service_token: str) -> None:
         raise AppError("UNAUTHORIZED_SERVICE", "Invalid service token.", status_code=401)
 
 
+def _lock_idempotency_key(db: Session, namespace: str, key: str) -> None:
+    """Serialize callback check+insert by a stable, transaction-scoped key."""
+    digest = hashlib.sha256(f"{namespace}:{key}".encode()).digest()
+    lock_id = int.from_bytes(digest[:8], byteorder="big", signed=True)
+    db.scalar(select(func.pg_advisory_xact_lock(lock_id)))
+
+
 @router.post("/heartbeat", response_model=N8nHeartbeatResult)
 def workflow_heartbeat(
     body: N8nHeartbeatIn,
@@ -61,6 +69,7 @@ def workflow_heartbeat(
     _require_service_token(service_token)
     if body.workflow_name not in _CANONICAL_WORKFLOW_NAMES:
         raise AppError("UNKNOWN_WORKFLOW", "Unknown Fleet Ops workflow.", status_code=422)
+    _lock_idempotency_key(db, "n8n_workflow_heartbeat", f"{body.execution_id}:{body.status}")
     already_recorded = (
         db.scalar(
             select(AuditEvent.id).where(
@@ -109,9 +118,29 @@ def return_callback(
             "INVALID_IDEMPOTENCY_KEY", "Idempotency-Key must be the event's UUID.", status_code=422
         ) from exc
 
-    event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
+    event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update())
     if event is None:
         raise AppError("EVENT_NOT_FOUND", "No outbox event matches this event ID.", status_code=404)
+    if body.event_id != event_id:
+        raise AppError(
+            "CALLBACK_EVENT_MISMATCH",
+            "Callback event_id does not match Idempotency-Key.",
+            status_code=409,
+        )
+    try:
+        expected_correlation_id = uuid.UUID(str(event.payload_json["correlation_id"]))
+    except (KeyError, TypeError, ValueError) as exc:
+        raise AppError(
+            "INVALID_EVENT_CORRELATION",
+            "The stored outbox event has no valid correlation ID.",
+            status_code=409,
+        ) from exc
+    if body.correlation_id != expected_correlation_id:
+        raise AppError(
+            "CALLBACK_CORRELATION_MISMATCH",
+            "Callback correlation_id does not match the outbox event.",
+            status_code=409,
+        )
 
     # Idempotent by event ID: n8n or our own dispatcher may redeliver the same event
     # (e.g. a lost response after a timeout), so this callback must not double-record.
@@ -131,7 +160,7 @@ def return_callback(
             actor_label="n8n",
             action="n8n_return_followup_recorded",
             entity_type="booking",
-            correlation_id=body.correlation_id,
+            correlation_id=expected_correlation_id,
             after={"follow_up": body.follow_up, "summary": body.summary},
             metadata={"event_id": str(event_id)},
         )
@@ -170,6 +199,7 @@ def workflow_error(
     other Fleet Ops n8n workflow. Idempotent on execution_id: n8n may redeliver the same
     error report (e.g. after a timed-out response), so this must not double-record."""
     _require_service_token(service_token)
+    _lock_idempotency_key(db, "n8n_workflow_failure", body.execution_id)
 
     already_recorded = (
         db.scalar(
@@ -181,19 +211,13 @@ def workflow_error(
         is not None
     )
     if not already_recorded:
-        correlation_id: uuid.UUID | None = None
-        if body.correlation_id:
-            try:
-                correlation_id = uuid.UUID(body.correlation_id)
-            except ValueError:
-                correlation_id = None
         record_audit_event(
             db,
             actor_type="service",
             actor_label="n8n error handler",
             action="n8n_workflow_failure_registered",
             entity_type="automation",
-            correlation_id=correlation_id,
+            correlation_id=body.correlation_id,
             after={
                 "workflow_id": body.workflow_id,
                 "workflow_name": body.workflow_name,
@@ -248,6 +272,7 @@ def procedures_sync_result(
     RAGcore Procedure Sync" workflow once it finishes uploading procedures to RAGcore.
     Idempotent on execution_id, matching the workflow-error and return-callback pattern."""
     _require_service_token(service_token)
+    _lock_idempotency_key(db, "n8n_procedure_sync", body.execution_id)
 
     already_recorded = (
         db.scalar(

backend/app/api/routers/knowledge.py

@@ -53,7 +53,9 @@ def ask_question(
         client_ip = (
             forwarded.split(",")[-1].strip()
             if forwarded
-            else request.client.host if request.client else "unknown"
+            else request.client.host
+            if request.client
+            else "unknown"
         )
         token = request.cookies.get(settings.session_cookie_name, "")
         session_key = hashlib.sha256(token.encode("utf-8")).hexdigest()

backend/app/api/routers/vehicles.py

@@ -27,6 +27,7 @@ from app.schemas import (
     VehiclePageOut,
 )
 from app.services.audit import record_audit_event
+from app.services.data_quality import open_odometer_regression_issue
 
 router = APIRouter(prefix="/api/v1/vehicles", tags=["vehicles"])
 
@@ -143,7 +144,7 @@ def list_vehicles(
 def get_vehicle(
     public_ref: str,
     db: Session = Depends(get_db),
-    _user: CurrentUser = Depends(get_current_user),
+    user: CurrentUser = Depends(get_current_user),
 ) -> VehicleDetailOut:
     vehicle = db.scalar(select(Vehicle).where(Vehicle.public_ref == public_ref))
     if vehicle is None:
@@ -163,11 +164,21 @@ def get_vehicle(
         .where(MaintenanceRecord.vehicle_id == vehicle.id)
         .order_by(MaintenanceRecord.occurred_at.desc())
     ).all()
-    issues = db.scalars(
-        select(DataQualityIssue)
-        .where(DataQualityIssue.entity_type == "vehicle", DataQualityIssue.entity_id == vehicle.id)
-        .order_by(DataQualityIssue.detected_at.desc())
-    ).all()
+    # Detailed data-quality evidence is an operations-manager surface. Employees still
+    # get the operational vehicle record they need, but never receive hidden evidence in
+    # the payload merely because the frontend omits the Quality tab.
+    issues = (
+        db.scalars(
+            select(DataQualityIssue)
+            .where(
+                DataQualityIssue.entity_type == "vehicle",
+                DataQualityIssue.entity_id == vehicle.id,
+            )
+            .order_by(DataQualityIssue.detected_at.desc())
+        ).all()
+        if user.role == "operations_manager"
+        else []
+    )
 
     booking_by_id = {b.id: b.public_ref for b in bookings}
     next_booking = next(
@@ -276,6 +287,12 @@ def create_maintenance_record(
     vehicle = db.scalar(select(Vehicle).where(Vehicle.public_ref == public_ref).with_for_update())
     if vehicle is None:
         raise HTTPException(status_code=404, detail="Vehicle not found")
+    correlation_id = uuid.uuid4()
+    before_vehicle = {
+        "operational_status": vehicle.operational_status,
+        "odometer_km": vehicle.odometer_km,
+        "next_service_km": vehicle.next_service_km,
+    }
     record = MaintenanceRecord(
         public_ref=f"MAINT-{uuid.uuid4().hex[:8].upper()}",
         vehicle_id=vehicle.id,
@@ -285,6 +302,17 @@ def create_maintenance_record(
         summary=body.summary.strip(),
     )
     db.add(record)
+    open_odometer_regression_issue(
+        db,
+        vehicle=vehicle,
+        reading_ref=record.public_ref,
+        reading_km=body.odometer_km,
+        canonical_km=vehicle.odometer_km,
+        source_type="maintenance",
+        related_refs=[record.public_ref],
+        actor_label=user.display_name,
+        correlation_id=correlation_id,
+    )
     vehicle.odometer_km = max(vehicle.odometer_km, body.odometer_km)
     if body.next_service_km is not None:
         if body.next_service_km < vehicle.odometer_km:
@@ -301,7 +329,14 @@ def create_maintenance_record(
         action="maintenance_record_created",
         entity_type="vehicle",
         entity_id=vehicle.id,
-        after={"maintenance_ref": record.public_ref, "status": vehicle.operational_status},
+        correlation_id=correlation_id,
+        before=before_vehicle,
+        after={
+            "maintenance_ref": record.public_ref,
+            "operational_status": vehicle.operational_status,
+            "odometer_km": vehicle.odometer_km,
+            "next_service_km": vehicle.next_service_km,
+        },
     )
     db.commit()
     return MaintenanceOut(