M54: harden operations and demo resilience
MobilityOps acceptance / backend (push) Failing after 19s
MobilityOps acceptance / frontend (push) Successful in 25s
MobilityOps acceptance / e2e (push) Skipped

This commit is contained in:
NuklearRabbit
2026-08-24 03:31:03 +02:00
parent b0706989db
commit 81e3fd63bd
101 changed files with 5641 additions and 828 deletions
+51 -3
View File
@@ -30,6 +30,7 @@ from app.schemas import (
ReturnPreviewResult,
)
from app.services.audit import record_audit_event
from app.services.data_quality import open_odometer_regression_issue
from app.services.returns import preview_vehicle_return, register_vehicle_return
router = APIRouter(prefix="/api/v1/bookings", tags=["bookings"])
@@ -146,8 +147,14 @@ def create_booking(
) -> BookingOut:
if body.ends_at <= body.starts_at:
raise HTTPException(status_code=422, detail="Booking end must be after its start")
customer = db.scalar(select(Customer).where(Customer.public_ref == body.customer_ref))
if customer is None or customer.merged_into_customer_id is not None:
customer = db.scalar(
select(Customer).where(Customer.public_ref == body.customer_ref).with_for_update()
)
if (
customer is None
or customer.merged_into_customer_id is not None
or customer.anonymized_at is not None
):
raise HTTPException(status_code=422, detail="Customer is unavailable for booking")
# Serialise booking creation per vehicle. The overlap check must run after
# acquiring this lock, otherwise two concurrent requests can both pass it.
@@ -179,7 +186,9 @@ def create_booking(
status="reserved",
start_odometer_km=None,
end_odometer_km=None,
requirements_complete=body.requirements_complete,
# Requirements are intentionally confirmed in a separate, audited action.
# Never allow booking creation to bypass that operational checkpoint.
requirements_complete=False,
)
db.add(booking)
db.flush()
@@ -225,6 +234,15 @@ def checkout_booking(
if active_conflict is not None:
raise HTTPException(status_code=409, detail="Vehicle already has an active booking")
correlation_id = uuid.uuid4()
before_booking = {
"status": booking.status,
"start_odometer_km": booking.start_odometer_km,
}
before_vehicle = {
"operational_status": vehicle.operational_status,
"odometer_km": vehicle.odometer_km,
}
attention_reasons: list[str] = []
if body.start_odometer_km < vehicle.odometer_km:
attention_reasons.append("odometer_regression")
@@ -250,6 +268,18 @@ def checkout_booking(
completed_by=user.display_name,
)
db.add(inspection)
if "odometer_regression" in attention_reasons:
open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=inspection.public_ref,
reading_km=body.start_odometer_km,
canonical_km=vehicle.odometer_km,
source_type="checkout",
related_refs=[booking.public_ref, inspection.public_ref],
actor_label=user.display_name,
correlation_id=correlation_id,
)
if attention_reasons:
booking.status = "blocked"
vehicle.operational_status = (
@@ -269,13 +299,31 @@ def checkout_booking(
action="booking_checkout_recorded",
entity_type="booking",
entity_id=booking.id,
correlation_id=correlation_id,
before=before_booking,
after={
"inspection_ref": inspection.public_ref,
"booking_status": booking.status,
"start_odometer_km": booking.start_odometer_km,
"vehicle_status": vehicle.operational_status,
"attention_reasons": attention_reasons,
},
)
record_audit_event(
db,
actor_type="user",
actor_label=user.display_name,
action="vehicle_status_changed",
entity_type="vehicle",
entity_id=vehicle.id,
correlation_id=correlation_id,
before=before_vehicle,
after={
"operational_status": vehicle.operational_status,
"odometer_km": vehicle.odometer_km,
},
metadata={"booking_ref": booking.public_ref, "inspection_ref": inspection.public_ref},
)
db.commit()
return CheckoutBookingResult(
booking_ref=booking.public_ref,
+1
View File
@@ -21,6 +21,7 @@ def search_customers(
select(Customer)
.where(
Customer.merged_into_customer_id.is_(None),
Customer.anonymized_at.is_(None),
or_(
Customer.public_ref.ilike(term),
Customer.first_name.ilike(term),
+14
View File
@@ -11,6 +11,7 @@ from app.models.booking import Booking
from app.models.customer import Customer
from app.models.data_quality import DataQualityIssue
from app.models.inspection import Inspection
from app.models.maintenance import MaintenanceRecord
from app.models.user import User
from app.models.vehicle import Vehicle
from app.schemas import (
@@ -224,6 +225,8 @@ _PREFIX_TO_TYPE = {
"MO-": "vehicle",
"BK-": "booking",
"INSP-": "inspection",
"MAINT-": "maintenance",
"MNT-": "maintenance",
}
@@ -292,6 +295,17 @@ def _snapshot(entity_type: str, ref: str, db: Session) -> dict | None:
"completed_at": inspection.completed_at.isoformat(),
"booking_ref": booking.public_ref if booking else None,
}
if entity_type == "maintenance":
record = db.scalar(select(MaintenanceRecord).where(MaintenanceRecord.public_ref == ref))
if record is None:
return None
return {
"entity_type": "maintenance",
"public_ref": record.public_ref,
"odometer_km": record.odometer_km,
"occurred_at": record.occurred_at.isoformat(),
"category": record.category,
}
return None
+33 -9
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import threading
import time
import uuid
from datetime import UTC, datetime
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from sqlalchemy import func, select
@@ -10,7 +11,9 @@ from sqlalchemy.orm import Session
from app.api.deps import get_current_user, get_db, require_operations_manager
from app.core.config import get_settings
from app.core.db import begin_exclusive_demo_reset, end_exclusive_demo_reset, engine
from app.core.security import SessionPayload, create_session_token, read_session_token
from app.models.audit import AuditEvent
from app.models.user import User
from app.schemas import CurrentUser, DemoLoginRequest, DemoManifestOut
from app.seed_loader import reset_and_seed
@@ -21,7 +24,6 @@ from app.services.sessions import revoke_session
router = APIRouter(prefix="/api/v1/demo", tags=["demo"])
settings = get_settings()
_reset_guard = threading.Lock()
_last_reset_monotonic = 0.0
_RESET_ADVISORY_LOCK_ID = 706_533_149
@@ -110,7 +112,6 @@ def demo_reset(
db: Session = Depends(get_db),
user: CurrentUser = Depends(require_operations_manager),
) -> dict:
global _last_reset_monotonic
if not settings.mobilityops_demo_mode:
# Outside demo mode the reset endpoint must not exist at all: it wipes
# operational data and replaces it with synthetic records.
@@ -122,19 +123,35 @@ def demo_reset(
)
if not _reset_guard.acquire(blocking=False):
raise HTTPException(status_code=409, detail="A demo reset is already running.")
replica_lock_connection = None
try:
elapsed = time.monotonic() - _last_reset_monotonic
if _last_reset_monotonic and elapsed < settings.demo_reset_cooldown_seconds:
# A session-level lock on its own connection rejects another replica immediately;
# the main DB session can then safely end its auth read transaction and wait on
# the normal shared/exclusive data barrier without releasing this replica guard.
replica_lock_connection = engine.connect()
locked = replica_lock_connection.scalar(
select(func.pg_try_advisory_lock(_RESET_ADVISORY_LOCK_ID))
)
if not locked:
raise HTTPException(status_code=409, detail="A demo reset is already running.")
begin_exclusive_demo_reset(db)
# The audit timestamp is shared by every replica. A process-local monotonic
# timestamp cannot protect a multi-replica deployment.
last_reset_at = db.scalar(
select(AuditEvent.occurred_at)
.where(AuditEvent.action == "demo_reset")
.order_by(AuditEvent.occurred_at.desc())
.limit(1)
)
elapsed = (datetime.now(UTC) - last_reset_at).total_seconds() if last_reset_at else None
if elapsed is not None and elapsed < settings.demo_reset_cooldown_seconds:
retry_after = max(1, int(settings.demo_reset_cooldown_seconds - elapsed + 0.999))
raise HTTPException(
status_code=429,
detail=f"Demo reset is cooling down. Retry in {retry_after} seconds.",
headers={"Retry-After": str(retry_after)},
)
locked = db.scalar(select(func.pg_try_advisory_xact_lock(_RESET_ADVISORY_LOCK_ID)))
if not locked:
raise HTTPException(status_code=409, detail="A demo reset is already running.")
result = reset_and_seed(db, preserve_integration_telemetry=True)
result = reset_and_seed(db, preserve_integration_telemetry=True, commit=False)
integrity = scenario_integrity_report(db)
record_audit_event(
db,
@@ -149,8 +166,15 @@ def demo_reset(
},
)
db.commit()
_last_reset_monotonic = time.monotonic()
end_exclusive_demo_reset(db)
finally:
if replica_lock_connection is not None:
try:
replica_lock_connection.scalar(
select(func.pg_advisory_unlock(_RESET_ADVISORY_LOCK_ID))
)
finally:
replica_lock_connection.close()
_reset_guard.release()
response.delete_cookie(settings.session_cookie_name)
return {
+35 -10
View File
@@ -1,12 +1,13 @@
from __future__ import annotations
import hashlib
import hmac
import uuid
from datetime import UTC, datetime
from pathlib import Path
from fastapi import APIRouter, Depends, Header
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.api.deps import get_db
@@ -51,6 +52,13 @@ def _require_service_token(service_token: str) -> None:
raise AppError("UNAUTHORIZED_SERVICE", "Invalid service token.", status_code=401)
def _lock_idempotency_key(db: Session, namespace: str, key: str) -> None:
"""Serialize callback check+insert by a stable, transaction-scoped key."""
digest = hashlib.sha256(f"{namespace}:{key}".encode()).digest()
lock_id = int.from_bytes(digest[:8], byteorder="big", signed=True)
db.scalar(select(func.pg_advisory_xact_lock(lock_id)))
@router.post("/heartbeat", response_model=N8nHeartbeatResult)
def workflow_heartbeat(
body: N8nHeartbeatIn,
@@ -61,6 +69,7 @@ def workflow_heartbeat(
_require_service_token(service_token)
if body.workflow_name not in _CANONICAL_WORKFLOW_NAMES:
raise AppError("UNKNOWN_WORKFLOW", "Unknown Fleet Ops workflow.", status_code=422)
_lock_idempotency_key(db, "n8n_workflow_heartbeat", f"{body.execution_id}:{body.status}")
already_recorded = (
db.scalar(
select(AuditEvent.id).where(
@@ -109,9 +118,29 @@ def return_callback(
"INVALID_IDEMPOTENCY_KEY", "Idempotency-Key must be the event's UUID.", status_code=422
) from exc
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update())
if event is None:
raise AppError("EVENT_NOT_FOUND", "No outbox event matches this event ID.", status_code=404)
if body.event_id != event_id:
raise AppError(
"CALLBACK_EVENT_MISMATCH",
"Callback event_id does not match Idempotency-Key.",
status_code=409,
)
try:
expected_correlation_id = uuid.UUID(str(event.payload_json["correlation_id"]))
except (KeyError, TypeError, ValueError) as exc:
raise AppError(
"INVALID_EVENT_CORRELATION",
"The stored outbox event has no valid correlation ID.",
status_code=409,
) from exc
if body.correlation_id != expected_correlation_id:
raise AppError(
"CALLBACK_CORRELATION_MISMATCH",
"Callback correlation_id does not match the outbox event.",
status_code=409,
)
# Idempotent by event ID: n8n or our own dispatcher may redeliver the same event
# (e.g. a lost response after a timeout), so this callback must not double-record.
@@ -131,7 +160,7 @@ def return_callback(
actor_label="n8n",
action="n8n_return_followup_recorded",
entity_type="booking",
correlation_id=body.correlation_id,
correlation_id=expected_correlation_id,
after={"follow_up": body.follow_up, "summary": body.summary},
metadata={"event_id": str(event_id)},
)
@@ -170,6 +199,7 @@ def workflow_error(
other Fleet Ops n8n workflow. Idempotent on execution_id: n8n may redeliver the same
error report (e.g. after a timed-out response), so this must not double-record."""
_require_service_token(service_token)
_lock_idempotency_key(db, "n8n_workflow_failure", body.execution_id)
already_recorded = (
db.scalar(
@@ -181,19 +211,13 @@ def workflow_error(
is not None
)
if not already_recorded:
correlation_id: uuid.UUID | None = None
if body.correlation_id:
try:
correlation_id = uuid.UUID(body.correlation_id)
except ValueError:
correlation_id = None
record_audit_event(
db,
actor_type="service",
actor_label="n8n error handler",
action="n8n_workflow_failure_registered",
entity_type="automation",
correlation_id=correlation_id,
correlation_id=body.correlation_id,
after={
"workflow_id": body.workflow_id,
"workflow_name": body.workflow_name,
@@ -248,6 +272,7 @@ def procedures_sync_result(
RAGcore Procedure Sync" workflow once it finishes uploading procedures to RAGcore.
Idempotent on execution_id, matching the workflow-error and return-callback pattern."""
_require_service_token(service_token)
_lock_idempotency_key(db, "n8n_procedure_sync", body.execution_id)
already_recorded = (
db.scalar(
+3 -1
View File
@@ -53,7 +53,9 @@ def ask_question(
client_ip = (
forwarded.split(",")[-1].strip()
if forwarded
else request.client.host if request.client else "unknown"
else request.client.host
if request.client
else "unknown"
)
token = request.cookies.get(settings.session_cookie_name, "")
session_key = hashlib.sha256(token.encode("utf-8")).hexdigest()
+42 -7
View File
@@ -27,6 +27,7 @@ from app.schemas import (
VehiclePageOut,
)
from app.services.audit import record_audit_event
from app.services.data_quality import open_odometer_regression_issue
router = APIRouter(prefix="/api/v1/vehicles", tags=["vehicles"])
@@ -143,7 +144,7 @@ def list_vehicles(
def get_vehicle(
public_ref: str,
db: Session = Depends(get_db),
_user: CurrentUser = Depends(get_current_user),
user: CurrentUser = Depends(get_current_user),
) -> VehicleDetailOut:
vehicle = db.scalar(select(Vehicle).where(Vehicle.public_ref == public_ref))
if vehicle is None:
@@ -163,11 +164,21 @@ def get_vehicle(
.where(MaintenanceRecord.vehicle_id == vehicle.id)
.order_by(MaintenanceRecord.occurred_at.desc())
).all()
issues = db.scalars(
select(DataQualityIssue)
.where(DataQualityIssue.entity_type == "vehicle", DataQualityIssue.entity_id == vehicle.id)
.order_by(DataQualityIssue.detected_at.desc())
).all()
# Detailed data-quality evidence is an operations-manager surface. Employees still
# get the operational vehicle record they need, but never receive hidden evidence in
# the payload merely because the frontend omits the Quality tab.
issues = (
db.scalars(
select(DataQualityIssue)
.where(
DataQualityIssue.entity_type == "vehicle",
DataQualityIssue.entity_id == vehicle.id,
)
.order_by(DataQualityIssue.detected_at.desc())
).all()
if user.role == "operations_manager"
else []
)
booking_by_id = {b.id: b.public_ref for b in bookings}
next_booking = next(
@@ -276,6 +287,12 @@ def create_maintenance_record(
vehicle = db.scalar(select(Vehicle).where(Vehicle.public_ref == public_ref).with_for_update())
if vehicle is None:
raise HTTPException(status_code=404, detail="Vehicle not found")
correlation_id = uuid.uuid4()
before_vehicle = {
"operational_status": vehicle.operational_status,
"odometer_km": vehicle.odometer_km,
"next_service_km": vehicle.next_service_km,
}
record = MaintenanceRecord(
public_ref=f"MAINT-{uuid.uuid4().hex[:8].upper()}",
vehicle_id=vehicle.id,
@@ -285,6 +302,17 @@ def create_maintenance_record(
summary=body.summary.strip(),
)
db.add(record)
open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=record.public_ref,
reading_km=body.odometer_km,
canonical_km=vehicle.odometer_km,
source_type="maintenance",
related_refs=[record.public_ref],
actor_label=user.display_name,
correlation_id=correlation_id,
)
vehicle.odometer_km = max(vehicle.odometer_km, body.odometer_km)
if body.next_service_km is not None:
if body.next_service_km < vehicle.odometer_km:
@@ -301,7 +329,14 @@ def create_maintenance_record(
action="maintenance_record_created",
entity_type="vehicle",
entity_id=vehicle.id,
after={"maintenance_ref": record.public_ref, "status": vehicle.operational_status},
correlation_id=correlation_id,
before=before_vehicle,
after={
"maintenance_ref": record.public_ref,
"operational_status": vehicle.operational_status,
"odometer_km": vehicle.odometer_km,
"next_service_km": vehicle.next_service_km,
},
)
db.commit()
return MaintenanceOut(
+5 -1
View File
@@ -33,7 +33,11 @@ class Settings(BaseSettings):
n8n_callback_token: str = "replace-me-n8n-callback-token"
n8n_dispatch_enabled: bool = True
n8n_dispatch_interval_seconds: float = 3.0
n8n_http_timeout_seconds: float = 5.0
# The synchronous n8n workflow performs two bounded, retried callbacks before it
# acknowledges an event. Keep this above that complete workflow budget, while the
# delivery lease remains the wider crash-recovery boundary (enforced by the contract
# check in scripts/check-contracts.py).
n8n_http_timeout_seconds: float = 15.0
n8n_max_attempts: int = 5
n8n_delivery_lease_seconds: float = 120.0
app_secret: str = "replace-in-production"
+32 -1
View File
@@ -1,6 +1,6 @@
from collections.abc import Generator
from sqlalchemy import create_engine
from sqlalchemy import create_engine, event, func, select
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
from app.core.config import get_settings
@@ -10,6 +10,37 @@ settings = get_settings()
engine = create_engine(settings.database_url, pool_pre_ping=True, future=True)
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
# Every SQLAlchemy transaction participates in a shared database-wide barrier. Normal
# reads/writes coexist; the short demo reset takes the exclusive form so it can never
# interleave deletes/inserts with an API request, scanner, callback, or dispatcher cycle.
DEMO_DATA_BARRIER_LOCK_ID = 5_344_725_149_212_793_901
_EXCLUSIVE_RESET_INFO_KEY = "mobilityops_demo_reset_exclusive"
@event.listens_for(Session, "after_begin")
def _acquire_demo_data_barrier(session: Session, _transaction, connection) -> None:
if connection.dialect.name != "postgresql":
return
lock = (
func.pg_advisory_xact_lock(DEMO_DATA_BARRIER_LOCK_ID)
if session.info.get(_EXCLUSIVE_RESET_INFO_KEY)
else func.pg_advisory_xact_lock_shared(DEMO_DATA_BARRIER_LOCK_ID)
)
connection.execute(select(lock))
def begin_exclusive_demo_reset(db: Session) -> None:
"""Make the session's next transaction the exclusive side of the reset barrier."""
if db.in_transaction():
# Auth normally already read the user under a shared barrier. End that read-only
# transaction before requesting exclusive; in-place lock upgrades can deadlock.
db.rollback()
db.info[_EXCLUSIVE_RESET_INFO_KEY] = True
def end_exclusive_demo_reset(db: Session) -> None:
db.info.pop(_EXCLUSIVE_RESET_INFO_KEY, None)
class Base(DeclarativeBase):
pass
+12 -8
View File
@@ -4,9 +4,13 @@ import uuid
from datetime import datetime
from typing import Annotated, Any, Literal
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
Role = Literal["operations_manager", "rental_employee"]
NonBlankOperationalReason = Annotated[
str,
StringConstraints(strip_whitespace=True, min_length=3, max_length=500),
]
class DemoLoginRequest(BaseModel):
@@ -93,17 +97,16 @@ class CreateBookingRequest(BaseModel):
vehicle_ref: str = Field(min_length=3, max_length=20)
starts_at: datetime
ends_at: datetime
requirements_complete: bool = False
class CompleteBookingRequirementsRequest(BaseModel):
confirmation: str = Field(min_length=3, max_length=500)
confirmation: NonBlankOperationalReason
class RescheduleBookingRequest(BaseModel):
starts_at: datetime
ends_at: datetime
reason: str = Field(min_length=3, max_length=500)
reason: NonBlankOperationalReason
class CustomerOptionOut(BaseModel):
@@ -122,7 +125,7 @@ class AvailableVehicleOut(BaseModel):
class CancelBookingRequest(BaseModel):
reason: str = Field(min_length=3, max_length=500)
reason: NonBlankOperationalReason
class CheckoutBookingRequest(BaseModel):
@@ -225,7 +228,7 @@ class CreateMaintenanceRequest(BaseModel):
class ReleaseVehicleRequest(BaseModel):
reason: str = Field(min_length=3, max_length=500)
reason: NonBlankOperationalReason
class DataQualityIssueOut(BaseModel):
@@ -295,7 +298,7 @@ class WorkflowErrorReportIn(BaseModel):
]
error_summary: str = Field(max_length=500)
trigger_context: str | None = Field(default=None, max_length=200)
correlation_id: str | None = None
correlation_id: uuid.UUID | None = None
attempt: int = Field(default=1, ge=1, le=1000)
retry_action: str | None = Field(default=None, max_length=200)
@@ -433,7 +436,8 @@ class ReturnCallbackIn(BaseModel):
model_config = ConfigDict(extra="ignore")
correlation_id: uuid.UUID | None = None
event_id: uuid.UUID
correlation_id: uuid.UUID
follow_up: str | None = Field(default=None, max_length=200)
summary: str | None = Field(default=None, max_length=2000)
+51 -11
View File
@@ -11,6 +11,7 @@ from sqlalchemy import delete, insert, update
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.db import begin_exclusive_demo_reset, end_exclusive_demo_reset
from app.models.audit import AuditEvent
from app.models.booking import Booking
from app.models.customer import Customer
@@ -201,7 +202,11 @@ def load_seed(db: Session) -> SeedResult:
counts["bookings"] = len(booking_rows)
inspection_rows = []
for row in _read_csv("inspections.csv"):
inspection_source_rows = _read_csv("inspections.csv")
return_inspection_row_by_booking_ref = {
row["booking_ref"]: row for row in inspection_source_rows if row["type"] == "return"
}
for row in inspection_source_rows:
inspection_rows.append(
{
"id": uuid.uuid4(),
@@ -259,16 +264,18 @@ def load_seed(db: Session) -> SeedResult:
}
def _odometer_regression_signal(later_ref: str, earlier_ref: str) -> list[dict]:
later = booking_row_by_ref[later_ref]
earlier = booking_row_by_ref[earlier_ref]
later = return_inspection_row_by_booking_ref[later_ref]
earlier = return_inspection_row_by_booking_ref[earlier_ref]
return [
{
"code": "odometer.regression",
"source_type": "return",
"params": {
"later_ref": later_ref,
"later_km": later["end_odometer_km"],
"earlier_ref": earlier_ref,
"earlier_km": earlier["end_odometer_km"],
"later_ref": later["public_ref"],
"later_km": int(later["odometer_km"]),
"earlier_ref": earlier["public_ref"],
"earlier_km": int(earlier["odometer_km"]),
"booking_ref": later_ref,
},
}
]
@@ -354,6 +361,27 @@ def load_seed(db: Session) -> SeedResult:
entity_type, entity_id = resolve_entity(row["entity_ref"])
related_ref = row.get("related_ref") or ""
related_refs = related_ref.split("|") if related_ref else []
signals = _seed_signals(row["public_ref"], row["entity_ref"], related_refs)
evidence_extra: dict[str, object] = {}
if row["rule_type"] == "odometer_regression" and signals:
params = signals[0].get("params", {})
later_ref = params.get("later_ref")
earlier_ref = params.get("earlier_ref")
booking_ref = params.get("booking_ref")
if (
isinstance(earlier_ref, str)
and isinstance(later_ref, str)
and isinstance(booking_ref, str)
):
# The authored CSV prose predates structured references. Preserve the
# two real source bookings so the detail page can show evidence, while
# only the later (regressing) reading is eligible for correction.
related_refs = [earlier_ref, booking_ref, later_ref]
evidence_extra = {
"source_type": signals[0].get("source_type", "return"),
"source_types": [signals[0].get("source_type", "return")],
"correctable_booking_refs": [booking_ref],
}
severity_due_delta = {
"high": timedelta(hours=4),
"medium": timedelta(days=1),
@@ -372,7 +400,8 @@ def load_seed(db: Session) -> SeedResult:
"summary": row["evidence"],
"entity_ref": row["entity_ref"],
"related_refs": related_refs,
"signals": _seed_signals(row["public_ref"], row["entity_ref"], related_refs),
"signals": signals,
**evidence_extra,
},
"proposed_action_json": {},
"detected_at": now,
@@ -449,12 +478,23 @@ def load_seed(db: Session) -> SeedResult:
return SeedResult(counts=counts, anchor_date=today, seeded_at=seeded_at)
def reset_and_seed(db: Session, *, preserve_integration_telemetry: bool = False) -> SeedResult:
def reset_and_seed(
db: Session,
*,
preserve_integration_telemetry: bool = False,
commit: bool = True,
) -> SeedResult:
from app.services.data_quality import run_scan
if not db.info.get("mobilityops_demo_reset_exclusive"):
begin_exclusive_demo_reset(db)
clear_all(db, preserve_integration_telemetry=preserve_integration_telemetry)
result = load_seed(db)
db.commit()
scan = run_scan(db)
scan = run_scan(db, commit=False)
result.counts["data_quality_issues"] += sum(scan.created.values())
if commit:
db.commit()
end_exclusive_demo_reset(db)
else:
db.flush()
return result
+122 -202
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import uuid
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from datetime import UTC, datetime
from sqlalchemy import func, select, update
from sqlalchemy.orm import Session
@@ -11,10 +11,22 @@ from app.core.errors import AppError
from app.models.booking import Booking
from app.models.customer import Customer
from app.models.data_quality import DataQualityIssue
from app.models.inspection import Inspection
from app.models.maintenance import MaintenanceRecord
from app.models.vehicle import Vehicle
from app.schemas import CurrentUser, ResolveOdometerRegressionRequest
from app.schemas import CurrentUser
from app.services.audit import record_audit_event
from app.services.data_quality_common import has_open_issue as _has_open_issue
from app.services.data_quality_common import issue_due_at
from app.services.data_quality_common import load_open_issue as _load_open_issue
from app.services.data_quality_common import new_scan_ref as _new_scan_ref
from app.services.data_quality_duplicate_scan import scan_duplicate_customers
from app.services.data_quality_odometer import (
open_odometer_regression_issue as open_odometer_regression_issue,
)
from app.services.data_quality_odometer import (
resolve_odometer_regression as resolve_odometer_regression,
)
from app.services.vehicle_status import (
RECOMMENDATION_CODE_NO_CONFLICT,
VehicleStatusRecommendation,
@@ -28,15 +40,6 @@ REQUIRED_VEHICLE_FIELDS = ("registration_number", "make", "model", "location")
DATA_QUALITY_SCAN_LOCK_ID = 6_138_493_717_091_029_491
def issue_due_at(detected_at: datetime, severity: str) -> datetime:
"""Return the local operational SLA deadline for a newly detected issue."""
return detected_at + {
"high": timedelta(hours=4),
"medium": timedelta(days=1),
"low": timedelta(days=3),
}.get(severity, timedelta(days=1))
@dataclass
class ScanResult:
created: dict[str, int] = field(default_factory=dict)
@@ -45,25 +48,6 @@ class ScanResult:
self.created[rule_type] = self.created.get(rule_type, 0) + 1
def _has_open_issue(db: Session, rule_type: str, entity_type: str, entity_id: uuid.UUID) -> bool:
return (
db.scalar(
select(DataQualityIssue.id).where(
DataQualityIssue.rule_type == rule_type,
DataQualityIssue.entity_type == entity_type,
DataQualityIssue.entity_id == entity_id,
DataQualityIssue.status == "open",
)
)
is not None
)
def _new_scan_ref(prefix: str) -> str:
"""Generate a stable human-readable prefix with a concurrent-safe suffix."""
return f"{prefix}-{uuid.uuid4().hex[:10].upper()}"
def _open_issue(
db: Session,
scan: ScanResult,
@@ -76,6 +60,7 @@ def _open_issue(
entity_ref: str,
related_refs: list[str],
signals: list[dict] | None = None,
evidence_extra: dict | None = None,
) -> None:
if _has_open_issue(db, rule_type, entity_type, entity_id):
return
@@ -103,6 +88,7 @@ def _open_issue(
"related_refs": related_refs,
"signals": signals or [],
}
evidence.update(evidence_extra or {})
if previous is not None:
evidence["reopened_from"] = previous.public_ref
evidence["previous_decision"] = previous.status
@@ -235,57 +221,110 @@ def _scan_vehicle_status_conflicts(db: Session, scan: ScanResult) -> None:
)
def _scan_odometer_regressions(db: Session, scan: ScanResult) -> None:
# The seed dataset's vehicle.odometer_km is generated independently of booking
# history, so comparing every historical booking against it produces near-universal
# false positives. Instead check the booking sequence's own internal consistency:
# each vehicle's completed bookings should show a non-decreasing odometer reading.
vehicles = {v.id: v for v in db.scalars(select(Vehicle)).all()}
bookings_by_vehicle: dict[uuid.UUID, list[Booking]] = {}
for booking in db.scalars(
def _scan_odometer_regressions(
db: Session,
scan: ScanResult,
*,
actor_label: str | None,
actor_type: str,
) -> None:
# Compare the chronological history with itself rather than every historical reading
# to today's canonical value. That detects imported checkout/return/maintenance
# regressions without flagging every legitimate older reading.
# Runtime checkout/return/maintenance mutations take the vehicle lock before they
# inspect or append DQ-03 evidence. Taking the same lock here makes scan-vs-command
# check/merge atomic and prevents a partial-unique race for the open issue.
vehicles = {
v.id: v for v in db.scalars(select(Vehicle).order_by(Vehicle.id).with_for_update()).all()
}
readings_by_vehicle: dict[uuid.UUID, list[tuple[datetime, str, int, str, str | None]]] = {}
inspections = db.scalars(select(Inspection)).all()
return_inspection_booking_ids = {
inspection.booking_id for inspection in inspections if inspection.type == "return"
}
returned_bookings = db.scalars(
select(Booking).where(Booking.status == "returned", Booking.end_odometer_km.is_not(None))
).all():
bookings_by_vehicle.setdefault(booking.vehicle_id, []).append(booking)
).all()
booking_ref_by_id = {booking.id: booking.public_ref for booking in returned_bookings}
for booking in returned_bookings:
# A real return inspection owns the actual reading timestamp. Using the booking's
# planned ends_at as a duplicate second reading can make an early return appear to
# go backwards after a newer real inspection. Keep booking data only as the legacy
# import fallback when no return inspection exists.
if booking.id in return_inspection_booking_ids:
continue
assert booking.end_odometer_km is not None
readings_by_vehicle.setdefault(booking.vehicle_id, []).append(
(
booking.ends_at,
booking.public_ref,
booking.end_odometer_km,
"booking",
booking.public_ref,
)
)
for inspection in inspections:
readings_by_vehicle.setdefault(inspection.vehicle_id, []).append(
(
inspection.completed_at,
inspection.public_ref,
inspection.odometer_km,
inspection.type,
(
booking_ref_by_id.get(inspection.booking_id)
if inspection.type == "return"
else None
),
)
)
for record in db.scalars(select(MaintenanceRecord)).all():
readings_by_vehicle.setdefault(record.vehicle_id, []).append(
(record.occurred_at, record.public_ref, record.odometer_km, "maintenance", None)
)
for vehicle_id, bookings in bookings_by_vehicle.items():
bookings.sort(key=lambda b: b.ends_at)
for earlier, later in zip(bookings, bookings[1:], strict=False):
# The query above filters end_odometer_km IS NOT NULL, so both are ints here.
assert earlier.end_odometer_km is not None
assert later.end_odometer_km is not None
if later.end_odometer_km < earlier.end_odometer_km:
for vehicle_id, readings in readings_by_vehicle.items():
readings.sort(key=lambda reading: (reading[0], reading[1]))
highest = readings[0] if readings else None
for later in readings[1:]:
assert highest is not None
if later[2] < highest[2]:
vehicle = vehicles[vehicle_id]
_open_issue(
had_open_issue = _has_open_issue(
db,
scan,
rule_type="odometer_regression",
entity_type="vehicle",
entity_id=vehicle_id,
severity="medium",
summary=(
f"Booking {later.public_ref} recorded {later.end_odometer_km} km, "
f"below the {earlier.end_odometer_km} km recorded by earlier "
f"booking {earlier.public_ref}."
),
entity_ref=vehicle.public_ref,
related_refs=[earlier.public_ref, later.public_ref],
signals=[
{
"code": "odometer.regression",
"params": {
"later_ref": later.public_ref,
"later_km": later.end_odometer_km,
"earlier_ref": earlier.public_ref,
"earlier_km": earlier.end_odometer_km,
},
}
],
"odometer_regression",
"vehicle",
vehicle_id,
)
break
issue = open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=later[1],
reading_km=later[2],
canonical_km=highest[2],
canonical_ref=highest[1],
source_type=later[3],
related_refs=list(
dict.fromkeys(
[highest[1], later[1], *([later[4]] if later[4] is not None else [])]
)
),
correctable_booking_refs=[later[4]] if later[4] is not None else [],
public_ref=_new_scan_ref("DQ-SCAN"),
actor_label=actor_label,
actor_type=actor_type,
)
if issue is not None and not had_open_issue:
scan.bump("odometer_regression")
if later[2] > highest[2]:
highest = later
def run_scan(
db: Session, *, actor_label: str | None = None, actor_type: str = "user"
db: Session,
*,
actor_label: str | None = None,
actor_type: str = "user",
commit: bool = True,
) -> ScanResult:
# The check-then-insert work below spans several rules. Serialise whole scans at the
# database boundary so API and n8n triggers cannot both observe an empty condition.
@@ -293,7 +332,12 @@ def run_scan(
scan = ScanResult()
scan_duplicate_customers(db, scan, _open_issue)
_scan_missing_required_fields(db, scan)
_scan_odometer_regressions(db, scan)
_scan_odometer_regressions(
db,
scan,
actor_label=actor_label,
actor_type=actor_type,
)
_scan_booking_overlaps(db, scan)
_scan_vehicle_status_conflicts(db, scan)
if actor_label is not None:
@@ -305,28 +349,13 @@ def run_scan(
entity_type="system",
metadata={"created": scan.created},
)
db.commit()
if commit:
db.commit()
else:
db.flush()
return scan
def _load_open_issue(
db: Session, public_ref: str, *, lock: bool = True
) -> DataQualityIssue:
statement = select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref)
if lock:
statement = statement.with_for_update()
issue = db.scalar(statement)
if issue is None:
raise AppError("ISSUE_NOT_FOUND", "Data quality issue not found.", status_code=404)
if issue.status != "open":
raise AppError(
"ISSUE_NOT_OPEN",
f"Issue is '{issue.status}', not 'open'.",
status_code=409,
)
return issue
def defer_issue(db: Session, public_ref: str, actor: CurrentUser) -> DataQualityIssue:
issue = _load_open_issue(db, public_ref)
issue.status = "deferred"
@@ -456,115 +485,6 @@ def provide_missing_fields(
return issue
def resolve_odometer_regression(
db: Session, public_ref: str, body: ResolveOdometerRegressionRequest, actor: CurrentUser
) -> DataQualityIssue:
issue = _load_open_issue(db, public_ref)
if issue.rule_type != "odometer_regression":
raise AppError(
"NOT_AN_ODOMETER_ISSUE",
"This issue is not an odometer_regression issue.",
status_code=409,
)
# Lock order is booking -> vehicle everywhere (checkout, return, reschedule); taking
# the vehicle lock first here would be a deadlock waiting to happen under concurrency.
booking: Booking | None = None
if body.decision != "retain_canonical":
related_refs = issue.evidence_json.get("related_refs", [])
if body.booking_ref not in related_refs:
raise AppError(
"INVALID_BOOKING_REFERENCE",
"booking_ref must be one of this issue's related bookings.",
status_code=422,
)
if body.corrected_odometer_km is None:
raise AppError(
"CORRECTED_VALUE_REQUIRED",
"corrected_odometer_km is required when correcting a reading.",
status_code=422,
)
booking = db.scalar(
select(Booking).where(Booking.public_ref == body.booking_ref).with_for_update()
)
if booking is None:
raise AppError(
"BOOKING_NOT_FOUND", "The booking to correct was not found.", status_code=404
)
vehicle = db.scalar(select(Vehicle).where(Vehicle.id == issue.entity_id).with_for_update())
if vehicle is None:
raise AppError(
"VEHICLE_NOT_FOUND", "The vehicle for this issue was not found.", status_code=404
)
correlation_id = uuid.uuid4()
if body.decision == "retain_canonical":
record_audit_event(
db,
actor_type="user",
actor_label=actor.display_name,
action="data_quality_odometer_retained",
entity_type="vehicle",
entity_id=vehicle.id,
correlation_id=correlation_id,
metadata={"issue_ref": issue.public_ref, "canonical_odometer_km": vehicle.odometer_km},
)
else:
assert booking is not None and body.corrected_odometer_km is not None
# Never silently lower the canonical odometer: a correction must be at or above
# the current canonical value, otherwise it would just create a new regression.
if body.corrected_odometer_km < vehicle.odometer_km:
raise AppError(
"CORRECTION_BELOW_CANONICAL",
(
f"Corrected value {body.corrected_odometer_km} km is still below the "
f"canonical {vehicle.odometer_km} km; it would not resolve the regression."
),
status_code=422,
)
before = {
"booking_end_odometer_km": booking.end_odometer_km,
"vehicle_odometer_km": vehicle.odometer_km,
}
booking.end_odometer_km = body.corrected_odometer_km
vehicle.odometer_km = body.corrected_odometer_km
vehicle.version += 1
record_audit_event(
db,
actor_type="user",
actor_label=actor.display_name,
action="data_quality_odometer_corrected",
entity_type="vehicle",
entity_id=vehicle.id,
correlation_id=correlation_id,
before=before,
after={
"booking_end_odometer_km": booking.end_odometer_km,
"vehicle_odometer_km": vehicle.odometer_km,
},
metadata={"issue_ref": issue.public_ref, "booking_ref": booking.public_ref},
)
issue.status = "resolved"
issue.resolved_at = datetime.now(UTC)
issue.resolved_by = actor.display_name
record_audit_event(
db,
actor_type="user",
actor_label=actor.display_name,
action="data_quality_issue_resolved",
entity_type="data_quality_issue",
entity_id=issue.id,
correlation_id=correlation_id,
before={"status": "open"},
after={"status": "resolved"},
metadata={"decision": body.decision, "note": body.note},
)
db.commit()
return issue
def resolve_booking_overlap(
db: Session, public_ref: str, booking_ref: str, note: str | None, actor: CurrentUser
) -> DataQualityIssue:
@@ -0,0 +1,59 @@
from __future__ import annotations
import uuid
from datetime import datetime, timedelta
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models.data_quality import DataQualityIssue
def issue_due_at(detected_at: datetime, severity: str) -> datetime:
"""Return the local operational SLA deadline for a newly detected issue."""
return detected_at + {
"high": timedelta(hours=4),
"medium": timedelta(days=1),
"low": timedelta(days=3),
}.get(severity, timedelta(days=1))
def has_open_issue(
db: Session,
rule_type: str,
entity_type: str,
entity_id: uuid.UUID,
) -> bool:
return (
db.scalar(
select(DataQualityIssue.id).where(
DataQualityIssue.rule_type == rule_type,
DataQualityIssue.entity_type == entity_type,
DataQualityIssue.entity_id == entity_id,
DataQualityIssue.status == "open",
)
)
is not None
)
def new_scan_ref(prefix: str) -> str:
"""Generate a stable human-readable prefix with a concurrent-safe suffix."""
return f"{prefix}-{uuid.uuid4().hex[:10].upper()}"
def load_open_issue(db: Session, public_ref: str, *, lock: bool = True) -> DataQualityIssue:
statement = select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref)
if lock:
statement = statement.with_for_update().execution_options(populate_existing=True)
issue = db.scalar(statement)
if issue is None:
raise AppError("ISSUE_NOT_FOUND", "Data quality issue not found.", status_code=404)
if issue.status != "open":
raise AppError(
"ISSUE_NOT_OPEN",
f"Issue is '{issue.status}', not 'open'.",
status_code=409,
)
return issue
@@ -90,9 +90,7 @@ def scan_duplicate_customers[ScanType: ScanAccumulator](
ratio = SequenceMatcher(None, name_a, name_b).ratio()
if ratio >= 0.5:
score += round(ratio * 30)
signals.append(
{"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}}
)
signals.append({"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}})
summary_parts.append("similar name")
if score >= DUPLICATE_THRESHOLD:
@@ -0,0 +1,507 @@
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models.booking import Booking
from app.models.data_quality import DataQualityIssue
from app.models.inspection import Inspection
from app.models.vehicle import Vehicle
from app.schemas import CurrentUser, ResolveOdometerRegressionRequest
from app.services.audit import record_audit_event
from app.services.data_quality_common import issue_due_at, load_open_issue, new_scan_ref
def _odometer_fingerprint(
*,
source_type: str,
later_ref: str,
later_km: int,
) -> dict[str, str | int]:
"""Stable identity for one reviewed regression, independent of issue IDs."""
return {
"source_type": source_type,
"later_ref": later_ref,
"later_km": later_km,
}
def _was_odometer_regression_retained(
db: Session,
*,
vehicle_id: uuid.UUID,
fingerprint: dict[str, str | int],
) -> bool:
reviewed = db.scalars(
select(DataQualityIssue).where(
DataQualityIssue.rule_type == "odometer_regression",
DataQualityIssue.entity_type == "vehicle",
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.status == "resolved",
)
).all()
return any(
issue.evidence_json.get("resolution_decision") == "retain_canonical"
and fingerprint in issue.evidence_json.get("retained_odometer_fingerprints", [])
for issue in reviewed
)
def open_odometer_regression_issue(
db: Session,
*,
vehicle: Vehicle,
reading_ref: str,
reading_km: int,
canonical_km: int,
source_type: str,
related_refs: list[str],
correctable_booking_refs: list[str] | None = None,
canonical_ref: str | None = None,
detected_at: datetime | None = None,
public_ref: str | None = None,
actor_label: str | None = None,
actor_type: str = "user",
correlation_id: uuid.UUID | None = None,
) -> DataQualityIssue | None:
"""Open one explainable DQ-03 issue while the caller holds the vehicle lock."""
if reading_km >= canonical_km:
return None
earlier_ref = canonical_ref or vehicle.public_ref
fingerprint = _odometer_fingerprint(
source_type=source_type,
later_ref=reading_ref,
later_km=reading_km,
)
# A manager's explicit "retain canonical" decision acknowledges this exact source
# fact. Reopening it on every scheduled scan would create churn; only changed or new
# evidence (and therefore a different fingerprint) is actionable again.
if _was_odometer_regression_retained(
db,
vehicle_id=vehicle.id,
fingerprint=fingerprint,
):
return None
existing = db.scalar(
select(DataQualityIssue)
.where(
DataQualityIssue.rule_type == "odometer_regression",
DataQualityIssue.entity_type == "vehicle",
DataQualityIssue.entity_id == vehicle.id,
DataQualityIssue.status == "open",
)
.with_for_update()
.execution_options(populate_existing=True)
)
if existing is not None:
before_related_refs = list(existing.evidence_json.get("related_refs", []))
before_correctable_refs = list(existing.evidence_json.get("correctable_booking_refs", []))
new_signal = {
"code": "odometer.regression",
"source_type": source_type,
"params": {
"later_ref": reading_ref,
"later_km": reading_km,
"earlier_ref": earlier_ref,
"earlier_km": canonical_km,
},
}
existing_signals = list(existing.evidence_json.get("signals", []))
signal_was_new = not any(
isinstance(signal, dict)
and isinstance(signal.get("params"), dict)
and _odometer_fingerprint(
source_type=str(
signal.get("source_type", existing.evidence_json.get("source_type"))
),
later_ref=str(signal["params"].get("later_ref")),
later_km=signal["params"].get("later_km"),
)
== fingerprint
for signal in existing_signals
if isinstance(signal, dict)
and isinstance(signal.get("params"), dict)
and isinstance(signal["params"].get("later_km"), int)
)
if signal_was_new:
existing_signals.append(new_signal)
merged_related_refs = list(
dict.fromkeys([*before_related_refs, *(related_refs if signal_was_new else [])])
)
merged_correctable_refs = list(
dict.fromkeys([*before_correctable_refs, *(correctable_booking_refs or [])])
)
if (
not signal_was_new
and merged_related_refs == before_related_refs
and merged_correctable_refs == before_correctable_refs
):
return existing
source_types = list(existing.evidence_json.get("source_types", []))
previous_source_type = existing.evidence_json.get("source_type")
if (
isinstance(previous_source_type, str)
and previous_source_type != "multiple"
and previous_source_type not in source_types
):
source_types.append(previous_source_type)
if source_type not in source_types:
source_types.append(source_type)
existing.evidence_json = {
**existing.evidence_json,
"summary": (
f"{source_type.title()} {reading_ref} recorded {reading_km} km, below "
f"the canonical {canonical_km} km for {vehicle.public_ref}."
),
"related_refs": merged_related_refs,
"correctable_booking_refs": merged_correctable_refs,
"source_type": source_type if len(source_types) == 1 else "multiple",
"source_types": source_types,
"signals": existing_signals,
}
if actor_label is not None and (
merged_related_refs != before_related_refs
or merged_correctable_refs != before_correctable_refs
or signal_was_new
):
record_audit_event(
db,
actor_type=actor_type,
actor_label=actor_label,
action="data_quality_issue_evidence_updated",
entity_type="data_quality_issue",
entity_id=existing.id,
correlation_id=correlation_id,
before={
"related_refs": before_related_refs,
"correctable_booking_refs": before_correctable_refs,
},
after={
"related_refs": merged_related_refs,
"correctable_booking_refs": merged_correctable_refs,
},
metadata={"vehicle_ref": vehicle.public_ref, "reading_ref": reading_ref},
)
return existing
now = detected_at or datetime.now(UTC)
previous = db.scalar(
select(DataQualityIssue)
.where(
DataQualityIssue.rule_type == "odometer_regression",
DataQualityIssue.entity_type == "vehicle",
DataQualityIssue.entity_id == vehicle.id,
DataQualityIssue.status != "open",
)
.order_by(DataQualityIssue.detected_at.desc())
)
evidence = {
"summary": (
f"{source_type.title()} {reading_ref} recorded {reading_km} km, below "
f"the canonical {canonical_km} km for {vehicle.public_ref}."
),
"entity_ref": vehicle.public_ref,
"related_refs": related_refs,
"correctable_booking_refs": correctable_booking_refs or [],
"source_type": source_type,
"source_types": [source_type],
"signals": [
{
"code": "odometer.regression",
"source_type": source_type,
"params": {
"later_ref": reading_ref,
"later_km": reading_km,
"earlier_ref": earlier_ref,
"earlier_km": canonical_km,
},
}
],
}
if previous is not None:
evidence["reopened_from"] = previous.public_ref
evidence["previous_decision"] = previous.status
issue = DataQualityIssue(
public_ref=public_ref or new_scan_ref("DQ-ODO"),
rule_type="odometer_regression",
entity_type="vehicle",
entity_id=vehicle.id,
severity="medium",
status="open",
evidence_json=evidence,
proposed_action_json={},
detected_at=now,
due_at=issue_due_at(now, "medium"),
)
db.add(issue)
db.flush()
if actor_label is not None:
record_audit_event(
db,
actor_type=actor_type,
actor_label=actor_label,
action="data_quality_issue_created",
entity_type="data_quality_issue",
entity_id=issue.id,
correlation_id=correlation_id,
after={"status": "open", "rule_type": "odometer_regression"},
metadata={"vehicle_ref": vehicle.public_ref, "reading_ref": reading_ref},
)
return issue
def resolve_odometer_regression(
db: Session, public_ref: str, body: ResolveOdometerRegressionRequest, actor: CurrentUser
) -> DataQualityIssue:
# Read the routing data without a row lock first. The canonical mutation order is
# booking -> vehicle -> issue -> inspection everywhere, matching checkout/return.
# Locking the issue before the booking creates a resolver-vs-return deadlock.
issue_snapshot = load_open_issue(db, public_ref, lock=False)
if issue_snapshot.rule_type != "odometer_regression":
raise AppError(
"NOT_AN_ODOMETER_ISSUE",
"This issue is not an odometer_regression issue.",
status_code=409,
)
def correctable_refs(issue: DataQualityIssue) -> list[str]:
if "correctable_booking_refs" in issue.evidence_json:
refs = issue.evidence_json.get("correctable_booking_refs", [])
else:
# Backward compatibility for issues created before source-aware evidence.
refs = [
ref
for ref in issue.evidence_json.get("related_refs", [])
if isinstance(ref, str) and ref.startswith("BK-")
]
return [ref for ref in refs if isinstance(ref, str)]
booking: Booking | None = None
if body.decision != "retain_canonical":
if body.booking_ref not in correctable_refs(issue_snapshot):
raise AppError(
"INVALID_BOOKING_REFERENCE",
"booking_ref must be one of this issue's related bookings.",
status_code=422,
)
if body.corrected_odometer_km is None:
raise AppError(
"CORRECTED_VALUE_REQUIRED",
"corrected_odometer_km is required when correcting a reading.",
status_code=422,
)
booking = db.scalar(
select(Booking).where(Booking.public_ref == body.booking_ref).with_for_update()
)
if booking is None:
raise AppError(
"BOOKING_NOT_FOUND", "The booking to correct was not found.", status_code=404
)
vehicle = db.scalar(
select(Vehicle).where(Vehicle.id == issue_snapshot.entity_id).with_for_update()
)
if vehicle is None:
raise AppError(
"VEHICLE_NOT_FOUND", "The vehicle for this issue was not found.", status_code=404
)
# A return or scan may have appended evidence while we waited for the domain locks.
# Lock and refresh the issue only now, then revalidate every decision against that
# current evidence instead of resolving a stale snapshot.
issue = load_open_issue(db, public_ref)
if issue.rule_type != "odometer_regression" or issue.entity_id != vehicle.id:
raise AppError(
"ISSUE_CHANGED",
"The issue changed while the correction was being prepared. Review it again.",
status_code=409,
)
if booking is not None:
if booking.vehicle_id != vehicle.id or booking.public_ref not in correctable_refs(issue):
raise AppError(
"INVALID_BOOKING_REFERENCE",
"booking_ref must be one of this issue's related bookings.",
status_code=422,
)
correlation_id = uuid.uuid4()
if body.decision == "retain_canonical":
retained_fingerprints: list[dict[str, str | int]] = []
fallback_source_type = str(issue.evidence_json.get("source_type", "unknown"))
for signal in issue.evidence_json.get("signals", []):
if not isinstance(signal, dict) or signal.get("code") != "odometer.regression":
continue
params = signal.get("params")
if not isinstance(params, dict):
continue
source_type = signal.get("source_type", fallback_source_type)
later_ref = params.get("later_ref")
later_km = params.get("later_km")
earlier_ref = params.get("earlier_ref")
earlier_km = params.get("earlier_km")
if (
isinstance(source_type, str)
and isinstance(later_ref, str)
and isinstance(later_km, int)
and isinstance(earlier_ref, str)
and isinstance(earlier_km, int)
):
retained_fingerprints.append(
_odometer_fingerprint(
source_type=source_type,
later_ref=later_ref,
later_km=later_km,
)
)
issue.evidence_json = {
**issue.evidence_json,
"resolution_decision": "retain_canonical",
"retained_odometer_fingerprints": retained_fingerprints,
}
record_audit_event(
db,
actor_type="user",
actor_label=actor.display_name,
action="data_quality_odometer_retained",
entity_type="vehicle",
entity_id=vehicle.id,
correlation_id=correlation_id,
metadata={
"issue_ref": issue.public_ref,
"canonical_odometer_km": vehicle.odometer_km,
"retained_fingerprint_count": len(retained_fingerprints),
},
)
else:
assert booking is not None and body.corrected_odometer_km is not None
# Never silently lower the canonical odometer: a correction must be at or above
# the current canonical value, otherwise it would just create a new regression.
if body.corrected_odometer_km < vehicle.odometer_km:
raise AppError(
"CORRECTION_BELOW_CANONICAL",
(
f"Corrected value {body.corrected_odometer_km} km is still below the "
f"canonical {vehicle.odometer_km} km; it would not resolve the regression."
),
status_code=422,
)
# A returned booking has two persisted representations of the same reading.
# Correct every return inspection for the selected booking, including historical
# seed rows whose old issue evidence did not yet cite the inspection explicitly.
related_inspections = db.scalars(
select(Inspection)
.where(Inspection.booking_id == booking.id, Inspection.type == "return")
.with_for_update()
).all()
before = {
"booking_end_odometer_km": booking.end_odometer_km,
"vehicle_odometer_km": vehicle.odometer_km,
"inspection_odometer_km": {
inspection.public_ref: inspection.odometer_km for inspection in related_inspections
},
}
booking.end_odometer_km = body.corrected_odometer_km
for inspection in related_inspections:
inspection.odometer_km = body.corrected_odometer_km
vehicle.odometer_km = body.corrected_odometer_km
vehicle.version += 1
record_audit_event(
db,
actor_type="user",
actor_label=actor.display_name,
action="data_quality_odometer_corrected",
entity_type="vehicle",
entity_id=vehicle.id,
correlation_id=correlation_id,
before=before,
after={
"booking_end_odometer_km": booking.end_odometer_km,
"vehicle_odometer_km": vehicle.odometer_km,
"inspection_odometer_km": {
inspection.public_ref: inspection.odometer_km
for inspection in related_inspections
},
},
metadata={"issue_ref": issue.public_ref, "booking_ref": booking.public_ref},
)
has_remaining_evidence = False
if booking is not None:
target_refs = {booking.public_ref, *(item.public_ref for item in related_inspections)}
before_related = [
ref for ref in issue.evidence_json.get("related_refs", []) if isinstance(ref, str)
]
before_correctable = correctable_refs(issue)
before_signals = [
signal for signal in issue.evidence_json.get("signals", []) if isinstance(signal, dict)
]
remaining_signals = [
signal
for signal in before_signals
if not (
isinstance(signal.get("params"), dict)
and signal["params"].get("later_ref") in target_refs
)
]
remaining_related = [ref for ref in before_related if ref not in target_refs]
remaining_correctable = [ref for ref in before_correctable if ref != booking.public_ref]
has_remaining_evidence = bool(remaining_signals or remaining_correctable)
issue.evidence_json = {
**issue.evidence_json,
"summary": (
"Additional odometer regression evidence remains for review."
if has_remaining_evidence
else issue.evidence_json.get("summary", "Odometer reading corrected.")
),
"related_refs": remaining_related,
"correctable_booking_refs": remaining_correctable,
"signals": remaining_signals,
}
if has_remaining_evidence:
record_audit_event(
db,
actor_type="user",
actor_label=actor.display_name,
action="data_quality_issue_partially_resolved",
entity_type="data_quality_issue",
entity_id=issue.id,
correlation_id=correlation_id,
before={
"related_refs": before_related,
"correctable_booking_refs": before_correctable,
"signal_count": len(before_signals),
},
after={
"related_refs": remaining_related,
"correctable_booking_refs": remaining_correctable,
"signal_count": len(remaining_signals),
},
metadata={
"decision": body.decision,
"booking_ref": booking.public_ref,
"note": body.note,
},
)
if not has_remaining_evidence:
issue.status = "resolved"
issue.resolved_at = datetime.now(UTC)
issue.resolved_by = actor.display_name
record_audit_event(
db,
actor_type="user",
actor_label=actor.display_name,
action="data_quality_issue_resolved",
entity_type="data_quality_issue",
entity_id=issue.id,
correlation_id=correlation_id,
before={"status": "open"},
after={"status": "resolved"},
metadata={"decision": body.decision, "note": body.note},
)
db.commit()
return issue
+23 -10
View File
@@ -9,16 +9,28 @@ from app.core.config import get_settings
from app.models.audit import AuditEvent
from app.models.booking import Booking
from app.models.data_quality import DataQualityIssue
from app.models.outbox import OutboxEvent
from app.models.outbox import OutboxEvent, is_demo_scenario_failure
from app.schemas import DemoIntegrationSummaryOut, DemoManifestOut, DemoScenarioOut
from app.services.integration_status import derive_mcp_hub_status, derive_n8n_status
from app.services.knowledge import get_knowledge_provider
from app.services.knowledge import KnowledgeHealth, get_knowledge_provider
settings = get_settings()
_FAILED_DEMO_EVENT_ID = "00000000-0000-4000-8000-000000000020"
def _knowledge_scenario_ready(health: KnowledgeHealth) -> bool:
"""Require a reachable provider and independently verified indexed documents.
``source_document_count`` describes local Markdown files, while a sync report only
describes an upload attempt. Neither proves that the active provider can retrieve a
corpus, so an unavailable verification count remains honestly not ready.
"""
return bool(
health.available and health.document_count is not None and health.document_count > 0
)
def _last_reset(db: Session) -> tuple[datetime | None, str | None]:
marker = db.scalar(
select(AuditEvent)
@@ -49,7 +61,8 @@ def _scenarios(db: Session) -> list[DemoScenarioOut]:
return_ready = bool(booking and booking.status == "active" and booking.end_odometer_km is None)
duplicate_ready = bool(duplicate_issue and duplicate_issue.status == "open")
overlap_ready = bool(overlap_issue and overlap_issue.status == "open")
automation_ready = bool(failed_run and failed_run.delivery_status == "failed")
automation_ready = bool(failed_run and is_demo_scenario_failure(failed_run))
knowledge_ready = _knowledge_scenario_ready(knowledge_health)
return [
DemoScenarioOut(
@@ -119,8 +132,8 @@ def _scenarios(db: Session) -> list[DemoScenarioOut]:
estimated_minutes=2,
required_roles=["rental_employee", "operations_manager"],
start_path="/knowledge",
ready=knowledge_health.available,
blocked_reason_code=None if knowledge_health.available else "knowledgeUnavailable",
ready=knowledge_ready,
blocked_reason_code=None if knowledge_ready else "knowledgeUnavailable",
),
]
@@ -145,9 +158,11 @@ def _integrations(db: Session) -> list[DemoIntegrationSummaryOut]:
key="ragcore",
status_code=(
"operational"
if knowledge_health.provider == "ragcore" and knowledge_health.available
if knowledge_health.provider == "ragcore"
and _knowledge_scenario_ready(knowledge_health)
else "demoMode"
if knowledge_health.provider == "demo" and knowledge_health.available
if knowledge_health.provider == "demo"
and _knowledge_scenario_ready(knowledge_health)
else "unavailable"
),
detail_code=(
@@ -169,9 +184,7 @@ def _integrations(db: Session) -> list[DemoIntegrationSummaryOut]:
# "operational" -- same evidence rule the integration status page uses.
status_code="operational" if mcp_hub.state == "operational" else "notConnected",
detail_code=(
"mcpDetailOperational"
if mcp_hub.state == "operational"
else "mcpDetailPrepared"
"mcpDetailOperational" if mcp_hub.state == "operational" else "mcpDetailPrepared"
),
detail_params={},
),
+59 -5
View File
@@ -77,6 +77,14 @@ def _claim_due_events(batch_size: int = 5) -> list[uuid.UUID]:
for row in rows:
row.delivery_status = "delivering"
row.next_attempt_at = lease_deadline
# The token is stored inside the internal payload (the wire envelope below
# explicitly selects only contract fields). It lets the outcome transaction
# prove that this is still the same lease after network I/O. A stale worker
# must never overwrite a later reclaim/retry or an idempotent callback.
row.payload_json = {
**row.payload_json,
"_delivery_claim_token": str(uuid.uuid4()),
}
db.commit()
return claimed_ids
finally:
@@ -109,6 +117,9 @@ def _deliver_one(event_id: uuid.UUID) -> None:
wire_event = None
payload_error = f"Malformed outbox payload, missing key {exc}"
attempts = event.attempts
claim_token = event.payload_json.get("_delivery_claim_token")
if event.delivery_status != "delivering" or not isinstance(claim_token, str):
return
finally:
db.close()
@@ -130,9 +141,33 @@ def _deliver_one(event_id: uuid.UUID) -> None:
except ValueError:
body = None
if isinstance(body, dict):
success = bool(body.get("ok", True))
error = None if success else f"n8n reported failure: {body}"
error_code = None if success else "remoteReportedFailure"
acknowledged = body.get("ok") is True
response_event_id = body.get("event_id")
event_id_matches = response_event_id == str(event_id)
result = body.get("result")
execution_id = result.get("execution_id") if isinstance(result, dict) else None
execution_id_valid = isinstance(execution_id, str) and bool(execution_id.strip())
success = acknowledged and event_id_matches and execution_id_valid
if success:
error = None
error_code = None
elif not acknowledged:
error = (
"n8n response did not explicitly acknowledge the event with ok=true: "
f"{body}"
)
error_code = (
"remoteReportedFailure" if body.get("ok") is False else "malformedResponse"
)
elif not event_id_matches:
error = (
"n8n acknowledged a different event ID "
f"(expected {event_id}, received {response_event_id!r})"
)
error_code = "mismatchedEventId"
else:
error = "n8n response omitted a valid result.execution_id"
error_code = "malformedResponse"
else:
# A 2xx status with a non-object (or unparsable) body means the workflow
# itself errored before its "Respond to Webhook" node ran -- n8n's default
@@ -152,16 +187,35 @@ def _deliver_one(event_id: uuid.UUID) -> None:
db = SessionLocal()
try:
event = db.get(OutboxEvent, event_id)
event = db.scalar(
select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update()
)
if event is None:
return
if (
event.delivery_status != "delivering"
or event.attempts != attempts
or event.payload_json.get("_delivery_claim_token") != claim_token
):
logger.info(
"Ignoring stale delivery outcome for event %s because lease ownership changed",
event_id,
)
return
event.attempts = attempts + 1
event.payload_json = {
key: value
for key, value in event.payload_json.items()
if key != "_delivery_claim_token"
}
if success:
event.delivery_status = "succeeded"
event.last_error = None
event.last_error_code = None
event.next_attempt_at = None
event.external_run_id = str((body or {}).get("event_id", event_id))
result = (body or {}).get("result")
execution_id = result.get("execution_id") if isinstance(result, dict) else None
event.external_run_id = execution_id if isinstance(execution_id, str) else None
else:
event.last_error = (error or "delivery failed")[:2000]
event.last_error_code = error_code or "unknownError"
+45 -8
View File
@@ -7,7 +7,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any, Literal
import httpx
from sqlalchemy import Row, func, select
from sqlalchemy import Row, func, or_, select
from sqlalchemy.orm import Session
from app.core.config import get_settings
@@ -93,7 +93,10 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
latest_failure_at = db.scalar(
select(func.max(OutboxEvent.updated_at)).where(
OutboxEvent.delivery_status == "failed",
OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
or_(
OutboxEvent.last_error_code.is_(None),
OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
),
)
)
latest_demo_scenario_at = db.scalar(
@@ -106,11 +109,17 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
state: Literal["disabled", "unavailable", "degraded", "operational", "no_evidence"]
if not settings.n8n_dispatch_enabled:
state = "disabled"
elif not settings.n8n_webhook_url:
# Persisted history does not make a currently unconfigured dispatcher green.
state = "unavailable"
elif unexpected_failed > 0 and succeeded == 0:
state = "unavailable"
elif unexpected_failed > 0:
state = "degraded"
elif succeeded > 0 or pending > 0 or delivering > 0:
# Queued/in-flight work proves only that MobilityOps has work for the dispatcher;
# it does not prove that n8n has ever accepted a delivery. A green state requires
# at least one persisted successful round trip.
elif succeeded > 0:
state = "operational"
else:
state = "no_evidence"
@@ -128,8 +137,14 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
# it finishes uploading procedures to RAGcore (app/api/routers/integrations.py::
# procedures_sync_result), the same "the workflow's own callback is the evidence"
# pattern the scheduled scan and error handler already use below.
latest_procedure_sync_at = db.scalar(
select(func.max(AuditEvent.occurred_at)).where(AuditEvent.action == "n8n_procedures_synced")
latest_procedure_sync_row = db.execute(
select(AuditEvent.occurred_at, AuditEvent.after_json, AuditEvent.metadata_json)
.where(AuditEvent.action == "n8n_procedures_synced")
.order_by(AuditEvent.occurred_at.desc())
.limit(1)
).first()
latest_procedure_sync_at = (
latest_procedure_sync_row[0] if latest_procedure_sync_row is not None else None
)
# Error handler evidence: registrations posted by the "Fleet Ops — Workflow Error
@@ -198,6 +213,24 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
seen_at = failure_signal[0]
last_status = "failed"
execution_id = failure_signal[1]
if name == "Fleet Ops — RAGcore Procedure Sync" and latest_procedure_sync_row:
sync_at, sync_result, sync_metadata = latest_procedure_sync_row
synced = (sync_result or {}).get("synced", 0)
failed_syncs = (sync_result or {}).get("failed", 0)
# The result callback is authoritative for corpus delivery. A generic
# succeeded heartbeat cannot turn a zero/partial upload green.
sync_result_failed = (
not isinstance(synced, int)
or not isinstance(failed_syncs, int)
or synced <= 0
or failed_syncs > 0
)
if sync_result_failed:
last_status = "failed"
if seen_at is None or sync_at > seen_at:
seen_at = sync_at
if sync_result_failed:
execution_id = (sync_metadata or {}).get("execution_id")
workflow_state: Literal["no_evidence", "healthy", "stale", "failed"]
if seen_at is None:
workflow_state = "no_evidence"
@@ -265,16 +298,20 @@ def derive_mcp_hub_status(db: Session) -> McpHubIntegrationStatus:
last_client = latest_call_row[1] if latest_call_row else None
last_tool = (latest_call_row[2] or {}).get("tool") if latest_call_row else None
hub_reachable = _check_hub_reachable()
state: Literal["not_configured", "no_evidence", "operational"]
if not settings.mcp_hub_registration_enabled:
state = "not_configured"
elif total_calls > 0:
elif total_calls > 0 and hub_reachable is not False:
state = "operational"
else:
# Historical tool calls remain useful telemetry, but cannot support a current
# operational claim when the configured Hub health endpoint is unreachable.
# ``hub_reachable`` stays available separately so consumers can distinguish
# this from a provider that simply has no call evidence yet.
state = "no_evidence"
hub_reachable = _check_hub_reachable()
return McpHubIntegrationStatus(
registration_enabled=settings.mcp_hub_registration_enabled,
state=state,
+167 -45
View File
@@ -1,5 +1,8 @@
from __future__ import annotations
import hashlib
import re
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from threading import Lock
@@ -13,6 +16,10 @@ from app.services.knowledge import EvidenceState, GroundedAnswer, KnowledgeHealt
from app.services.knowledge.procedures import ProcedureDocument, iter_procedure_documents
_GROUNDED_ANSWERABILITY = {"answerable", "partially_answerable"}
_ANSWERABILITY_STATES = _GROUNDED_ANSWERABILITY | {
"not_answerable",
"conflicting_evidence",
}
# Mirrors DemoKnowledgeProvider's own extractive template in spirit: a real cited
# excerpt wrapped in a fixed sentence, never a generated summary. Used only as a
@@ -47,6 +54,11 @@ _DOMAIN_CONCEPTS: dict[str, tuple[str, ...]] = {
"availability": ("available", "availability", "beschikbaar", "disponible", "disponibilité"),
"technical": ("technical", "warning", "technisch", "waarschuwing", "technique", "alerte"),
}
_ANSWERABLE_INTENT_CONCEPTS = frozenset(_DOMAIN_CONCEPTS) - {"vehicle", "customer"}
def _normalize_evidence_text(value: str) -> str:
return " ".join(re.findall(r"\w+", value.casefold()))
def _question_concepts(question: str) -> set[str]:
@@ -93,7 +105,9 @@ def _rank_sources_for_concepts(sources: list[SourceCard], concepts: set[str]) ->
)
def _retrieval_score(result: dict) -> float | None:
def _retrieval_score(result: object) -> float | None:
if not isinstance(result, dict):
return None
scores = result.get("scores")
if not isinstance(scores, dict):
return None
@@ -124,23 +138,81 @@ class RAGcoreKnowledgeProvider:
excerpt RAGcore's own search actually found, wrapped in the same fixed citation
template `DemoKnowledgeProvider` uses -- never a fabricated summary.
Known gap, not fixable from this side: RAGcore's ingest pipeline currently tags every
chunk's `language` payload field as `"en"` regardless of actual document language (the
`/v1/uploads` contract has no per-file language field for a caller to set correctly).
Filtering search/answer requests by requested UI language would therefore silently
exclude genuinely-relevant nl-BE/fr-BE content, so this adapter deliberately does not
filter by language -- retrieval relies on the embedding model's cross-lingual matching.
Retrieval is scoped to the stable RAGcore ``source_id`` values owned by Fleet Ops for
the requested UI language. Returned internal document/version UUIDs are deliberately
not treated as Fleet Ops identifiers: every citation must instead prove the complete
managed-source chain (source id, URI, locator, checksum and extractive local text).
"""
name = "ragcore"
def __init__(self) -> None:
self._settings = get_settings()
self._managed_documents_by_source_id: dict[str, ProcedureDocument] = {}
for document in iter_procedure_documents(Path(self._settings.knowledge_dir)):
self._managed_documents_by_source_id[document.source_id] = document
self._verification_cache: dict[str, tuple[float, int]] = {}
self._verification_lock = Lock()
self._answers_circuit_lock = Lock()
self._answers_circuit_open_until = 0.0
def _managed_source(self, citation: object, language: str) -> SourceCard | None:
if not isinstance(citation, dict):
return None
source_id = citation.get("source_id")
if not isinstance(source_id, str):
return None
document = self._managed_documents_by_source_id.get(str(source_id))
if document is None or document.language != language:
return None
if citation.get("locator") != f"{document.document_id}.md":
return None
if citation.get("source_uri") != f"ragcore://source/{document.source_id}":
return None
for field_name in ("id", "document_id", "document_version_id"):
value = citation.get(field_name)
if not isinstance(value, str):
return None
try:
parsed = uuid.UUID(value)
except (ValueError, TypeError, AttributeError):
return None
if parsed.int == 0:
return None
if not isinstance(citation.get("title"), str):
return None
section = citation.get("section")
if section is not None and not isinstance(section, str):
return None
excerpt = citation.get("excerpt")
if not isinstance(excerpt, str) or not excerpt.strip():
return None
expected_hash = hashlib.sha256(excerpt.encode("utf-8")).hexdigest()
if citation.get("excerpt_sha256") != expected_hash:
return None
# RAGcore owns chunking, but the cited excerpt must still be extractive evidence
# from the authoritative local procedure. Token normalization tolerates Markdown
# punctuation/whitespace while rejecting provider text that was never uploaded.
normalized_excerpt = _normalize_evidence_text(excerpt)
if not normalized_excerpt or normalized_excerpt not in _normalize_evidence_text(
document.content
):
return None
return SourceCard(
document_id=document.document_id,
title=document.title,
version=document.version,
section=section or "",
excerpt=excerpt,
)
def _managed_source_ids(self, language: str) -> list[str]:
return [
document.source_id
for document in self._managed_documents_by_source_id.values()
if document.language == language
]
def _answers_circuit_is_open(self) -> bool:
with self._answers_circuit_lock:
return monotonic() < self._answers_circuit_open_until
@@ -284,7 +356,7 @@ class RAGcoreKnowledgeProvider:
if self._answers_circuit_is_open():
KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "circuit_open").inc()
else:
answered = self._ask_via_answers(question, correlation_id)
answered = self._ask_via_answers(question, correlation_id, language)
if answered is not None:
return answered
# /v1/answers itself is unavailable (non-2xx or unreachable) -- fall back to
@@ -295,7 +367,9 @@ class RAGcoreKnowledgeProvider:
# generation step.
return self._ask_via_search_fallback(question, correlation_id, language)
def _ask_via_answers(self, question: str, correlation_id: str) -> GroundedAnswer | None:
def _ask_via_answers(
self, question: str, correlation_id: str, language: str
) -> GroundedAnswer | None:
"""Returns None (not a GroundedAnswer) when /v1/answers itself is unavailable,
so the caller can fall back to search -- as opposed to a real 200 response
classifying the question as insufficiently answerable, which is a genuine,
@@ -307,6 +381,7 @@ class RAGcoreKnowledgeProvider:
json={
"query": question,
"requested_space_ids": [self._settings.ragcore_space_id],
"filters": {"source_ids": self._managed_source_ids(language)},
},
)
if response.status_code != 200:
@@ -320,25 +395,73 @@ class RAGcoreKnowledgeProvider:
return None
try:
citations = {c["id"]: c for c in body.get("citations", [])}
sources = [
SourceCard(
document_id=str(citation["document_id"]),
title=citation["title"],
version=str(citation["document_version_id"]),
section=citation.get("section") or "",
excerpt=citation["excerpt"],
)
for citation in citations.values()
if not isinstance(body, dict):
raise TypeError("answer response must be an object")
# Validate the provider's complete AnswerResponse contract before trusting
# generated prose. These IDs and claim bindings are the evidence that RAGcore
# ran its deterministic claim/citation validator; one unrelated but otherwise
# valid citation must never make arbitrary answer text appear grounded.
uuid.UUID(str(body["answer_id"]))
uuid.UUID(str(body["retrieval_run_id"]))
raw_citations = body.get("citations", [])
if not isinstance(raw_citations, list):
raise TypeError("citations must be a list")
citation_ids: set[uuid.UUID] = set()
for citation in raw_citations:
if not isinstance(citation, dict):
raise TypeError("citation must be an object")
citation_ids.add(uuid.UUID(str(citation["id"])))
raw_claims = body["claims"]
if not isinstance(raw_claims, list):
raise TypeError("claims must be a list")
claims_are_bound = bool(raw_claims)
answer_text = body.get("answer")
for claim in raw_claims:
if not isinstance(claim, dict):
raise TypeError("claim must be an object")
claim_text = claim.get("text")
claim_citation_ids = claim.get("citation_ids")
if (
not isinstance(claim_text, str)
or not claim_text.strip()
or not isinstance(answer_text, str)
or claim_text.strip() not in answer_text
or not isinstance(claim_citation_ids, list)
or not claim_citation_ids
):
claims_are_bound = False
continue
try:
bound_ids = {uuid.UUID(str(item)) for item in claim_citation_ids}
except (TypeError, ValueError):
claims_are_bound = False
continue
if not bound_ids.issubset(citation_ids):
claims_are_bound = False
mapped_sources = [
source
for citation in raw_citations
if (source := self._managed_source(citation, language)) is not None
]
sources = _deduplicate_sources(sources)
citations_are_managed = len(mapped_sources) == len(raw_citations)
sources = _deduplicate_sources(mapped_sources)
answerability = body.get("answerability", "not_answerable")
is_grounded = answerability in _GROUNDED_ANSWERABILITY and sources
if answerability not in _ANSWERABILITY_STATES:
raise TypeError("answerability is invalid")
if not isinstance(answer_text, str):
raise TypeError("answer must be a string")
is_grounded = (
answerability in _GROUNDED_ANSWERABILITY
and bool(sources)
and citations_are_managed
and claims_are_bound
and bool(answer_text.strip())
)
evidence_state: EvidenceState = "grounded" if is_grounded else "insufficient"
KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", evidence_state).inc()
self._close_answers_circuit()
return GroundedAnswer(
answer=body.get("answer", "") if evidence_state == "grounded" else "",
answer=answer_text if evidence_state == "grounded" else "",
evidence_state=evidence_state,
sources=sources if evidence_state == "grounded" else [],
provider=self.name,
@@ -366,6 +489,7 @@ class RAGcoreKnowledgeProvider:
json={
"query": question,
"requested_space_ids": [self._settings.ragcore_space_id],
"filters": {"source_ids": self._managed_source_ids(language)},
"max_results": 5,
},
)
@@ -378,24 +502,24 @@ class RAGcoreKnowledgeProvider:
return unavailable
try:
if not isinstance(body, dict):
raise TypeError("search response must be an object")
results = body.get("results", [])
if not isinstance(results, list):
raise TypeError("results must be a list")
sources = [
SourceCard(
document_id=str(result["citation"]["document_id"]),
title=result["citation"]["title"],
version=str(result["citation"]["document_version_id"]),
section=result["citation"].get("section") or "",
excerpt=result["citation"]["excerpt"],
)
source
for result in results
if isinstance(result, dict)
and (source := self._managed_source(result.get("citation"), language)) is not None
]
except (TypeError, KeyError, ValueError):
KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "malformed").inc()
return unavailable
for result in results:
if not isinstance(result, dict):
continue
score = _retrieval_score(result)
if score is not None:
KNOWLEDGE_RETRIEVAL_SCORE.observe(score)
@@ -403,15 +527,11 @@ class RAGcoreKnowledgeProvider:
all_sources = _deduplicate_sources(sources)
concepts = _question_concepts(question)
qualified_sources = [
SourceCard(
document_id=str(result["citation"]["document_id"]),
title=result["citation"]["title"],
version=str(result["citation"]["document_version_id"]),
section=result["citation"].get("section") or "",
excerpt=result["citation"]["excerpt"],
)
source
for result in results
if (_retrieval_score(result) or 0.0) >= self._settings.ragcore_min_search_score
if isinstance(result, dict)
and (_retrieval_score(result) or 0.0) >= self._settings.ragcore_min_search_score
and (source := self._managed_source(result.get("citation"), language)) is not None
]
sources = _rank_sources_for_concepts(_deduplicate_sources(qualified_sources), concepts)
if not all_sources:
@@ -424,15 +544,17 @@ class RAGcoreKnowledgeProvider:
correlation_id=correlation_id,
)
damage_evidence = any(
term
in (
f"{source.document_id} {source.title} {source.section} {source.excerpt}"
).casefold()
required_concepts = concepts & _ANSWERABLE_INTENT_CONCEPTS
evidence_text = " ".join(
f"{source.document_id} {source.title} {source.section} {source.excerpt}"
for source in sources
for term in _DOMAIN_CONCEPTS["damage"]
)
if not concepts or not sources or ("damage" in concepts and not damage_evidence):
).casefold()
covered_concepts = {
concept
for concept in required_concepts
if any(term in evidence_text for term in _DOMAIN_CONCEPTS[concept])
}
if not required_concepts or not sources or covered_concepts != required_concepts:
KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "insufficient").inc()
return GroundedAnswer(
answer="",
+15 -22
View File
@@ -4,7 +4,7 @@ import hashlib
import json
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from datetime import UTC, datetime
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
@@ -12,13 +12,13 @@ from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models.booking import Booking
from app.models.data_quality import DataQualityIssue
from app.models.idempotency import IdempotencyRecord
from app.models.inspection import Inspection
from app.models.outbox import OutboxEvent
from app.models.vehicle import Vehicle
from app.schemas import CurrentUser, RegisterReturnRequest
from app.services.audit import record_audit_event
from app.services.data_quality import open_odometer_regression_issue
def _new_inspection_ref() -> str:
@@ -261,28 +261,21 @@ def register_vehicle_return(
quality_issue_ref: str | None = None
if evaluation.odometer_regression:
issue = DataQualityIssue(
public_ref=f"DQ-RET-{str(inspection.public_ref).split('-')[-1]}",
rule_type="odometer_regression",
entity_type="vehicle",
entity_id=vehicle.id,
severity="medium",
status="open",
evidence_json={
"summary": (
f"Return submitted {body.end_odometer_km} km, below canonical "
f"{evaluation.canonical_odometer_km} km."
),
"entity_ref": vehicle.public_ref,
"related_refs": [booking.public_ref, inspection.public_ref],
},
proposed_action_json={},
issue = open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=inspection.public_ref,
reading_km=body.end_odometer_km,
canonical_km=evaluation.canonical_odometer_km,
source_type="return",
related_refs=[booking.public_ref, inspection.public_ref],
correctable_booking_refs=[booking.public_ref],
detected_at=now,
due_at=now + timedelta(days=1),
public_ref=f"DQ-RET-{str(inspection.public_ref).split('-')[-1]}",
actor_label=actor.display_name,
correlation_id=correlation_id,
)
db.add(issue)
db.flush()
quality_issue_ref = issue.public_ref
quality_issue_ref = issue.public_ref if issue is not None else None
resulting_status = evaluation.resulting_vehicle_status
vehicle.odometer_km = evaluation.resulting_odometer_km
+105 -2
View File
@@ -1,3 +1,15 @@
import threading
import pytest
from sqlalchemy import delete, func, select
from app.core.db import SessionLocal, engine
from app.models.audit import AuditEvent
from app.models.vehicle import Vehicle
from app.seed_loader import reset_and_seed
from app.services.audit import record_audit_event
def test_unauthenticated_dashboard_is_rejected(client):
response = client.get("/api/v1/dashboard")
assert response.status_code == 401
@@ -101,15 +113,106 @@ def test_demo_reset_rejects_concurrent_rebuild(ops_client):
assert response.status_code == 409
def test_demo_reset_database_lock_blocks_another_api_replica(ops_client):
import app.api.routers.demo as demo_router
# Use a raw connection: an ordinary Session intentionally participates in the
# shared mutation barrier and would make an exclusive reset wait before it can test
# this separate non-blocking replica lock.
with engine.connect() as lock_connection:
lock_connection.scalar(select(func.pg_advisory_lock(demo_router._RESET_ADVISORY_LOCK_ID)))
try:
response = ops_client.post("/api/v1/demo/reset")
finally:
lock_connection.scalar(
select(func.pg_advisory_unlock(demo_router._RESET_ADVISORY_LOCK_ID))
)
assert response.status_code == 409
def test_demo_reset_waits_for_active_mutation_and_then_replaces_it_atomically():
started = threading.Event()
completed = threading.Event()
errors: list[BaseException] = []
def reset_in_other_session() -> None:
started.set()
try:
with SessionLocal() as reset_db:
reset_and_seed(reset_db)
except BaseException as exc: # pragma: no cover - assertion reports thread failures
errors.append(exc)
finally:
completed.set()
with SessionLocal() as mutation_db:
vehicle = mutation_db.scalar(
select(Vehicle).where(Vehicle.public_ref == "MO-001").with_for_update()
)
vehicle.location = "Concurrent mutation marker"
mutation_db.flush()
worker = threading.Thread(target=reset_in_other_session, daemon=True)
worker.start()
assert started.wait(timeout=1)
assert not completed.wait(timeout=0.2)
mutation_db.commit()
worker.join(timeout=10)
assert not worker.is_alive(), "reset deadlocked behind the active mutation"
assert errors == []
with SessionLocal() as db:
restored = db.scalar(select(Vehicle).where(Vehicle.public_ref == "MO-001"))
assert restored.location != "Concurrent mutation marker"
def test_demo_reset_cooldown_returns_retry_after(ops_client, monkeypatch):
import app.api.routers.demo as demo_router
monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 60)
monkeypatch.setattr(demo_router, "_last_reset_monotonic", demo_router.time.monotonic())
with SessionLocal() as db:
cooldown_event = record_audit_event(
db,
actor_type="user",
actor_label="Cooldown test",
action="demo_reset",
entity_type="system",
)
db.commit()
cooldown_event_id = cooldown_event.id
response = ops_client.post("/api/v1/demo/reset")
assert response.status_code == 429
assert int(response.headers["retry-after"]) >= 1
monkeypatch.setattr(demo_router, "_last_reset_monotonic", 0.0)
with SessionLocal() as db:
db.execute(delete(AuditEvent).where(AuditEvent.id == cooldown_event_id))
db.commit()
def test_demo_reset_rolls_back_every_change_when_integrity_check_fails(ops_client, monkeypatch):
import app.api.routers.demo as demo_router
monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 0)
with SessionLocal() as db:
probe = record_audit_event(
db,
actor_type="system",
actor_label="Atomic reset test",
action="reset_atomicity_probe",
entity_type="system",
)
db.commit()
probe_id = probe.id
def fail_integrity_check(_db):
raise RuntimeError("Injected integrity-check failure")
monkeypatch.setattr(demo_router, "scenario_integrity_report", fail_integrity_check)
with pytest.raises(RuntimeError, match="Injected integrity-check failure"):
ops_client.post("/api/v1/demo/reset")
with SessionLocal() as db:
assert db.scalar(select(AuditEvent).where(AuditEvent.id == probe_id)) is not None
db.execute(delete(AuditEvent).where(AuditEvent.id == probe_id))
db.commit()
def test_reset_is_rejected_when_demo_allow_reset_is_disabled(ops_client, monkeypatch):
+127 -1
View File
@@ -95,6 +95,16 @@ def test_booking_requirements_are_explicit_and_audited(ops_client):
)
assert checkout.status_code == 409
whitespace_only = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": " "},
)
assert whitespace_only.status_code == 422
assert (
ops_client.get(f"/api/v1/bookings/{booking['public_ref']}").json()["requirements_complete"]
is False
)
confirmed = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
@@ -106,6 +116,29 @@ def test_booking_requirements_are_explicit_and_audited(ops_client):
assert any(item["entity_ref"] == booking["public_ref"] for item in audit.json())
def test_booking_creation_cannot_preconfirm_requirements(ops_client):
window = {"starts_at": "2033-09-01T10:00:00Z", "ends_at": "2033-09-02T12:00:00Z"}
vehicle = ops_client.get("/api/v1/bookings/availability", params=window).json()[0]
created = ops_client.post(
"/api/v1/bookings",
json={
"customer_ref": "CUS-0001",
"vehicle_ref": vehicle["public_ref"],
"requirements_complete": True,
**window,
},
)
assert created.status_code == 201
assert created.json()["requirements_complete"] is False
audit = ops_client.get(
"/api/v1/audit",
params={"action": "booking_requirements_completed"},
)
assert all(item["entity_ref"] != created.json()["public_ref"] for item in audit.json())
def test_customer_search_returns_canonical_customers(ops_client):
response = ops_client.get("/api/v1/customers", params={"query": "CUS-"})
assert response.status_code == 200
@@ -221,10 +254,14 @@ def test_checkout_records_inspection_and_activates_safe_booking(ops_client):
json={
"customer_ref": "CUS-0001",
"vehicle_ref": vehicle["public_ref"],
"requirements_complete": True,
**window,
},
).json()
confirmed = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
)
assert confirmed.status_code == 200
response = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/checkout",
json={
@@ -241,6 +278,95 @@ def test_checkout_records_inspection_and_activates_safe_booking(ops_client):
updated_vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
assert updated_vehicle["operational_status"] == "rented"
assert any(item["type"] == "checkout" for item in updated_vehicle["inspections"])
checkout_audit = ops_client.get(
"/api/v1/audit",
params={"action": "booking_checkout_recorded"},
).json()
booking_event = next(
item for item in checkout_audit if item["entity_ref"] == booking["public_ref"]
)
vehicle_audit = ops_client.get(
"/api/v1/audit",
params={
"action": "vehicle_status_changed",
"correlation_id": booking_event["correlation_id"],
},
).json()
assert len(vehicle_audit) == 1
assert vehicle_audit[0]["entity_ref"] == vehicle["public_ref"]
assert vehicle_audit[0]["before"]["operational_status"] == "available"
assert vehicle_audit[0]["after"]["operational_status"] == "rented"
def test_checkout_odometer_regression_keeps_canonical_and_opens_bounded_quality_issue(
ops_client,
):
window = {"starts_at": "2052-09-01T10:00:00Z", "ends_at": "2052-09-02T12:00:00Z"}
existing_issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
unavailable_refs = {issue["entity_ref"] for issue in existing_issues}
available = ops_client.get("/api/v1/bookings/availability", params=window).json()
vehicle_option = next(
item
for item in available
if item["operational_status"] == "available" and item["public_ref"] not in unavailable_refs
)
vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle_option['public_ref']}").json()
assert vehicle["odometer_km"] > 0
booking = ops_client.post(
"/api/v1/bookings",
json={
"customer_ref": "CUS-0003",
"vehicle_ref": vehicle["public_ref"],
**window,
},
).json()
ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
)
checkout = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/checkout",
json={
"start_odometer_km": vehicle["odometer_km"] - 1,
"fuel_level_percent": 90,
"cleanliness_ok": True,
"damage_reported": False,
"technical_warning": False,
},
)
assert checkout.status_code == 200
assert checkout.json()["booking_status"] == "blocked"
updated_vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
assert updated_vehicle["odometer_km"] == vehicle["odometer_km"]
issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
issue = next(item for item in issues if item["entity_ref"] == vehicle["public_ref"])
assert issue["evidence"]["source_type"] == "checkout"
assert issue["evidence"]["correctable_booking_refs"] == []
detail = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
assert any(snapshot["entity_type"] == "inspection" for snapshot in detail["related_snapshots"])
invalid_correction = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": booking["public_ref"],
"corrected_odometer_km": vehicle["odometer_km"],
},
)
assert invalid_correction.status_code == 422
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={"decision": "retain_canonical"},
)
assert retained.status_code == 200
assert retained.json()["status"] == "resolved"
def test_get_booking_detail(ops_client):
+43
View File
@@ -1,3 +1,9 @@
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.vehicle import Vehicle
def test_dashboard_metrics_are_persisted_counts(ops_client):
response = ops_client.get("/api/v1/dashboard")
assert response.status_code == 200
@@ -15,6 +21,43 @@ def test_dashboard_metrics_are_persisted_counts(ops_client):
assert metrics["pending_or_failed_workflows"] >= 1
def test_dashboard_metrics_change_when_persisted_fleet_data_changes(ops_client):
before = ops_client.get("/api/v1/dashboard").json()["metrics"]
db = SessionLocal()
probe = Vehicle(
public_ref="MO-METRIC-PROBE",
make="Synthetic",
model="Metric probe",
model_year=2026,
registration_number="TEST-METRIC-PROBE",
location="Test fixture",
operational_status="blocked",
odometer_km=0,
next_service_km=1,
active=True,
version=1,
)
try:
db.add(probe)
db.commit()
after_response = ops_client.get("/api/v1/dashboard")
assert after_response.status_code == 200
after = after_response.json()["metrics"]
assert after["blocked"] == before["blocked"] + 1
fleet_statuses = ("available", "rented", "cleaning", "maintenance", "blocked")
assert (
sum(after[status] for status in fleet_statuses)
== sum(before[status] for status in fleet_statuses) + 1
)
finally:
persisted_probe = db.scalar(select(Vehicle).where(Vehicle.public_ref == "MO-METRIC-PROBE"))
if persisted_probe is not None:
db.delete(persisted_probe)
db.commit()
db.close()
def test_dashboard_attention_items_link_to_records(ops_client):
response = ops_client.get("/api/v1/dashboard")
body = response.json()
+818 -4
View File
@@ -1,8 +1,25 @@
from sqlalchemy import select
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from threading import Barrier
from sqlalchemy import delete, select, text
from app.core.db import SessionLocal
from app.models.audit import AuditEvent
from app.models.booking import Booking
from app.models.customer import Customer
from app.models.data_quality import DataQualityIssue
from app.models.idempotency import IdempotencyRecord
from app.models.inspection import Inspection
from app.models.maintenance import MaintenanceRecord
from app.models.outbox import OutboxEvent
from app.models.vehicle import Vehicle
from app.schemas import CurrentUser, RegisterReturnRequest, ResolveOdometerRegressionRequest
from app.services.data_quality import (
open_odometer_regression_issue,
resolve_odometer_regression,
)
from app.services.returns import register_vehicle_return
def _activate_booking(vehicle_ref: str, start_odometer_km: int) -> str:
@@ -21,6 +38,30 @@ def _activate_booking(vehicle_ref: str, start_odometer_km: int) -> str:
db.close()
def _cleanup_odometer_scenario(vehicle_id, customer_id) -> None:
"""Remove one isolated DQ-03 scenario in foreign-key-safe order."""
with SessionLocal() as db:
booking_ids = db.scalars(select(Booking.id).where(Booking.vehicle_id == vehicle_id)).all()
issue_ids = db.scalars(
select(DataQualityIssue.id).where(DataQualityIssue.entity_id == vehicle_id)
).all()
audit_entity_ids = [vehicle_id, *booking_ids, *issue_ids]
if audit_entity_ids:
db.execute(delete(AuditEvent).where(AuditEvent.entity_id.in_(audit_entity_ids)))
if booking_ids:
db.execute(
delete(IdempotencyRecord).where(IdempotencyRecord.booking_id.in_(booking_ids))
)
db.execute(delete(OutboxEvent).where(OutboxEvent.aggregate_id.in_(booking_ids)))
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.execute(delete(Customer).where(Customer.id == customer_id))
db.commit()
def test_list_includes_all_five_rule_types(ops_client):
response = ops_client.get("/api/v1/data-quality/issues")
assert response.status_code == 200
@@ -43,6 +84,84 @@ def test_scan_is_idempotent_once_seeded(ops_client):
assert response.json()["created"] == {}
def test_scan_detects_imported_maintenance_odometer_regression(ops_client):
with SessionLocal() as db:
vehicle = Vehicle(
public_ref="MO-DQ-SCAN",
make="Synthetic",
model="Scanner",
model_year=2026,
registration_number="DQ-SCAN-01",
location="Brussels",
operational_status="available",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add(vehicle)
db.flush()
vehicle_id = vehicle.id
db.add_all(
[
MaintenanceRecord(
public_ref="MAINT-DQ-SCAN-A",
vehicle_id=vehicle.id,
occurred_at=datetime(2045, 1, 1, tzinfo=UTC),
odometer_km=20_000,
category="inspection",
summary="Synthetic scan baseline",
),
MaintenanceRecord(
public_ref="MAINT-DQ-SCAN-B",
vehicle_id=vehicle.id,
occurred_at=datetime(2045, 2, 1, tzinfo=UTC),
odometer_km=19_000,
category="inspection",
summary="Synthetic imported regression",
),
]
)
db.commit()
try:
response = ops_client.post("/api/v1/data-quality/scan")
assert response.status_code == 200
assert response.json()["created"]["odometer_regression"] >= 1
issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
issue = next(item for item in issues if item["entity_ref"] == "MO-DQ-SCAN")
assert issue["evidence"]["source_type"] == "maintenance"
assert issue["evidence"]["correctable_booking_refs"] == []
# A later imported regression must enrich the existing open issue instead of
# being silently dropped by the generic check-then-return scanner path.
with SessionLocal() as db:
db.add(
MaintenanceRecord(
public_ref="MAINT-DQ-SCAN-C",
vehicle_id=vehicle_id,
occurred_at=datetime(2045, 3, 1, tzinfo=UTC),
odometer_km=18_000,
category="inspection",
summary="Second synthetic imported regression",
)
)
db.commit()
rescanned = ops_client.post("/api/v1/data-quality/scan")
assert rescanned.status_code == 200
enriched = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
later_refs = {signal["params"]["later_ref"] for signal in enriched["evidence"]["signals"]}
assert {"MAINT-DQ-SCAN-B", "MAINT-DQ-SCAN-C"}.issubset(later_refs)
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.commit()
def test_scan_requires_operations_manager(employee_client):
response = employee_client.post("/api/v1/data-quality/scan")
assert response.status_code == 403
@@ -440,9 +559,8 @@ def test_resolve_odometer_regression_correction_below_canonical_is_rejected_then
def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_client):
# The seeded odometer_regression issues carry no related booking (CSV-only rows).
# Create a fresh one with a real related booking via a live regression return, so
# the "correct_reading" path has an actual booking_ref to target.
# Create a live return regression to prove the complete correction chain updates
# both persisted representations of that reading (booking + return inspection).
booking_ref = _activate_booking("MO-018", start_odometer_km=12000)
vehicle_before = ops_client.get("/api/v1/vehicles/MO-018").json()
low_reading = vehicle_before["odometer_km"] - 200
@@ -459,6 +577,7 @@ def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_clien
)
assert returned.status_code == 201
issue_ref = returned.json()["quality_issue_ref"]
inspection_ref = returned.json()["inspection_ref"]
assert issue_ref is not None
corrected = vehicle_before["odometer_km"] + 500
@@ -477,6 +596,701 @@ def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_clien
assert vehicle["odometer_km"] == corrected
booking = ops_client.get(f"/api/v1/bookings/{booking_ref}").json()
assert booking["end_odometer_km"] == corrected
with SessionLocal() as db:
inspection = db.scalar(select(Inspection).where(Inspection.public_ref == inspection_ref))
assert inspection is not None
assert inspection.odometer_km == corrected
def test_correcting_one_of_two_regressions_keeps_the_other_open(ops_client):
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-MULTI",
first_name="Synthetic",
last_name="Multi",
email="dq-multi@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-MULTI",
make="Synthetic",
model="Multi",
model_year=2026,
registration_number="DQ-MULTI",
location="Brussels",
operational_status="available",
odometer_km=1_000,
next_service_km=2_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
bookings: list[Booking] = []
inspections: list[Inspection] = []
for index, reading in enumerate((900, 800), start=1):
booking = Booking(
public_ref=f"BK-DQ-MULTI-{index}",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=datetime(2046, index, 1, tzinfo=UTC),
ends_at=datetime(2046, index, 2, tzinfo=UTC),
status="returned",
start_odometer_km=reading - 10,
end_odometer_km=reading,
requirements_complete=True,
)
db.add(booking)
db.flush()
inspection = Inspection(
public_ref=f"INSP-DQ-M-{index}",
booking_id=booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=reading,
completed_at=booking.ends_at,
)
db.add(inspection)
db.flush()
open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=inspection.public_ref,
reading_km=reading,
canonical_km=vehicle.odometer_km,
source_type="return",
related_refs=[booking.public_ref, inspection.public_ref],
correctable_booking_refs=[booking.public_ref],
public_ref="DQ-MULTI-SOURCE" if index == 1 else None,
)
bookings.append(booking)
inspections.append(inspection)
booking_refs = [booking.public_ref for booking in bookings]
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
first = ops_client.post(
"/api/v1/data-quality/issues/DQ-MULTI-SOURCE/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": booking_refs[0],
"corrected_odometer_km": 1_100,
},
)
assert first.status_code == 200, first.text
assert first.json()["status"] == "open"
assert first.json()["evidence"]["correctable_booking_refs"] == [booking_refs[1]]
second = ops_client.post(
"/api/v1/data-quality/issues/DQ-MULTI-SOURCE/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": booking_refs[1],
"corrected_odometer_km": 1_200,
},
)
assert second.status_code == 200, second.text
assert second.json()["status"] == "resolved"
with SessionLocal() as db:
persisted = db.scalars(
select(Inspection)
.where(Inspection.vehicle_id == vehicle_id)
.order_by(Inspection.public_ref)
).all()
assert [item.odometer_km for item in persisted] == [1_100, 1_200]
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.execute(delete(Customer).where(Customer.id == customer_id))
db.commit()
def test_retained_regression_is_suppressed_until_source_reading_changes(ops_client):
with SessionLocal() as db:
vehicle = Vehicle(
public_ref="MO-DQ-RETAIN",
make="Synthetic",
model="Retain",
model_year=2026,
registration_number="DQ-RETAIN",
location="Brussels",
operational_status="available",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add(vehicle)
db.flush()
vehicle_id = vehicle.id
db.add_all(
[
MaintenanceRecord(
public_ref="MAINT-DQ-RET-A",
vehicle_id=vehicle.id,
occurred_at=datetime(2047, 1, 1, tzinfo=UTC),
odometer_km=20_000,
category="inspection",
summary="Retain baseline",
),
MaintenanceRecord(
public_ref="MAINT-DQ-RET-B",
vehicle_id=vehicle.id,
occurred_at=datetime(2047, 2, 1, tzinfo=UTC),
odometer_km=19_000,
category="inspection",
summary="Retained source reading",
),
]
)
db.commit()
try:
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
issue = next(
item
for item in ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
if item["entity_ref"] == "MO-DQ-RETAIN"
)
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={"decision": "retain_canonical", "note": "Verified source entry"},
)
assert retained.status_code == 200
assert retained.json()["status"] == "resolved"
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
with SessionLocal() as db:
assert (
db.scalar(
select(DataQualityIssue).where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.status == "open",
)
)
is None
)
changed = db.scalar(
select(MaintenanceRecord).where(MaintenanceRecord.public_ref == "MAINT-DQ-RET-B")
)
changed.odometer_km = 18_999
db.commit()
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
reopened = next(
item
for item in ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
if item["entity_ref"] == "MO-DQ-RETAIN"
)
assert reopened["evidence"]["reopened_from"] == issue["public_ref"]
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.commit()
def test_live_return_retained_as_canonical_is_not_reopened_by_scan(ops_client):
now = datetime.now(UTC)
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-LIVE-RET",
first_name="Synthetic",
last_name="Retained return",
email="dq-live-retain@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-LIVE-RET",
make="Synthetic",
model="Retained return",
model_year=2026,
registration_number="DQ-LIVE-RET",
location="Brussels",
operational_status="rented",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
booking = Booking(
public_ref="BK-DQ-LIVE-RET",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=2),
ends_at=now + timedelta(days=1),
status="active",
start_odometer_km=19_500,
end_odometer_km=None,
requirements_complete=True,
)
db.add_all(
[
booking,
MaintenanceRecord(
public_ref="MNT-DQ-LIVE-RET",
vehicle_id=vehicle.id,
occurred_at=now - timedelta(days=3),
odometer_km=20_000,
category="inspection",
summary="Synthetic canonical baseline",
),
]
)
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
returned = ops_client.post(
"/api/v1/bookings/BK-DQ-LIVE-RET/return",
json={
"end_odometer_km": 19_000,
"fuel_level_percent": 50,
"cleanliness_ok": True,
"damage_reported": False,
"technical_warning": False,
},
headers={"Idempotency-Key": "test-dq-live-retain-001"},
)
assert returned.status_code == 201, returned.text
issue_ref = returned.json()["quality_issue_ref"]
inspection_ref = returned.json()["inspection_ref"]
assert issue_ref is not None
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue_ref}/resolve-odometer-regression",
json={"decision": "retain_canonical", "note": "Source reading verified as wrong."},
)
assert retained.status_code == 200, retained.text
assert retained.json()["status"] == "resolved"
assert retained.json()["evidence"]["retained_odometer_fingerprints"] == [
{
"source_type": "return",
"later_ref": inspection_ref,
"later_km": 19_000,
}
]
scanned = ops_client.post("/api/v1/data-quality/scan")
assert scanned.status_code == 200, scanned.text
with SessionLocal() as db:
issues = db.scalars(
select(DataQualityIssue)
.where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.rule_type == "odometer_regression",
)
.order_by(DataQualityIssue.detected_at)
).all()
assert [(issue.public_ref, issue.status) for issue in issues] == [
(issue_ref, "resolved")
]
finally:
_cleanup_odometer_scenario(vehicle_id, customer_id)
def test_scanned_return_regression_can_correct_booking_all_inspections_and_vehicle(ops_client):
now = datetime.now(UTC)
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-SCAN-RET",
first_name="Synthetic",
last_name="Scanned return",
email="dq-scan-return@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-SCAN-RET",
make="Synthetic",
model="Scanned return",
model_year=2026,
registration_number="DQ-SCAN-RET",
location="Brussels",
operational_status="available",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
booking = Booking(
public_ref="BK-DQ-SCAN-RET",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=3),
ends_at=now - timedelta(hours=12),
status="returned",
start_odometer_km=19_500,
end_odometer_km=18_500,
requirements_complete=True,
)
db.add(booking)
db.flush()
db.add_all(
[
MaintenanceRecord(
public_ref="MNT-DQ-SCAN-RET",
vehicle_id=vehicle.id,
occurred_at=now - timedelta(days=4),
odometer_km=20_000,
category="inspection",
summary="Synthetic canonical baseline",
),
Inspection(
public_ref="INSP-DQ-SCAN-R1",
booking_id=booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=19_000,
completed_at=now - timedelta(days=1),
),
Inspection(
public_ref="INSP-DQ-SCAN-R2",
booking_id=booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=18_500,
completed_at=now - timedelta(hours=12),
),
]
)
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
scanned = ops_client.post("/api/v1/data-quality/scan")
assert scanned.status_code == 200, scanned.text
issue = next(
item
for item in ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
if item["entity_ref"] == "MO-DQ-SCAN-RET"
)
assert issue["evidence"]["correctable_booking_refs"] == ["BK-DQ-SCAN-RET"]
later_refs = {signal["params"]["later_ref"] for signal in issue["evidence"]["signals"]}
assert later_refs == {"INSP-DQ-SCAN-R1", "INSP-DQ-SCAN-R2"}
corrected = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": "BK-DQ-SCAN-RET",
"corrected_odometer_km": 20_500,
},
)
assert corrected.status_code == 200, corrected.text
assert corrected.json()["status"] == "resolved"
with SessionLocal() as db:
persisted_vehicle = db.get(Vehicle, vehicle_id)
persisted_booking = db.scalar(
select(Booking).where(Booking.public_ref == "BK-DQ-SCAN-RET")
)
assert persisted_booking is not None
persisted_inspections = db.scalars(
select(Inspection)
.where(Inspection.booking_id == persisted_booking.id, Inspection.type == "return")
.order_by(Inspection.public_ref)
).all()
assert persisted_vehicle is not None
assert persisted_vehicle.odometer_km == 20_500
assert persisted_booking.end_odometer_km == 20_500
assert [inspection.odometer_km for inspection in persisted_inspections] == [
20_500,
20_500,
]
finally:
_cleanup_odometer_scenario(vehicle_id, customer_id)
def test_resolver_and_concurrent_return_finish_without_deadlock_or_lost_evidence():
now = datetime.now(UTC)
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-RACE",
first_name="Synthetic",
last_name="Race",
email="dq-race@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-RACE",
make="Synthetic",
model="Race",
model_year=2026,
registration_number="DQ-RACE",
location="Brussels",
operational_status="rented",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
corrected_booking = Booking(
public_ref="BK-DQ-RACE-OLD",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=4),
ends_at=now - timedelta(days=3),
status="returned",
start_odometer_km=19_500,
end_odometer_km=19_000,
requirements_complete=True,
)
concurrent_booking = Booking(
public_ref="BK-DQ-RACE-NEW",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=1),
ends_at=now + timedelta(days=1),
status="active",
start_odometer_km=19_000,
end_odometer_km=None,
requirements_complete=True,
)
db.add_all([corrected_booking, concurrent_booking])
db.flush()
old_inspection = Inspection(
public_ref="INSP-DQ-RACE-OLD",
booking_id=corrected_booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=19_000,
completed_at=corrected_booking.ends_at,
)
db.add(old_inspection)
db.flush()
issue = open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=old_inspection.public_ref,
reading_km=19_000,
canonical_km=20_000,
source_type="return",
related_refs=[corrected_booking.public_ref, old_inspection.public_ref],
correctable_booking_refs=[corrected_booking.public_ref],
public_ref="DQ-ODO-RACE",
)
assert issue is not None
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
actor = CurrentUser(
public_ref="USR-DQ-RACE",
display_name="DQ Race Manager",
role="operations_manager",
)
start = Barrier(2)
def correct_existing_reading() -> str:
with SessionLocal() as db:
db.execute(text("SET LOCAL lock_timeout = '5s'"))
start.wait(timeout=5)
resolved = resolve_odometer_regression(
db,
"DQ-ODO-RACE",
ResolveOdometerRegressionRequest(
decision="correct_reading",
booking_ref="BK-DQ-RACE-OLD",
corrected_odometer_km=20_500,
),
actor,
)
return resolved.status
def return_other_booking() -> dict:
with SessionLocal() as db:
db.execute(text("SET LOCAL lock_timeout = '5s'"))
start.wait(timeout=5)
_status, response = register_vehicle_return(
db,
"BK-DQ-RACE-NEW",
RegisterReturnRequest(
end_odometer_km=18_000,
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
),
"test-dq-resolve-return-race-001",
actor,
)
return response
try:
with ThreadPoolExecutor(max_workers=2) as executor:
correction_future = executor.submit(correct_existing_reading)
return_future = executor.submit(return_other_booking)
correction_status = correction_future.result(timeout=15)
return_result = return_future.result(timeout=15)
assert correction_status in {"open", "resolved"}
assert return_result["quality_issue_ref"] is not None
with SessionLocal() as db:
persisted_vehicle = db.get(Vehicle, vehicle_id)
bookings = {
booking.public_ref: booking
for booking in db.scalars(
select(Booking).where(Booking.vehicle_id == vehicle_id)
).all()
}
old_inspections = db.scalars(
select(Inspection).where(Inspection.booking_id == bookings["BK-DQ-RACE-OLD"].id)
).all()
open_issues = db.scalars(
select(DataQualityIssue).where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.rule_type == "odometer_regression",
DataQualityIssue.status == "open",
)
).all()
assert persisted_vehicle is not None
assert persisted_vehicle.odometer_km == 20_500
assert bookings["BK-DQ-RACE-OLD"].end_odometer_km == 20_500
assert [inspection.odometer_km for inspection in old_inspections] == [20_500]
assert bookings["BK-DQ-RACE-NEW"].status == "returned"
assert bookings["BK-DQ-RACE-NEW"].end_odometer_km == 18_000
assert len(open_issues) == 1
assert open_issues[0].evidence_json["correctable_booking_refs"] == ["BK-DQ-RACE-NEW"]
later_refs = {
signal["params"]["later_ref"] for signal in open_issues[0].evidence_json["signals"]
}
assert later_refs == {return_result["inspection_ref"]}
finally:
_cleanup_odometer_scenario(vehicle_id, customer_id)
def test_early_return_uses_inspection_time_without_duplicate_booking_regression(ops_client):
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-EARLY",
first_name="Synthetic",
last_name="Early",
email="dq-early@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-EARLY",
make="Synthetic",
model="Early",
model_year=2026,
registration_number="DQ-EARLY",
location="Brussels",
operational_status="available",
odometer_km=200,
next_service_km=10_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
early = Booking(
public_ref="BK-DQ-EARLY-A",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=datetime(2048, 1, 1, tzinfo=UTC),
ends_at=datetime(2048, 3, 1, tzinfo=UTC),
status="returned",
start_odometer_km=90,
end_odometer_km=100,
requirements_complete=True,
)
later = Booking(
public_ref="BK-DQ-EARLY-B",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=datetime(2048, 1, 10, tzinfo=UTC),
ends_at=datetime(2048, 2, 1, tzinfo=UTC),
status="returned",
start_odometer_km=100,
end_odometer_km=200,
requirements_complete=True,
)
db.add_all([early, later])
db.flush()
db.add_all(
[
Inspection(
public_ref="INSP-DQ-EARLY-A",
booking_id=early.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=100,
completed_at=datetime(2048, 1, 2, tzinfo=UTC),
),
Inspection(
public_ref="INSP-DQ-EARLY-B",
booking_id=later.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=200,
completed_at=datetime(2048, 2, 1, tzinfo=UTC),
),
]
)
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
with SessionLocal() as db:
assert (
db.scalar(
select(DataQualityIssue).where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.rule_type == "odometer_regression",
)
)
is None
)
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.execute(delete(Customer).where(Customer.id == customer_id))
db.commit()
def test_manual_scan_records_audit_event(ops_client):
+104
View File
@@ -1,5 +1,10 @@
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.outbox import OutboxEvent
from app.seed_loader import reset_and_seed
from app.services import demo_manifest
from app.services.knowledge import KnowledgeHealth
def test_demo_manifest_is_public(client):
@@ -52,3 +57,102 @@ def test_demo_manifest_ragcore_labelled_as_demo_mode_not_live(client):
body = client.get("/api/v1/demo/manifest").json()
ragcore = next(i for i in body["integrations"] if i["key"] == "ragcore")
assert ragcore["status_code"] == "demoMode"
def test_automation_scenario_requires_the_exact_prepared_failure():
db = SessionLocal()
try:
reset_and_seed(db)
event = db.scalar(
select(OutboxEvent).where(OutboxEvent.event_id == demo_manifest._FAILED_DEMO_EVENT_ID)
)
assert event is not None
event.last_error_code = "connectionError"
db.flush()
scenario = next(
item for item in demo_manifest._scenarios(db) if item.id == "automation-retry"
)
assert scenario.ready is False
finally:
db.rollback()
db.close()
def test_knowledge_scenario_rejects_available_provider_with_verified_empty_corpus(
client, monkeypatch
):
class EmptyKnowledgeProvider:
def health(self, language="en-GB"):
return KnowledgeHealth(
provider="ragcore",
available=True,
detail="Ready, but no indexed documents.",
tenant="fleet-ops",
workspace="mobilityops",
collection="procedures",
document_count=0,
source_document_count=4,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="verified",
)
monkeypatch.setattr(demo_manifest, "get_knowledge_provider", EmptyKnowledgeProvider)
body = client.get("/api/v1/demo/manifest").json()
scenario = next(item for item in body["scenarios"] if item["id"] == "knowledge-question")
assert scenario["ready"] is False
assert scenario["blocked_reason_code"] == "knowledgeUnavailable"
ragcore = next(item for item in body["integrations"] if item["key"] == "ragcore")
assert ragcore["status_code"] == "unavailable"
def test_knowledge_scenario_rejects_local_sources_when_index_count_is_unverified(
client, monkeypatch
):
class SourceAwareKnowledgeProvider:
def health(self, language="en-GB"):
return KnowledgeHealth(
provider="ragcore",
available=True,
detail="Ready; index count endpoint is unavailable.",
tenant="fleet-ops",
workspace="mobilityops",
collection="procedures",
document_count=None,
source_document_count=4,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="not_reported",
)
monkeypatch.setattr(demo_manifest, "get_knowledge_provider", SourceAwareKnowledgeProvider)
body = client.get("/api/v1/demo/manifest").json()
scenario = next(item for item in body["scenarios"] if item["id"] == "knowledge-question")
assert scenario["ready"] is False
assert scenario["blocked_reason_code"] == "knowledgeUnavailable"
ragcore = next(item for item in body["integrations"] if item["key"] == "ragcore")
assert ragcore["status_code"] == "unavailable"
def test_knowledge_scenario_requires_provider_availability_even_with_documents():
health = KnowledgeHealth(
provider="ragcore",
available=False,
detail="Provider is unreachable.",
tenant="fleet-ops",
workspace="mobilityops",
collection="procedures",
document_count=4,
source_document_count=4,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="verified",
)
assert demo_manifest._knowledge_scenario_ready(health) is False
+120 -5
View File
@@ -4,6 +4,7 @@ import uuid
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
import pytest
from sqlalchemy import select
from app.core.config import get_settings
@@ -65,12 +66,17 @@ def test_claim_marks_events_delivering():
def test_deliver_one_success(monkeypatch):
event_id = _make_pending_event("MO-002")
execution_id = "n8n-execution-123"
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": execution_id},
},
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
@@ -79,7 +85,7 @@ def test_deliver_one_success(monkeypatch):
event = _get_event(event_id)
assert event.delivery_status == "succeeded"
assert event.attempts == 1
assert event.external_run_id == str(event_id)
assert event.external_run_id == execution_id
assert event.last_error is None
assert event.last_error_code is None
@@ -129,6 +135,106 @@ def test_deliver_one_treats_empty_2xx_body_as_failure(monkeypatch):
assert event.last_error_code == "malformedResponse"
def test_deliver_one_rejects_json_object_without_explicit_success_ack(monkeypatch):
event_id = _make_pending_event("MO-010")
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "pending"
assert event.attempts == 1
assert event.last_error_code == "malformedResponse"
assert event.external_run_id is None
@pytest.mark.parametrize("result", [{}, {"execution_id": 123}, {"execution_id": " "}])
def test_deliver_one_rejects_ack_without_valid_execution_id(monkeypatch, result):
event_id = _make_pending_event("MO-EXEC-ID")
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": result},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "pending"
assert event.last_error_code == "malformedResponse"
assert event.external_run_id is None
def test_deliver_one_rejects_success_ack_for_a_different_event(monkeypatch):
event_id = _make_pending_event("MO-011")
dispatcher._claim_due_events()
other_event_id = uuid.uuid4()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(other_event_id)},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "pending"
assert event.attempts == 1
assert event.last_error_code == "mismatchedEventId"
assert event.external_run_id is None
def test_late_delivery_outcome_cannot_overwrite_a_newer_lease(monkeypatch):
event_id = _make_pending_event("MO-LEASE-RACE")
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
# Simulate lease expiry/reclaim while the original worker is still in network
# I/O. The later claimant owns a different token and its state must win.
with SessionLocal() as db:
event = db.scalar(
select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update()
)
event.payload_json = {
**event.payload_json,
"_delivery_claim_token": "newer-worker-token",
}
event.next_attempt_at = datetime.now(UTC) + timedelta(minutes=1)
db.commit()
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": "stale-worker-execution"},
},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "delivering"
assert event.attempts == 0
assert event.external_run_id is None
assert event.payload_json["_delivery_claim_token"] == "newer-worker-token"
def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
event_id = _make_pending_event("MO-004")
settings = get_settings()
@@ -139,7 +245,6 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
for _ in range(settings.n8n_max_attempts):
dispatcher._claim_due_events()
db = SessionLocal()
try:
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
@@ -147,6 +252,8 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
db.commit()
finally:
db.close()
claimed = dispatcher._claim_due_events(batch_size=1000)
assert event_id in claimed
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
@@ -259,7 +366,11 @@ def test_run_dispatch_cycle_recovers_a_stale_lease_before_claiming(monkeypatch):
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": "n8n-recovered-execution"},
},
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
@@ -275,7 +386,11 @@ def test_run_dispatch_cycle_end_to_end(monkeypatch):
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": "n8n-cycle-execution"},
},
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+8 -3
View File
@@ -174,11 +174,12 @@ def test_return_idempotency_key_rejects_different_body(ops_client):
def test_return_callback_rejects_malformed_correlation_id(client):
event_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning", "correlation_id": "nope"},
json={"event_id": event_id, "follow_up": "cleaning", "correlation_id": "nope"},
headers={
"Idempotency-Key": str(uuid.uuid4()),
"Idempotency-Key": event_id,
"X-Service-Token": get_settings().n8n_callback_token,
},
)
@@ -195,10 +196,14 @@ def test_blocked_checkout_booking_can_be_cancelled(ops_client):
json={
"customer_ref": "CUS-0002",
"vehicle_ref": vehicle["public_ref"],
"requirements_complete": True,
**window,
},
).json()
confirmed = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
)
assert confirmed.status_code == 200
checkout = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/checkout",
json={
+102 -1
View File
@@ -1,8 +1,14 @@
from sqlalchemy import select
import uuid
from datetime import UTC, datetime
import pytest
from sqlalchemy import delete, select
from app.core.db import SessionLocal
from app.models.audit import AuditEvent
from app.models.outbox import OutboxEvent
from app.seed_loader import reset_and_seed
from app.services import integration_status
def _reseed() -> None:
@@ -108,6 +114,101 @@ def test_integration_status_is_operational_once_all_failed_events_resolved(ops_c
assert body["state"] == "operational"
@pytest.mark.parametrize("delivery_status", ["pending", "delivering"])
def test_queued_work_without_any_success_is_not_reported_operational(delivery_status):
_reseed()
db = SessionLocal()
try:
for event in db.scalars(select(OutboxEvent)).all():
event.delivery_status = delivery_status
event.last_error = None
event.last_error_code = None
db.execute(
delete(AuditEvent).where(
AuditEvent.action.in_(
{
"data_quality_scan_run",
"n8n_procedures_synced",
"n8n_workflow_failure_registered",
"n8n_workflow_heartbeat",
}
)
)
)
db.flush()
status = integration_status.derive_n8n_status(db)
assert status.succeeded == 0
assert getattr(status, delivery_status) > 0
assert status.state == "no_evidence"
finally:
db.rollback()
db.close()
def test_historical_success_is_not_green_when_webhook_is_unconfigured(monkeypatch):
_reseed()
with SessionLocal() as db:
monkeypatch.setattr(integration_status.settings, "n8n_dispatch_enabled", True)
monkeypatch.setattr(integration_status.settings, "n8n_webhook_url", "")
status = integration_status.derive_n8n_status(db)
assert status.configured is False
assert status.succeeded > 0
assert status.state == "unavailable"
def test_null_coded_real_failure_sets_latest_failure_timestamp():
_reseed()
with SessionLocal() as db:
event = db.scalar(
select(OutboxEvent).where(OutboxEvent.delivery_status == "succeeded").limit(1)
)
event.delivery_status = "failed"
event.last_error_code = None
db.flush()
status = integration_status.derive_n8n_status(db)
assert status.unexpected_failed == 1
assert status.latest_failure_at is not None
db.rollback()
def test_unreachable_hub_invalidates_historical_operational_claim(monkeypatch):
db = SessionLocal()
try:
db.add(
AuditEvent(
actor_type="service",
actor_id=None,
actor_label="itworx-mcp-hub",
action="mcp_tool_request",
entity_type="integration",
entity_id=None,
correlation_id=uuid.uuid4(),
before_json=None,
after_json=None,
metadata_json={"tool": "operations_summary"},
occurred_at=datetime.now(UTC),
)
)
db.flush()
monkeypatch.setattr(integration_status.settings, "mcp_hub_registration_enabled", True)
monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: False)
unreachable = integration_status.derive_mcp_hub_status(db)
assert unreachable.total_calls > 0
assert unreachable.hub_reachable is False
assert unreachable.state == "no_evidence"
monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: True)
reachable = integration_status.derive_mcp_hub_status(db)
assert reachable.state == "operational"
finally:
db.rollback()
db.close()
def test_integration_status_lists_all_four_canonical_workflows(ops_client):
body = ops_client.get("/api/v1/integrations/status").json()["n8n"]
assert body["expected_workflow_count"] == 4
+158 -7
View File
@@ -1,4 +1,5 @@
import uuid
from concurrent.futures import ThreadPoolExecutor
from app.core.config import get_settings
@@ -12,19 +13,29 @@ def _callback_headers(event_id: str, token: str | None = None):
def test_callback_rejects_wrong_service_token(client):
event_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning"},
headers=_callback_headers(str(uuid.uuid4()), token="wrong-token"),
json={
"event_id": event_id,
"correlation_id": str(uuid.uuid4()),
"follow_up": "cleaning",
},
headers=_callback_headers(event_id, token="wrong-token"),
)
assert response.status_code == 401
def test_callback_unknown_event_returns_404(client):
event_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning"},
headers=_callback_headers(str(uuid.uuid4())),
json={
"event_id": event_id,
"correlation_id": str(uuid.uuid4()),
"follow_up": "cleaning",
},
headers=_callback_headers(event_id),
)
assert response.status_code == 404
@@ -39,13 +50,14 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
db = SessionLocal()
try:
booking = db.scalar(select(Booking).limit(1))
correlation_id = uuid.uuid4()
event = OutboxEvent(
event_id=uuid.uuid4(),
event_type="vehicle.returned.v1",
aggregate_type="booking",
aggregate_id=booking.id,
payload_json={
"correlation_id": str(uuid.uuid4()),
"correlation_id": str(correlation_id),
"aggregate": {
"type": "booking",
"id": str(booking.id),
@@ -66,12 +78,22 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
first = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning", "summary": "test"},
json={
"event_id": event_id,
"correlation_id": str(correlation_id),
"follow_up": "cleaning",
"summary": "test",
},
headers=_callback_headers(event_id),
)
second = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning", "summary": "test"},
json={
"event_id": event_id,
"correlation_id": str(correlation_id),
"follow_up": "cleaning",
"summary": "test",
},
headers=_callback_headers(event_id),
)
assert first.status_code == 200
@@ -82,6 +104,79 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
).json()
matching = [e for e in audit_events if e["metadata"]["event_id"] == event_id]
assert len(matching) == 1
assert matching[0]["correlation_id"] == str(correlation_id)
def test_callback_rejects_crossed_event_or_correlation(client):
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.outbox import OutboxEvent
with SessionLocal() as db:
event = db.scalar(select(OutboxEvent).limit(1))
event_id = str(event.event_id)
correlation_id = event.payload_json["correlation_id"]
crossed_event = client.post(
"/api/v1/integrations/n8n/return-callback",
json={
"event_id": str(uuid.uuid4()),
"correlation_id": correlation_id,
"follow_up": "cleaning",
},
headers=_callback_headers(event_id),
)
assert crossed_event.status_code == 409
assert crossed_event.json()["error"]["code"] == "CALLBACK_EVENT_MISMATCH"
crossed_correlation = client.post(
"/api/v1/integrations/n8n/return-callback",
json={
"event_id": event_id,
"correlation_id": str(uuid.uuid4()),
"follow_up": "cleaning",
},
headers=_callback_headers(event_id),
)
assert crossed_correlation.status_code == 409
assert crossed_correlation.json()["error"]["code"] == "CALLBACK_CORRELATION_MISMATCH"
def test_callback_concurrent_retries_record_one_audit(client, ops_client):
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.outbox import OutboxEvent
with SessionLocal() as db:
event = db.scalar(select(OutboxEvent).limit(1))
event_id = str(event.event_id)
correlation_id = event.payload_json["correlation_id"]
body = {
"event_id": event_id,
"correlation_id": correlation_id,
"follow_up": "cleaning",
}
def post_callback(_index: int):
return client.post(
"/api/v1/integrations/n8n/return-callback",
json=body,
headers=_callback_headers(event_id),
)
with ThreadPoolExecutor(max_workers=2) as pool:
responses = list(pool.map(post_callback, range(2)))
assert [response.status_code for response in responses] == [200, 200]
matching = [
event
for event in ops_client.get(
"/api/v1/audit", params={"action": "n8n_return_followup_recorded"}
).json()
if event["metadata"]["event_id"] == event_id
]
assert len(matching) == 1
def test_scheduled_scan_rejects_wrong_service_token(client):
@@ -183,6 +278,42 @@ def test_workflow_error_registers_and_is_idempotent_by_execution_id(client, ops_
assert matching[0]["after"]["retry_action"] == "n8n will retry automatically"
def test_workflow_error_preserves_correlation_and_rejects_malformed(client, ops_client):
settings = get_settings()
headers = {"X-Service-Token": settings.n8n_callback_token}
execution_id = str(uuid.uuid4())
correlation_id = str(uuid.uuid4())
accepted = client.post(
"/api/v1/integrations/n8n/workflow-error",
json=_workflow_error_body(execution_id, correlation_id=correlation_id),
headers=headers,
)
assert accepted.status_code == 200
matching = [
event
for event in ops_client.get(
"/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
).json()
if event["metadata"]["execution_id"] == execution_id
]
assert len(matching) == 1
assert matching[0]["correlation_id"] == correlation_id
malformed_execution = str(uuid.uuid4())
malformed = client.post(
"/api/v1/integrations/n8n/workflow-error",
json=_workflow_error_body(malformed_execution, correlation_id="not-a-uuid"),
headers=headers,
)
assert malformed.status_code == 422
assert all(
event["metadata"]["execution_id"] != malformed_execution
for event in ops_client.get(
"/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
).json()
)
def test_workflow_error_bounds_summary_length(client):
settings = get_settings()
response = client.post(
@@ -266,6 +397,26 @@ def test_procedures_sync_result_registers_and_is_idempotent(client, ops_client):
assert ragcore_sync["last_seen_at"] is not None
def test_failed_or_partial_procedure_sync_is_never_reported_healthy(client, ops_client):
settings = get_settings()
headers = {"X-Service-Token": settings.n8n_callback_token}
for synced, failed in ((0, 33), (32, 1)):
execution_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/procedures-sync-result",
json={"execution_id": execution_id, "synced": synced, "failed": failed},
headers=headers,
)
assert response.status_code == 200
status = ops_client.get("/api/v1/integrations/status").json()["n8n"]
workflow = next(w for w in status["workflows"] if "RAGcore" in w["name"])
assert workflow["state"] == "failed"
assert workflow["last_status"] == "failed"
assert workflow["last_execution_id"] == execution_id
assert status["state"] == "degraded"
def test_workflow_heartbeat_is_idempotent_and_drives_live_status(client, ops_client):
settings = get_settings()
execution_id = str(uuid.uuid4())
+409 -70
View File
@@ -1,9 +1,12 @@
from __future__ import annotations
import hashlib
import re
import uuid
from pathlib import Path
import httpx
import pytest
from app.api.routers import knowledge as knowledge_router
from app.core.config import get_settings
@@ -244,11 +247,11 @@ def test_ragcore_status_preserves_stronger_verified_index_evidence(client, ops_c
class _FakeResponse:
def __init__(self, status_code: int, body: dict):
def __init__(self, status_code: int, body: object):
self.status_code = status_code
self._body = body
def json(self) -> dict:
def json(self) -> object:
return self._body
@@ -260,6 +263,7 @@ class _FakeClient:
post_responses=None,
raise_on=None,
get_handler=None,
requests=None,
):
self._get_response = get_response
self._post_response = post_response
@@ -270,6 +274,7 @@ class _FakeClient:
self._post_responses = post_responses or {}
self._raise_on = raise_on
self._get_handler = get_handler
self.requests = requests if requests is not None else []
def __enter__(self):
return self
@@ -289,6 +294,7 @@ class _FakeClient:
def post(self, path, json=None):
if self._raise_on == "post":
raise httpx.ConnectError("no ragcore in this environment")
self.requests.append((path, json))
return self._post_responses.get(path, self._post_response)
@@ -393,22 +399,107 @@ def test_ragcore_provider_health_degrades_on_connection_error(monkeypatch):
assert "unavailable" in health.detail.lower()
def _answers_body(**overrides) -> dict:
_EXCERPTS = {
("en-GB", "damage-procedure"): (
"When a vehicle returns with visible or reported damage, mark damage in the "
"return inspection, add a concise factual description and keep the vehicle "
"blocked."
),
("en-GB", "vehicle-return-procedure"): (
"Open the active booking and record the ending odometer, fuel level, cleanliness, "
"visible damage, technical warnings and relevant notes."
),
("en-GB", "cleaning-checklist"): (
"Cleaning completion alone does not make a blocked or maintenance vehicle "
"available. Fleet Ops derives availability from all active restrictions."
),
("en-GB", "maintenance-escalation"): (
"Escalate when a technical warning is reported, the service threshold is reached, "
"a safety-related defect is observed or an existing maintenance block remains "
"unresolved."
),
("nl-BE", "damage-procedure"): (
"Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schade "
"in de retourinspectie, voeg een beknopte feitelijke beschrijving toe en houd het "
"voertuig geblokkeerd."
),
("nl-BE", "vehicle-return-procedure"): (
"Open de actieve boeking en registreer de eindkilometerstand, het brandstofniveau, "
"de netheid, zichtbare schade, technische waarschuwingen en relevante notities."
),
}
def _managed_document(provider: RAGcoreKnowledgeProvider, document_id: str, language: str):
return next(
document
for document in provider._managed_documents_by_source_id.values()
if document.document_id == document_id and document.language == language
)
def _citation(
provider: RAGcoreKnowledgeProvider,
document_id: str,
*,
language: str = "en-GB",
section: str = "Procedure",
seed: str = "1",
) -> dict:
document = _managed_document(provider, document_id, language)
excerpt = _EXCERPTS[(language, document_id)]
return {
"id": str(uuid.uuid5(uuid.NAMESPACE_URL, f"citation:{language}:{document_id}:{seed}")),
# These are deliberately opaque RAGcore-owned UUIDs, not Fleet Ops' stable
# human-readable procedure id.
"document_id": str(
uuid.uuid5(uuid.NAMESPACE_URL, f"ragcore-document:{language}:{document_id}")
),
"document_version_id": str(
uuid.uuid5(
uuid.NAMESPACE_URL,
f"ragcore-document-version:{language}:{document_id}:{seed}",
)
),
"title": "untrusted-provider-title.md",
"section": section,
"excerpt": excerpt,
"excerpt_sha256": hashlib.sha256(excerpt.encode("utf-8")).hexdigest(),
"source_uri": f"ragcore://source/{document.source_id}",
"source_id": document.source_id,
"locator": f"{document.document_id}.md",
}
def _answers_body(provider: RAGcoreKnowledgeProvider, **overrides) -> dict:
citation = _citation(provider, "damage-procedure", section="Detection")
answer = "Report damage and route the vehicle to maintenance."
body = {
"answer": "Report damage and route the vehicle to maintenance.",
"answer_id": str(uuid.uuid4()),
"retrieval_run_id": str(uuid.uuid4()),
"answer": answer,
"answerability": "answerable",
"citations": [
{
"id": "cite-1",
"document_id": "doc-1",
"document_version_id": "version-1",
"title": "Damage handling procedure",
"section": "Detection",
"excerpt": "Inspect the vehicle for visible damage.",
}
],
"citations": [citation],
"claims": [{"text": answer, "citation_ids": [citation["id"]]}],
}
body.update(overrides)
if "claims" not in overrides:
effective_answer = body.get("answer")
effective_citations = body.get("citations")
body["claims"] = (
[
{
"text": effective_answer,
"citation_ids": [item["id"] for item in effective_citations],
}
]
if isinstance(effective_answer, str)
and effective_answer
and isinstance(effective_citations, list)
and effective_citations
and all(isinstance(item, dict) and "id" in item for item in effective_citations)
else []
)
return body
@@ -418,47 +509,178 @@ def test_ragcore_provider_grounded_answer_maps_citations_to_sources(monkeypatch)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body())),
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(provider))),
)
answer = provider.ask("What must I do about damage?", "test-correlation-grounded")
assert answer.evidence_state == "grounded"
assert answer.answer
assert len(answer.sources) == 1
source = answer.sources[0]
assert source.document_id == "doc-1"
assert source.document_id == "damage-procedure"
assert source.title == "Damage handling procedure"
assert source.version == "version-1"
assert source.version == "1.3"
assert source.section == "Detection"
assert source.excerpt
@pytest.mark.parametrize(
"mutate",
[
lambda body: body.pop("claims"),
lambda body: body.update(claims=[]),
lambda body: body["claims"][0].update(citation_ids=[str(uuid.uuid4())]),
lambda body: body.pop("answer_id"),
lambda body: body.update(retrieval_run_id="not-a-uuid"),
],
)
def test_ragcore_answer_requires_valid_claim_citation_contract(monkeypatch, mutate):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
body = _answers_body(provider)
mutate(body)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, body)),
)
answer = provider.ask("What must I do about damage?", "test-invalid-claim-contract")
assert answer.evidence_state != "grounded"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_requests_are_scoped_to_managed_sources_for_requested_language(
monkeypatch,
):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
requests = []
fake_client = _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
},
requests=requests,
)
monkeypatch.setattr(provider, "_client", lambda: fake_client)
answer = provider.ask(
"Wat is de procedure voor een voertuigretour?",
"language-filter",
"nl-BE",
)
assert answer.evidence_state == "grounded"
assert [path for path, _payload in requests] == ["/v1/answers", "/v1/search"]
expected_source_ids = provider._managed_source_ids("nl-BE")
assert len(expected_source_ids) == 11
assert all(uuid.UUID(source_id) for source_id in expected_source_ids)
for _path, payload in requests:
assert payload["filters"] == {"source_ids": expected_source_ids}
assert payload["requested_space_ids"] == ["space-1"]
def test_ragcore_sources_deduplicate_reuploaded_versions_and_cap_cards(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
citations = []
for index in range(5):
document_ids = [
"damage-procedure",
"damage-procedure",
"vehicle-return-procedure",
"cleaning-checklist",
"maintenance-escalation",
]
for index, document_id in enumerate(document_ids):
citations.append(
{
"id": f"cite-{index}",
"document_id": f"doc-{index}",
"document_version_id": f"version-{index}",
"title": "Damage procedure" if index < 2 else f"Procedure {index}",
"section": "Return",
"excerpt": (
f"Record visible damage before release, chunk {index}."
if index < 2
else f"Unique procedure evidence {index}."
),
}
_citation(
provider,
document_id,
section="Return",
seed=str(index),
)
)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(citations=citations))),
lambda: _FakeClient(
post_response=_FakeResponse(200, _answers_body(provider, citations=citations))
),
)
answer = provider.ask("What must I do about vehicle damage?", "dedupe-test")
assert len(answer.sources) == 3
assert sum(source.title == "Damage procedure" for source in answer.sources) == 1
assert sum(source.title == "Damage handling procedure" for source in answer.sources) == 1
def test_ragcore_answer_rejects_unknown_document_citation(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200,
_answers_body(
provider,
answer="A plausible-looking but unmanaged answer.",
citations=[
{
**_citation(provider, "damage-procedure"),
"source_id": str(uuid.uuid4()),
}
],
),
)
),
)
answer = provider.ask("What must I do about damage?", "unknown-answer-citation")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
@pytest.mark.parametrize(
("field_name", "bad_value"),
[
("id", "not-a-uuid"),
("document_id", "not-a-uuid"),
("document_version_id", "00000000-0000-0000-0000-000000000000"),
("source_id", "00000000-0000-4000-8000-000000000099"),
("source_uri", "ragcore://source/00000000-0000-4000-8000-000000000099"),
("locator", "another-procedure.md"),
("excerpt_sha256", "0" * 64),
("title", None),
("section", {"not": "a string"}),
],
)
def test_ragcore_rejects_broken_managed_source_provenance(field_name, bad_value):
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure")
citation[field_name] = bad_value
assert provider._managed_source(citation, "en-GB") is None
def test_ragcore_rejects_fabricated_excerpt_even_with_matching_hash():
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure")
fabricated = "Invent a repair price and promise it to the customer."
citation["excerpt"] = fabricated
citation["excerpt_sha256"] = hashlib.sha256(fabricated.encode("utf-8")).hexdigest()
assert provider._managed_source(citation, "en-GB") is None
def test_ragcore_never_relabels_an_english_source_as_dutch():
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure", language="en-GB")
assert provider._managed_source(citation, "nl-BE") is None
def test_knowledge_feedback_is_audited_and_can_be_changed(ops_client):
@@ -503,6 +725,7 @@ def test_ragcore_provider_not_answerable_is_insufficient_and_never_fabricates(mo
post_response=_FakeResponse(
200,
_answers_body(
provider,
answer="This should never be shown.",
answerability="not_answerable",
citations=[],
@@ -524,7 +747,8 @@ def test_ragcore_provider_answerable_without_citations_is_insufficient(monkeypat
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200, _answers_body(answerability="answerable", citations=[])
200,
_answers_body(provider, answerability="answerable", citations=[]),
)
),
)
@@ -565,19 +789,66 @@ def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch):
assert answer.evidence_state == "unavailable"
def _search_body(**overrides) -> dict:
def test_ragcore_provider_malformed_top_level_bodies_do_not_escape(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(200, ["not", "an", "object"]),
"/v1/search": _FakeResponse(200, None),
}
),
)
answer = provider.ask("Anything?", "test-correlation-malformed-top-level")
assert answer.evidence_state == "unavailable"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_answer_with_malformed_citation_is_insufficient(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200,
_answers_body(provider, citations=[None]),
)
),
)
answer = provider.ask("What must I do about damage?", "malformed-answer-citation")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def _search_body(
provider: RAGcoreKnowledgeProvider,
language: str = "en-GB",
**overrides,
) -> dict:
body = {
"retrieval_run_id": str(uuid.uuid4()),
"effective_space_ids": ["00000000-0000-4000-8000-000000000001"],
"results": [
{
"chunk_id": "chunk-1",
"citation": {
"id": "cite-1",
"document_id": "doc-1",
"document_version_id": "version-1",
"title": "Vehicle return procedure",
"section": "Return",
"excerpt": "Register the return odometer reading before releasing the vehicle.",
},
"chunk_id": str(uuid.uuid4()),
"text": _EXCERPTS[(language, "vehicle-return-procedure")],
"citation": _citation(
provider,
"vehicle-return-procedure",
language=language,
section="Return",
),
"rank": 1,
"scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None},
}
@@ -588,6 +859,31 @@ def _search_body(**overrides) -> dict:
return body
def test_ragcore_search_ignores_malformed_result_and_citation_bodies(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
malformed_search = _search_body(
provider,
results=[None, {"citation": ["not-an-object"], "scores": "not-an-object"}],
)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, malformed_search),
}
),
)
answer = provider.ask("What is the vehicle return procedure?", "malformed-search-items")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch):
"""/v1/answers itself failing (a real RAGcore-side outage in its generation step,
not a real 'insufficient evidence' classification) must not silently degrade
@@ -601,26 +897,24 @@ def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypa
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider)),
}
),
)
answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback")
assert answer.evidence_state == "grounded"
assert "Register the return odometer reading" in answer.answer
assert "Open the active booking and record the ending odometer" in answer.answer
assert "Vehicle return procedure" in answer.answer
assert len(answer.sources) == 1
assert answer.sources[0].title == "Vehicle return procedure"
assert answer.sources[0].excerpt == (
"Register the return odometer reading before releasing the vehicle."
)
assert answer.sources[0].excerpt == _EXCERPTS[("en-GB", "vehicle-return-procedure")]
def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0)
search_response = _FakeResponse(200, _search_body())
search_response = _FakeResponse(200, _search_body(provider))
monkeypatch.setattr(
provider,
"_client",
@@ -632,8 +926,7 @@ def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
),
)
assert (
provider.ask("What is the vehicle return procedure?", "first").evidence_state
== "grounded"
provider.ask("What is the vehicle return procedure?", "first").evidence_state == "grounded"
)
monkeypatch.setattr(
@@ -654,7 +947,7 @@ def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
}
),
)
@@ -676,7 +969,7 @@ def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkey
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body(results=[])),
"/v1/search": _FakeResponse(200, _search_body(provider, results=[])),
}
),
)
@@ -695,7 +988,7 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider)),
}
),
)
@@ -707,19 +1000,66 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
assert answer.answer == ""
def test_ragcore_search_fallback_rejects_unknown_document_citation(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body(provider)
search["results"][0]["citation"]["source_id"] = str(uuid.uuid4())
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, search),
}
),
)
answer = provider.ask(
"What is the vehicle return procedure?",
"unknown-search-citation",
)
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_search_fallback_rejects_vehicle_colour_despite_high_score(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body(provider)
search["results"][0]["scores"]["fused"] = 1.0
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, search),
}
),
)
answer = provider.ask("What colour is this vehicle?", "vehicle-colour")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body()
search = _search_body(provider, "nl-BE", results=[])
search["results"].append(
{
"citation": {
"document_id": "damage-procedure",
"document_version_id": "version-2",
"title": "damage-procedure.md",
"section": "Damage",
"excerpt": "Record damage and keep the vehicle blocked.",
},
"citation": _citation(
provider,
"damage-procedure",
language="nl-BE",
section="Damage",
),
"rank": 2,
"scores": {"fused": 0.01},
}
@@ -743,16 +1083,15 @@ def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
def test_ragcore_search_fallback_accepts_top_ranked_ragcore_damage_evidence(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body()
search = _search_body(provider, "nl-BE", results=[])
search["results"].append(
{
"citation": {
"document_id": "damage-procedure",
"document_version_id": "version-2",
"title": "damage-procedure.md",
"section": "Damage",
"excerpt": "Record damage and keep the vehicle blocked.",
},
"citation": _citation(
provider,
"damage-procedure",
language="nl-BE",
section="Damage",
),
"rank": 1,
# RAGcore uses reciprocal-rank fusion; a genuine rank-one result is about
# 1 / (60 + 1), not a normalized 0..1 relevance score.
+31
View File
@@ -1,4 +1,12 @@
from app.core.config import get_settings
from app.main import app
MCP_ROUTE_ALLOWLIST = {
("GET", "/api/v1/integrations/mcp/operations-summary"),
("GET", "/api/v1/integrations/mcp/attention-vehicles"),
("GET", "/api/v1/integrations/mcp/vehicles/{vehicle_ref}"),
("POST", "/api/v1/integrations/mcp/search-knowledge"),
}
def _headers(token: str | None = None, client_id: str = "test-mcp-client"):
@@ -154,3 +162,26 @@ def test_no_write_endpoints_exist_under_mcp_namespace(client):
]:
response = getattr(client, method)(path, headers=_headers())
assert response.status_code in (404, 405)
def test_mcp_namespace_matches_exact_route_allowlist_and_rejects_lookalikes(client):
implemented = {
(method.upper(), path)
for path, operations in app.openapi()["paths"].items()
if path.startswith("/api/v1/integrations/mcp/")
for method in operations
if method in {"get", "post", "put", "patch", "delete"}
}
assert implemented == MCP_ROUTE_ALLOWLIST
lookalikes = [
("get", "/api/v1/integrations/mcp-extra/operations-summary"),
("get", "/api/v1/integrations/mcp/operations-summary-extra"),
("get", "/api/v1/integrations/mcp/prefix/attention-vehicles"),
("get", "/api/v1/integrations/mcp/attention-vehicles/suffix"),
("post", "/api/v1/integrations/mcp/search-knowledge-extra"),
("post", "/api/v1/integrations/mcp/prefix/search-knowledge"),
]
for method, path in lookalikes:
response = client.request(method, path, headers=_headers(), json={})
assert response.status_code == 404, path
+18
View File
@@ -63,6 +63,24 @@ def test_eligible_customer_is_irreversibly_anonymized_without_pii_in_audit(ops_c
)
assert response.status_code == 200
assert response.json()["status"] == "anonymized"
search = ops_client.get("/api/v1/customers", params={"query": "CUS-PRIVACY"})
assert search.status_code == 200
assert search.json() == []
window = {
"starts_at": "2090-01-01T10:00:00Z",
"ends_at": "2090-01-02T10:00:00Z",
}
available = ops_client.get("/api/v1/bookings/availability", params=window).json()
assert available
booking = ops_client.post(
"/api/v1/bookings",
json={
"customer_ref": "CUS-PRIVACY",
"vehicle_ref": available[0]["public_ref"],
**window,
},
)
assert booking.status_code == 422
with SessionLocal() as db:
customer = db.scalar(select(Customer).where(Customer.id == customer_id))
assert customer is not None
+17
View File
@@ -128,6 +128,23 @@ def test_seed_scenario_s4_booking_overlap():
db.close()
def test_seed_odometer_issues_expose_only_the_regressing_booking_as_correctable():
db = SessionLocal()
try:
reset_and_seed(db)
issue = _by_ref(db, DataQualityIssue, "DQ-0007")
assert issue is not None
assert issue.evidence_json["source_type"] == "return"
assert issue.evidence_json["related_refs"] == [
"INSP-0057",
"BK-H-0007",
"INSP-0007",
]
assert issue.evidence_json["correctable_booking_refs"] == ["BK-H-0007"]
finally:
db.close()
def test_seed_scenario_s5_failed_workflow_run():
"""S5: one seeded outbox event is durably 'failed' (terminal, retryable), not merely
pending, so the background dispatcher never silently auto-heals it away."""
+73
View File
@@ -0,0 +1,73 @@
from __future__ import annotations
import csv
import subprocess
import sys
from pathlib import Path
REPOSITORY_ROOT = next(
parent
for parent in Path(__file__).resolve().parents
if (parent / "seed" / "generate_seed.py").is_file()
)
SEED_DIRECTORY = REPOSITORY_ROOT / "seed"
def _rows_by_ref(path: Path) -> dict[str, dict[str, str]]:
with path.open(newline="", encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
assert all(None not in row for row in rows), f"malformed CSV row in {path.name}"
return {row["public_ref"]: row for row in rows}
def test_generator_reproduces_committed_seed_snapshot_byte_for_byte(tmp_path: Path):
subprocess.run(
[
sys.executable,
str(SEED_DIRECTORY / "generate_seed.py"),
"--anchor",
"2026-08-01",
"--seed",
"20260801",
"--out",
str(tmp_path),
],
cwd=REPOSITORY_ROOT,
check=True,
capture_output=True,
text=True,
)
committed_files = sorted(SEED_DIRECTORY.glob("*.csv"))
generated_files = sorted(tmp_path.glob("*.csv"))
assert [path.name for path in generated_files] == [path.name for path in committed_files]
for committed in committed_files:
generated = tmp_path / committed.name
assert generated.read_bytes() == committed.read_bytes(), (
f"{committed.name} differs from the deterministic generated snapshot"
)
bookings = _rows_by_ref(tmp_path / "bookings.csv")
inspections = _rows_by_ref(tmp_path / "inspections.csv")
issues = _rows_by_ref(tmp_path / "data_quality_issues.csv")
assert int(bookings["BK-H-0007"]["end_odometer_km"]) < int(
bookings["BK-H-0057"]["end_odometer_km"]
)
assert int(bookings["BK-H-0010"]["end_odometer_km"]) < int(
bookings["BK-H-0060"]["end_odometer_km"]
)
assert inspections["INSP-0007"]["odometer_km"] == bookings["BK-H-0007"]["end_odometer_km"]
assert inspections["INSP-0010"]["odometer_km"] == bookings["BK-H-0010"]["end_odometer_km"]
assert inspections["INSP-0001"]["completed_at"] == bookings["BK-H-0001"]["ends_at"]
expected_issue_refs = {f"DQ-{index:04d}" for index in range(5, 22)}
assert expected_issue_refs <= issues.keys()
assert all(
issues[public_ref]["evidence"] != "Synthetic deterministic seed issue"
for public_ref in expected_issue_refs
)
assert "INSP-0007" in issues["DQ-0007"]["evidence"]
assert "INSP-0057" in issues["DQ-0007"]["evidence"]
assert "INSP-0010" in issues["DQ-0010"]["evidence"]
assert "INSP-0060" in issues["DQ-0010"]["evidence"]
+19
View File
@@ -23,3 +23,22 @@ def test_manager_can_create_and_update_user(ops_client):
def test_employee_cannot_manage_users(employee_client):
assert employee_client.get("/api/v1/users").status_code == 403
assert (
employee_client.post(
"/api/v1/users",
json={
"email": "unauthorised@example.test",
"display_name": "Unauthorised User",
"role": "rental_employee",
"password": "a-secure-demo-password",
},
).status_code
== 403
)
assert (
employee_client.patch(
"/api/v1/users/USR-OPS",
json={"display_name": "Unauthorised Change"},
).status_code
== 403
)
+53
View File
@@ -46,6 +46,12 @@ def test_vehicle_detail_includes_related_records(ops_client):
assert len(body["quality_issues"]) >= 1
def test_employee_vehicle_detail_never_exposes_quality_evidence(employee_client):
response = employee_client.get("/api/v1/vehicles/MO-016")
assert response.status_code == 200
assert response.json()["quality_issues"] == []
def test_vehicle_detail_404_for_unknown_ref(ops_client):
response = ops_client.get("/api/v1/vehicles/MO-999")
assert response.status_code == 404
@@ -78,6 +84,53 @@ def test_manager_can_record_maintenance_and_release_vehicle(ops_client):
assert released.json()["operational_status"] == "available"
def test_low_maintenance_reading_keeps_canonical_and_opens_quality_issue(ops_client):
existing_issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
unavailable_refs = {issue["entity_ref"] for issue in existing_issues}
vehicles = ops_client.get("/api/v1/vehicles", params={"status": "available"}).json()
vehicle = next(
candidate
for candidate in vehicles
if candidate["odometer_km"] > 0 and candidate["public_ref"] not in unavailable_refs
)
response = ops_client.post(
f"/api/v1/vehicles/{vehicle['public_ref']}/maintenance",
json={
"occurred_at": "2053-01-15T12:00:00Z",
"odometer_km": vehicle["odometer_km"] - 1,
"category": "inspection",
"summary": "Imported workshop reading requires verification.",
"mark_maintenance": True,
},
)
assert response.status_code == 201
detail = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
assert detail["odometer_km"] == vehicle["odometer_km"]
issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
issue = next(item for item in issues if item["entity_ref"] == vehicle["public_ref"])
assert issue["evidence"]["source_type"] == "maintenance"
assert issue["evidence"]["correctable_booking_refs"] == []
issue_detail = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
assert any(
snapshot["entity_type"] == "maintenance"
and snapshot["public_ref"] == response.json()["public_ref"]
for snapshot in issue_detail["related_snapshots"]
)
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={"decision": "retain_canonical"},
)
assert retained.status_code == 200
def test_employee_cannot_record_maintenance(employee_client):
response = employee_client.post(
"/api/v1/vehicles/MO-001/maintenance",