M54: harden operations and demo resilience
This commit is contained in:
@@ -30,6 +30,7 @@ from app.schemas import (
|
||||
ReturnPreviewResult,
|
||||
)
|
||||
from app.services.audit import record_audit_event
|
||||
from app.services.data_quality import open_odometer_regression_issue
|
||||
from app.services.returns import preview_vehicle_return, register_vehicle_return
|
||||
|
||||
router = APIRouter(prefix="/api/v1/bookings", tags=["bookings"])
|
||||
@@ -146,8 +147,14 @@ def create_booking(
|
||||
) -> BookingOut:
|
||||
if body.ends_at <= body.starts_at:
|
||||
raise HTTPException(status_code=422, detail="Booking end must be after its start")
|
||||
customer = db.scalar(select(Customer).where(Customer.public_ref == body.customer_ref))
|
||||
if customer is None or customer.merged_into_customer_id is not None:
|
||||
customer = db.scalar(
|
||||
select(Customer).where(Customer.public_ref == body.customer_ref).with_for_update()
|
||||
)
|
||||
if (
|
||||
customer is None
|
||||
or customer.merged_into_customer_id is not None
|
||||
or customer.anonymized_at is not None
|
||||
):
|
||||
raise HTTPException(status_code=422, detail="Customer is unavailable for booking")
|
||||
# Serialise booking creation per vehicle. The overlap check must run after
|
||||
# acquiring this lock, otherwise two concurrent requests can both pass it.
|
||||
@@ -179,7 +186,9 @@ def create_booking(
|
||||
status="reserved",
|
||||
start_odometer_km=None,
|
||||
end_odometer_km=None,
|
||||
requirements_complete=body.requirements_complete,
|
||||
# Requirements are intentionally confirmed in a separate, audited action.
|
||||
# Never allow booking creation to bypass that operational checkpoint.
|
||||
requirements_complete=False,
|
||||
)
|
||||
db.add(booking)
|
||||
db.flush()
|
||||
@@ -225,6 +234,15 @@ def checkout_booking(
|
||||
if active_conflict is not None:
|
||||
raise HTTPException(status_code=409, detail="Vehicle already has an active booking")
|
||||
|
||||
correlation_id = uuid.uuid4()
|
||||
before_booking = {
|
||||
"status": booking.status,
|
||||
"start_odometer_km": booking.start_odometer_km,
|
||||
}
|
||||
before_vehicle = {
|
||||
"operational_status": vehicle.operational_status,
|
||||
"odometer_km": vehicle.odometer_km,
|
||||
}
|
||||
attention_reasons: list[str] = []
|
||||
if body.start_odometer_km < vehicle.odometer_km:
|
||||
attention_reasons.append("odometer_regression")
|
||||
@@ -250,6 +268,18 @@ def checkout_booking(
|
||||
completed_by=user.display_name,
|
||||
)
|
||||
db.add(inspection)
|
||||
if "odometer_regression" in attention_reasons:
|
||||
open_odometer_regression_issue(
|
||||
db,
|
||||
vehicle=vehicle,
|
||||
reading_ref=inspection.public_ref,
|
||||
reading_km=body.start_odometer_km,
|
||||
canonical_km=vehicle.odometer_km,
|
||||
source_type="checkout",
|
||||
related_refs=[booking.public_ref, inspection.public_ref],
|
||||
actor_label=user.display_name,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
if attention_reasons:
|
||||
booking.status = "blocked"
|
||||
vehicle.operational_status = (
|
||||
@@ -269,13 +299,31 @@ def checkout_booking(
|
||||
action="booking_checkout_recorded",
|
||||
entity_type="booking",
|
||||
entity_id=booking.id,
|
||||
correlation_id=correlation_id,
|
||||
before=before_booking,
|
||||
after={
|
||||
"inspection_ref": inspection.public_ref,
|
||||
"booking_status": booking.status,
|
||||
"start_odometer_km": booking.start_odometer_km,
|
||||
"vehicle_status": vehicle.operational_status,
|
||||
"attention_reasons": attention_reasons,
|
||||
},
|
||||
)
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=user.display_name,
|
||||
action="vehicle_status_changed",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
correlation_id=correlation_id,
|
||||
before=before_vehicle,
|
||||
after={
|
||||
"operational_status": vehicle.operational_status,
|
||||
"odometer_km": vehicle.odometer_km,
|
||||
},
|
||||
metadata={"booking_ref": booking.public_ref, "inspection_ref": inspection.public_ref},
|
||||
)
|
||||
db.commit()
|
||||
return CheckoutBookingResult(
|
||||
booking_ref=booking.public_ref,
|
||||
|
||||
@@ -21,6 +21,7 @@ def search_customers(
|
||||
select(Customer)
|
||||
.where(
|
||||
Customer.merged_into_customer_id.is_(None),
|
||||
Customer.anonymized_at.is_(None),
|
||||
or_(
|
||||
Customer.public_ref.ilike(term),
|
||||
Customer.first_name.ilike(term),
|
||||
|
||||
@@ -11,6 +11,7 @@ from app.models.booking import Booking
|
||||
from app.models.customer import Customer
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.inspection import Inspection
|
||||
from app.models.maintenance import MaintenanceRecord
|
||||
from app.models.user import User
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.schemas import (
|
||||
@@ -224,6 +225,8 @@ _PREFIX_TO_TYPE = {
|
||||
"MO-": "vehicle",
|
||||
"BK-": "booking",
|
||||
"INSP-": "inspection",
|
||||
"MAINT-": "maintenance",
|
||||
"MNT-": "maintenance",
|
||||
}
|
||||
|
||||
|
||||
@@ -292,6 +295,17 @@ def _snapshot(entity_type: str, ref: str, db: Session) -> dict | None:
|
||||
"completed_at": inspection.completed_at.isoformat(),
|
||||
"booking_ref": booking.public_ref if booking else None,
|
||||
}
|
||||
if entity_type == "maintenance":
|
||||
record = db.scalar(select(MaintenanceRecord).where(MaintenanceRecord.public_ref == ref))
|
||||
if record is None:
|
||||
return None
|
||||
return {
|
||||
"entity_type": "maintenance",
|
||||
"public_ref": record.public_ref,
|
||||
"odometer_km": record.odometer_km,
|
||||
"occurred_at": record.occurred_at.isoformat(),
|
||||
"category": record.category,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from sqlalchemy import func, select
|
||||
@@ -10,7 +11,9 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import get_current_user, get_db, require_operations_manager
|
||||
from app.core.config import get_settings
|
||||
from app.core.db import begin_exclusive_demo_reset, end_exclusive_demo_reset, engine
|
||||
from app.core.security import SessionPayload, create_session_token, read_session_token
|
||||
from app.models.audit import AuditEvent
|
||||
from app.models.user import User
|
||||
from app.schemas import CurrentUser, DemoLoginRequest, DemoManifestOut
|
||||
from app.seed_loader import reset_and_seed
|
||||
@@ -21,7 +24,6 @@ from app.services.sessions import revoke_session
|
||||
router = APIRouter(prefix="/api/v1/demo", tags=["demo"])
|
||||
settings = get_settings()
|
||||
_reset_guard = threading.Lock()
|
||||
_last_reset_monotonic = 0.0
|
||||
_RESET_ADVISORY_LOCK_ID = 706_533_149
|
||||
|
||||
|
||||
@@ -110,7 +112,6 @@ def demo_reset(
|
||||
db: Session = Depends(get_db),
|
||||
user: CurrentUser = Depends(require_operations_manager),
|
||||
) -> dict:
|
||||
global _last_reset_monotonic
|
||||
if not settings.mobilityops_demo_mode:
|
||||
# Outside demo mode the reset endpoint must not exist at all: it wipes
|
||||
# operational data and replaces it with synthetic records.
|
||||
@@ -122,19 +123,35 @@ def demo_reset(
|
||||
)
|
||||
if not _reset_guard.acquire(blocking=False):
|
||||
raise HTTPException(status_code=409, detail="A demo reset is already running.")
|
||||
replica_lock_connection = None
|
||||
try:
|
||||
elapsed = time.monotonic() - _last_reset_monotonic
|
||||
if _last_reset_monotonic and elapsed < settings.demo_reset_cooldown_seconds:
|
||||
# A session-level lock on its own connection rejects another replica immediately;
|
||||
# the main DB session can then safely end its auth read transaction and wait on
|
||||
# the normal shared/exclusive data barrier without releasing this replica guard.
|
||||
replica_lock_connection = engine.connect()
|
||||
locked = replica_lock_connection.scalar(
|
||||
select(func.pg_try_advisory_lock(_RESET_ADVISORY_LOCK_ID))
|
||||
)
|
||||
if not locked:
|
||||
raise HTTPException(status_code=409, detail="A demo reset is already running.")
|
||||
begin_exclusive_demo_reset(db)
|
||||
# The audit timestamp is shared by every replica. A process-local monotonic
|
||||
# timestamp cannot protect a multi-replica deployment.
|
||||
last_reset_at = db.scalar(
|
||||
select(AuditEvent.occurred_at)
|
||||
.where(AuditEvent.action == "demo_reset")
|
||||
.order_by(AuditEvent.occurred_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
elapsed = (datetime.now(UTC) - last_reset_at).total_seconds() if last_reset_at else None
|
||||
if elapsed is not None and elapsed < settings.demo_reset_cooldown_seconds:
|
||||
retry_after = max(1, int(settings.demo_reset_cooldown_seconds - elapsed + 0.999))
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"Demo reset is cooling down. Retry in {retry_after} seconds.",
|
||||
headers={"Retry-After": str(retry_after)},
|
||||
)
|
||||
locked = db.scalar(select(func.pg_try_advisory_xact_lock(_RESET_ADVISORY_LOCK_ID)))
|
||||
if not locked:
|
||||
raise HTTPException(status_code=409, detail="A demo reset is already running.")
|
||||
result = reset_and_seed(db, preserve_integration_telemetry=True)
|
||||
result = reset_and_seed(db, preserve_integration_telemetry=True, commit=False)
|
||||
integrity = scenario_integrity_report(db)
|
||||
record_audit_event(
|
||||
db,
|
||||
@@ -149,8 +166,15 @@ def demo_reset(
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
_last_reset_monotonic = time.monotonic()
|
||||
end_exclusive_demo_reset(db)
|
||||
finally:
|
||||
if replica_lock_connection is not None:
|
||||
try:
|
||||
replica_lock_connection.scalar(
|
||||
select(func.pg_advisory_unlock(_RESET_ADVISORY_LOCK_ID))
|
||||
)
|
||||
finally:
|
||||
replica_lock_connection.close()
|
||||
_reset_guard.release()
|
||||
response.delete_cookie(settings.session_cookie_name)
|
||||
return {
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, Header
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import get_db
|
||||
@@ -51,6 +52,13 @@ def _require_service_token(service_token: str) -> None:
|
||||
raise AppError("UNAUTHORIZED_SERVICE", "Invalid service token.", status_code=401)
|
||||
|
||||
|
||||
def _lock_idempotency_key(db: Session, namespace: str, key: str) -> None:
|
||||
"""Serialize callback check+insert by a stable, transaction-scoped key."""
|
||||
digest = hashlib.sha256(f"{namespace}:{key}".encode()).digest()
|
||||
lock_id = int.from_bytes(digest[:8], byteorder="big", signed=True)
|
||||
db.scalar(select(func.pg_advisory_xact_lock(lock_id)))
|
||||
|
||||
|
||||
@router.post("/heartbeat", response_model=N8nHeartbeatResult)
|
||||
def workflow_heartbeat(
|
||||
body: N8nHeartbeatIn,
|
||||
@@ -61,6 +69,7 @@ def workflow_heartbeat(
|
||||
_require_service_token(service_token)
|
||||
if body.workflow_name not in _CANONICAL_WORKFLOW_NAMES:
|
||||
raise AppError("UNKNOWN_WORKFLOW", "Unknown Fleet Ops workflow.", status_code=422)
|
||||
_lock_idempotency_key(db, "n8n_workflow_heartbeat", f"{body.execution_id}:{body.status}")
|
||||
already_recorded = (
|
||||
db.scalar(
|
||||
select(AuditEvent.id).where(
|
||||
@@ -109,9 +118,29 @@ def return_callback(
|
||||
"INVALID_IDEMPOTENCY_KEY", "Idempotency-Key must be the event's UUID.", status_code=422
|
||||
) from exc
|
||||
|
||||
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
|
||||
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update())
|
||||
if event is None:
|
||||
raise AppError("EVENT_NOT_FOUND", "No outbox event matches this event ID.", status_code=404)
|
||||
if body.event_id != event_id:
|
||||
raise AppError(
|
||||
"CALLBACK_EVENT_MISMATCH",
|
||||
"Callback event_id does not match Idempotency-Key.",
|
||||
status_code=409,
|
||||
)
|
||||
try:
|
||||
expected_correlation_id = uuid.UUID(str(event.payload_json["correlation_id"]))
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise AppError(
|
||||
"INVALID_EVENT_CORRELATION",
|
||||
"The stored outbox event has no valid correlation ID.",
|
||||
status_code=409,
|
||||
) from exc
|
||||
if body.correlation_id != expected_correlation_id:
|
||||
raise AppError(
|
||||
"CALLBACK_CORRELATION_MISMATCH",
|
||||
"Callback correlation_id does not match the outbox event.",
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
# Idempotent by event ID: n8n or our own dispatcher may redeliver the same event
|
||||
# (e.g. a lost response after a timeout), so this callback must not double-record.
|
||||
@@ -131,7 +160,7 @@ def return_callback(
|
||||
actor_label="n8n",
|
||||
action="n8n_return_followup_recorded",
|
||||
entity_type="booking",
|
||||
correlation_id=body.correlation_id,
|
||||
correlation_id=expected_correlation_id,
|
||||
after={"follow_up": body.follow_up, "summary": body.summary},
|
||||
metadata={"event_id": str(event_id)},
|
||||
)
|
||||
@@ -170,6 +199,7 @@ def workflow_error(
|
||||
other Fleet Ops n8n workflow. Idempotent on execution_id: n8n may redeliver the same
|
||||
error report (e.g. after a timed-out response), so this must not double-record."""
|
||||
_require_service_token(service_token)
|
||||
_lock_idempotency_key(db, "n8n_workflow_failure", body.execution_id)
|
||||
|
||||
already_recorded = (
|
||||
db.scalar(
|
||||
@@ -181,19 +211,13 @@ def workflow_error(
|
||||
is not None
|
||||
)
|
||||
if not already_recorded:
|
||||
correlation_id: uuid.UUID | None = None
|
||||
if body.correlation_id:
|
||||
try:
|
||||
correlation_id = uuid.UUID(body.correlation_id)
|
||||
except ValueError:
|
||||
correlation_id = None
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="service",
|
||||
actor_label="n8n error handler",
|
||||
action="n8n_workflow_failure_registered",
|
||||
entity_type="automation",
|
||||
correlation_id=correlation_id,
|
||||
correlation_id=body.correlation_id,
|
||||
after={
|
||||
"workflow_id": body.workflow_id,
|
||||
"workflow_name": body.workflow_name,
|
||||
@@ -248,6 +272,7 @@ def procedures_sync_result(
|
||||
RAGcore Procedure Sync" workflow once it finishes uploading procedures to RAGcore.
|
||||
Idempotent on execution_id, matching the workflow-error and return-callback pattern."""
|
||||
_require_service_token(service_token)
|
||||
_lock_idempotency_key(db, "n8n_procedure_sync", body.execution_id)
|
||||
|
||||
already_recorded = (
|
||||
db.scalar(
|
||||
|
||||
@@ -53,7 +53,9 @@ def ask_question(
|
||||
client_ip = (
|
||||
forwarded.split(",")[-1].strip()
|
||||
if forwarded
|
||||
else request.client.host if request.client else "unknown"
|
||||
else request.client.host
|
||||
if request.client
|
||||
else "unknown"
|
||||
)
|
||||
token = request.cookies.get(settings.session_cookie_name, "")
|
||||
session_key = hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
@@ -27,6 +27,7 @@ from app.schemas import (
|
||||
VehiclePageOut,
|
||||
)
|
||||
from app.services.audit import record_audit_event
|
||||
from app.services.data_quality import open_odometer_regression_issue
|
||||
|
||||
router = APIRouter(prefix="/api/v1/vehicles", tags=["vehicles"])
|
||||
|
||||
@@ -143,7 +144,7 @@ def list_vehicles(
|
||||
def get_vehicle(
|
||||
public_ref: str,
|
||||
db: Session = Depends(get_db),
|
||||
_user: CurrentUser = Depends(get_current_user),
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> VehicleDetailOut:
|
||||
vehicle = db.scalar(select(Vehicle).where(Vehicle.public_ref == public_ref))
|
||||
if vehicle is None:
|
||||
@@ -163,11 +164,21 @@ def get_vehicle(
|
||||
.where(MaintenanceRecord.vehicle_id == vehicle.id)
|
||||
.order_by(MaintenanceRecord.occurred_at.desc())
|
||||
).all()
|
||||
issues = db.scalars(
|
||||
select(DataQualityIssue)
|
||||
.where(DataQualityIssue.entity_type == "vehicle", DataQualityIssue.entity_id == vehicle.id)
|
||||
.order_by(DataQualityIssue.detected_at.desc())
|
||||
).all()
|
||||
# Detailed data-quality evidence is an operations-manager surface. Employees still
|
||||
# get the operational vehicle record they need, but never receive hidden evidence in
|
||||
# the payload merely because the frontend omits the Quality tab.
|
||||
issues = (
|
||||
db.scalars(
|
||||
select(DataQualityIssue)
|
||||
.where(
|
||||
DataQualityIssue.entity_type == "vehicle",
|
||||
DataQualityIssue.entity_id == vehicle.id,
|
||||
)
|
||||
.order_by(DataQualityIssue.detected_at.desc())
|
||||
).all()
|
||||
if user.role == "operations_manager"
|
||||
else []
|
||||
)
|
||||
|
||||
booking_by_id = {b.id: b.public_ref for b in bookings}
|
||||
next_booking = next(
|
||||
@@ -276,6 +287,12 @@ def create_maintenance_record(
|
||||
vehicle = db.scalar(select(Vehicle).where(Vehicle.public_ref == public_ref).with_for_update())
|
||||
if vehicle is None:
|
||||
raise HTTPException(status_code=404, detail="Vehicle not found")
|
||||
correlation_id = uuid.uuid4()
|
||||
before_vehicle = {
|
||||
"operational_status": vehicle.operational_status,
|
||||
"odometer_km": vehicle.odometer_km,
|
||||
"next_service_km": vehicle.next_service_km,
|
||||
}
|
||||
record = MaintenanceRecord(
|
||||
public_ref=f"MAINT-{uuid.uuid4().hex[:8].upper()}",
|
||||
vehicle_id=vehicle.id,
|
||||
@@ -285,6 +302,17 @@ def create_maintenance_record(
|
||||
summary=body.summary.strip(),
|
||||
)
|
||||
db.add(record)
|
||||
open_odometer_regression_issue(
|
||||
db,
|
||||
vehicle=vehicle,
|
||||
reading_ref=record.public_ref,
|
||||
reading_km=body.odometer_km,
|
||||
canonical_km=vehicle.odometer_km,
|
||||
source_type="maintenance",
|
||||
related_refs=[record.public_ref],
|
||||
actor_label=user.display_name,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
vehicle.odometer_km = max(vehicle.odometer_km, body.odometer_km)
|
||||
if body.next_service_km is not None:
|
||||
if body.next_service_km < vehicle.odometer_km:
|
||||
@@ -301,7 +329,14 @@ def create_maintenance_record(
|
||||
action="maintenance_record_created",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
after={"maintenance_ref": record.public_ref, "status": vehicle.operational_status},
|
||||
correlation_id=correlation_id,
|
||||
before=before_vehicle,
|
||||
after={
|
||||
"maintenance_ref": record.public_ref,
|
||||
"operational_status": vehicle.operational_status,
|
||||
"odometer_km": vehicle.odometer_km,
|
||||
"next_service_km": vehicle.next_service_km,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
return MaintenanceOut(
|
||||
|
||||
@@ -33,7 +33,11 @@ class Settings(BaseSettings):
|
||||
n8n_callback_token: str = "replace-me-n8n-callback-token"
|
||||
n8n_dispatch_enabled: bool = True
|
||||
n8n_dispatch_interval_seconds: float = 3.0
|
||||
n8n_http_timeout_seconds: float = 5.0
|
||||
# The synchronous n8n workflow performs two bounded, retried callbacks before it
|
||||
# acknowledges an event. Keep this above that complete workflow budget, while the
|
||||
# delivery lease remains the wider crash-recovery boundary (enforced by the contract
|
||||
# check in scripts/check-contracts.py).
|
||||
n8n_http_timeout_seconds: float = 15.0
|
||||
n8n_max_attempts: int = 5
|
||||
n8n_delivery_lease_seconds: float = 120.0
|
||||
app_secret: str = "replace-in-production"
|
||||
|
||||
+32
-1
@@ -1,6 +1,6 @@
|
||||
from collections.abc import Generator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, event, func, select
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.core.config import get_settings
|
||||
@@ -10,6 +10,37 @@ settings = get_settings()
|
||||
engine = create_engine(settings.database_url, pool_pre_ping=True, future=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
|
||||
|
||||
# Every SQLAlchemy transaction participates in a shared database-wide barrier. Normal
|
||||
# reads/writes coexist; the short demo reset takes the exclusive form so it can never
|
||||
# interleave deletes/inserts with an API request, scanner, callback, or dispatcher cycle.
|
||||
DEMO_DATA_BARRIER_LOCK_ID = 5_344_725_149_212_793_901
|
||||
_EXCLUSIVE_RESET_INFO_KEY = "mobilityops_demo_reset_exclusive"
|
||||
|
||||
|
||||
@event.listens_for(Session, "after_begin")
|
||||
def _acquire_demo_data_barrier(session: Session, _transaction, connection) -> None:
|
||||
if connection.dialect.name != "postgresql":
|
||||
return
|
||||
lock = (
|
||||
func.pg_advisory_xact_lock(DEMO_DATA_BARRIER_LOCK_ID)
|
||||
if session.info.get(_EXCLUSIVE_RESET_INFO_KEY)
|
||||
else func.pg_advisory_xact_lock_shared(DEMO_DATA_BARRIER_LOCK_ID)
|
||||
)
|
||||
connection.execute(select(lock))
|
||||
|
||||
|
||||
def begin_exclusive_demo_reset(db: Session) -> None:
|
||||
"""Make the session's next transaction the exclusive side of the reset barrier."""
|
||||
if db.in_transaction():
|
||||
# Auth normally already read the user under a shared barrier. End that read-only
|
||||
# transaction before requesting exclusive; in-place lock upgrades can deadlock.
|
||||
db.rollback()
|
||||
db.info[_EXCLUSIVE_RESET_INFO_KEY] = True
|
||||
|
||||
|
||||
def end_exclusive_demo_reset(db: Session) -> None:
|
||||
db.info.pop(_EXCLUSIVE_RESET_INFO_KEY, None)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
+12
-8
@@ -4,9 +4,13 @@ import uuid
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
||||
|
||||
Role = Literal["operations_manager", "rental_employee"]
|
||||
NonBlankOperationalReason = Annotated[
|
||||
str,
|
||||
StringConstraints(strip_whitespace=True, min_length=3, max_length=500),
|
||||
]
|
||||
|
||||
|
||||
class DemoLoginRequest(BaseModel):
|
||||
@@ -93,17 +97,16 @@ class CreateBookingRequest(BaseModel):
|
||||
vehicle_ref: str = Field(min_length=3, max_length=20)
|
||||
starts_at: datetime
|
||||
ends_at: datetime
|
||||
requirements_complete: bool = False
|
||||
|
||||
|
||||
class CompleteBookingRequirementsRequest(BaseModel):
|
||||
confirmation: str = Field(min_length=3, max_length=500)
|
||||
confirmation: NonBlankOperationalReason
|
||||
|
||||
|
||||
class RescheduleBookingRequest(BaseModel):
|
||||
starts_at: datetime
|
||||
ends_at: datetime
|
||||
reason: str = Field(min_length=3, max_length=500)
|
||||
reason: NonBlankOperationalReason
|
||||
|
||||
|
||||
class CustomerOptionOut(BaseModel):
|
||||
@@ -122,7 +125,7 @@ class AvailableVehicleOut(BaseModel):
|
||||
|
||||
|
||||
class CancelBookingRequest(BaseModel):
|
||||
reason: str = Field(min_length=3, max_length=500)
|
||||
reason: NonBlankOperationalReason
|
||||
|
||||
|
||||
class CheckoutBookingRequest(BaseModel):
|
||||
@@ -225,7 +228,7 @@ class CreateMaintenanceRequest(BaseModel):
|
||||
|
||||
|
||||
class ReleaseVehicleRequest(BaseModel):
|
||||
reason: str = Field(min_length=3, max_length=500)
|
||||
reason: NonBlankOperationalReason
|
||||
|
||||
|
||||
class DataQualityIssueOut(BaseModel):
|
||||
@@ -295,7 +298,7 @@ class WorkflowErrorReportIn(BaseModel):
|
||||
]
|
||||
error_summary: str = Field(max_length=500)
|
||||
trigger_context: str | None = Field(default=None, max_length=200)
|
||||
correlation_id: str | None = None
|
||||
correlation_id: uuid.UUID | None = None
|
||||
attempt: int = Field(default=1, ge=1, le=1000)
|
||||
retry_action: str | None = Field(default=None, max_length=200)
|
||||
|
||||
@@ -433,7 +436,8 @@ class ReturnCallbackIn(BaseModel):
|
||||
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
correlation_id: uuid.UUID | None = None
|
||||
event_id: uuid.UUID
|
||||
correlation_id: uuid.UUID
|
||||
follow_up: str | None = Field(default=None, max_length=200)
|
||||
summary: str | None = Field(default=None, max_length=2000)
|
||||
|
||||
|
||||
+51
-11
@@ -11,6 +11,7 @@ from sqlalchemy import delete, insert, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.db import begin_exclusive_demo_reset, end_exclusive_demo_reset
|
||||
from app.models.audit import AuditEvent
|
||||
from app.models.booking import Booking
|
||||
from app.models.customer import Customer
|
||||
@@ -201,7 +202,11 @@ def load_seed(db: Session) -> SeedResult:
|
||||
counts["bookings"] = len(booking_rows)
|
||||
|
||||
inspection_rows = []
|
||||
for row in _read_csv("inspections.csv"):
|
||||
inspection_source_rows = _read_csv("inspections.csv")
|
||||
return_inspection_row_by_booking_ref = {
|
||||
row["booking_ref"]: row for row in inspection_source_rows if row["type"] == "return"
|
||||
}
|
||||
for row in inspection_source_rows:
|
||||
inspection_rows.append(
|
||||
{
|
||||
"id": uuid.uuid4(),
|
||||
@@ -259,16 +264,18 @@ def load_seed(db: Session) -> SeedResult:
|
||||
}
|
||||
|
||||
def _odometer_regression_signal(later_ref: str, earlier_ref: str) -> list[dict]:
|
||||
later = booking_row_by_ref[later_ref]
|
||||
earlier = booking_row_by_ref[earlier_ref]
|
||||
later = return_inspection_row_by_booking_ref[later_ref]
|
||||
earlier = return_inspection_row_by_booking_ref[earlier_ref]
|
||||
return [
|
||||
{
|
||||
"code": "odometer.regression",
|
||||
"source_type": "return",
|
||||
"params": {
|
||||
"later_ref": later_ref,
|
||||
"later_km": later["end_odometer_km"],
|
||||
"earlier_ref": earlier_ref,
|
||||
"earlier_km": earlier["end_odometer_km"],
|
||||
"later_ref": later["public_ref"],
|
||||
"later_km": int(later["odometer_km"]),
|
||||
"earlier_ref": earlier["public_ref"],
|
||||
"earlier_km": int(earlier["odometer_km"]),
|
||||
"booking_ref": later_ref,
|
||||
},
|
||||
}
|
||||
]
|
||||
@@ -354,6 +361,27 @@ def load_seed(db: Session) -> SeedResult:
|
||||
entity_type, entity_id = resolve_entity(row["entity_ref"])
|
||||
related_ref = row.get("related_ref") or ""
|
||||
related_refs = related_ref.split("|") if related_ref else []
|
||||
signals = _seed_signals(row["public_ref"], row["entity_ref"], related_refs)
|
||||
evidence_extra: dict[str, object] = {}
|
||||
if row["rule_type"] == "odometer_regression" and signals:
|
||||
params = signals[0].get("params", {})
|
||||
later_ref = params.get("later_ref")
|
||||
earlier_ref = params.get("earlier_ref")
|
||||
booking_ref = params.get("booking_ref")
|
||||
if (
|
||||
isinstance(earlier_ref, str)
|
||||
and isinstance(later_ref, str)
|
||||
and isinstance(booking_ref, str)
|
||||
):
|
||||
# The authored CSV prose predates structured references. Preserve the
|
||||
# two real source bookings so the detail page can show evidence, while
|
||||
# only the later (regressing) reading is eligible for correction.
|
||||
related_refs = [earlier_ref, booking_ref, later_ref]
|
||||
evidence_extra = {
|
||||
"source_type": signals[0].get("source_type", "return"),
|
||||
"source_types": [signals[0].get("source_type", "return")],
|
||||
"correctable_booking_refs": [booking_ref],
|
||||
}
|
||||
severity_due_delta = {
|
||||
"high": timedelta(hours=4),
|
||||
"medium": timedelta(days=1),
|
||||
@@ -372,7 +400,8 @@ def load_seed(db: Session) -> SeedResult:
|
||||
"summary": row["evidence"],
|
||||
"entity_ref": row["entity_ref"],
|
||||
"related_refs": related_refs,
|
||||
"signals": _seed_signals(row["public_ref"], row["entity_ref"], related_refs),
|
||||
"signals": signals,
|
||||
**evidence_extra,
|
||||
},
|
||||
"proposed_action_json": {},
|
||||
"detected_at": now,
|
||||
@@ -449,12 +478,23 @@ def load_seed(db: Session) -> SeedResult:
|
||||
return SeedResult(counts=counts, anchor_date=today, seeded_at=seeded_at)
|
||||
|
||||
|
||||
def reset_and_seed(db: Session, *, preserve_integration_telemetry: bool = False) -> SeedResult:
|
||||
def reset_and_seed(
|
||||
db: Session,
|
||||
*,
|
||||
preserve_integration_telemetry: bool = False,
|
||||
commit: bool = True,
|
||||
) -> SeedResult:
|
||||
from app.services.data_quality import run_scan
|
||||
|
||||
if not db.info.get("mobilityops_demo_reset_exclusive"):
|
||||
begin_exclusive_demo_reset(db)
|
||||
clear_all(db, preserve_integration_telemetry=preserve_integration_telemetry)
|
||||
result = load_seed(db)
|
||||
db.commit()
|
||||
scan = run_scan(db)
|
||||
scan = run_scan(db, commit=False)
|
||||
result.counts["data_quality_issues"] += sum(scan.created.values())
|
||||
if commit:
|
||||
db.commit()
|
||||
end_exclusive_demo_reset(db)
|
||||
else:
|
||||
db.flush()
|
||||
return result
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -11,10 +11,22 @@ from app.core.errors import AppError
|
||||
from app.models.booking import Booking
|
||||
from app.models.customer import Customer
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.inspection import Inspection
|
||||
from app.models.maintenance import MaintenanceRecord
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.schemas import CurrentUser, ResolveOdometerRegressionRequest
|
||||
from app.schemas import CurrentUser
|
||||
from app.services.audit import record_audit_event
|
||||
from app.services.data_quality_common import has_open_issue as _has_open_issue
|
||||
from app.services.data_quality_common import issue_due_at
|
||||
from app.services.data_quality_common import load_open_issue as _load_open_issue
|
||||
from app.services.data_quality_common import new_scan_ref as _new_scan_ref
|
||||
from app.services.data_quality_duplicate_scan import scan_duplicate_customers
|
||||
from app.services.data_quality_odometer import (
|
||||
open_odometer_regression_issue as open_odometer_regression_issue,
|
||||
)
|
||||
from app.services.data_quality_odometer import (
|
||||
resolve_odometer_regression as resolve_odometer_regression,
|
||||
)
|
||||
from app.services.vehicle_status import (
|
||||
RECOMMENDATION_CODE_NO_CONFLICT,
|
||||
VehicleStatusRecommendation,
|
||||
@@ -28,15 +40,6 @@ REQUIRED_VEHICLE_FIELDS = ("registration_number", "make", "model", "location")
|
||||
DATA_QUALITY_SCAN_LOCK_ID = 6_138_493_717_091_029_491
|
||||
|
||||
|
||||
def issue_due_at(detected_at: datetime, severity: str) -> datetime:
|
||||
"""Return the local operational SLA deadline for a newly detected issue."""
|
||||
return detected_at + {
|
||||
"high": timedelta(hours=4),
|
||||
"medium": timedelta(days=1),
|
||||
"low": timedelta(days=3),
|
||||
}.get(severity, timedelta(days=1))
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
created: dict[str, int] = field(default_factory=dict)
|
||||
@@ -45,25 +48,6 @@ class ScanResult:
|
||||
self.created[rule_type] = self.created.get(rule_type, 0) + 1
|
||||
|
||||
|
||||
def _has_open_issue(db: Session, rule_type: str, entity_type: str, entity_id: uuid.UUID) -> bool:
|
||||
return (
|
||||
db.scalar(
|
||||
select(DataQualityIssue.id).where(
|
||||
DataQualityIssue.rule_type == rule_type,
|
||||
DataQualityIssue.entity_type == entity_type,
|
||||
DataQualityIssue.entity_id == entity_id,
|
||||
DataQualityIssue.status == "open",
|
||||
)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def _new_scan_ref(prefix: str) -> str:
|
||||
"""Generate a stable human-readable prefix with a concurrent-safe suffix."""
|
||||
return f"{prefix}-{uuid.uuid4().hex[:10].upper()}"
|
||||
|
||||
|
||||
def _open_issue(
|
||||
db: Session,
|
||||
scan: ScanResult,
|
||||
@@ -76,6 +60,7 @@ def _open_issue(
|
||||
entity_ref: str,
|
||||
related_refs: list[str],
|
||||
signals: list[dict] | None = None,
|
||||
evidence_extra: dict | None = None,
|
||||
) -> None:
|
||||
if _has_open_issue(db, rule_type, entity_type, entity_id):
|
||||
return
|
||||
@@ -103,6 +88,7 @@ def _open_issue(
|
||||
"related_refs": related_refs,
|
||||
"signals": signals or [],
|
||||
}
|
||||
evidence.update(evidence_extra or {})
|
||||
if previous is not None:
|
||||
evidence["reopened_from"] = previous.public_ref
|
||||
evidence["previous_decision"] = previous.status
|
||||
@@ -235,57 +221,110 @@ def _scan_vehicle_status_conflicts(db: Session, scan: ScanResult) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _scan_odometer_regressions(db: Session, scan: ScanResult) -> None:
|
||||
# The seed dataset's vehicle.odometer_km is generated independently of booking
|
||||
# history, so comparing every historical booking against it produces near-universal
|
||||
# false positives. Instead check the booking sequence's own internal consistency:
|
||||
# each vehicle's completed bookings should show a non-decreasing odometer reading.
|
||||
vehicles = {v.id: v for v in db.scalars(select(Vehicle)).all()}
|
||||
bookings_by_vehicle: dict[uuid.UUID, list[Booking]] = {}
|
||||
for booking in db.scalars(
|
||||
def _scan_odometer_regressions(
|
||||
db: Session,
|
||||
scan: ScanResult,
|
||||
*,
|
||||
actor_label: str | None,
|
||||
actor_type: str,
|
||||
) -> None:
|
||||
# Compare the chronological history with itself rather than every historical reading
|
||||
# to today's canonical value. That detects imported checkout/return/maintenance
|
||||
# regressions without flagging every legitimate older reading.
|
||||
# Runtime checkout/return/maintenance mutations take the vehicle lock before they
|
||||
# inspect or append DQ-03 evidence. Taking the same lock here makes scan-vs-command
|
||||
# check/merge atomic and prevents a partial-unique race for the open issue.
|
||||
vehicles = {
|
||||
v.id: v for v in db.scalars(select(Vehicle).order_by(Vehicle.id).with_for_update()).all()
|
||||
}
|
||||
readings_by_vehicle: dict[uuid.UUID, list[tuple[datetime, str, int, str, str | None]]] = {}
|
||||
inspections = db.scalars(select(Inspection)).all()
|
||||
return_inspection_booking_ids = {
|
||||
inspection.booking_id for inspection in inspections if inspection.type == "return"
|
||||
}
|
||||
returned_bookings = db.scalars(
|
||||
select(Booking).where(Booking.status == "returned", Booking.end_odometer_km.is_not(None))
|
||||
).all():
|
||||
bookings_by_vehicle.setdefault(booking.vehicle_id, []).append(booking)
|
||||
).all()
|
||||
booking_ref_by_id = {booking.id: booking.public_ref for booking in returned_bookings}
|
||||
for booking in returned_bookings:
|
||||
# A real return inspection owns the actual reading timestamp. Using the booking's
|
||||
# planned ends_at as a duplicate second reading can make an early return appear to
|
||||
# go backwards after a newer real inspection. Keep booking data only as the legacy
|
||||
# import fallback when no return inspection exists.
|
||||
if booking.id in return_inspection_booking_ids:
|
||||
continue
|
||||
assert booking.end_odometer_km is not None
|
||||
readings_by_vehicle.setdefault(booking.vehicle_id, []).append(
|
||||
(
|
||||
booking.ends_at,
|
||||
booking.public_ref,
|
||||
booking.end_odometer_km,
|
||||
"booking",
|
||||
booking.public_ref,
|
||||
)
|
||||
)
|
||||
for inspection in inspections:
|
||||
readings_by_vehicle.setdefault(inspection.vehicle_id, []).append(
|
||||
(
|
||||
inspection.completed_at,
|
||||
inspection.public_ref,
|
||||
inspection.odometer_km,
|
||||
inspection.type,
|
||||
(
|
||||
booking_ref_by_id.get(inspection.booking_id)
|
||||
if inspection.type == "return"
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
for record in db.scalars(select(MaintenanceRecord)).all():
|
||||
readings_by_vehicle.setdefault(record.vehicle_id, []).append(
|
||||
(record.occurred_at, record.public_ref, record.odometer_km, "maintenance", None)
|
||||
)
|
||||
|
||||
for vehicle_id, bookings in bookings_by_vehicle.items():
|
||||
bookings.sort(key=lambda b: b.ends_at)
|
||||
for earlier, later in zip(bookings, bookings[1:], strict=False):
|
||||
# The query above filters end_odometer_km IS NOT NULL, so both are ints here.
|
||||
assert earlier.end_odometer_km is not None
|
||||
assert later.end_odometer_km is not None
|
||||
if later.end_odometer_km < earlier.end_odometer_km:
|
||||
for vehicle_id, readings in readings_by_vehicle.items():
|
||||
readings.sort(key=lambda reading: (reading[0], reading[1]))
|
||||
highest = readings[0] if readings else None
|
||||
for later in readings[1:]:
|
||||
assert highest is not None
|
||||
if later[2] < highest[2]:
|
||||
vehicle = vehicles[vehicle_id]
|
||||
_open_issue(
|
||||
had_open_issue = _has_open_issue(
|
||||
db,
|
||||
scan,
|
||||
rule_type="odometer_regression",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle_id,
|
||||
severity="medium",
|
||||
summary=(
|
||||
f"Booking {later.public_ref} recorded {later.end_odometer_km} km, "
|
||||
f"below the {earlier.end_odometer_km} km recorded by earlier "
|
||||
f"booking {earlier.public_ref}."
|
||||
),
|
||||
entity_ref=vehicle.public_ref,
|
||||
related_refs=[earlier.public_ref, later.public_ref],
|
||||
signals=[
|
||||
{
|
||||
"code": "odometer.regression",
|
||||
"params": {
|
||||
"later_ref": later.public_ref,
|
||||
"later_km": later.end_odometer_km,
|
||||
"earlier_ref": earlier.public_ref,
|
||||
"earlier_km": earlier.end_odometer_km,
|
||||
},
|
||||
}
|
||||
],
|
||||
"odometer_regression",
|
||||
"vehicle",
|
||||
vehicle_id,
|
||||
)
|
||||
break
|
||||
issue = open_odometer_regression_issue(
|
||||
db,
|
||||
vehicle=vehicle,
|
||||
reading_ref=later[1],
|
||||
reading_km=later[2],
|
||||
canonical_km=highest[2],
|
||||
canonical_ref=highest[1],
|
||||
source_type=later[3],
|
||||
related_refs=list(
|
||||
dict.fromkeys(
|
||||
[highest[1], later[1], *([later[4]] if later[4] is not None else [])]
|
||||
)
|
||||
),
|
||||
correctable_booking_refs=[later[4]] if later[4] is not None else [],
|
||||
public_ref=_new_scan_ref("DQ-SCAN"),
|
||||
actor_label=actor_label,
|
||||
actor_type=actor_type,
|
||||
)
|
||||
if issue is not None and not had_open_issue:
|
||||
scan.bump("odometer_regression")
|
||||
if later[2] > highest[2]:
|
||||
highest = later
|
||||
|
||||
|
||||
def run_scan(
|
||||
db: Session, *, actor_label: str | None = None, actor_type: str = "user"
|
||||
db: Session,
|
||||
*,
|
||||
actor_label: str | None = None,
|
||||
actor_type: str = "user",
|
||||
commit: bool = True,
|
||||
) -> ScanResult:
|
||||
# The check-then-insert work below spans several rules. Serialise whole scans at the
|
||||
# database boundary so API and n8n triggers cannot both observe an empty condition.
|
||||
@@ -293,7 +332,12 @@ def run_scan(
|
||||
scan = ScanResult()
|
||||
scan_duplicate_customers(db, scan, _open_issue)
|
||||
_scan_missing_required_fields(db, scan)
|
||||
_scan_odometer_regressions(db, scan)
|
||||
_scan_odometer_regressions(
|
||||
db,
|
||||
scan,
|
||||
actor_label=actor_label,
|
||||
actor_type=actor_type,
|
||||
)
|
||||
_scan_booking_overlaps(db, scan)
|
||||
_scan_vehicle_status_conflicts(db, scan)
|
||||
if actor_label is not None:
|
||||
@@ -305,28 +349,13 @@ def run_scan(
|
||||
entity_type="system",
|
||||
metadata={"created": scan.created},
|
||||
)
|
||||
db.commit()
|
||||
if commit:
|
||||
db.commit()
|
||||
else:
|
||||
db.flush()
|
||||
return scan
|
||||
|
||||
|
||||
def _load_open_issue(
|
||||
db: Session, public_ref: str, *, lock: bool = True
|
||||
) -> DataQualityIssue:
|
||||
statement = select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref)
|
||||
if lock:
|
||||
statement = statement.with_for_update()
|
||||
issue = db.scalar(statement)
|
||||
if issue is None:
|
||||
raise AppError("ISSUE_NOT_FOUND", "Data quality issue not found.", status_code=404)
|
||||
if issue.status != "open":
|
||||
raise AppError(
|
||||
"ISSUE_NOT_OPEN",
|
||||
f"Issue is '{issue.status}', not 'open'.",
|
||||
status_code=409,
|
||||
)
|
||||
return issue
|
||||
|
||||
|
||||
def defer_issue(db: Session, public_ref: str, actor: CurrentUser) -> DataQualityIssue:
|
||||
issue = _load_open_issue(db, public_ref)
|
||||
issue.status = "deferred"
|
||||
@@ -456,115 +485,6 @@ def provide_missing_fields(
|
||||
return issue
|
||||
|
||||
|
||||
def resolve_odometer_regression(
|
||||
db: Session, public_ref: str, body: ResolveOdometerRegressionRequest, actor: CurrentUser
|
||||
) -> DataQualityIssue:
|
||||
issue = _load_open_issue(db, public_ref)
|
||||
if issue.rule_type != "odometer_regression":
|
||||
raise AppError(
|
||||
"NOT_AN_ODOMETER_ISSUE",
|
||||
"This issue is not an odometer_regression issue.",
|
||||
status_code=409,
|
||||
)
|
||||
# Lock order is booking -> vehicle everywhere (checkout, return, reschedule); taking
|
||||
# the vehicle lock first here would be a deadlock waiting to happen under concurrency.
|
||||
booking: Booking | None = None
|
||||
if body.decision != "retain_canonical":
|
||||
related_refs = issue.evidence_json.get("related_refs", [])
|
||||
if body.booking_ref not in related_refs:
|
||||
raise AppError(
|
||||
"INVALID_BOOKING_REFERENCE",
|
||||
"booking_ref must be one of this issue's related bookings.",
|
||||
status_code=422,
|
||||
)
|
||||
if body.corrected_odometer_km is None:
|
||||
raise AppError(
|
||||
"CORRECTED_VALUE_REQUIRED",
|
||||
"corrected_odometer_km is required when correcting a reading.",
|
||||
status_code=422,
|
||||
)
|
||||
booking = db.scalar(
|
||||
select(Booking).where(Booking.public_ref == body.booking_ref).with_for_update()
|
||||
)
|
||||
if booking is None:
|
||||
raise AppError(
|
||||
"BOOKING_NOT_FOUND", "The booking to correct was not found.", status_code=404
|
||||
)
|
||||
vehicle = db.scalar(select(Vehicle).where(Vehicle.id == issue.entity_id).with_for_update())
|
||||
if vehicle is None:
|
||||
raise AppError(
|
||||
"VEHICLE_NOT_FOUND", "The vehicle for this issue was not found.", status_code=404
|
||||
)
|
||||
|
||||
correlation_id = uuid.uuid4()
|
||||
|
||||
if body.decision == "retain_canonical":
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_odometer_retained",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
correlation_id=correlation_id,
|
||||
metadata={"issue_ref": issue.public_ref, "canonical_odometer_km": vehicle.odometer_km},
|
||||
)
|
||||
else:
|
||||
assert booking is not None and body.corrected_odometer_km is not None
|
||||
# Never silently lower the canonical odometer: a correction must be at or above
|
||||
# the current canonical value, otherwise it would just create a new regression.
|
||||
if body.corrected_odometer_km < vehicle.odometer_km:
|
||||
raise AppError(
|
||||
"CORRECTION_BELOW_CANONICAL",
|
||||
(
|
||||
f"Corrected value {body.corrected_odometer_km} km is still below the "
|
||||
f"canonical {vehicle.odometer_km} km; it would not resolve the regression."
|
||||
),
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
before = {
|
||||
"booking_end_odometer_km": booking.end_odometer_km,
|
||||
"vehicle_odometer_km": vehicle.odometer_km,
|
||||
}
|
||||
booking.end_odometer_km = body.corrected_odometer_km
|
||||
vehicle.odometer_km = body.corrected_odometer_km
|
||||
vehicle.version += 1
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_odometer_corrected",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
correlation_id=correlation_id,
|
||||
before=before,
|
||||
after={
|
||||
"booking_end_odometer_km": booking.end_odometer_km,
|
||||
"vehicle_odometer_km": vehicle.odometer_km,
|
||||
},
|
||||
metadata={"issue_ref": issue.public_ref, "booking_ref": booking.public_ref},
|
||||
)
|
||||
|
||||
issue.status = "resolved"
|
||||
issue.resolved_at = datetime.now(UTC)
|
||||
issue.resolved_by = actor.display_name
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_issue_resolved",
|
||||
entity_type="data_quality_issue",
|
||||
entity_id=issue.id,
|
||||
correlation_id=correlation_id,
|
||||
before={"status": "open"},
|
||||
after={"status": "resolved"},
|
||||
metadata={"decision": body.decision, "note": body.note},
|
||||
)
|
||||
db.commit()
|
||||
return issue
|
||||
|
||||
|
||||
def resolve_booking_overlap(
|
||||
db: Session, public_ref: str, booking_ref: str, note: str | None, actor: CurrentUser
|
||||
) -> DataQualityIssue:
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
|
||||
|
||||
def issue_due_at(detected_at: datetime, severity: str) -> datetime:
|
||||
"""Return the local operational SLA deadline for a newly detected issue."""
|
||||
return detected_at + {
|
||||
"high": timedelta(hours=4),
|
||||
"medium": timedelta(days=1),
|
||||
"low": timedelta(days=3),
|
||||
}.get(severity, timedelta(days=1))
|
||||
|
||||
|
||||
def has_open_issue(
|
||||
db: Session,
|
||||
rule_type: str,
|
||||
entity_type: str,
|
||||
entity_id: uuid.UUID,
|
||||
) -> bool:
|
||||
return (
|
||||
db.scalar(
|
||||
select(DataQualityIssue.id).where(
|
||||
DataQualityIssue.rule_type == rule_type,
|
||||
DataQualityIssue.entity_type == entity_type,
|
||||
DataQualityIssue.entity_id == entity_id,
|
||||
DataQualityIssue.status == "open",
|
||||
)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def new_scan_ref(prefix: str) -> str:
|
||||
"""Generate a stable human-readable prefix with a concurrent-safe suffix."""
|
||||
return f"{prefix}-{uuid.uuid4().hex[:10].upper()}"
|
||||
|
||||
|
||||
def load_open_issue(db: Session, public_ref: str, *, lock: bool = True) -> DataQualityIssue:
|
||||
statement = select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref)
|
||||
if lock:
|
||||
statement = statement.with_for_update().execution_options(populate_existing=True)
|
||||
issue = db.scalar(statement)
|
||||
if issue is None:
|
||||
raise AppError("ISSUE_NOT_FOUND", "Data quality issue not found.", status_code=404)
|
||||
if issue.status != "open":
|
||||
raise AppError(
|
||||
"ISSUE_NOT_OPEN",
|
||||
f"Issue is '{issue.status}', not 'open'.",
|
||||
status_code=409,
|
||||
)
|
||||
return issue
|
||||
@@ -90,9 +90,7 @@ def scan_duplicate_customers[ScanType: ScanAccumulator](
|
||||
ratio = SequenceMatcher(None, name_a, name_b).ratio()
|
||||
if ratio >= 0.5:
|
||||
score += round(ratio * 30)
|
||||
signals.append(
|
||||
{"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}}
|
||||
)
|
||||
signals.append({"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}})
|
||||
summary_parts.append("similar name")
|
||||
|
||||
if score >= DUPLICATE_THRESHOLD:
|
||||
|
||||
@@ -0,0 +1,507 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models.booking import Booking
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.inspection import Inspection
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.schemas import CurrentUser, ResolveOdometerRegressionRequest
|
||||
from app.services.audit import record_audit_event
|
||||
from app.services.data_quality_common import issue_due_at, load_open_issue, new_scan_ref
|
||||
|
||||
|
||||
def _odometer_fingerprint(
|
||||
*,
|
||||
source_type: str,
|
||||
later_ref: str,
|
||||
later_km: int,
|
||||
) -> dict[str, str | int]:
|
||||
"""Stable identity for one reviewed regression, independent of issue IDs."""
|
||||
return {
|
||||
"source_type": source_type,
|
||||
"later_ref": later_ref,
|
||||
"later_km": later_km,
|
||||
}
|
||||
|
||||
|
||||
def _was_odometer_regression_retained(
|
||||
db: Session,
|
||||
*,
|
||||
vehicle_id: uuid.UUID,
|
||||
fingerprint: dict[str, str | int],
|
||||
) -> bool:
|
||||
reviewed = db.scalars(
|
||||
select(DataQualityIssue).where(
|
||||
DataQualityIssue.rule_type == "odometer_regression",
|
||||
DataQualityIssue.entity_type == "vehicle",
|
||||
DataQualityIssue.entity_id == vehicle_id,
|
||||
DataQualityIssue.status == "resolved",
|
||||
)
|
||||
).all()
|
||||
return any(
|
||||
issue.evidence_json.get("resolution_decision") == "retain_canonical"
|
||||
and fingerprint in issue.evidence_json.get("retained_odometer_fingerprints", [])
|
||||
for issue in reviewed
|
||||
)
|
||||
|
||||
|
||||
def open_odometer_regression_issue(
|
||||
db: Session,
|
||||
*,
|
||||
vehicle: Vehicle,
|
||||
reading_ref: str,
|
||||
reading_km: int,
|
||||
canonical_km: int,
|
||||
source_type: str,
|
||||
related_refs: list[str],
|
||||
correctable_booking_refs: list[str] | None = None,
|
||||
canonical_ref: str | None = None,
|
||||
detected_at: datetime | None = None,
|
||||
public_ref: str | None = None,
|
||||
actor_label: str | None = None,
|
||||
actor_type: str = "user",
|
||||
correlation_id: uuid.UUID | None = None,
|
||||
) -> DataQualityIssue | None:
|
||||
"""Open one explainable DQ-03 issue while the caller holds the vehicle lock."""
|
||||
if reading_km >= canonical_km:
|
||||
return None
|
||||
earlier_ref = canonical_ref or vehicle.public_ref
|
||||
fingerprint = _odometer_fingerprint(
|
||||
source_type=source_type,
|
||||
later_ref=reading_ref,
|
||||
later_km=reading_km,
|
||||
)
|
||||
# A manager's explicit "retain canonical" decision acknowledges this exact source
|
||||
# fact. Reopening it on every scheduled scan would create churn; only changed or new
|
||||
# evidence (and therefore a different fingerprint) is actionable again.
|
||||
if _was_odometer_regression_retained(
|
||||
db,
|
||||
vehicle_id=vehicle.id,
|
||||
fingerprint=fingerprint,
|
||||
):
|
||||
return None
|
||||
existing = db.scalar(
|
||||
select(DataQualityIssue)
|
||||
.where(
|
||||
DataQualityIssue.rule_type == "odometer_regression",
|
||||
DataQualityIssue.entity_type == "vehicle",
|
||||
DataQualityIssue.entity_id == vehicle.id,
|
||||
DataQualityIssue.status == "open",
|
||||
)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
if existing is not None:
|
||||
before_related_refs = list(existing.evidence_json.get("related_refs", []))
|
||||
before_correctable_refs = list(existing.evidence_json.get("correctable_booking_refs", []))
|
||||
new_signal = {
|
||||
"code": "odometer.regression",
|
||||
"source_type": source_type,
|
||||
"params": {
|
||||
"later_ref": reading_ref,
|
||||
"later_km": reading_km,
|
||||
"earlier_ref": earlier_ref,
|
||||
"earlier_km": canonical_km,
|
||||
},
|
||||
}
|
||||
existing_signals = list(existing.evidence_json.get("signals", []))
|
||||
signal_was_new = not any(
|
||||
isinstance(signal, dict)
|
||||
and isinstance(signal.get("params"), dict)
|
||||
and _odometer_fingerprint(
|
||||
source_type=str(
|
||||
signal.get("source_type", existing.evidence_json.get("source_type"))
|
||||
),
|
||||
later_ref=str(signal["params"].get("later_ref")),
|
||||
later_km=signal["params"].get("later_km"),
|
||||
)
|
||||
== fingerprint
|
||||
for signal in existing_signals
|
||||
if isinstance(signal, dict)
|
||||
and isinstance(signal.get("params"), dict)
|
||||
and isinstance(signal["params"].get("later_km"), int)
|
||||
)
|
||||
if signal_was_new:
|
||||
existing_signals.append(new_signal)
|
||||
merged_related_refs = list(
|
||||
dict.fromkeys([*before_related_refs, *(related_refs if signal_was_new else [])])
|
||||
)
|
||||
merged_correctable_refs = list(
|
||||
dict.fromkeys([*before_correctable_refs, *(correctable_booking_refs or [])])
|
||||
)
|
||||
if (
|
||||
not signal_was_new
|
||||
and merged_related_refs == before_related_refs
|
||||
and merged_correctable_refs == before_correctable_refs
|
||||
):
|
||||
return existing
|
||||
source_types = list(existing.evidence_json.get("source_types", []))
|
||||
previous_source_type = existing.evidence_json.get("source_type")
|
||||
if (
|
||||
isinstance(previous_source_type, str)
|
||||
and previous_source_type != "multiple"
|
||||
and previous_source_type not in source_types
|
||||
):
|
||||
source_types.append(previous_source_type)
|
||||
if source_type not in source_types:
|
||||
source_types.append(source_type)
|
||||
existing.evidence_json = {
|
||||
**existing.evidence_json,
|
||||
"summary": (
|
||||
f"{source_type.title()} {reading_ref} recorded {reading_km} km, below "
|
||||
f"the canonical {canonical_km} km for {vehicle.public_ref}."
|
||||
),
|
||||
"related_refs": merged_related_refs,
|
||||
"correctable_booking_refs": merged_correctable_refs,
|
||||
"source_type": source_type if len(source_types) == 1 else "multiple",
|
||||
"source_types": source_types,
|
||||
"signals": existing_signals,
|
||||
}
|
||||
if actor_label is not None and (
|
||||
merged_related_refs != before_related_refs
|
||||
or merged_correctable_refs != before_correctable_refs
|
||||
or signal_was_new
|
||||
):
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type=actor_type,
|
||||
actor_label=actor_label,
|
||||
action="data_quality_issue_evidence_updated",
|
||||
entity_type="data_quality_issue",
|
||||
entity_id=existing.id,
|
||||
correlation_id=correlation_id,
|
||||
before={
|
||||
"related_refs": before_related_refs,
|
||||
"correctable_booking_refs": before_correctable_refs,
|
||||
},
|
||||
after={
|
||||
"related_refs": merged_related_refs,
|
||||
"correctable_booking_refs": merged_correctable_refs,
|
||||
},
|
||||
metadata={"vehicle_ref": vehicle.public_ref, "reading_ref": reading_ref},
|
||||
)
|
||||
return existing
|
||||
now = detected_at or datetime.now(UTC)
|
||||
previous = db.scalar(
|
||||
select(DataQualityIssue)
|
||||
.where(
|
||||
DataQualityIssue.rule_type == "odometer_regression",
|
||||
DataQualityIssue.entity_type == "vehicle",
|
||||
DataQualityIssue.entity_id == vehicle.id,
|
||||
DataQualityIssue.status != "open",
|
||||
)
|
||||
.order_by(DataQualityIssue.detected_at.desc())
|
||||
)
|
||||
evidence = {
|
||||
"summary": (
|
||||
f"{source_type.title()} {reading_ref} recorded {reading_km} km, below "
|
||||
f"the canonical {canonical_km} km for {vehicle.public_ref}."
|
||||
),
|
||||
"entity_ref": vehicle.public_ref,
|
||||
"related_refs": related_refs,
|
||||
"correctable_booking_refs": correctable_booking_refs or [],
|
||||
"source_type": source_type,
|
||||
"source_types": [source_type],
|
||||
"signals": [
|
||||
{
|
||||
"code": "odometer.regression",
|
||||
"source_type": source_type,
|
||||
"params": {
|
||||
"later_ref": reading_ref,
|
||||
"later_km": reading_km,
|
||||
"earlier_ref": earlier_ref,
|
||||
"earlier_km": canonical_km,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
if previous is not None:
|
||||
evidence["reopened_from"] = previous.public_ref
|
||||
evidence["previous_decision"] = previous.status
|
||||
issue = DataQualityIssue(
|
||||
public_ref=public_ref or new_scan_ref("DQ-ODO"),
|
||||
rule_type="odometer_regression",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
severity="medium",
|
||||
status="open",
|
||||
evidence_json=evidence,
|
||||
proposed_action_json={},
|
||||
detected_at=now,
|
||||
due_at=issue_due_at(now, "medium"),
|
||||
)
|
||||
db.add(issue)
|
||||
db.flush()
|
||||
if actor_label is not None:
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type=actor_type,
|
||||
actor_label=actor_label,
|
||||
action="data_quality_issue_created",
|
||||
entity_type="data_quality_issue",
|
||||
entity_id=issue.id,
|
||||
correlation_id=correlation_id,
|
||||
after={"status": "open", "rule_type": "odometer_regression"},
|
||||
metadata={"vehicle_ref": vehicle.public_ref, "reading_ref": reading_ref},
|
||||
)
|
||||
return issue
|
||||
|
||||
|
||||
def resolve_odometer_regression(
|
||||
db: Session, public_ref: str, body: ResolveOdometerRegressionRequest, actor: CurrentUser
|
||||
) -> DataQualityIssue:
|
||||
# Read the routing data without a row lock first. The canonical mutation order is
|
||||
# booking -> vehicle -> issue -> inspection everywhere, matching checkout/return.
|
||||
# Locking the issue before the booking creates a resolver-vs-return deadlock.
|
||||
issue_snapshot = load_open_issue(db, public_ref, lock=False)
|
||||
if issue_snapshot.rule_type != "odometer_regression":
|
||||
raise AppError(
|
||||
"NOT_AN_ODOMETER_ISSUE",
|
||||
"This issue is not an odometer_regression issue.",
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
def correctable_refs(issue: DataQualityIssue) -> list[str]:
|
||||
if "correctable_booking_refs" in issue.evidence_json:
|
||||
refs = issue.evidence_json.get("correctable_booking_refs", [])
|
||||
else:
|
||||
# Backward compatibility for issues created before source-aware evidence.
|
||||
refs = [
|
||||
ref
|
||||
for ref in issue.evidence_json.get("related_refs", [])
|
||||
if isinstance(ref, str) and ref.startswith("BK-")
|
||||
]
|
||||
return [ref for ref in refs if isinstance(ref, str)]
|
||||
|
||||
booking: Booking | None = None
|
||||
if body.decision != "retain_canonical":
|
||||
if body.booking_ref not in correctable_refs(issue_snapshot):
|
||||
raise AppError(
|
||||
"INVALID_BOOKING_REFERENCE",
|
||||
"booking_ref must be one of this issue's related bookings.",
|
||||
status_code=422,
|
||||
)
|
||||
if body.corrected_odometer_km is None:
|
||||
raise AppError(
|
||||
"CORRECTED_VALUE_REQUIRED",
|
||||
"corrected_odometer_km is required when correcting a reading.",
|
||||
status_code=422,
|
||||
)
|
||||
booking = db.scalar(
|
||||
select(Booking).where(Booking.public_ref == body.booking_ref).with_for_update()
|
||||
)
|
||||
if booking is None:
|
||||
raise AppError(
|
||||
"BOOKING_NOT_FOUND", "The booking to correct was not found.", status_code=404
|
||||
)
|
||||
vehicle = db.scalar(
|
||||
select(Vehicle).where(Vehicle.id == issue_snapshot.entity_id).with_for_update()
|
||||
)
|
||||
if vehicle is None:
|
||||
raise AppError(
|
||||
"VEHICLE_NOT_FOUND", "The vehicle for this issue was not found.", status_code=404
|
||||
)
|
||||
|
||||
# A return or scan may have appended evidence while we waited for the domain locks.
|
||||
# Lock and refresh the issue only now, then revalidate every decision against that
|
||||
# current evidence instead of resolving a stale snapshot.
|
||||
issue = load_open_issue(db, public_ref)
|
||||
if issue.rule_type != "odometer_regression" or issue.entity_id != vehicle.id:
|
||||
raise AppError(
|
||||
"ISSUE_CHANGED",
|
||||
"The issue changed while the correction was being prepared. Review it again.",
|
||||
status_code=409,
|
||||
)
|
||||
if booking is not None:
|
||||
if booking.vehicle_id != vehicle.id or booking.public_ref not in correctable_refs(issue):
|
||||
raise AppError(
|
||||
"INVALID_BOOKING_REFERENCE",
|
||||
"booking_ref must be one of this issue's related bookings.",
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
correlation_id = uuid.uuid4()
|
||||
|
||||
if body.decision == "retain_canonical":
|
||||
retained_fingerprints: list[dict[str, str | int]] = []
|
||||
fallback_source_type = str(issue.evidence_json.get("source_type", "unknown"))
|
||||
for signal in issue.evidence_json.get("signals", []):
|
||||
if not isinstance(signal, dict) or signal.get("code") != "odometer.regression":
|
||||
continue
|
||||
params = signal.get("params")
|
||||
if not isinstance(params, dict):
|
||||
continue
|
||||
source_type = signal.get("source_type", fallback_source_type)
|
||||
later_ref = params.get("later_ref")
|
||||
later_km = params.get("later_km")
|
||||
earlier_ref = params.get("earlier_ref")
|
||||
earlier_km = params.get("earlier_km")
|
||||
if (
|
||||
isinstance(source_type, str)
|
||||
and isinstance(later_ref, str)
|
||||
and isinstance(later_km, int)
|
||||
and isinstance(earlier_ref, str)
|
||||
and isinstance(earlier_km, int)
|
||||
):
|
||||
retained_fingerprints.append(
|
||||
_odometer_fingerprint(
|
||||
source_type=source_type,
|
||||
later_ref=later_ref,
|
||||
later_km=later_km,
|
||||
)
|
||||
)
|
||||
issue.evidence_json = {
|
||||
**issue.evidence_json,
|
||||
"resolution_decision": "retain_canonical",
|
||||
"retained_odometer_fingerprints": retained_fingerprints,
|
||||
}
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_odometer_retained",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
correlation_id=correlation_id,
|
||||
metadata={
|
||||
"issue_ref": issue.public_ref,
|
||||
"canonical_odometer_km": vehicle.odometer_km,
|
||||
"retained_fingerprint_count": len(retained_fingerprints),
|
||||
},
|
||||
)
|
||||
else:
|
||||
assert booking is not None and body.corrected_odometer_km is not None
|
||||
# Never silently lower the canonical odometer: a correction must be at or above
|
||||
# the current canonical value, otherwise it would just create a new regression.
|
||||
if body.corrected_odometer_km < vehicle.odometer_km:
|
||||
raise AppError(
|
||||
"CORRECTION_BELOW_CANONICAL",
|
||||
(
|
||||
f"Corrected value {body.corrected_odometer_km} km is still below the "
|
||||
f"canonical {vehicle.odometer_km} km; it would not resolve the regression."
|
||||
),
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
# A returned booking has two persisted representations of the same reading.
|
||||
# Correct every return inspection for the selected booking, including historical
|
||||
# seed rows whose old issue evidence did not yet cite the inspection explicitly.
|
||||
related_inspections = db.scalars(
|
||||
select(Inspection)
|
||||
.where(Inspection.booking_id == booking.id, Inspection.type == "return")
|
||||
.with_for_update()
|
||||
).all()
|
||||
before = {
|
||||
"booking_end_odometer_km": booking.end_odometer_km,
|
||||
"vehicle_odometer_km": vehicle.odometer_km,
|
||||
"inspection_odometer_km": {
|
||||
inspection.public_ref: inspection.odometer_km for inspection in related_inspections
|
||||
},
|
||||
}
|
||||
booking.end_odometer_km = body.corrected_odometer_km
|
||||
for inspection in related_inspections:
|
||||
inspection.odometer_km = body.corrected_odometer_km
|
||||
vehicle.odometer_km = body.corrected_odometer_km
|
||||
vehicle.version += 1
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_odometer_corrected",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
correlation_id=correlation_id,
|
||||
before=before,
|
||||
after={
|
||||
"booking_end_odometer_km": booking.end_odometer_km,
|
||||
"vehicle_odometer_km": vehicle.odometer_km,
|
||||
"inspection_odometer_km": {
|
||||
inspection.public_ref: inspection.odometer_km
|
||||
for inspection in related_inspections
|
||||
},
|
||||
},
|
||||
metadata={"issue_ref": issue.public_ref, "booking_ref": booking.public_ref},
|
||||
)
|
||||
|
||||
has_remaining_evidence = False
|
||||
if booking is not None:
|
||||
target_refs = {booking.public_ref, *(item.public_ref for item in related_inspections)}
|
||||
before_related = [
|
||||
ref for ref in issue.evidence_json.get("related_refs", []) if isinstance(ref, str)
|
||||
]
|
||||
before_correctable = correctable_refs(issue)
|
||||
before_signals = [
|
||||
signal for signal in issue.evidence_json.get("signals", []) if isinstance(signal, dict)
|
||||
]
|
||||
remaining_signals = [
|
||||
signal
|
||||
for signal in before_signals
|
||||
if not (
|
||||
isinstance(signal.get("params"), dict)
|
||||
and signal["params"].get("later_ref") in target_refs
|
||||
)
|
||||
]
|
||||
remaining_related = [ref for ref in before_related if ref not in target_refs]
|
||||
remaining_correctable = [ref for ref in before_correctable if ref != booking.public_ref]
|
||||
has_remaining_evidence = bool(remaining_signals or remaining_correctable)
|
||||
issue.evidence_json = {
|
||||
**issue.evidence_json,
|
||||
"summary": (
|
||||
"Additional odometer regression evidence remains for review."
|
||||
if has_remaining_evidence
|
||||
else issue.evidence_json.get("summary", "Odometer reading corrected.")
|
||||
),
|
||||
"related_refs": remaining_related,
|
||||
"correctable_booking_refs": remaining_correctable,
|
||||
"signals": remaining_signals,
|
||||
}
|
||||
if has_remaining_evidence:
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_issue_partially_resolved",
|
||||
entity_type="data_quality_issue",
|
||||
entity_id=issue.id,
|
||||
correlation_id=correlation_id,
|
||||
before={
|
||||
"related_refs": before_related,
|
||||
"correctable_booking_refs": before_correctable,
|
||||
"signal_count": len(before_signals),
|
||||
},
|
||||
after={
|
||||
"related_refs": remaining_related,
|
||||
"correctable_booking_refs": remaining_correctable,
|
||||
"signal_count": len(remaining_signals),
|
||||
},
|
||||
metadata={
|
||||
"decision": body.decision,
|
||||
"booking_ref": booking.public_ref,
|
||||
"note": body.note,
|
||||
},
|
||||
)
|
||||
|
||||
if not has_remaining_evidence:
|
||||
issue.status = "resolved"
|
||||
issue.resolved_at = datetime.now(UTC)
|
||||
issue.resolved_by = actor.display_name
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_issue_resolved",
|
||||
entity_type="data_quality_issue",
|
||||
entity_id=issue.id,
|
||||
correlation_id=correlation_id,
|
||||
before={"status": "open"},
|
||||
after={"status": "resolved"},
|
||||
metadata={"decision": body.decision, "note": body.note},
|
||||
)
|
||||
db.commit()
|
||||
return issue
|
||||
@@ -9,16 +9,28 @@ from app.core.config import get_settings
|
||||
from app.models.audit import AuditEvent
|
||||
from app.models.booking import Booking
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.outbox import OutboxEvent, is_demo_scenario_failure
|
||||
from app.schemas import DemoIntegrationSummaryOut, DemoManifestOut, DemoScenarioOut
|
||||
from app.services.integration_status import derive_mcp_hub_status, derive_n8n_status
|
||||
from app.services.knowledge import get_knowledge_provider
|
||||
from app.services.knowledge import KnowledgeHealth, get_knowledge_provider
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
_FAILED_DEMO_EVENT_ID = "00000000-0000-4000-8000-000000000020"
|
||||
|
||||
|
||||
def _knowledge_scenario_ready(health: KnowledgeHealth) -> bool:
|
||||
"""Require a reachable provider and independently verified indexed documents.
|
||||
|
||||
``source_document_count`` describes local Markdown files, while a sync report only
|
||||
describes an upload attempt. Neither proves that the active provider can retrieve a
|
||||
corpus, so an unavailable verification count remains honestly not ready.
|
||||
"""
|
||||
return bool(
|
||||
health.available and health.document_count is not None and health.document_count > 0
|
||||
)
|
||||
|
||||
|
||||
def _last_reset(db: Session) -> tuple[datetime | None, str | None]:
|
||||
marker = db.scalar(
|
||||
select(AuditEvent)
|
||||
@@ -49,7 +61,8 @@ def _scenarios(db: Session) -> list[DemoScenarioOut]:
|
||||
return_ready = bool(booking and booking.status == "active" and booking.end_odometer_km is None)
|
||||
duplicate_ready = bool(duplicate_issue and duplicate_issue.status == "open")
|
||||
overlap_ready = bool(overlap_issue and overlap_issue.status == "open")
|
||||
automation_ready = bool(failed_run and failed_run.delivery_status == "failed")
|
||||
automation_ready = bool(failed_run and is_demo_scenario_failure(failed_run))
|
||||
knowledge_ready = _knowledge_scenario_ready(knowledge_health)
|
||||
|
||||
return [
|
||||
DemoScenarioOut(
|
||||
@@ -119,8 +132,8 @@ def _scenarios(db: Session) -> list[DemoScenarioOut]:
|
||||
estimated_minutes=2,
|
||||
required_roles=["rental_employee", "operations_manager"],
|
||||
start_path="/knowledge",
|
||||
ready=knowledge_health.available,
|
||||
blocked_reason_code=None if knowledge_health.available else "knowledgeUnavailable",
|
||||
ready=knowledge_ready,
|
||||
blocked_reason_code=None if knowledge_ready else "knowledgeUnavailable",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -145,9 +158,11 @@ def _integrations(db: Session) -> list[DemoIntegrationSummaryOut]:
|
||||
key="ragcore",
|
||||
status_code=(
|
||||
"operational"
|
||||
if knowledge_health.provider == "ragcore" and knowledge_health.available
|
||||
if knowledge_health.provider == "ragcore"
|
||||
and _knowledge_scenario_ready(knowledge_health)
|
||||
else "demoMode"
|
||||
if knowledge_health.provider == "demo" and knowledge_health.available
|
||||
if knowledge_health.provider == "demo"
|
||||
and _knowledge_scenario_ready(knowledge_health)
|
||||
else "unavailable"
|
||||
),
|
||||
detail_code=(
|
||||
@@ -169,9 +184,7 @@ def _integrations(db: Session) -> list[DemoIntegrationSummaryOut]:
|
||||
# "operational" -- same evidence rule the integration status page uses.
|
||||
status_code="operational" if mcp_hub.state == "operational" else "notConnected",
|
||||
detail_code=(
|
||||
"mcpDetailOperational"
|
||||
if mcp_hub.state == "operational"
|
||||
else "mcpDetailPrepared"
|
||||
"mcpDetailOperational" if mcp_hub.state == "operational" else "mcpDetailPrepared"
|
||||
),
|
||||
detail_params={},
|
||||
),
|
||||
|
||||
@@ -77,6 +77,14 @@ def _claim_due_events(batch_size: int = 5) -> list[uuid.UUID]:
|
||||
for row in rows:
|
||||
row.delivery_status = "delivering"
|
||||
row.next_attempt_at = lease_deadline
|
||||
# The token is stored inside the internal payload (the wire envelope below
|
||||
# explicitly selects only contract fields). It lets the outcome transaction
|
||||
# prove that this is still the same lease after network I/O. A stale worker
|
||||
# must never overwrite a later reclaim/retry or an idempotent callback.
|
||||
row.payload_json = {
|
||||
**row.payload_json,
|
||||
"_delivery_claim_token": str(uuid.uuid4()),
|
||||
}
|
||||
db.commit()
|
||||
return claimed_ids
|
||||
finally:
|
||||
@@ -109,6 +117,9 @@ def _deliver_one(event_id: uuid.UUID) -> None:
|
||||
wire_event = None
|
||||
payload_error = f"Malformed outbox payload, missing key {exc}"
|
||||
attempts = event.attempts
|
||||
claim_token = event.payload_json.get("_delivery_claim_token")
|
||||
if event.delivery_status != "delivering" or not isinstance(claim_token, str):
|
||||
return
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -130,9 +141,33 @@ def _deliver_one(event_id: uuid.UUID) -> None:
|
||||
except ValueError:
|
||||
body = None
|
||||
if isinstance(body, dict):
|
||||
success = bool(body.get("ok", True))
|
||||
error = None if success else f"n8n reported failure: {body}"
|
||||
error_code = None if success else "remoteReportedFailure"
|
||||
acknowledged = body.get("ok") is True
|
||||
response_event_id = body.get("event_id")
|
||||
event_id_matches = response_event_id == str(event_id)
|
||||
result = body.get("result")
|
||||
execution_id = result.get("execution_id") if isinstance(result, dict) else None
|
||||
execution_id_valid = isinstance(execution_id, str) and bool(execution_id.strip())
|
||||
success = acknowledged and event_id_matches and execution_id_valid
|
||||
if success:
|
||||
error = None
|
||||
error_code = None
|
||||
elif not acknowledged:
|
||||
error = (
|
||||
"n8n response did not explicitly acknowledge the event with ok=true: "
|
||||
f"{body}"
|
||||
)
|
||||
error_code = (
|
||||
"remoteReportedFailure" if body.get("ok") is False else "malformedResponse"
|
||||
)
|
||||
elif not event_id_matches:
|
||||
error = (
|
||||
"n8n acknowledged a different event ID "
|
||||
f"(expected {event_id}, received {response_event_id!r})"
|
||||
)
|
||||
error_code = "mismatchedEventId"
|
||||
else:
|
||||
error = "n8n response omitted a valid result.execution_id"
|
||||
error_code = "malformedResponse"
|
||||
else:
|
||||
# A 2xx status with a non-object (or unparsable) body means the workflow
|
||||
# itself errored before its "Respond to Webhook" node ran -- n8n's default
|
||||
@@ -152,16 +187,35 @@ def _deliver_one(event_id: uuid.UUID) -> None:
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
event = db.get(OutboxEvent, event_id)
|
||||
event = db.scalar(
|
||||
select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update()
|
||||
)
|
||||
if event is None:
|
||||
return
|
||||
if (
|
||||
event.delivery_status != "delivering"
|
||||
or event.attempts != attempts
|
||||
or event.payload_json.get("_delivery_claim_token") != claim_token
|
||||
):
|
||||
logger.info(
|
||||
"Ignoring stale delivery outcome for event %s because lease ownership changed",
|
||||
event_id,
|
||||
)
|
||||
return
|
||||
event.attempts = attempts + 1
|
||||
event.payload_json = {
|
||||
key: value
|
||||
for key, value in event.payload_json.items()
|
||||
if key != "_delivery_claim_token"
|
||||
}
|
||||
if success:
|
||||
event.delivery_status = "succeeded"
|
||||
event.last_error = None
|
||||
event.last_error_code = None
|
||||
event.next_attempt_at = None
|
||||
event.external_run_id = str((body or {}).get("event_id", event_id))
|
||||
result = (body or {}).get("result")
|
||||
execution_id = result.get("execution_id") if isinstance(result, dict) else None
|
||||
event.external_run_id = execution_id if isinstance(execution_id, str) else None
|
||||
else:
|
||||
event.last_error = (error or "delivery failed")[:2000]
|
||||
event.last_error_code = error_code or "unknownError"
|
||||
|
||||
@@ -7,7 +7,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import Row, func, select
|
||||
from sqlalchemy import Row, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
@@ -93,7 +93,10 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
|
||||
latest_failure_at = db.scalar(
|
||||
select(func.max(OutboxEvent.updated_at)).where(
|
||||
OutboxEvent.delivery_status == "failed",
|
||||
OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
|
||||
or_(
|
||||
OutboxEvent.last_error_code.is_(None),
|
||||
OutboxEvent.last_error_code != DEMO_SCENARIO_ERROR_CODE,
|
||||
),
|
||||
)
|
||||
)
|
||||
latest_demo_scenario_at = db.scalar(
|
||||
@@ -106,11 +109,17 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
|
||||
state: Literal["disabled", "unavailable", "degraded", "operational", "no_evidence"]
|
||||
if not settings.n8n_dispatch_enabled:
|
||||
state = "disabled"
|
||||
elif not settings.n8n_webhook_url:
|
||||
# Persisted history does not make a currently unconfigured dispatcher green.
|
||||
state = "unavailable"
|
||||
elif unexpected_failed > 0 and succeeded == 0:
|
||||
state = "unavailable"
|
||||
elif unexpected_failed > 0:
|
||||
state = "degraded"
|
||||
elif succeeded > 0 or pending > 0 or delivering > 0:
|
||||
# Queued/in-flight work proves only that MobilityOps has work for the dispatcher;
|
||||
# it does not prove that n8n has ever accepted a delivery. A green state requires
|
||||
# at least one persisted successful round trip.
|
||||
elif succeeded > 0:
|
||||
state = "operational"
|
||||
else:
|
||||
state = "no_evidence"
|
||||
@@ -128,8 +137,14 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
|
||||
# it finishes uploading procedures to RAGcore (app/api/routers/integrations.py::
|
||||
# procedures_sync_result), the same "the workflow's own callback is the evidence"
|
||||
# pattern the scheduled scan and error handler already use below.
|
||||
latest_procedure_sync_at = db.scalar(
|
||||
select(func.max(AuditEvent.occurred_at)).where(AuditEvent.action == "n8n_procedures_synced")
|
||||
latest_procedure_sync_row = db.execute(
|
||||
select(AuditEvent.occurred_at, AuditEvent.after_json, AuditEvent.metadata_json)
|
||||
.where(AuditEvent.action == "n8n_procedures_synced")
|
||||
.order_by(AuditEvent.occurred_at.desc())
|
||||
.limit(1)
|
||||
).first()
|
||||
latest_procedure_sync_at = (
|
||||
latest_procedure_sync_row[0] if latest_procedure_sync_row is not None else None
|
||||
)
|
||||
|
||||
# Error handler evidence: registrations posted by the "Fleet Ops — Workflow Error
|
||||
@@ -198,6 +213,24 @@ def derive_n8n_status(db: Session) -> N8nIntegrationStatus:
|
||||
seen_at = failure_signal[0]
|
||||
last_status = "failed"
|
||||
execution_id = failure_signal[1]
|
||||
if name == "Fleet Ops — RAGcore Procedure Sync" and latest_procedure_sync_row:
|
||||
sync_at, sync_result, sync_metadata = latest_procedure_sync_row
|
||||
synced = (sync_result or {}).get("synced", 0)
|
||||
failed_syncs = (sync_result or {}).get("failed", 0)
|
||||
# The result callback is authoritative for corpus delivery. A generic
|
||||
# succeeded heartbeat cannot turn a zero/partial upload green.
|
||||
sync_result_failed = (
|
||||
not isinstance(synced, int)
|
||||
or not isinstance(failed_syncs, int)
|
||||
or synced <= 0
|
||||
or failed_syncs > 0
|
||||
)
|
||||
if sync_result_failed:
|
||||
last_status = "failed"
|
||||
if seen_at is None or sync_at > seen_at:
|
||||
seen_at = sync_at
|
||||
if sync_result_failed:
|
||||
execution_id = (sync_metadata or {}).get("execution_id")
|
||||
workflow_state: Literal["no_evidence", "healthy", "stale", "failed"]
|
||||
if seen_at is None:
|
||||
workflow_state = "no_evidence"
|
||||
@@ -265,16 +298,20 @@ def derive_mcp_hub_status(db: Session) -> McpHubIntegrationStatus:
|
||||
last_client = latest_call_row[1] if latest_call_row else None
|
||||
last_tool = (latest_call_row[2] or {}).get("tool") if latest_call_row else None
|
||||
|
||||
hub_reachable = _check_hub_reachable()
|
||||
|
||||
state: Literal["not_configured", "no_evidence", "operational"]
|
||||
if not settings.mcp_hub_registration_enabled:
|
||||
state = "not_configured"
|
||||
elif total_calls > 0:
|
||||
elif total_calls > 0 and hub_reachable is not False:
|
||||
state = "operational"
|
||||
else:
|
||||
# Historical tool calls remain useful telemetry, but cannot support a current
|
||||
# operational claim when the configured Hub health endpoint is unreachable.
|
||||
# ``hub_reachable`` stays available separately so consumers can distinguish
|
||||
# this from a provider that simply has no call evidence yet.
|
||||
state = "no_evidence"
|
||||
|
||||
hub_reachable = _check_hub_reachable()
|
||||
|
||||
return McpHubIntegrationStatus(
|
||||
registration_enabled=settings.mcp_hub_registration_enabled,
|
||||
state=state,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
@@ -13,6 +16,10 @@ from app.services.knowledge import EvidenceState, GroundedAnswer, KnowledgeHealt
|
||||
from app.services.knowledge.procedures import ProcedureDocument, iter_procedure_documents
|
||||
|
||||
_GROUNDED_ANSWERABILITY = {"answerable", "partially_answerable"}
|
||||
_ANSWERABILITY_STATES = _GROUNDED_ANSWERABILITY | {
|
||||
"not_answerable",
|
||||
"conflicting_evidence",
|
||||
}
|
||||
|
||||
# Mirrors DemoKnowledgeProvider's own extractive template in spirit: a real cited
|
||||
# excerpt wrapped in a fixed sentence, never a generated summary. Used only as a
|
||||
@@ -47,6 +54,11 @@ _DOMAIN_CONCEPTS: dict[str, tuple[str, ...]] = {
|
||||
"availability": ("available", "availability", "beschikbaar", "disponible", "disponibilité"),
|
||||
"technical": ("technical", "warning", "technisch", "waarschuwing", "technique", "alerte"),
|
||||
}
|
||||
_ANSWERABLE_INTENT_CONCEPTS = frozenset(_DOMAIN_CONCEPTS) - {"vehicle", "customer"}
|
||||
|
||||
|
||||
def _normalize_evidence_text(value: str) -> str:
|
||||
return " ".join(re.findall(r"\w+", value.casefold()))
|
||||
|
||||
|
||||
def _question_concepts(question: str) -> set[str]:
|
||||
@@ -93,7 +105,9 @@ def _rank_sources_for_concepts(sources: list[SourceCard], concepts: set[str]) ->
|
||||
)
|
||||
|
||||
|
||||
def _retrieval_score(result: dict) -> float | None:
|
||||
def _retrieval_score(result: object) -> float | None:
|
||||
if not isinstance(result, dict):
|
||||
return None
|
||||
scores = result.get("scores")
|
||||
if not isinstance(scores, dict):
|
||||
return None
|
||||
@@ -124,23 +138,81 @@ class RAGcoreKnowledgeProvider:
|
||||
excerpt RAGcore's own search actually found, wrapped in the same fixed citation
|
||||
template `DemoKnowledgeProvider` uses -- never a fabricated summary.
|
||||
|
||||
Known gap, not fixable from this side: RAGcore's ingest pipeline currently tags every
|
||||
chunk's `language` payload field as `"en"` regardless of actual document language (the
|
||||
`/v1/uploads` contract has no per-file language field for a caller to set correctly).
|
||||
Filtering search/answer requests by requested UI language would therefore silently
|
||||
exclude genuinely-relevant nl-BE/fr-BE content, so this adapter deliberately does not
|
||||
filter by language -- retrieval relies on the embedding model's cross-lingual matching.
|
||||
Retrieval is scoped to the stable RAGcore ``source_id`` values owned by Fleet Ops for
|
||||
the requested UI language. Returned internal document/version UUIDs are deliberately
|
||||
not treated as Fleet Ops identifiers: every citation must instead prove the complete
|
||||
managed-source chain (source id, URI, locator, checksum and extractive local text).
|
||||
"""
|
||||
|
||||
name = "ragcore"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._settings = get_settings()
|
||||
self._managed_documents_by_source_id: dict[str, ProcedureDocument] = {}
|
||||
for document in iter_procedure_documents(Path(self._settings.knowledge_dir)):
|
||||
self._managed_documents_by_source_id[document.source_id] = document
|
||||
self._verification_cache: dict[str, tuple[float, int]] = {}
|
||||
self._verification_lock = Lock()
|
||||
self._answers_circuit_lock = Lock()
|
||||
self._answers_circuit_open_until = 0.0
|
||||
|
||||
def _managed_source(self, citation: object, language: str) -> SourceCard | None:
|
||||
if not isinstance(citation, dict):
|
||||
return None
|
||||
source_id = citation.get("source_id")
|
||||
if not isinstance(source_id, str):
|
||||
return None
|
||||
document = self._managed_documents_by_source_id.get(str(source_id))
|
||||
if document is None or document.language != language:
|
||||
return None
|
||||
if citation.get("locator") != f"{document.document_id}.md":
|
||||
return None
|
||||
if citation.get("source_uri") != f"ragcore://source/{document.source_id}":
|
||||
return None
|
||||
for field_name in ("id", "document_id", "document_version_id"):
|
||||
value = citation.get(field_name)
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
parsed = uuid.UUID(value)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return None
|
||||
if parsed.int == 0:
|
||||
return None
|
||||
if not isinstance(citation.get("title"), str):
|
||||
return None
|
||||
section = citation.get("section")
|
||||
if section is not None and not isinstance(section, str):
|
||||
return None
|
||||
excerpt = citation.get("excerpt")
|
||||
if not isinstance(excerpt, str) or not excerpt.strip():
|
||||
return None
|
||||
expected_hash = hashlib.sha256(excerpt.encode("utf-8")).hexdigest()
|
||||
if citation.get("excerpt_sha256") != expected_hash:
|
||||
return None
|
||||
# RAGcore owns chunking, but the cited excerpt must still be extractive evidence
|
||||
# from the authoritative local procedure. Token normalization tolerates Markdown
|
||||
# punctuation/whitespace while rejecting provider text that was never uploaded.
|
||||
normalized_excerpt = _normalize_evidence_text(excerpt)
|
||||
if not normalized_excerpt or normalized_excerpt not in _normalize_evidence_text(
|
||||
document.content
|
||||
):
|
||||
return None
|
||||
return SourceCard(
|
||||
document_id=document.document_id,
|
||||
title=document.title,
|
||||
version=document.version,
|
||||
section=section or "",
|
||||
excerpt=excerpt,
|
||||
)
|
||||
|
||||
def _managed_source_ids(self, language: str) -> list[str]:
|
||||
return [
|
||||
document.source_id
|
||||
for document in self._managed_documents_by_source_id.values()
|
||||
if document.language == language
|
||||
]
|
||||
|
||||
def _answers_circuit_is_open(self) -> bool:
|
||||
with self._answers_circuit_lock:
|
||||
return monotonic() < self._answers_circuit_open_until
|
||||
@@ -284,7 +356,7 @@ class RAGcoreKnowledgeProvider:
|
||||
if self._answers_circuit_is_open():
|
||||
KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "circuit_open").inc()
|
||||
else:
|
||||
answered = self._ask_via_answers(question, correlation_id)
|
||||
answered = self._ask_via_answers(question, correlation_id, language)
|
||||
if answered is not None:
|
||||
return answered
|
||||
# /v1/answers itself is unavailable (non-2xx or unreachable) -- fall back to
|
||||
@@ -295,7 +367,9 @@ class RAGcoreKnowledgeProvider:
|
||||
# generation step.
|
||||
return self._ask_via_search_fallback(question, correlation_id, language)
|
||||
|
||||
def _ask_via_answers(self, question: str, correlation_id: str) -> GroundedAnswer | None:
|
||||
def _ask_via_answers(
|
||||
self, question: str, correlation_id: str, language: str
|
||||
) -> GroundedAnswer | None:
|
||||
"""Returns None (not a GroundedAnswer) when /v1/answers itself is unavailable,
|
||||
so the caller can fall back to search -- as opposed to a real 200 response
|
||||
classifying the question as insufficiently answerable, which is a genuine,
|
||||
@@ -307,6 +381,7 @@ class RAGcoreKnowledgeProvider:
|
||||
json={
|
||||
"query": question,
|
||||
"requested_space_ids": [self._settings.ragcore_space_id],
|
||||
"filters": {"source_ids": self._managed_source_ids(language)},
|
||||
},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
@@ -320,25 +395,73 @@ class RAGcoreKnowledgeProvider:
|
||||
return None
|
||||
|
||||
try:
|
||||
citations = {c["id"]: c for c in body.get("citations", [])}
|
||||
sources = [
|
||||
SourceCard(
|
||||
document_id=str(citation["document_id"]),
|
||||
title=citation["title"],
|
||||
version=str(citation["document_version_id"]),
|
||||
section=citation.get("section") or "",
|
||||
excerpt=citation["excerpt"],
|
||||
)
|
||||
for citation in citations.values()
|
||||
if not isinstance(body, dict):
|
||||
raise TypeError("answer response must be an object")
|
||||
# Validate the provider's complete AnswerResponse contract before trusting
|
||||
# generated prose. These IDs and claim bindings are the evidence that RAGcore
|
||||
# ran its deterministic claim/citation validator; one unrelated but otherwise
|
||||
# valid citation must never make arbitrary answer text appear grounded.
|
||||
uuid.UUID(str(body["answer_id"]))
|
||||
uuid.UUID(str(body["retrieval_run_id"]))
|
||||
raw_citations = body.get("citations", [])
|
||||
if not isinstance(raw_citations, list):
|
||||
raise TypeError("citations must be a list")
|
||||
citation_ids: set[uuid.UUID] = set()
|
||||
for citation in raw_citations:
|
||||
if not isinstance(citation, dict):
|
||||
raise TypeError("citation must be an object")
|
||||
citation_ids.add(uuid.UUID(str(citation["id"])))
|
||||
raw_claims = body["claims"]
|
||||
if not isinstance(raw_claims, list):
|
||||
raise TypeError("claims must be a list")
|
||||
claims_are_bound = bool(raw_claims)
|
||||
answer_text = body.get("answer")
|
||||
for claim in raw_claims:
|
||||
if not isinstance(claim, dict):
|
||||
raise TypeError("claim must be an object")
|
||||
claim_text = claim.get("text")
|
||||
claim_citation_ids = claim.get("citation_ids")
|
||||
if (
|
||||
not isinstance(claim_text, str)
|
||||
or not claim_text.strip()
|
||||
or not isinstance(answer_text, str)
|
||||
or claim_text.strip() not in answer_text
|
||||
or not isinstance(claim_citation_ids, list)
|
||||
or not claim_citation_ids
|
||||
):
|
||||
claims_are_bound = False
|
||||
continue
|
||||
try:
|
||||
bound_ids = {uuid.UUID(str(item)) for item in claim_citation_ids}
|
||||
except (TypeError, ValueError):
|
||||
claims_are_bound = False
|
||||
continue
|
||||
if not bound_ids.issubset(citation_ids):
|
||||
claims_are_bound = False
|
||||
mapped_sources = [
|
||||
source
|
||||
for citation in raw_citations
|
||||
if (source := self._managed_source(citation, language)) is not None
|
||||
]
|
||||
sources = _deduplicate_sources(sources)
|
||||
citations_are_managed = len(mapped_sources) == len(raw_citations)
|
||||
sources = _deduplicate_sources(mapped_sources)
|
||||
answerability = body.get("answerability", "not_answerable")
|
||||
is_grounded = answerability in _GROUNDED_ANSWERABILITY and sources
|
||||
if answerability not in _ANSWERABILITY_STATES:
|
||||
raise TypeError("answerability is invalid")
|
||||
if not isinstance(answer_text, str):
|
||||
raise TypeError("answer must be a string")
|
||||
is_grounded = (
|
||||
answerability in _GROUNDED_ANSWERABILITY
|
||||
and bool(sources)
|
||||
and citations_are_managed
|
||||
and claims_are_bound
|
||||
and bool(answer_text.strip())
|
||||
)
|
||||
evidence_state: EvidenceState = "grounded" if is_grounded else "insufficient"
|
||||
KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", evidence_state).inc()
|
||||
self._close_answers_circuit()
|
||||
return GroundedAnswer(
|
||||
answer=body.get("answer", "") if evidence_state == "grounded" else "",
|
||||
answer=answer_text if evidence_state == "grounded" else "",
|
||||
evidence_state=evidence_state,
|
||||
sources=sources if evidence_state == "grounded" else [],
|
||||
provider=self.name,
|
||||
@@ -366,6 +489,7 @@ class RAGcoreKnowledgeProvider:
|
||||
json={
|
||||
"query": question,
|
||||
"requested_space_ids": [self._settings.ragcore_space_id],
|
||||
"filters": {"source_ids": self._managed_source_ids(language)},
|
||||
"max_results": 5,
|
||||
},
|
||||
)
|
||||
@@ -378,24 +502,24 @@ class RAGcoreKnowledgeProvider:
|
||||
return unavailable
|
||||
|
||||
try:
|
||||
if not isinstance(body, dict):
|
||||
raise TypeError("search response must be an object")
|
||||
results = body.get("results", [])
|
||||
if not isinstance(results, list):
|
||||
raise TypeError("results must be a list")
|
||||
sources = [
|
||||
SourceCard(
|
||||
document_id=str(result["citation"]["document_id"]),
|
||||
title=result["citation"]["title"],
|
||||
version=str(result["citation"]["document_version_id"]),
|
||||
section=result["citation"].get("section") or "",
|
||||
excerpt=result["citation"]["excerpt"],
|
||||
)
|
||||
source
|
||||
for result in results
|
||||
if isinstance(result, dict)
|
||||
and (source := self._managed_source(result.get("citation"), language)) is not None
|
||||
]
|
||||
except (TypeError, KeyError, ValueError):
|
||||
KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "malformed").inc()
|
||||
return unavailable
|
||||
|
||||
for result in results:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
score = _retrieval_score(result)
|
||||
if score is not None:
|
||||
KNOWLEDGE_RETRIEVAL_SCORE.observe(score)
|
||||
@@ -403,15 +527,11 @@ class RAGcoreKnowledgeProvider:
|
||||
all_sources = _deduplicate_sources(sources)
|
||||
concepts = _question_concepts(question)
|
||||
qualified_sources = [
|
||||
SourceCard(
|
||||
document_id=str(result["citation"]["document_id"]),
|
||||
title=result["citation"]["title"],
|
||||
version=str(result["citation"]["document_version_id"]),
|
||||
section=result["citation"].get("section") or "",
|
||||
excerpt=result["citation"]["excerpt"],
|
||||
)
|
||||
source
|
||||
for result in results
|
||||
if (_retrieval_score(result) or 0.0) >= self._settings.ragcore_min_search_score
|
||||
if isinstance(result, dict)
|
||||
and (_retrieval_score(result) or 0.0) >= self._settings.ragcore_min_search_score
|
||||
and (source := self._managed_source(result.get("citation"), language)) is not None
|
||||
]
|
||||
sources = _rank_sources_for_concepts(_deduplicate_sources(qualified_sources), concepts)
|
||||
if not all_sources:
|
||||
@@ -424,15 +544,17 @@ class RAGcoreKnowledgeProvider:
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
|
||||
damage_evidence = any(
|
||||
term
|
||||
in (
|
||||
f"{source.document_id} {source.title} {source.section} {source.excerpt}"
|
||||
).casefold()
|
||||
required_concepts = concepts & _ANSWERABLE_INTENT_CONCEPTS
|
||||
evidence_text = " ".join(
|
||||
f"{source.document_id} {source.title} {source.section} {source.excerpt}"
|
||||
for source in sources
|
||||
for term in _DOMAIN_CONCEPTS["damage"]
|
||||
)
|
||||
if not concepts or not sources or ("damage" in concepts and not damage_evidence):
|
||||
).casefold()
|
||||
covered_concepts = {
|
||||
concept
|
||||
for concept in required_concepts
|
||||
if any(term in evidence_text for term in _DOMAIN_CONCEPTS[concept])
|
||||
}
|
||||
if not required_concepts or not sources or covered_concepts != required_concepts:
|
||||
KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "insufficient").inc()
|
||||
return GroundedAnswer(
|
||||
answer="",
|
||||
|
||||
@@ -4,7 +4,7 @@ import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -12,13 +12,13 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models.booking import Booking
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.idempotency import IdempotencyRecord
|
||||
from app.models.inspection import Inspection
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.schemas import CurrentUser, RegisterReturnRequest
|
||||
from app.services.audit import record_audit_event
|
||||
from app.services.data_quality import open_odometer_regression_issue
|
||||
|
||||
|
||||
def _new_inspection_ref() -> str:
|
||||
@@ -261,28 +261,21 @@ def register_vehicle_return(
|
||||
|
||||
quality_issue_ref: str | None = None
|
||||
if evaluation.odometer_regression:
|
||||
issue = DataQualityIssue(
|
||||
public_ref=f"DQ-RET-{str(inspection.public_ref).split('-')[-1]}",
|
||||
rule_type="odometer_regression",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
severity="medium",
|
||||
status="open",
|
||||
evidence_json={
|
||||
"summary": (
|
||||
f"Return submitted {body.end_odometer_km} km, below canonical "
|
||||
f"{evaluation.canonical_odometer_km} km."
|
||||
),
|
||||
"entity_ref": vehicle.public_ref,
|
||||
"related_refs": [booking.public_ref, inspection.public_ref],
|
||||
},
|
||||
proposed_action_json={},
|
||||
issue = open_odometer_regression_issue(
|
||||
db,
|
||||
vehicle=vehicle,
|
||||
reading_ref=inspection.public_ref,
|
||||
reading_km=body.end_odometer_km,
|
||||
canonical_km=evaluation.canonical_odometer_km,
|
||||
source_type="return",
|
||||
related_refs=[booking.public_ref, inspection.public_ref],
|
||||
correctable_booking_refs=[booking.public_ref],
|
||||
detected_at=now,
|
||||
due_at=now + timedelta(days=1),
|
||||
public_ref=f"DQ-RET-{str(inspection.public_ref).split('-')[-1]}",
|
||||
actor_label=actor.display_name,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
db.add(issue)
|
||||
db.flush()
|
||||
quality_issue_ref = issue.public_ref
|
||||
quality_issue_ref = issue.public_ref if issue is not None else None
|
||||
|
||||
resulting_status = evaluation.resulting_vehicle_status
|
||||
vehicle.odometer_km = evaluation.resulting_odometer_km
|
||||
|
||||
+105
-2
@@ -1,3 +1,15 @@
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import delete, func, select
|
||||
|
||||
from app.core.db import SessionLocal, engine
|
||||
from app.models.audit import AuditEvent
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.seed_loader import reset_and_seed
|
||||
from app.services.audit import record_audit_event
|
||||
|
||||
|
||||
def test_unauthenticated_dashboard_is_rejected(client):
|
||||
response = client.get("/api/v1/dashboard")
|
||||
assert response.status_code == 401
|
||||
@@ -101,15 +113,106 @@ def test_demo_reset_rejects_concurrent_rebuild(ops_client):
|
||||
assert response.status_code == 409
|
||||
|
||||
|
||||
def test_demo_reset_database_lock_blocks_another_api_replica(ops_client):
|
||||
import app.api.routers.demo as demo_router
|
||||
|
||||
# Use a raw connection: an ordinary Session intentionally participates in the
|
||||
# shared mutation barrier and would make an exclusive reset wait before it can test
|
||||
# this separate non-blocking replica lock.
|
||||
with engine.connect() as lock_connection:
|
||||
lock_connection.scalar(select(func.pg_advisory_lock(demo_router._RESET_ADVISORY_LOCK_ID)))
|
||||
try:
|
||||
response = ops_client.post("/api/v1/demo/reset")
|
||||
finally:
|
||||
lock_connection.scalar(
|
||||
select(func.pg_advisory_unlock(demo_router._RESET_ADVISORY_LOCK_ID))
|
||||
)
|
||||
assert response.status_code == 409
|
||||
|
||||
|
||||
def test_demo_reset_waits_for_active_mutation_and_then_replaces_it_atomically():
|
||||
started = threading.Event()
|
||||
completed = threading.Event()
|
||||
errors: list[BaseException] = []
|
||||
|
||||
def reset_in_other_session() -> None:
|
||||
started.set()
|
||||
try:
|
||||
with SessionLocal() as reset_db:
|
||||
reset_and_seed(reset_db)
|
||||
except BaseException as exc: # pragma: no cover - assertion reports thread failures
|
||||
errors.append(exc)
|
||||
finally:
|
||||
completed.set()
|
||||
|
||||
with SessionLocal() as mutation_db:
|
||||
vehicle = mutation_db.scalar(
|
||||
select(Vehicle).where(Vehicle.public_ref == "MO-001").with_for_update()
|
||||
)
|
||||
vehicle.location = "Concurrent mutation marker"
|
||||
mutation_db.flush()
|
||||
worker = threading.Thread(target=reset_in_other_session, daemon=True)
|
||||
worker.start()
|
||||
assert started.wait(timeout=1)
|
||||
assert not completed.wait(timeout=0.2)
|
||||
mutation_db.commit()
|
||||
|
||||
worker.join(timeout=10)
|
||||
assert not worker.is_alive(), "reset deadlocked behind the active mutation"
|
||||
assert errors == []
|
||||
with SessionLocal() as db:
|
||||
restored = db.scalar(select(Vehicle).where(Vehicle.public_ref == "MO-001"))
|
||||
assert restored.location != "Concurrent mutation marker"
|
||||
|
||||
|
||||
def test_demo_reset_cooldown_returns_retry_after(ops_client, monkeypatch):
|
||||
import app.api.routers.demo as demo_router
|
||||
|
||||
monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 60)
|
||||
monkeypatch.setattr(demo_router, "_last_reset_monotonic", demo_router.time.monotonic())
|
||||
with SessionLocal() as db:
|
||||
cooldown_event = record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label="Cooldown test",
|
||||
action="demo_reset",
|
||||
entity_type="system",
|
||||
)
|
||||
db.commit()
|
||||
cooldown_event_id = cooldown_event.id
|
||||
response = ops_client.post("/api/v1/demo/reset")
|
||||
assert response.status_code == 429
|
||||
assert int(response.headers["retry-after"]) >= 1
|
||||
monkeypatch.setattr(demo_router, "_last_reset_monotonic", 0.0)
|
||||
with SessionLocal() as db:
|
||||
db.execute(delete(AuditEvent).where(AuditEvent.id == cooldown_event_id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_demo_reset_rolls_back_every_change_when_integrity_check_fails(ops_client, monkeypatch):
|
||||
import app.api.routers.demo as demo_router
|
||||
|
||||
monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 0)
|
||||
with SessionLocal() as db:
|
||||
probe = record_audit_event(
|
||||
db,
|
||||
actor_type="system",
|
||||
actor_label="Atomic reset test",
|
||||
action="reset_atomicity_probe",
|
||||
entity_type="system",
|
||||
)
|
||||
db.commit()
|
||||
probe_id = probe.id
|
||||
|
||||
def fail_integrity_check(_db):
|
||||
raise RuntimeError("Injected integrity-check failure")
|
||||
|
||||
monkeypatch.setattr(demo_router, "scenario_integrity_report", fail_integrity_check)
|
||||
with pytest.raises(RuntimeError, match="Injected integrity-check failure"):
|
||||
ops_client.post("/api/v1/demo/reset")
|
||||
|
||||
with SessionLocal() as db:
|
||||
assert db.scalar(select(AuditEvent).where(AuditEvent.id == probe_id)) is not None
|
||||
db.execute(delete(AuditEvent).where(AuditEvent.id == probe_id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_reset_is_rejected_when_demo_allow_reset_is_disabled(ops_client, monkeypatch):
|
||||
|
||||
@@ -95,6 +95,16 @@ def test_booking_requirements_are_explicit_and_audited(ops_client):
|
||||
)
|
||||
assert checkout.status_code == 409
|
||||
|
||||
whitespace_only = ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
|
||||
json={"confirmation": " "},
|
||||
)
|
||||
assert whitespace_only.status_code == 422
|
||||
assert (
|
||||
ops_client.get(f"/api/v1/bookings/{booking['public_ref']}").json()["requirements_complete"]
|
||||
is False
|
||||
)
|
||||
|
||||
confirmed = ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
|
||||
json={"confirmation": "Licence and rental conditions checked"},
|
||||
@@ -106,6 +116,29 @@ def test_booking_requirements_are_explicit_and_audited(ops_client):
|
||||
assert any(item["entity_ref"] == booking["public_ref"] for item in audit.json())
|
||||
|
||||
|
||||
def test_booking_creation_cannot_preconfirm_requirements(ops_client):
|
||||
window = {"starts_at": "2033-09-01T10:00:00Z", "ends_at": "2033-09-02T12:00:00Z"}
|
||||
vehicle = ops_client.get("/api/v1/bookings/availability", params=window).json()[0]
|
||||
|
||||
created = ops_client.post(
|
||||
"/api/v1/bookings",
|
||||
json={
|
||||
"customer_ref": "CUS-0001",
|
||||
"vehicle_ref": vehicle["public_ref"],
|
||||
"requirements_complete": True,
|
||||
**window,
|
||||
},
|
||||
)
|
||||
|
||||
assert created.status_code == 201
|
||||
assert created.json()["requirements_complete"] is False
|
||||
audit = ops_client.get(
|
||||
"/api/v1/audit",
|
||||
params={"action": "booking_requirements_completed"},
|
||||
)
|
||||
assert all(item["entity_ref"] != created.json()["public_ref"] for item in audit.json())
|
||||
|
||||
|
||||
def test_customer_search_returns_canonical_customers(ops_client):
|
||||
response = ops_client.get("/api/v1/customers", params={"query": "CUS-"})
|
||||
assert response.status_code == 200
|
||||
@@ -221,10 +254,14 @@ def test_checkout_records_inspection_and_activates_safe_booking(ops_client):
|
||||
json={
|
||||
"customer_ref": "CUS-0001",
|
||||
"vehicle_ref": vehicle["public_ref"],
|
||||
"requirements_complete": True,
|
||||
**window,
|
||||
},
|
||||
).json()
|
||||
confirmed = ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
|
||||
json={"confirmation": "Licence and rental conditions checked"},
|
||||
)
|
||||
assert confirmed.status_code == 200
|
||||
response = ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/checkout",
|
||||
json={
|
||||
@@ -241,6 +278,95 @@ def test_checkout_records_inspection_and_activates_safe_booking(ops_client):
|
||||
updated_vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
|
||||
assert updated_vehicle["operational_status"] == "rented"
|
||||
assert any(item["type"] == "checkout" for item in updated_vehicle["inspections"])
|
||||
checkout_audit = ops_client.get(
|
||||
"/api/v1/audit",
|
||||
params={"action": "booking_checkout_recorded"},
|
||||
).json()
|
||||
booking_event = next(
|
||||
item for item in checkout_audit if item["entity_ref"] == booking["public_ref"]
|
||||
)
|
||||
vehicle_audit = ops_client.get(
|
||||
"/api/v1/audit",
|
||||
params={
|
||||
"action": "vehicle_status_changed",
|
||||
"correlation_id": booking_event["correlation_id"],
|
||||
},
|
||||
).json()
|
||||
assert len(vehicle_audit) == 1
|
||||
assert vehicle_audit[0]["entity_ref"] == vehicle["public_ref"]
|
||||
assert vehicle_audit[0]["before"]["operational_status"] == "available"
|
||||
assert vehicle_audit[0]["after"]["operational_status"] == "rented"
|
||||
|
||||
|
||||
def test_checkout_odometer_regression_keeps_canonical_and_opens_bounded_quality_issue(
|
||||
ops_client,
|
||||
):
|
||||
window = {"starts_at": "2052-09-01T10:00:00Z", "ends_at": "2052-09-02T12:00:00Z"}
|
||||
existing_issues = ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
unavailable_refs = {issue["entity_ref"] for issue in existing_issues}
|
||||
available = ops_client.get("/api/v1/bookings/availability", params=window).json()
|
||||
vehicle_option = next(
|
||||
item
|
||||
for item in available
|
||||
if item["operational_status"] == "available" and item["public_ref"] not in unavailable_refs
|
||||
)
|
||||
vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle_option['public_ref']}").json()
|
||||
assert vehicle["odometer_km"] > 0
|
||||
booking = ops_client.post(
|
||||
"/api/v1/bookings",
|
||||
json={
|
||||
"customer_ref": "CUS-0003",
|
||||
"vehicle_ref": vehicle["public_ref"],
|
||||
**window,
|
||||
},
|
||||
).json()
|
||||
ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
|
||||
json={"confirmation": "Licence and rental conditions checked"},
|
||||
)
|
||||
|
||||
checkout = ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/checkout",
|
||||
json={
|
||||
"start_odometer_km": vehicle["odometer_km"] - 1,
|
||||
"fuel_level_percent": 90,
|
||||
"cleanliness_ok": True,
|
||||
"damage_reported": False,
|
||||
"technical_warning": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert checkout.status_code == 200
|
||||
assert checkout.json()["booking_status"] == "blocked"
|
||||
updated_vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
|
||||
assert updated_vehicle["odometer_km"] == vehicle["odometer_km"]
|
||||
issues = ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
issue = next(item for item in issues if item["entity_ref"] == vehicle["public_ref"])
|
||||
assert issue["evidence"]["source_type"] == "checkout"
|
||||
assert issue["evidence"]["correctable_booking_refs"] == []
|
||||
detail = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
|
||||
assert any(snapshot["entity_type"] == "inspection" for snapshot in detail["related_snapshots"])
|
||||
invalid_correction = ops_client.post(
|
||||
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
|
||||
json={
|
||||
"decision": "correct_reading",
|
||||
"booking_ref": booking["public_ref"],
|
||||
"corrected_odometer_km": vehicle["odometer_km"],
|
||||
},
|
||||
)
|
||||
assert invalid_correction.status_code == 422
|
||||
retained = ops_client.post(
|
||||
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
|
||||
json={"decision": "retain_canonical"},
|
||||
)
|
||||
assert retained.status_code == 200
|
||||
assert retained.json()["status"] == "resolved"
|
||||
|
||||
|
||||
def test_get_booking_detail(ops_client):
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.models.vehicle import Vehicle
|
||||
|
||||
|
||||
def test_dashboard_metrics_are_persisted_counts(ops_client):
|
||||
response = ops_client.get("/api/v1/dashboard")
|
||||
assert response.status_code == 200
|
||||
@@ -15,6 +21,43 @@ def test_dashboard_metrics_are_persisted_counts(ops_client):
|
||||
assert metrics["pending_or_failed_workflows"] >= 1
|
||||
|
||||
|
||||
def test_dashboard_metrics_change_when_persisted_fleet_data_changes(ops_client):
|
||||
before = ops_client.get("/api/v1/dashboard").json()["metrics"]
|
||||
db = SessionLocal()
|
||||
probe = Vehicle(
|
||||
public_ref="MO-METRIC-PROBE",
|
||||
make="Synthetic",
|
||||
model="Metric probe",
|
||||
model_year=2026,
|
||||
registration_number="TEST-METRIC-PROBE",
|
||||
location="Test fixture",
|
||||
operational_status="blocked",
|
||||
odometer_km=0,
|
||||
next_service_km=1,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
try:
|
||||
db.add(probe)
|
||||
db.commit()
|
||||
|
||||
after_response = ops_client.get("/api/v1/dashboard")
|
||||
assert after_response.status_code == 200
|
||||
after = after_response.json()["metrics"]
|
||||
assert after["blocked"] == before["blocked"] + 1
|
||||
fleet_statuses = ("available", "rented", "cleaning", "maintenance", "blocked")
|
||||
assert (
|
||||
sum(after[status] for status in fleet_statuses)
|
||||
== sum(before[status] for status in fleet_statuses) + 1
|
||||
)
|
||||
finally:
|
||||
persisted_probe = db.scalar(select(Vehicle).where(Vehicle.public_ref == "MO-METRIC-PROBE"))
|
||||
if persisted_probe is not None:
|
||||
db.delete(persisted_probe)
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
def test_dashboard_attention_items_link_to_records(ops_client):
|
||||
response = ops_client.get("/api/v1/dashboard")
|
||||
body = response.json()
|
||||
|
||||
@@ -1,8 +1,25 @@
|
||||
from sqlalchemy import select
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from threading import Barrier
|
||||
|
||||
from sqlalchemy import delete, select, text
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.models.audit import AuditEvent
|
||||
from app.models.booking import Booking
|
||||
from app.models.customer import Customer
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.idempotency import IdempotencyRecord
|
||||
from app.models.inspection import Inspection
|
||||
from app.models.maintenance import MaintenanceRecord
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.schemas import CurrentUser, RegisterReturnRequest, ResolveOdometerRegressionRequest
|
||||
from app.services.data_quality import (
|
||||
open_odometer_regression_issue,
|
||||
resolve_odometer_regression,
|
||||
)
|
||||
from app.services.returns import register_vehicle_return
|
||||
|
||||
|
||||
def _activate_booking(vehicle_ref: str, start_odometer_km: int) -> str:
|
||||
@@ -21,6 +38,30 @@ def _activate_booking(vehicle_ref: str, start_odometer_km: int) -> str:
|
||||
db.close()
|
||||
|
||||
|
||||
def _cleanup_odometer_scenario(vehicle_id, customer_id) -> None:
|
||||
"""Remove one isolated DQ-03 scenario in foreign-key-safe order."""
|
||||
with SessionLocal() as db:
|
||||
booking_ids = db.scalars(select(Booking.id).where(Booking.vehicle_id == vehicle_id)).all()
|
||||
issue_ids = db.scalars(
|
||||
select(DataQualityIssue.id).where(DataQualityIssue.entity_id == vehicle_id)
|
||||
).all()
|
||||
audit_entity_ids = [vehicle_id, *booking_ids, *issue_ids]
|
||||
if audit_entity_ids:
|
||||
db.execute(delete(AuditEvent).where(AuditEvent.entity_id.in_(audit_entity_ids)))
|
||||
if booking_ids:
|
||||
db.execute(
|
||||
delete(IdempotencyRecord).where(IdempotencyRecord.booking_id.in_(booking_ids))
|
||||
)
|
||||
db.execute(delete(OutboxEvent).where(OutboxEvent.aggregate_id.in_(booking_ids)))
|
||||
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
|
||||
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
|
||||
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
|
||||
db.execute(delete(Customer).where(Customer.id == customer_id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_list_includes_all_five_rule_types(ops_client):
|
||||
response = ops_client.get("/api/v1/data-quality/issues")
|
||||
assert response.status_code == 200
|
||||
@@ -43,6 +84,84 @@ def test_scan_is_idempotent_once_seeded(ops_client):
|
||||
assert response.json()["created"] == {}
|
||||
|
||||
|
||||
def test_scan_detects_imported_maintenance_odometer_regression(ops_client):
|
||||
with SessionLocal() as db:
|
||||
vehicle = Vehicle(
|
||||
public_ref="MO-DQ-SCAN",
|
||||
make="Synthetic",
|
||||
model="Scanner",
|
||||
model_year=2026,
|
||||
registration_number="DQ-SCAN-01",
|
||||
location="Brussels",
|
||||
operational_status="available",
|
||||
odometer_km=20_000,
|
||||
next_service_km=30_000,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
db.add(vehicle)
|
||||
db.flush()
|
||||
vehicle_id = vehicle.id
|
||||
db.add_all(
|
||||
[
|
||||
MaintenanceRecord(
|
||||
public_ref="MAINT-DQ-SCAN-A",
|
||||
vehicle_id=vehicle.id,
|
||||
occurred_at=datetime(2045, 1, 1, tzinfo=UTC),
|
||||
odometer_km=20_000,
|
||||
category="inspection",
|
||||
summary="Synthetic scan baseline",
|
||||
),
|
||||
MaintenanceRecord(
|
||||
public_ref="MAINT-DQ-SCAN-B",
|
||||
vehicle_id=vehicle.id,
|
||||
occurred_at=datetime(2045, 2, 1, tzinfo=UTC),
|
||||
odometer_km=19_000,
|
||||
category="inspection",
|
||||
summary="Synthetic imported regression",
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
try:
|
||||
response = ops_client.post("/api/v1/data-quality/scan")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["created"]["odometer_regression"] >= 1
|
||||
issues = ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
issue = next(item for item in issues if item["entity_ref"] == "MO-DQ-SCAN")
|
||||
assert issue["evidence"]["source_type"] == "maintenance"
|
||||
assert issue["evidence"]["correctable_booking_refs"] == []
|
||||
|
||||
# A later imported regression must enrich the existing open issue instead of
|
||||
# being silently dropped by the generic check-then-return scanner path.
|
||||
with SessionLocal() as db:
|
||||
db.add(
|
||||
MaintenanceRecord(
|
||||
public_ref="MAINT-DQ-SCAN-C",
|
||||
vehicle_id=vehicle_id,
|
||||
occurred_at=datetime(2045, 3, 1, tzinfo=UTC),
|
||||
odometer_km=18_000,
|
||||
category="inspection",
|
||||
summary="Second synthetic imported regression",
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
rescanned = ops_client.post("/api/v1/data-quality/scan")
|
||||
assert rescanned.status_code == 200
|
||||
enriched = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
|
||||
later_refs = {signal["params"]["later_ref"] for signal in enriched["evidence"]["signals"]}
|
||||
assert {"MAINT-DQ-SCAN-B", "MAINT-DQ-SCAN-C"}.issubset(later_refs)
|
||||
finally:
|
||||
with SessionLocal() as db:
|
||||
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
|
||||
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_scan_requires_operations_manager(employee_client):
|
||||
response = employee_client.post("/api/v1/data-quality/scan")
|
||||
assert response.status_code == 403
|
||||
@@ -440,9 +559,8 @@ def test_resolve_odometer_regression_correction_below_canonical_is_rejected_then
|
||||
|
||||
|
||||
def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_client):
|
||||
# The seeded odometer_regression issues carry no related booking (CSV-only rows).
|
||||
# Create a fresh one with a real related booking via a live regression return, so
|
||||
# the "correct_reading" path has an actual booking_ref to target.
|
||||
# Create a live return regression to prove the complete correction chain updates
|
||||
# both persisted representations of that reading (booking + return inspection).
|
||||
booking_ref = _activate_booking("MO-018", start_odometer_km=12000)
|
||||
vehicle_before = ops_client.get("/api/v1/vehicles/MO-018").json()
|
||||
low_reading = vehicle_before["odometer_km"] - 200
|
||||
@@ -459,6 +577,7 @@ def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_clien
|
||||
)
|
||||
assert returned.status_code == 201
|
||||
issue_ref = returned.json()["quality_issue_ref"]
|
||||
inspection_ref = returned.json()["inspection_ref"]
|
||||
assert issue_ref is not None
|
||||
|
||||
corrected = vehicle_before["odometer_km"] + 500
|
||||
@@ -477,6 +596,701 @@ def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_clien
|
||||
assert vehicle["odometer_km"] == corrected
|
||||
booking = ops_client.get(f"/api/v1/bookings/{booking_ref}").json()
|
||||
assert booking["end_odometer_km"] == corrected
|
||||
with SessionLocal() as db:
|
||||
inspection = db.scalar(select(Inspection).where(Inspection.public_ref == inspection_ref))
|
||||
assert inspection is not None
|
||||
assert inspection.odometer_km == corrected
|
||||
|
||||
|
||||
def test_correcting_one_of_two_regressions_keeps_the_other_open(ops_client):
|
||||
with SessionLocal() as db:
|
||||
customer = Customer(
|
||||
public_ref="CUS-DQ-MULTI",
|
||||
first_name="Synthetic",
|
||||
last_name="Multi",
|
||||
email="dq-multi@example.test",
|
||||
)
|
||||
vehicle = Vehicle(
|
||||
public_ref="MO-DQ-MULTI",
|
||||
make="Synthetic",
|
||||
model="Multi",
|
||||
model_year=2026,
|
||||
registration_number="DQ-MULTI",
|
||||
location="Brussels",
|
||||
operational_status="available",
|
||||
odometer_km=1_000,
|
||||
next_service_km=2_000,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
db.add_all([customer, vehicle])
|
||||
db.flush()
|
||||
bookings: list[Booking] = []
|
||||
inspections: list[Inspection] = []
|
||||
for index, reading in enumerate((900, 800), start=1):
|
||||
booking = Booking(
|
||||
public_ref=f"BK-DQ-MULTI-{index}",
|
||||
customer_id=customer.id,
|
||||
vehicle_id=vehicle.id,
|
||||
starts_at=datetime(2046, index, 1, tzinfo=UTC),
|
||||
ends_at=datetime(2046, index, 2, tzinfo=UTC),
|
||||
status="returned",
|
||||
start_odometer_km=reading - 10,
|
||||
end_odometer_km=reading,
|
||||
requirements_complete=True,
|
||||
)
|
||||
db.add(booking)
|
||||
db.flush()
|
||||
inspection = Inspection(
|
||||
public_ref=f"INSP-DQ-M-{index}",
|
||||
booking_id=booking.id,
|
||||
vehicle_id=vehicle.id,
|
||||
type="return",
|
||||
fuel_level_percent=50,
|
||||
cleanliness_ok=True,
|
||||
damage_reported=False,
|
||||
technical_warning=False,
|
||||
odometer_km=reading,
|
||||
completed_at=booking.ends_at,
|
||||
)
|
||||
db.add(inspection)
|
||||
db.flush()
|
||||
open_odometer_regression_issue(
|
||||
db,
|
||||
vehicle=vehicle,
|
||||
reading_ref=inspection.public_ref,
|
||||
reading_km=reading,
|
||||
canonical_km=vehicle.odometer_km,
|
||||
source_type="return",
|
||||
related_refs=[booking.public_ref, inspection.public_ref],
|
||||
correctable_booking_refs=[booking.public_ref],
|
||||
public_ref="DQ-MULTI-SOURCE" if index == 1 else None,
|
||||
)
|
||||
bookings.append(booking)
|
||||
inspections.append(inspection)
|
||||
booking_refs = [booking.public_ref for booking in bookings]
|
||||
vehicle_id = vehicle.id
|
||||
customer_id = customer.id
|
||||
db.commit()
|
||||
|
||||
try:
|
||||
first = ops_client.post(
|
||||
"/api/v1/data-quality/issues/DQ-MULTI-SOURCE/resolve-odometer-regression",
|
||||
json={
|
||||
"decision": "correct_reading",
|
||||
"booking_ref": booking_refs[0],
|
||||
"corrected_odometer_km": 1_100,
|
||||
},
|
||||
)
|
||||
assert first.status_code == 200, first.text
|
||||
assert first.json()["status"] == "open"
|
||||
assert first.json()["evidence"]["correctable_booking_refs"] == [booking_refs[1]]
|
||||
|
||||
second = ops_client.post(
|
||||
"/api/v1/data-quality/issues/DQ-MULTI-SOURCE/resolve-odometer-regression",
|
||||
json={
|
||||
"decision": "correct_reading",
|
||||
"booking_ref": booking_refs[1],
|
||||
"corrected_odometer_km": 1_200,
|
||||
},
|
||||
)
|
||||
assert second.status_code == 200, second.text
|
||||
assert second.json()["status"] == "resolved"
|
||||
with SessionLocal() as db:
|
||||
persisted = db.scalars(
|
||||
select(Inspection)
|
||||
.where(Inspection.vehicle_id == vehicle_id)
|
||||
.order_by(Inspection.public_ref)
|
||||
).all()
|
||||
assert [item.odometer_km for item in persisted] == [1_100, 1_200]
|
||||
finally:
|
||||
with SessionLocal() as db:
|
||||
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
|
||||
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
|
||||
db.execute(delete(Customer).where(Customer.id == customer_id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_retained_regression_is_suppressed_until_source_reading_changes(ops_client):
|
||||
with SessionLocal() as db:
|
||||
vehicle = Vehicle(
|
||||
public_ref="MO-DQ-RETAIN",
|
||||
make="Synthetic",
|
||||
model="Retain",
|
||||
model_year=2026,
|
||||
registration_number="DQ-RETAIN",
|
||||
location="Brussels",
|
||||
operational_status="available",
|
||||
odometer_km=20_000,
|
||||
next_service_km=30_000,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
db.add(vehicle)
|
||||
db.flush()
|
||||
vehicle_id = vehicle.id
|
||||
db.add_all(
|
||||
[
|
||||
MaintenanceRecord(
|
||||
public_ref="MAINT-DQ-RET-A",
|
||||
vehicle_id=vehicle.id,
|
||||
occurred_at=datetime(2047, 1, 1, tzinfo=UTC),
|
||||
odometer_km=20_000,
|
||||
category="inspection",
|
||||
summary="Retain baseline",
|
||||
),
|
||||
MaintenanceRecord(
|
||||
public_ref="MAINT-DQ-RET-B",
|
||||
vehicle_id=vehicle.id,
|
||||
occurred_at=datetime(2047, 2, 1, tzinfo=UTC),
|
||||
odometer_km=19_000,
|
||||
category="inspection",
|
||||
summary="Retained source reading",
|
||||
),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
try:
|
||||
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
|
||||
issue = next(
|
||||
item
|
||||
for item in ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
if item["entity_ref"] == "MO-DQ-RETAIN"
|
||||
)
|
||||
retained = ops_client.post(
|
||||
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
|
||||
json={"decision": "retain_canonical", "note": "Verified source entry"},
|
||||
)
|
||||
assert retained.status_code == 200
|
||||
assert retained.json()["status"] == "resolved"
|
||||
|
||||
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
|
||||
with SessionLocal() as db:
|
||||
assert (
|
||||
db.scalar(
|
||||
select(DataQualityIssue).where(
|
||||
DataQualityIssue.entity_id == vehicle_id,
|
||||
DataQualityIssue.status == "open",
|
||||
)
|
||||
)
|
||||
is None
|
||||
)
|
||||
changed = db.scalar(
|
||||
select(MaintenanceRecord).where(MaintenanceRecord.public_ref == "MAINT-DQ-RET-B")
|
||||
)
|
||||
changed.odometer_km = 18_999
|
||||
db.commit()
|
||||
|
||||
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
|
||||
reopened = next(
|
||||
item
|
||||
for item in ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
if item["entity_ref"] == "MO-DQ-RETAIN"
|
||||
)
|
||||
assert reopened["evidence"]["reopened_from"] == issue["public_ref"]
|
||||
finally:
|
||||
with SessionLocal() as db:
|
||||
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
|
||||
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_live_return_retained_as_canonical_is_not_reopened_by_scan(ops_client):
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as db:
|
||||
customer = Customer(
|
||||
public_ref="CUS-DQ-LIVE-RET",
|
||||
first_name="Synthetic",
|
||||
last_name="Retained return",
|
||||
email="dq-live-retain@example.test",
|
||||
)
|
||||
vehicle = Vehicle(
|
||||
public_ref="MO-DQ-LIVE-RET",
|
||||
make="Synthetic",
|
||||
model="Retained return",
|
||||
model_year=2026,
|
||||
registration_number="DQ-LIVE-RET",
|
||||
location="Brussels",
|
||||
operational_status="rented",
|
||||
odometer_km=20_000,
|
||||
next_service_km=30_000,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
db.add_all([customer, vehicle])
|
||||
db.flush()
|
||||
booking = Booking(
|
||||
public_ref="BK-DQ-LIVE-RET",
|
||||
customer_id=customer.id,
|
||||
vehicle_id=vehicle.id,
|
||||
starts_at=now - timedelta(days=2),
|
||||
ends_at=now + timedelta(days=1),
|
||||
status="active",
|
||||
start_odometer_km=19_500,
|
||||
end_odometer_km=None,
|
||||
requirements_complete=True,
|
||||
)
|
||||
db.add_all(
|
||||
[
|
||||
booking,
|
||||
MaintenanceRecord(
|
||||
public_ref="MNT-DQ-LIVE-RET",
|
||||
vehicle_id=vehicle.id,
|
||||
occurred_at=now - timedelta(days=3),
|
||||
odometer_km=20_000,
|
||||
category="inspection",
|
||||
summary="Synthetic canonical baseline",
|
||||
),
|
||||
]
|
||||
)
|
||||
vehicle_id = vehicle.id
|
||||
customer_id = customer.id
|
||||
db.commit()
|
||||
|
||||
try:
|
||||
returned = ops_client.post(
|
||||
"/api/v1/bookings/BK-DQ-LIVE-RET/return",
|
||||
json={
|
||||
"end_odometer_km": 19_000,
|
||||
"fuel_level_percent": 50,
|
||||
"cleanliness_ok": True,
|
||||
"damage_reported": False,
|
||||
"technical_warning": False,
|
||||
},
|
||||
headers={"Idempotency-Key": "test-dq-live-retain-001"},
|
||||
)
|
||||
assert returned.status_code == 201, returned.text
|
||||
issue_ref = returned.json()["quality_issue_ref"]
|
||||
inspection_ref = returned.json()["inspection_ref"]
|
||||
assert issue_ref is not None
|
||||
|
||||
retained = ops_client.post(
|
||||
f"/api/v1/data-quality/issues/{issue_ref}/resolve-odometer-regression",
|
||||
json={"decision": "retain_canonical", "note": "Source reading verified as wrong."},
|
||||
)
|
||||
assert retained.status_code == 200, retained.text
|
||||
assert retained.json()["status"] == "resolved"
|
||||
assert retained.json()["evidence"]["retained_odometer_fingerprints"] == [
|
||||
{
|
||||
"source_type": "return",
|
||||
"later_ref": inspection_ref,
|
||||
"later_km": 19_000,
|
||||
}
|
||||
]
|
||||
|
||||
scanned = ops_client.post("/api/v1/data-quality/scan")
|
||||
assert scanned.status_code == 200, scanned.text
|
||||
with SessionLocal() as db:
|
||||
issues = db.scalars(
|
||||
select(DataQualityIssue)
|
||||
.where(
|
||||
DataQualityIssue.entity_id == vehicle_id,
|
||||
DataQualityIssue.rule_type == "odometer_regression",
|
||||
)
|
||||
.order_by(DataQualityIssue.detected_at)
|
||||
).all()
|
||||
assert [(issue.public_ref, issue.status) for issue in issues] == [
|
||||
(issue_ref, "resolved")
|
||||
]
|
||||
finally:
|
||||
_cleanup_odometer_scenario(vehicle_id, customer_id)
|
||||
|
||||
|
||||
def test_scanned_return_regression_can_correct_booking_all_inspections_and_vehicle(ops_client):
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as db:
|
||||
customer = Customer(
|
||||
public_ref="CUS-DQ-SCAN-RET",
|
||||
first_name="Synthetic",
|
||||
last_name="Scanned return",
|
||||
email="dq-scan-return@example.test",
|
||||
)
|
||||
vehicle = Vehicle(
|
||||
public_ref="MO-DQ-SCAN-RET",
|
||||
make="Synthetic",
|
||||
model="Scanned return",
|
||||
model_year=2026,
|
||||
registration_number="DQ-SCAN-RET",
|
||||
location="Brussels",
|
||||
operational_status="available",
|
||||
odometer_km=20_000,
|
||||
next_service_km=30_000,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
db.add_all([customer, vehicle])
|
||||
db.flush()
|
||||
booking = Booking(
|
||||
public_ref="BK-DQ-SCAN-RET",
|
||||
customer_id=customer.id,
|
||||
vehicle_id=vehicle.id,
|
||||
starts_at=now - timedelta(days=3),
|
||||
ends_at=now - timedelta(hours=12),
|
||||
status="returned",
|
||||
start_odometer_km=19_500,
|
||||
end_odometer_km=18_500,
|
||||
requirements_complete=True,
|
||||
)
|
||||
db.add(booking)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
[
|
||||
MaintenanceRecord(
|
||||
public_ref="MNT-DQ-SCAN-RET",
|
||||
vehicle_id=vehicle.id,
|
||||
occurred_at=now - timedelta(days=4),
|
||||
odometer_km=20_000,
|
||||
category="inspection",
|
||||
summary="Synthetic canonical baseline",
|
||||
),
|
||||
Inspection(
|
||||
public_ref="INSP-DQ-SCAN-R1",
|
||||
booking_id=booking.id,
|
||||
vehicle_id=vehicle.id,
|
||||
type="return",
|
||||
fuel_level_percent=50,
|
||||
cleanliness_ok=True,
|
||||
damage_reported=False,
|
||||
technical_warning=False,
|
||||
odometer_km=19_000,
|
||||
completed_at=now - timedelta(days=1),
|
||||
),
|
||||
Inspection(
|
||||
public_ref="INSP-DQ-SCAN-R2",
|
||||
booking_id=booking.id,
|
||||
vehicle_id=vehicle.id,
|
||||
type="return",
|
||||
fuel_level_percent=50,
|
||||
cleanliness_ok=True,
|
||||
damage_reported=False,
|
||||
technical_warning=False,
|
||||
odometer_km=18_500,
|
||||
completed_at=now - timedelta(hours=12),
|
||||
),
|
||||
]
|
||||
)
|
||||
vehicle_id = vehicle.id
|
||||
customer_id = customer.id
|
||||
db.commit()
|
||||
|
||||
try:
|
||||
scanned = ops_client.post("/api/v1/data-quality/scan")
|
||||
assert scanned.status_code == 200, scanned.text
|
||||
issue = next(
|
||||
item
|
||||
for item in ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
if item["entity_ref"] == "MO-DQ-SCAN-RET"
|
||||
)
|
||||
assert issue["evidence"]["correctable_booking_refs"] == ["BK-DQ-SCAN-RET"]
|
||||
later_refs = {signal["params"]["later_ref"] for signal in issue["evidence"]["signals"]}
|
||||
assert later_refs == {"INSP-DQ-SCAN-R1", "INSP-DQ-SCAN-R2"}
|
||||
|
||||
corrected = ops_client.post(
|
||||
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
|
||||
json={
|
||||
"decision": "correct_reading",
|
||||
"booking_ref": "BK-DQ-SCAN-RET",
|
||||
"corrected_odometer_km": 20_500,
|
||||
},
|
||||
)
|
||||
assert corrected.status_code == 200, corrected.text
|
||||
assert corrected.json()["status"] == "resolved"
|
||||
|
||||
with SessionLocal() as db:
|
||||
persisted_vehicle = db.get(Vehicle, vehicle_id)
|
||||
persisted_booking = db.scalar(
|
||||
select(Booking).where(Booking.public_ref == "BK-DQ-SCAN-RET")
|
||||
)
|
||||
assert persisted_booking is not None
|
||||
persisted_inspections = db.scalars(
|
||||
select(Inspection)
|
||||
.where(Inspection.booking_id == persisted_booking.id, Inspection.type == "return")
|
||||
.order_by(Inspection.public_ref)
|
||||
).all()
|
||||
assert persisted_vehicle is not None
|
||||
assert persisted_vehicle.odometer_km == 20_500
|
||||
assert persisted_booking.end_odometer_km == 20_500
|
||||
assert [inspection.odometer_km for inspection in persisted_inspections] == [
|
||||
20_500,
|
||||
20_500,
|
||||
]
|
||||
finally:
|
||||
_cleanup_odometer_scenario(vehicle_id, customer_id)
|
||||
|
||||
|
||||
def test_resolver_and_concurrent_return_finish_without_deadlock_or_lost_evidence():
|
||||
now = datetime.now(UTC)
|
||||
with SessionLocal() as db:
|
||||
customer = Customer(
|
||||
public_ref="CUS-DQ-RACE",
|
||||
first_name="Synthetic",
|
||||
last_name="Race",
|
||||
email="dq-race@example.test",
|
||||
)
|
||||
vehicle = Vehicle(
|
||||
public_ref="MO-DQ-RACE",
|
||||
make="Synthetic",
|
||||
model="Race",
|
||||
model_year=2026,
|
||||
registration_number="DQ-RACE",
|
||||
location="Brussels",
|
||||
operational_status="rented",
|
||||
odometer_km=20_000,
|
||||
next_service_km=30_000,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
db.add_all([customer, vehicle])
|
||||
db.flush()
|
||||
corrected_booking = Booking(
|
||||
public_ref="BK-DQ-RACE-OLD",
|
||||
customer_id=customer.id,
|
||||
vehicle_id=vehicle.id,
|
||||
starts_at=now - timedelta(days=4),
|
||||
ends_at=now - timedelta(days=3),
|
||||
status="returned",
|
||||
start_odometer_km=19_500,
|
||||
end_odometer_km=19_000,
|
||||
requirements_complete=True,
|
||||
)
|
||||
concurrent_booking = Booking(
|
||||
public_ref="BK-DQ-RACE-NEW",
|
||||
customer_id=customer.id,
|
||||
vehicle_id=vehicle.id,
|
||||
starts_at=now - timedelta(days=1),
|
||||
ends_at=now + timedelta(days=1),
|
||||
status="active",
|
||||
start_odometer_km=19_000,
|
||||
end_odometer_km=None,
|
||||
requirements_complete=True,
|
||||
)
|
||||
db.add_all([corrected_booking, concurrent_booking])
|
||||
db.flush()
|
||||
old_inspection = Inspection(
|
||||
public_ref="INSP-DQ-RACE-OLD",
|
||||
booking_id=corrected_booking.id,
|
||||
vehicle_id=vehicle.id,
|
||||
type="return",
|
||||
fuel_level_percent=50,
|
||||
cleanliness_ok=True,
|
||||
damage_reported=False,
|
||||
technical_warning=False,
|
||||
odometer_km=19_000,
|
||||
completed_at=corrected_booking.ends_at,
|
||||
)
|
||||
db.add(old_inspection)
|
||||
db.flush()
|
||||
issue = open_odometer_regression_issue(
|
||||
db,
|
||||
vehicle=vehicle,
|
||||
reading_ref=old_inspection.public_ref,
|
||||
reading_km=19_000,
|
||||
canonical_km=20_000,
|
||||
source_type="return",
|
||||
related_refs=[corrected_booking.public_ref, old_inspection.public_ref],
|
||||
correctable_booking_refs=[corrected_booking.public_ref],
|
||||
public_ref="DQ-ODO-RACE",
|
||||
)
|
||||
assert issue is not None
|
||||
vehicle_id = vehicle.id
|
||||
customer_id = customer.id
|
||||
db.commit()
|
||||
|
||||
actor = CurrentUser(
|
||||
public_ref="USR-DQ-RACE",
|
||||
display_name="DQ Race Manager",
|
||||
role="operations_manager",
|
||||
)
|
||||
start = Barrier(2)
|
||||
|
||||
def correct_existing_reading() -> str:
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("SET LOCAL lock_timeout = '5s'"))
|
||||
start.wait(timeout=5)
|
||||
resolved = resolve_odometer_regression(
|
||||
db,
|
||||
"DQ-ODO-RACE",
|
||||
ResolveOdometerRegressionRequest(
|
||||
decision="correct_reading",
|
||||
booking_ref="BK-DQ-RACE-OLD",
|
||||
corrected_odometer_km=20_500,
|
||||
),
|
||||
actor,
|
||||
)
|
||||
return resolved.status
|
||||
|
||||
def return_other_booking() -> dict:
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("SET LOCAL lock_timeout = '5s'"))
|
||||
start.wait(timeout=5)
|
||||
_status, response = register_vehicle_return(
|
||||
db,
|
||||
"BK-DQ-RACE-NEW",
|
||||
RegisterReturnRequest(
|
||||
end_odometer_km=18_000,
|
||||
fuel_level_percent=50,
|
||||
cleanliness_ok=True,
|
||||
damage_reported=False,
|
||||
technical_warning=False,
|
||||
),
|
||||
"test-dq-resolve-return-race-001",
|
||||
actor,
|
||||
)
|
||||
return response
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
correction_future = executor.submit(correct_existing_reading)
|
||||
return_future = executor.submit(return_other_booking)
|
||||
correction_status = correction_future.result(timeout=15)
|
||||
return_result = return_future.result(timeout=15)
|
||||
|
||||
assert correction_status in {"open", "resolved"}
|
||||
assert return_result["quality_issue_ref"] is not None
|
||||
with SessionLocal() as db:
|
||||
persisted_vehicle = db.get(Vehicle, vehicle_id)
|
||||
bookings = {
|
||||
booking.public_ref: booking
|
||||
for booking in db.scalars(
|
||||
select(Booking).where(Booking.vehicle_id == vehicle_id)
|
||||
).all()
|
||||
}
|
||||
old_inspections = db.scalars(
|
||||
select(Inspection).where(Inspection.booking_id == bookings["BK-DQ-RACE-OLD"].id)
|
||||
).all()
|
||||
open_issues = db.scalars(
|
||||
select(DataQualityIssue).where(
|
||||
DataQualityIssue.entity_id == vehicle_id,
|
||||
DataQualityIssue.rule_type == "odometer_regression",
|
||||
DataQualityIssue.status == "open",
|
||||
)
|
||||
).all()
|
||||
assert persisted_vehicle is not None
|
||||
assert persisted_vehicle.odometer_km == 20_500
|
||||
assert bookings["BK-DQ-RACE-OLD"].end_odometer_km == 20_500
|
||||
assert [inspection.odometer_km for inspection in old_inspections] == [20_500]
|
||||
assert bookings["BK-DQ-RACE-NEW"].status == "returned"
|
||||
assert bookings["BK-DQ-RACE-NEW"].end_odometer_km == 18_000
|
||||
assert len(open_issues) == 1
|
||||
assert open_issues[0].evidence_json["correctable_booking_refs"] == ["BK-DQ-RACE-NEW"]
|
||||
later_refs = {
|
||||
signal["params"]["later_ref"] for signal in open_issues[0].evidence_json["signals"]
|
||||
}
|
||||
assert later_refs == {return_result["inspection_ref"]}
|
||||
finally:
|
||||
_cleanup_odometer_scenario(vehicle_id, customer_id)
|
||||
|
||||
|
||||
def test_early_return_uses_inspection_time_without_duplicate_booking_regression(ops_client):
|
||||
with SessionLocal() as db:
|
||||
customer = Customer(
|
||||
public_ref="CUS-DQ-EARLY",
|
||||
first_name="Synthetic",
|
||||
last_name="Early",
|
||||
email="dq-early@example.test",
|
||||
)
|
||||
vehicle = Vehicle(
|
||||
public_ref="MO-DQ-EARLY",
|
||||
make="Synthetic",
|
||||
model="Early",
|
||||
model_year=2026,
|
||||
registration_number="DQ-EARLY",
|
||||
location="Brussels",
|
||||
operational_status="available",
|
||||
odometer_km=200,
|
||||
next_service_km=10_000,
|
||||
active=True,
|
||||
version=1,
|
||||
)
|
||||
db.add_all([customer, vehicle])
|
||||
db.flush()
|
||||
early = Booking(
|
||||
public_ref="BK-DQ-EARLY-A",
|
||||
customer_id=customer.id,
|
||||
vehicle_id=vehicle.id,
|
||||
starts_at=datetime(2048, 1, 1, tzinfo=UTC),
|
||||
ends_at=datetime(2048, 3, 1, tzinfo=UTC),
|
||||
status="returned",
|
||||
start_odometer_km=90,
|
||||
end_odometer_km=100,
|
||||
requirements_complete=True,
|
||||
)
|
||||
later = Booking(
|
||||
public_ref="BK-DQ-EARLY-B",
|
||||
customer_id=customer.id,
|
||||
vehicle_id=vehicle.id,
|
||||
starts_at=datetime(2048, 1, 10, tzinfo=UTC),
|
||||
ends_at=datetime(2048, 2, 1, tzinfo=UTC),
|
||||
status="returned",
|
||||
start_odometer_km=100,
|
||||
end_odometer_km=200,
|
||||
requirements_complete=True,
|
||||
)
|
||||
db.add_all([early, later])
|
||||
db.flush()
|
||||
db.add_all(
|
||||
[
|
||||
Inspection(
|
||||
public_ref="INSP-DQ-EARLY-A",
|
||||
booking_id=early.id,
|
||||
vehicle_id=vehicle.id,
|
||||
type="return",
|
||||
fuel_level_percent=50,
|
||||
cleanliness_ok=True,
|
||||
damage_reported=False,
|
||||
technical_warning=False,
|
||||
odometer_km=100,
|
||||
completed_at=datetime(2048, 1, 2, tzinfo=UTC),
|
||||
),
|
||||
Inspection(
|
||||
public_ref="INSP-DQ-EARLY-B",
|
||||
booking_id=later.id,
|
||||
vehicle_id=vehicle.id,
|
||||
type="return",
|
||||
fuel_level_percent=50,
|
||||
cleanliness_ok=True,
|
||||
damage_reported=False,
|
||||
technical_warning=False,
|
||||
odometer_km=200,
|
||||
completed_at=datetime(2048, 2, 1, tzinfo=UTC),
|
||||
),
|
||||
]
|
||||
)
|
||||
vehicle_id = vehicle.id
|
||||
customer_id = customer.id
|
||||
db.commit()
|
||||
try:
|
||||
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
|
||||
with SessionLocal() as db:
|
||||
assert (
|
||||
db.scalar(
|
||||
select(DataQualityIssue).where(
|
||||
DataQualityIssue.entity_id == vehicle_id,
|
||||
DataQualityIssue.rule_type == "odometer_regression",
|
||||
)
|
||||
)
|
||||
is None
|
||||
)
|
||||
finally:
|
||||
with SessionLocal() as db:
|
||||
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
|
||||
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
|
||||
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
|
||||
db.execute(delete(Customer).where(Customer.id == customer_id))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_manual_scan_records_audit_event(ops_client):
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.seed_loader import reset_and_seed
|
||||
from app.services import demo_manifest
|
||||
from app.services.knowledge import KnowledgeHealth
|
||||
|
||||
|
||||
def test_demo_manifest_is_public(client):
|
||||
@@ -52,3 +57,102 @@ def test_demo_manifest_ragcore_labelled_as_demo_mode_not_live(client):
|
||||
body = client.get("/api/v1/demo/manifest").json()
|
||||
ragcore = next(i for i in body["integrations"] if i["key"] == "ragcore")
|
||||
assert ragcore["status_code"] == "demoMode"
|
||||
|
||||
|
||||
def test_automation_scenario_requires_the_exact_prepared_failure():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
reset_and_seed(db)
|
||||
event = db.scalar(
|
||||
select(OutboxEvent).where(OutboxEvent.event_id == demo_manifest._FAILED_DEMO_EVENT_ID)
|
||||
)
|
||||
assert event is not None
|
||||
event.last_error_code = "connectionError"
|
||||
db.flush()
|
||||
|
||||
scenario = next(
|
||||
item for item in demo_manifest._scenarios(db) if item.id == "automation-retry"
|
||||
)
|
||||
assert scenario.ready is False
|
||||
finally:
|
||||
db.rollback()
|
||||
db.close()
|
||||
|
||||
|
||||
def test_knowledge_scenario_rejects_available_provider_with_verified_empty_corpus(
|
||||
client, monkeypatch
|
||||
):
|
||||
class EmptyKnowledgeProvider:
|
||||
def health(self, language="en-GB"):
|
||||
return KnowledgeHealth(
|
||||
provider="ragcore",
|
||||
available=True,
|
||||
detail="Ready, but no indexed documents.",
|
||||
tenant="fleet-ops",
|
||||
workspace="mobilityops",
|
||||
collection="procedures",
|
||||
document_count=0,
|
||||
source_document_count=4,
|
||||
reported_synced_document_count=None,
|
||||
reported_failed_document_count=None,
|
||||
last_sync_at=None,
|
||||
statistics_state="verified",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(demo_manifest, "get_knowledge_provider", EmptyKnowledgeProvider)
|
||||
|
||||
body = client.get("/api/v1/demo/manifest").json()
|
||||
scenario = next(item for item in body["scenarios"] if item["id"] == "knowledge-question")
|
||||
assert scenario["ready"] is False
|
||||
assert scenario["blocked_reason_code"] == "knowledgeUnavailable"
|
||||
ragcore = next(item for item in body["integrations"] if item["key"] == "ragcore")
|
||||
assert ragcore["status_code"] == "unavailable"
|
||||
|
||||
|
||||
def test_knowledge_scenario_rejects_local_sources_when_index_count_is_unverified(
|
||||
client, monkeypatch
|
||||
):
|
||||
class SourceAwareKnowledgeProvider:
|
||||
def health(self, language="en-GB"):
|
||||
return KnowledgeHealth(
|
||||
provider="ragcore",
|
||||
available=True,
|
||||
detail="Ready; index count endpoint is unavailable.",
|
||||
tenant="fleet-ops",
|
||||
workspace="mobilityops",
|
||||
collection="procedures",
|
||||
document_count=None,
|
||||
source_document_count=4,
|
||||
reported_synced_document_count=None,
|
||||
reported_failed_document_count=None,
|
||||
last_sync_at=None,
|
||||
statistics_state="not_reported",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(demo_manifest, "get_knowledge_provider", SourceAwareKnowledgeProvider)
|
||||
|
||||
body = client.get("/api/v1/demo/manifest").json()
|
||||
scenario = next(item for item in body["scenarios"] if item["id"] == "knowledge-question")
|
||||
assert scenario["ready"] is False
|
||||
assert scenario["blocked_reason_code"] == "knowledgeUnavailable"
|
||||
ragcore = next(item for item in body["integrations"] if item["key"] == "ragcore")
|
||||
assert ragcore["status_code"] == "unavailable"
|
||||
|
||||
|
||||
def test_knowledge_scenario_requires_provider_availability_even_with_documents():
|
||||
health = KnowledgeHealth(
|
||||
provider="ragcore",
|
||||
available=False,
|
||||
detail="Provider is unreachable.",
|
||||
tenant="fleet-ops",
|
||||
workspace="mobilityops",
|
||||
collection="procedures",
|
||||
document_count=4,
|
||||
source_document_count=4,
|
||||
reported_synced_document_count=None,
|
||||
reported_failed_document_count=None,
|
||||
last_sync_at=None,
|
||||
statistics_state="verified",
|
||||
)
|
||||
|
||||
assert demo_manifest._knowledge_scenario_ready(health) is False
|
||||
|
||||
@@ -4,6 +4,7 @@ import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.config import get_settings
|
||||
@@ -65,12 +66,17 @@ def test_claim_marks_events_delivering():
|
||||
|
||||
def test_deliver_one_success(monkeypatch):
|
||||
event_id = _make_pending_event("MO-002")
|
||||
execution_id = "n8n-execution-123"
|
||||
dispatcher._claim_due_events()
|
||||
|
||||
def fake_post(url, json, headers, timeout):
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
|
||||
json=lambda: {
|
||||
"ok": True,
|
||||
"event_id": str(event_id),
|
||||
"result": {"execution_id": execution_id},
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
@@ -79,7 +85,7 @@ def test_deliver_one_success(monkeypatch):
|
||||
event = _get_event(event_id)
|
||||
assert event.delivery_status == "succeeded"
|
||||
assert event.attempts == 1
|
||||
assert event.external_run_id == str(event_id)
|
||||
assert event.external_run_id == execution_id
|
||||
assert event.last_error is None
|
||||
assert event.last_error_code is None
|
||||
|
||||
@@ -129,6 +135,106 @@ def test_deliver_one_treats_empty_2xx_body_as_failure(monkeypatch):
|
||||
assert event.last_error_code == "malformedResponse"
|
||||
|
||||
|
||||
def test_deliver_one_rejects_json_object_without_explicit_success_ack(monkeypatch):
|
||||
event_id = _make_pending_event("MO-010")
|
||||
dispatcher._claim_due_events()
|
||||
|
||||
def fake_post(url, json, headers, timeout):
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {},
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
dispatcher._deliver_one(event_id)
|
||||
|
||||
event = _get_event(event_id)
|
||||
assert event.delivery_status == "pending"
|
||||
assert event.attempts == 1
|
||||
assert event.last_error_code == "malformedResponse"
|
||||
assert event.external_run_id is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("result", [{}, {"execution_id": 123}, {"execution_id": " "}])
|
||||
def test_deliver_one_rejects_ack_without_valid_execution_id(monkeypatch, result):
|
||||
event_id = _make_pending_event("MO-EXEC-ID")
|
||||
dispatcher._claim_due_events()
|
||||
|
||||
def fake_post(url, json, headers, timeout):
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {"ok": True, "event_id": str(event_id), "result": result},
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
dispatcher._deliver_one(event_id)
|
||||
event = _get_event(event_id)
|
||||
assert event.delivery_status == "pending"
|
||||
assert event.last_error_code == "malformedResponse"
|
||||
assert event.external_run_id is None
|
||||
|
||||
|
||||
def test_deliver_one_rejects_success_ack_for_a_different_event(monkeypatch):
|
||||
event_id = _make_pending_event("MO-011")
|
||||
dispatcher._claim_due_events()
|
||||
other_event_id = uuid.uuid4()
|
||||
|
||||
def fake_post(url, json, headers, timeout):
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {"ok": True, "event_id": str(other_event_id)},
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
dispatcher._deliver_one(event_id)
|
||||
|
||||
event = _get_event(event_id)
|
||||
assert event.delivery_status == "pending"
|
||||
assert event.attempts == 1
|
||||
assert event.last_error_code == "mismatchedEventId"
|
||||
assert event.external_run_id is None
|
||||
|
||||
|
||||
def test_late_delivery_outcome_cannot_overwrite_a_newer_lease(monkeypatch):
|
||||
event_id = _make_pending_event("MO-LEASE-RACE")
|
||||
dispatcher._claim_due_events()
|
||||
|
||||
def fake_post(url, json, headers, timeout):
|
||||
# Simulate lease expiry/reclaim while the original worker is still in network
|
||||
# I/O. The later claimant owns a different token and its state must win.
|
||||
with SessionLocal() as db:
|
||||
event = db.scalar(
|
||||
select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update()
|
||||
)
|
||||
event.payload_json = {
|
||||
**event.payload_json,
|
||||
"_delivery_claim_token": "newer-worker-token",
|
||||
}
|
||||
event.next_attempt_at = datetime.now(UTC) + timedelta(minutes=1)
|
||||
db.commit()
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {
|
||||
"ok": True,
|
||||
"event_id": str(event_id),
|
||||
"result": {"execution_id": "stale-worker-execution"},
|
||||
},
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
dispatcher._deliver_one(event_id)
|
||||
|
||||
event = _get_event(event_id)
|
||||
assert event.delivery_status == "delivering"
|
||||
assert event.attempts == 0
|
||||
assert event.external_run_id is None
|
||||
assert event.payload_json["_delivery_claim_token"] == "newer-worker-token"
|
||||
|
||||
|
||||
def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
|
||||
event_id = _make_pending_event("MO-004")
|
||||
settings = get_settings()
|
||||
@@ -139,7 +245,6 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
|
||||
for _ in range(settings.n8n_max_attempts):
|
||||
dispatcher._claim_due_events()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
|
||||
@@ -147,6 +252,8 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
claimed = dispatcher._claim_due_events(batch_size=1000)
|
||||
assert event_id in claimed
|
||||
dispatcher._deliver_one(event_id)
|
||||
|
||||
event = _get_event(event_id)
|
||||
@@ -259,7 +366,11 @@ def test_run_dispatch_cycle_recovers_a_stale_lease_before_claiming(monkeypatch):
|
||||
def fake_post(url, json, headers, timeout):
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
|
||||
json=lambda: {
|
||||
"ok": True,
|
||||
"event_id": str(event_id),
|
||||
"result": {"execution_id": "n8n-recovered-execution"},
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
@@ -275,7 +386,11 @@ def test_run_dispatch_cycle_end_to_end(monkeypatch):
|
||||
def fake_post(url, json, headers, timeout):
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
|
||||
json=lambda: {
|
||||
"ok": True,
|
||||
"event_id": str(event_id),
|
||||
"result": {"execution_id": "n8n-cycle-execution"},
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
|
||||
|
||||
@@ -174,11 +174,12 @@ def test_return_idempotency_key_rejects_different_body(ops_client):
|
||||
|
||||
|
||||
def test_return_callback_rejects_malformed_correlation_id(client):
|
||||
event_id = str(uuid.uuid4())
|
||||
response = client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json={"follow_up": "cleaning", "correlation_id": "nope"},
|
||||
json={"event_id": event_id, "follow_up": "cleaning", "correlation_id": "nope"},
|
||||
headers={
|
||||
"Idempotency-Key": str(uuid.uuid4()),
|
||||
"Idempotency-Key": event_id,
|
||||
"X-Service-Token": get_settings().n8n_callback_token,
|
||||
},
|
||||
)
|
||||
@@ -195,10 +196,14 @@ def test_blocked_checkout_booking_can_be_cancelled(ops_client):
|
||||
json={
|
||||
"customer_ref": "CUS-0002",
|
||||
"vehicle_ref": vehicle["public_ref"],
|
||||
"requirements_complete": True,
|
||||
**window,
|
||||
},
|
||||
).json()
|
||||
confirmed = ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
|
||||
json={"confirmation": "Licence and rental conditions checked"},
|
||||
)
|
||||
assert confirmed.status_code == 200
|
||||
checkout = ops_client.post(
|
||||
f"/api/v1/bookings/{booking['public_ref']}/checkout",
|
||||
json={
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
from sqlalchemy import select
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.models.audit import AuditEvent
|
||||
from app.models.outbox import OutboxEvent
|
||||
from app.seed_loader import reset_and_seed
|
||||
from app.services import integration_status
|
||||
|
||||
|
||||
def _reseed() -> None:
|
||||
@@ -108,6 +114,101 @@ def test_integration_status_is_operational_once_all_failed_events_resolved(ops_c
|
||||
assert body["state"] == "operational"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("delivery_status", ["pending", "delivering"])
|
||||
def test_queued_work_without_any_success_is_not_reported_operational(delivery_status):
|
||||
_reseed()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
for event in db.scalars(select(OutboxEvent)).all():
|
||||
event.delivery_status = delivery_status
|
||||
event.last_error = None
|
||||
event.last_error_code = None
|
||||
db.execute(
|
||||
delete(AuditEvent).where(
|
||||
AuditEvent.action.in_(
|
||||
{
|
||||
"data_quality_scan_run",
|
||||
"n8n_procedures_synced",
|
||||
"n8n_workflow_failure_registered",
|
||||
"n8n_workflow_heartbeat",
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
status = integration_status.derive_n8n_status(db)
|
||||
|
||||
assert status.succeeded == 0
|
||||
assert getattr(status, delivery_status) > 0
|
||||
assert status.state == "no_evidence"
|
||||
finally:
|
||||
db.rollback()
|
||||
db.close()
|
||||
|
||||
|
||||
def test_historical_success_is_not_green_when_webhook_is_unconfigured(monkeypatch):
|
||||
_reseed()
|
||||
with SessionLocal() as db:
|
||||
monkeypatch.setattr(integration_status.settings, "n8n_dispatch_enabled", True)
|
||||
monkeypatch.setattr(integration_status.settings, "n8n_webhook_url", "")
|
||||
status = integration_status.derive_n8n_status(db)
|
||||
assert status.configured is False
|
||||
assert status.succeeded > 0
|
||||
assert status.state == "unavailable"
|
||||
|
||||
|
||||
def test_null_coded_real_failure_sets_latest_failure_timestamp():
|
||||
_reseed()
|
||||
with SessionLocal() as db:
|
||||
event = db.scalar(
|
||||
select(OutboxEvent).where(OutboxEvent.delivery_status == "succeeded").limit(1)
|
||||
)
|
||||
event.delivery_status = "failed"
|
||||
event.last_error_code = None
|
||||
db.flush()
|
||||
status = integration_status.derive_n8n_status(db)
|
||||
assert status.unexpected_failed == 1
|
||||
assert status.latest_failure_at is not None
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_unreachable_hub_invalidates_historical_operational_claim(monkeypatch):
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.add(
|
||||
AuditEvent(
|
||||
actor_type="service",
|
||||
actor_id=None,
|
||||
actor_label="itworx-mcp-hub",
|
||||
action="mcp_tool_request",
|
||||
entity_type="integration",
|
||||
entity_id=None,
|
||||
correlation_id=uuid.uuid4(),
|
||||
before_json=None,
|
||||
after_json=None,
|
||||
metadata_json={"tool": "operations_summary"},
|
||||
occurred_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
monkeypatch.setattr(integration_status.settings, "mcp_hub_registration_enabled", True)
|
||||
monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: False)
|
||||
|
||||
unreachable = integration_status.derive_mcp_hub_status(db)
|
||||
|
||||
assert unreachable.total_calls > 0
|
||||
assert unreachable.hub_reachable is False
|
||||
assert unreachable.state == "no_evidence"
|
||||
|
||||
monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: True)
|
||||
reachable = integration_status.derive_mcp_hub_status(db)
|
||||
assert reachable.state == "operational"
|
||||
finally:
|
||||
db.rollback()
|
||||
db.close()
|
||||
|
||||
|
||||
def test_integration_status_lists_all_four_canonical_workflows(ops_client):
|
||||
body = ops_client.get("/api/v1/integrations/status").json()["n8n"]
|
||||
assert body["expected_workflow_count"] == 4
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from app.core.config import get_settings
|
||||
|
||||
@@ -12,19 +13,29 @@ def _callback_headers(event_id: str, token: str | None = None):
|
||||
|
||||
|
||||
def test_callback_rejects_wrong_service_token(client):
|
||||
event_id = str(uuid.uuid4())
|
||||
response = client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json={"follow_up": "cleaning"},
|
||||
headers=_callback_headers(str(uuid.uuid4()), token="wrong-token"),
|
||||
json={
|
||||
"event_id": event_id,
|
||||
"correlation_id": str(uuid.uuid4()),
|
||||
"follow_up": "cleaning",
|
||||
},
|
||||
headers=_callback_headers(event_id, token="wrong-token"),
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
def test_callback_unknown_event_returns_404(client):
|
||||
event_id = str(uuid.uuid4())
|
||||
response = client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json={"follow_up": "cleaning"},
|
||||
headers=_callback_headers(str(uuid.uuid4())),
|
||||
json={
|
||||
"event_id": event_id,
|
||||
"correlation_id": str(uuid.uuid4()),
|
||||
"follow_up": "cleaning",
|
||||
},
|
||||
headers=_callback_headers(event_id),
|
||||
)
|
||||
assert response.status_code == 404
|
||||
|
||||
@@ -39,13 +50,14 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
|
||||
db = SessionLocal()
|
||||
try:
|
||||
booking = db.scalar(select(Booking).limit(1))
|
||||
correlation_id = uuid.uuid4()
|
||||
event = OutboxEvent(
|
||||
event_id=uuid.uuid4(),
|
||||
event_type="vehicle.returned.v1",
|
||||
aggregate_type="booking",
|
||||
aggregate_id=booking.id,
|
||||
payload_json={
|
||||
"correlation_id": str(uuid.uuid4()),
|
||||
"correlation_id": str(correlation_id),
|
||||
"aggregate": {
|
||||
"type": "booking",
|
||||
"id": str(booking.id),
|
||||
@@ -66,12 +78,22 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
|
||||
|
||||
first = client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json={"follow_up": "cleaning", "summary": "test"},
|
||||
json={
|
||||
"event_id": event_id,
|
||||
"correlation_id": str(correlation_id),
|
||||
"follow_up": "cleaning",
|
||||
"summary": "test",
|
||||
},
|
||||
headers=_callback_headers(event_id),
|
||||
)
|
||||
second = client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json={"follow_up": "cleaning", "summary": "test"},
|
||||
json={
|
||||
"event_id": event_id,
|
||||
"correlation_id": str(correlation_id),
|
||||
"follow_up": "cleaning",
|
||||
"summary": "test",
|
||||
},
|
||||
headers=_callback_headers(event_id),
|
||||
)
|
||||
assert first.status_code == 200
|
||||
@@ -82,6 +104,79 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
|
||||
).json()
|
||||
matching = [e for e in audit_events if e["metadata"]["event_id"] == event_id]
|
||||
assert len(matching) == 1
|
||||
assert matching[0]["correlation_id"] == str(correlation_id)
|
||||
|
||||
|
||||
def test_callback_rejects_crossed_event_or_correlation(client):
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.models.outbox import OutboxEvent
|
||||
|
||||
with SessionLocal() as db:
|
||||
event = db.scalar(select(OutboxEvent).limit(1))
|
||||
event_id = str(event.event_id)
|
||||
correlation_id = event.payload_json["correlation_id"]
|
||||
|
||||
crossed_event = client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json={
|
||||
"event_id": str(uuid.uuid4()),
|
||||
"correlation_id": correlation_id,
|
||||
"follow_up": "cleaning",
|
||||
},
|
||||
headers=_callback_headers(event_id),
|
||||
)
|
||||
assert crossed_event.status_code == 409
|
||||
assert crossed_event.json()["error"]["code"] == "CALLBACK_EVENT_MISMATCH"
|
||||
|
||||
crossed_correlation = client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json={
|
||||
"event_id": event_id,
|
||||
"correlation_id": str(uuid.uuid4()),
|
||||
"follow_up": "cleaning",
|
||||
},
|
||||
headers=_callback_headers(event_id),
|
||||
)
|
||||
assert crossed_correlation.status_code == 409
|
||||
assert crossed_correlation.json()["error"]["code"] == "CALLBACK_CORRELATION_MISMATCH"
|
||||
|
||||
|
||||
def test_callback_concurrent_retries_record_one_audit(client, ops_client):
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.models.outbox import OutboxEvent
|
||||
|
||||
with SessionLocal() as db:
|
||||
event = db.scalar(select(OutboxEvent).limit(1))
|
||||
event_id = str(event.event_id)
|
||||
correlation_id = event.payload_json["correlation_id"]
|
||||
body = {
|
||||
"event_id": event_id,
|
||||
"correlation_id": correlation_id,
|
||||
"follow_up": "cleaning",
|
||||
}
|
||||
|
||||
def post_callback(_index: int):
|
||||
return client.post(
|
||||
"/api/v1/integrations/n8n/return-callback",
|
||||
json=body,
|
||||
headers=_callback_headers(event_id),
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
responses = list(pool.map(post_callback, range(2)))
|
||||
assert [response.status_code for response in responses] == [200, 200]
|
||||
matching = [
|
||||
event
|
||||
for event in ops_client.get(
|
||||
"/api/v1/audit", params={"action": "n8n_return_followup_recorded"}
|
||||
).json()
|
||||
if event["metadata"]["event_id"] == event_id
|
||||
]
|
||||
assert len(matching) == 1
|
||||
|
||||
|
||||
def test_scheduled_scan_rejects_wrong_service_token(client):
|
||||
@@ -183,6 +278,42 @@ def test_workflow_error_registers_and_is_idempotent_by_execution_id(client, ops_
|
||||
assert matching[0]["after"]["retry_action"] == "n8n will retry automatically"
|
||||
|
||||
|
||||
def test_workflow_error_preserves_correlation_and_rejects_malformed(client, ops_client):
|
||||
settings = get_settings()
|
||||
headers = {"X-Service-Token": settings.n8n_callback_token}
|
||||
execution_id = str(uuid.uuid4())
|
||||
correlation_id = str(uuid.uuid4())
|
||||
accepted = client.post(
|
||||
"/api/v1/integrations/n8n/workflow-error",
|
||||
json=_workflow_error_body(execution_id, correlation_id=correlation_id),
|
||||
headers=headers,
|
||||
)
|
||||
assert accepted.status_code == 200
|
||||
matching = [
|
||||
event
|
||||
for event in ops_client.get(
|
||||
"/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
|
||||
).json()
|
||||
if event["metadata"]["execution_id"] == execution_id
|
||||
]
|
||||
assert len(matching) == 1
|
||||
assert matching[0]["correlation_id"] == correlation_id
|
||||
|
||||
malformed_execution = str(uuid.uuid4())
|
||||
malformed = client.post(
|
||||
"/api/v1/integrations/n8n/workflow-error",
|
||||
json=_workflow_error_body(malformed_execution, correlation_id="not-a-uuid"),
|
||||
headers=headers,
|
||||
)
|
||||
assert malformed.status_code == 422
|
||||
assert all(
|
||||
event["metadata"]["execution_id"] != malformed_execution
|
||||
for event in ops_client.get(
|
||||
"/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
|
||||
).json()
|
||||
)
|
||||
|
||||
|
||||
def test_workflow_error_bounds_summary_length(client):
|
||||
settings = get_settings()
|
||||
response = client.post(
|
||||
@@ -266,6 +397,26 @@ def test_procedures_sync_result_registers_and_is_idempotent(client, ops_client):
|
||||
assert ragcore_sync["last_seen_at"] is not None
|
||||
|
||||
|
||||
def test_failed_or_partial_procedure_sync_is_never_reported_healthy(client, ops_client):
|
||||
settings = get_settings()
|
||||
headers = {"X-Service-Token": settings.n8n_callback_token}
|
||||
for synced, failed in ((0, 33), (32, 1)):
|
||||
execution_id = str(uuid.uuid4())
|
||||
response = client.post(
|
||||
"/api/v1/integrations/n8n/procedures-sync-result",
|
||||
json={"execution_id": execution_id, "synced": synced, "failed": failed},
|
||||
headers=headers,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
status = ops_client.get("/api/v1/integrations/status").json()["n8n"]
|
||||
workflow = next(w for w in status["workflows"] if "RAGcore" in w["name"])
|
||||
assert workflow["state"] == "failed"
|
||||
assert workflow["last_status"] == "failed"
|
||||
assert workflow["last_execution_id"] == execution_id
|
||||
assert status["state"] == "degraded"
|
||||
|
||||
|
||||
def test_workflow_heartbeat_is_idempotent_and_drives_live_status(client, ops_client):
|
||||
settings = get_settings()
|
||||
execution_id = str(uuid.uuid4())
|
||||
|
||||
+409
-70
@@ -1,9 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.api.routers import knowledge as knowledge_router
|
||||
from app.core.config import get_settings
|
||||
@@ -244,11 +247,11 @@ def test_ragcore_status_preserves_stronger_verified_index_evidence(client, ops_c
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, status_code: int, body: dict):
|
||||
def __init__(self, status_code: int, body: object):
|
||||
self.status_code = status_code
|
||||
self._body = body
|
||||
|
||||
def json(self) -> dict:
|
||||
def json(self) -> object:
|
||||
return self._body
|
||||
|
||||
|
||||
@@ -260,6 +263,7 @@ class _FakeClient:
|
||||
post_responses=None,
|
||||
raise_on=None,
|
||||
get_handler=None,
|
||||
requests=None,
|
||||
):
|
||||
self._get_response = get_response
|
||||
self._post_response = post_response
|
||||
@@ -270,6 +274,7 @@ class _FakeClient:
|
||||
self._post_responses = post_responses or {}
|
||||
self._raise_on = raise_on
|
||||
self._get_handler = get_handler
|
||||
self.requests = requests if requests is not None else []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
@@ -289,6 +294,7 @@ class _FakeClient:
|
||||
def post(self, path, json=None):
|
||||
if self._raise_on == "post":
|
||||
raise httpx.ConnectError("no ragcore in this environment")
|
||||
self.requests.append((path, json))
|
||||
return self._post_responses.get(path, self._post_response)
|
||||
|
||||
|
||||
@@ -393,22 +399,107 @@ def test_ragcore_provider_health_degrades_on_connection_error(monkeypatch):
|
||||
assert "unavailable" in health.detail.lower()
|
||||
|
||||
|
||||
def _answers_body(**overrides) -> dict:
|
||||
_EXCERPTS = {
|
||||
("en-GB", "damage-procedure"): (
|
||||
"When a vehicle returns with visible or reported damage, mark damage in the "
|
||||
"return inspection, add a concise factual description and keep the vehicle "
|
||||
"blocked."
|
||||
),
|
||||
("en-GB", "vehicle-return-procedure"): (
|
||||
"Open the active booking and record the ending odometer, fuel level, cleanliness, "
|
||||
"visible damage, technical warnings and relevant notes."
|
||||
),
|
||||
("en-GB", "cleaning-checklist"): (
|
||||
"Cleaning completion alone does not make a blocked or maintenance vehicle "
|
||||
"available. Fleet Ops derives availability from all active restrictions."
|
||||
),
|
||||
("en-GB", "maintenance-escalation"): (
|
||||
"Escalate when a technical warning is reported, the service threshold is reached, "
|
||||
"a safety-related defect is observed or an existing maintenance block remains "
|
||||
"unresolved."
|
||||
),
|
||||
("nl-BE", "damage-procedure"): (
|
||||
"Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schade "
|
||||
"in de retourinspectie, voeg een beknopte feitelijke beschrijving toe en houd het "
|
||||
"voertuig geblokkeerd."
|
||||
),
|
||||
("nl-BE", "vehicle-return-procedure"): (
|
||||
"Open de actieve boeking en registreer de eindkilometerstand, het brandstofniveau, "
|
||||
"de netheid, zichtbare schade, technische waarschuwingen en relevante notities."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _managed_document(provider: RAGcoreKnowledgeProvider, document_id: str, language: str):
|
||||
return next(
|
||||
document
|
||||
for document in provider._managed_documents_by_source_id.values()
|
||||
if document.document_id == document_id and document.language == language
|
||||
)
|
||||
|
||||
|
||||
def _citation(
|
||||
provider: RAGcoreKnowledgeProvider,
|
||||
document_id: str,
|
||||
*,
|
||||
language: str = "en-GB",
|
||||
section: str = "Procedure",
|
||||
seed: str = "1",
|
||||
) -> dict:
|
||||
document = _managed_document(provider, document_id, language)
|
||||
excerpt = _EXCERPTS[(language, document_id)]
|
||||
return {
|
||||
"id": str(uuid.uuid5(uuid.NAMESPACE_URL, f"citation:{language}:{document_id}:{seed}")),
|
||||
# These are deliberately opaque RAGcore-owned UUIDs, not Fleet Ops' stable
|
||||
# human-readable procedure id.
|
||||
"document_id": str(
|
||||
uuid.uuid5(uuid.NAMESPACE_URL, f"ragcore-document:{language}:{document_id}")
|
||||
),
|
||||
"document_version_id": str(
|
||||
uuid.uuid5(
|
||||
uuid.NAMESPACE_URL,
|
||||
f"ragcore-document-version:{language}:{document_id}:{seed}",
|
||||
)
|
||||
),
|
||||
"title": "untrusted-provider-title.md",
|
||||
"section": section,
|
||||
"excerpt": excerpt,
|
||||
"excerpt_sha256": hashlib.sha256(excerpt.encode("utf-8")).hexdigest(),
|
||||
"source_uri": f"ragcore://source/{document.source_id}",
|
||||
"source_id": document.source_id,
|
||||
"locator": f"{document.document_id}.md",
|
||||
}
|
||||
|
||||
|
||||
def _answers_body(provider: RAGcoreKnowledgeProvider, **overrides) -> dict:
|
||||
citation = _citation(provider, "damage-procedure", section="Detection")
|
||||
answer = "Report damage and route the vehicle to maintenance."
|
||||
body = {
|
||||
"answer": "Report damage and route the vehicle to maintenance.",
|
||||
"answer_id": str(uuid.uuid4()),
|
||||
"retrieval_run_id": str(uuid.uuid4()),
|
||||
"answer": answer,
|
||||
"answerability": "answerable",
|
||||
"citations": [
|
||||
{
|
||||
"id": "cite-1",
|
||||
"document_id": "doc-1",
|
||||
"document_version_id": "version-1",
|
||||
"title": "Damage handling procedure",
|
||||
"section": "Detection",
|
||||
"excerpt": "Inspect the vehicle for visible damage.",
|
||||
}
|
||||
],
|
||||
"citations": [citation],
|
||||
"claims": [{"text": answer, "citation_ids": [citation["id"]]}],
|
||||
}
|
||||
body.update(overrides)
|
||||
if "claims" not in overrides:
|
||||
effective_answer = body.get("answer")
|
||||
effective_citations = body.get("citations")
|
||||
body["claims"] = (
|
||||
[
|
||||
{
|
||||
"text": effective_answer,
|
||||
"citation_ids": [item["id"] for item in effective_citations],
|
||||
}
|
||||
]
|
||||
if isinstance(effective_answer, str)
|
||||
and effective_answer
|
||||
and isinstance(effective_citations, list)
|
||||
and effective_citations
|
||||
and all(isinstance(item, dict) and "id" in item for item in effective_citations)
|
||||
else []
|
||||
)
|
||||
return body
|
||||
|
||||
|
||||
@@ -418,47 +509,178 @@ def test_ragcore_provider_grounded_answer_maps_citations_to_sources(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body())),
|
||||
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(provider))),
|
||||
)
|
||||
answer = provider.ask("What must I do about damage?", "test-correlation-grounded")
|
||||
assert answer.evidence_state == "grounded"
|
||||
assert answer.answer
|
||||
assert len(answer.sources) == 1
|
||||
source = answer.sources[0]
|
||||
assert source.document_id == "doc-1"
|
||||
assert source.document_id == "damage-procedure"
|
||||
assert source.title == "Damage handling procedure"
|
||||
assert source.version == "version-1"
|
||||
assert source.version == "1.3"
|
||||
assert source.section == "Detection"
|
||||
assert source.excerpt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda body: body.pop("claims"),
|
||||
lambda body: body.update(claims=[]),
|
||||
lambda body: body["claims"][0].update(citation_ids=[str(uuid.uuid4())]),
|
||||
lambda body: body.pop("answer_id"),
|
||||
lambda body: body.update(retrieval_run_id="not-a-uuid"),
|
||||
],
|
||||
)
|
||||
def test_ragcore_answer_requires_valid_claim_citation_contract(monkeypatch, mutate):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
body = _answers_body(provider)
|
||||
mutate(body)
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(post_response=_FakeResponse(200, body)),
|
||||
)
|
||||
|
||||
answer = provider.ask("What must I do about damage?", "test-invalid-claim-contract")
|
||||
|
||||
assert answer.evidence_state != "grounded"
|
||||
assert answer.answer == ""
|
||||
assert answer.sources == []
|
||||
|
||||
|
||||
def test_ragcore_requests_are_scoped_to_managed_sources_for_requested_language(
|
||||
monkeypatch,
|
||||
):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
requests = []
|
||||
fake_client = _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {}),
|
||||
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
|
||||
},
|
||||
requests=requests,
|
||||
)
|
||||
monkeypatch.setattr(provider, "_client", lambda: fake_client)
|
||||
|
||||
answer = provider.ask(
|
||||
"Wat is de procedure voor een voertuigretour?",
|
||||
"language-filter",
|
||||
"nl-BE",
|
||||
)
|
||||
|
||||
assert answer.evidence_state == "grounded"
|
||||
assert [path for path, _payload in requests] == ["/v1/answers", "/v1/search"]
|
||||
expected_source_ids = provider._managed_source_ids("nl-BE")
|
||||
assert len(expected_source_ids) == 11
|
||||
assert all(uuid.UUID(source_id) for source_id in expected_source_ids)
|
||||
for _path, payload in requests:
|
||||
assert payload["filters"] == {"source_ids": expected_source_ids}
|
||||
assert payload["requested_space_ids"] == ["space-1"]
|
||||
|
||||
|
||||
def test_ragcore_sources_deduplicate_reuploaded_versions_and_cap_cards(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
citations = []
|
||||
for index in range(5):
|
||||
document_ids = [
|
||||
"damage-procedure",
|
||||
"damage-procedure",
|
||||
"vehicle-return-procedure",
|
||||
"cleaning-checklist",
|
||||
"maintenance-escalation",
|
||||
]
|
||||
for index, document_id in enumerate(document_ids):
|
||||
citations.append(
|
||||
{
|
||||
"id": f"cite-{index}",
|
||||
"document_id": f"doc-{index}",
|
||||
"document_version_id": f"version-{index}",
|
||||
"title": "Damage procedure" if index < 2 else f"Procedure {index}",
|
||||
"section": "Return",
|
||||
"excerpt": (
|
||||
f"Record visible damage before release, chunk {index}."
|
||||
if index < 2
|
||||
else f"Unique procedure evidence {index}."
|
||||
),
|
||||
}
|
||||
_citation(
|
||||
provider,
|
||||
document_id,
|
||||
section="Return",
|
||||
seed=str(index),
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(citations=citations))),
|
||||
lambda: _FakeClient(
|
||||
post_response=_FakeResponse(200, _answers_body(provider, citations=citations))
|
||||
),
|
||||
)
|
||||
answer = provider.ask("What must I do about vehicle damage?", "dedupe-test")
|
||||
assert len(answer.sources) == 3
|
||||
assert sum(source.title == "Damage procedure" for source in answer.sources) == 1
|
||||
assert sum(source.title == "Damage handling procedure" for source in answer.sources) == 1
|
||||
|
||||
|
||||
def test_ragcore_answer_rejects_unknown_document_citation(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_response=_FakeResponse(
|
||||
200,
|
||||
_answers_body(
|
||||
provider,
|
||||
answer="A plausible-looking but unmanaged answer.",
|
||||
citations=[
|
||||
{
|
||||
**_citation(provider, "damage-procedure"),
|
||||
"source_id": str(uuid.uuid4()),
|
||||
}
|
||||
],
|
||||
),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
answer = provider.ask("What must I do about damage?", "unknown-answer-citation")
|
||||
|
||||
assert answer.evidence_state == "insufficient"
|
||||
assert answer.answer == ""
|
||||
assert answer.sources == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field_name", "bad_value"),
|
||||
[
|
||||
("id", "not-a-uuid"),
|
||||
("document_id", "not-a-uuid"),
|
||||
("document_version_id", "00000000-0000-0000-0000-000000000000"),
|
||||
("source_id", "00000000-0000-4000-8000-000000000099"),
|
||||
("source_uri", "ragcore://source/00000000-0000-4000-8000-000000000099"),
|
||||
("locator", "another-procedure.md"),
|
||||
("excerpt_sha256", "0" * 64),
|
||||
("title", None),
|
||||
("section", {"not": "a string"}),
|
||||
],
|
||||
)
|
||||
def test_ragcore_rejects_broken_managed_source_provenance(field_name, bad_value):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
citation = _citation(provider, "damage-procedure")
|
||||
citation[field_name] = bad_value
|
||||
|
||||
assert provider._managed_source(citation, "en-GB") is None
|
||||
|
||||
|
||||
def test_ragcore_rejects_fabricated_excerpt_even_with_matching_hash():
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
citation = _citation(provider, "damage-procedure")
|
||||
fabricated = "Invent a repair price and promise it to the customer."
|
||||
citation["excerpt"] = fabricated
|
||||
citation["excerpt_sha256"] = hashlib.sha256(fabricated.encode("utf-8")).hexdigest()
|
||||
|
||||
assert provider._managed_source(citation, "en-GB") is None
|
||||
|
||||
|
||||
def test_ragcore_never_relabels_an_english_source_as_dutch():
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
citation = _citation(provider, "damage-procedure", language="en-GB")
|
||||
|
||||
assert provider._managed_source(citation, "nl-BE") is None
|
||||
|
||||
|
||||
def test_knowledge_feedback_is_audited_and_can_be_changed(ops_client):
|
||||
@@ -503,6 +725,7 @@ def test_ragcore_provider_not_answerable_is_insufficient_and_never_fabricates(mo
|
||||
post_response=_FakeResponse(
|
||||
200,
|
||||
_answers_body(
|
||||
provider,
|
||||
answer="This should never be shown.",
|
||||
answerability="not_answerable",
|
||||
citations=[],
|
||||
@@ -524,7 +747,8 @@ def test_ragcore_provider_answerable_without_citations_is_insufficient(monkeypat
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_response=_FakeResponse(
|
||||
200, _answers_body(answerability="answerable", citations=[])
|
||||
200,
|
||||
_answers_body(provider, answerability="answerable", citations=[]),
|
||||
)
|
||||
),
|
||||
)
|
||||
@@ -565,19 +789,66 @@ def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch):
|
||||
assert answer.evidence_state == "unavailable"
|
||||
|
||||
|
||||
def _search_body(**overrides) -> dict:
|
||||
def test_ragcore_provider_malformed_top_level_bodies_do_not_escape(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(200, ["not", "an", "object"]),
|
||||
"/v1/search": _FakeResponse(200, None),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
answer = provider.ask("Anything?", "test-correlation-malformed-top-level")
|
||||
|
||||
assert answer.evidence_state == "unavailable"
|
||||
assert answer.answer == ""
|
||||
assert answer.sources == []
|
||||
|
||||
|
||||
def test_ragcore_answer_with_malformed_citation_is_insufficient(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_response=_FakeResponse(
|
||||
200,
|
||||
_answers_body(provider, citations=[None]),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
answer = provider.ask("What must I do about damage?", "malformed-answer-citation")
|
||||
|
||||
assert answer.evidence_state == "insufficient"
|
||||
assert answer.answer == ""
|
||||
assert answer.sources == []
|
||||
|
||||
|
||||
def _search_body(
|
||||
provider: RAGcoreKnowledgeProvider,
|
||||
language: str = "en-GB",
|
||||
**overrides,
|
||||
) -> dict:
|
||||
body = {
|
||||
"retrieval_run_id": str(uuid.uuid4()),
|
||||
"effective_space_ids": ["00000000-0000-4000-8000-000000000001"],
|
||||
"results": [
|
||||
{
|
||||
"chunk_id": "chunk-1",
|
||||
"citation": {
|
||||
"id": "cite-1",
|
||||
"document_id": "doc-1",
|
||||
"document_version_id": "version-1",
|
||||
"title": "Vehicle return procedure",
|
||||
"section": "Return",
|
||||
"excerpt": "Register the return odometer reading before releasing the vehicle.",
|
||||
},
|
||||
"chunk_id": str(uuid.uuid4()),
|
||||
"text": _EXCERPTS[(language, "vehicle-return-procedure")],
|
||||
"citation": _citation(
|
||||
provider,
|
||||
"vehicle-return-procedure",
|
||||
language=language,
|
||||
section="Return",
|
||||
),
|
||||
"rank": 1,
|
||||
"scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None},
|
||||
}
|
||||
@@ -588,6 +859,31 @@ def _search_body(**overrides) -> dict:
|
||||
return body
|
||||
|
||||
|
||||
def test_ragcore_search_ignores_malformed_result_and_citation_bodies(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
malformed_search = _search_body(
|
||||
provider,
|
||||
results=[None, {"citation": ["not-an-object"], "scores": "not-an-object"}],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {}),
|
||||
"/v1/search": _FakeResponse(200, malformed_search),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
answer = provider.ask("What is the vehicle return procedure?", "malformed-search-items")
|
||||
|
||||
assert answer.evidence_state == "insufficient"
|
||||
assert answer.answer == ""
|
||||
assert answer.sources == []
|
||||
|
||||
|
||||
def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch):
|
||||
"""/v1/answers itself failing (a real RAGcore-side outage in its generation step,
|
||||
not a real 'insufficient evidence' classification) must not silently degrade
|
||||
@@ -601,26 +897,24 @@ def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypa
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
||||
"/v1/search": _FakeResponse(200, _search_body()),
|
||||
"/v1/search": _FakeResponse(200, _search_body(provider)),
|
||||
}
|
||||
),
|
||||
)
|
||||
answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback")
|
||||
assert answer.evidence_state == "grounded"
|
||||
assert "Register the return odometer reading" in answer.answer
|
||||
assert "Open the active booking and record the ending odometer" in answer.answer
|
||||
assert "Vehicle return procedure" in answer.answer
|
||||
assert len(answer.sources) == 1
|
||||
assert answer.sources[0].title == "Vehicle return procedure"
|
||||
assert answer.sources[0].excerpt == (
|
||||
"Register the return odometer reading before releasing the vehicle."
|
||||
)
|
||||
assert answer.sources[0].excerpt == _EXCERPTS[("en-GB", "vehicle-return-procedure")]
|
||||
|
||||
|
||||
def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0)
|
||||
search_response = _FakeResponse(200, _search_body())
|
||||
search_response = _FakeResponse(200, _search_body(provider))
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
@@ -632,8 +926,7 @@ def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
|
||||
),
|
||||
)
|
||||
assert (
|
||||
provider.ask("What is the vehicle return procedure?", "first").evidence_state
|
||||
== "grounded"
|
||||
provider.ask("What is the vehicle return procedure?", "first").evidence_state == "grounded"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -654,7 +947,7 @@ def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
||||
"/v1/search": _FakeResponse(200, _search_body()),
|
||||
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
|
||||
}
|
||||
),
|
||||
)
|
||||
@@ -676,7 +969,7 @@ def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkey
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
||||
"/v1/search": _FakeResponse(200, _search_body(results=[])),
|
||||
"/v1/search": _FakeResponse(200, _search_body(provider, results=[])),
|
||||
}
|
||||
),
|
||||
)
|
||||
@@ -695,7 +988,7 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {}),
|
||||
"/v1/search": _FakeResponse(200, _search_body()),
|
||||
"/v1/search": _FakeResponse(200, _search_body(provider)),
|
||||
}
|
||||
),
|
||||
)
|
||||
@@ -707,19 +1000,66 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
|
||||
assert answer.answer == ""
|
||||
|
||||
|
||||
def test_ragcore_search_fallback_rejects_unknown_document_citation(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
search = _search_body(provider)
|
||||
search["results"][0]["citation"]["source_id"] = str(uuid.uuid4())
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {}),
|
||||
"/v1/search": _FakeResponse(200, search),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
answer = provider.ask(
|
||||
"What is the vehicle return procedure?",
|
||||
"unknown-search-citation",
|
||||
)
|
||||
|
||||
assert answer.evidence_state == "insufficient"
|
||||
assert answer.answer == ""
|
||||
assert answer.sources == []
|
||||
|
||||
|
||||
def test_ragcore_search_fallback_rejects_vehicle_colour_despite_high_score(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
search = _search_body(provider)
|
||||
search["results"][0]["scores"]["fused"] = 1.0
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {}),
|
||||
"/v1/search": _FakeResponse(200, search),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
answer = provider.ask("What colour is this vehicle?", "vehicle-colour")
|
||||
|
||||
assert answer.evidence_state == "insufficient"
|
||||
assert answer.answer == ""
|
||||
|
||||
|
||||
def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
search = _search_body()
|
||||
search = _search_body(provider, "nl-BE", results=[])
|
||||
search["results"].append(
|
||||
{
|
||||
"citation": {
|
||||
"document_id": "damage-procedure",
|
||||
"document_version_id": "version-2",
|
||||
"title": "damage-procedure.md",
|
||||
"section": "Damage",
|
||||
"excerpt": "Record damage and keep the vehicle blocked.",
|
||||
},
|
||||
"citation": _citation(
|
||||
provider,
|
||||
"damage-procedure",
|
||||
language="nl-BE",
|
||||
section="Damage",
|
||||
),
|
||||
"rank": 2,
|
||||
"scores": {"fused": 0.01},
|
||||
}
|
||||
@@ -743,16 +1083,15 @@ def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
|
||||
def test_ragcore_search_fallback_accepts_top_ranked_ragcore_damage_evidence(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
search = _search_body()
|
||||
search = _search_body(provider, "nl-BE", results=[])
|
||||
search["results"].append(
|
||||
{
|
||||
"citation": {
|
||||
"document_id": "damage-procedure",
|
||||
"document_version_id": "version-2",
|
||||
"title": "damage-procedure.md",
|
||||
"section": "Damage",
|
||||
"excerpt": "Record damage and keep the vehicle blocked.",
|
||||
},
|
||||
"citation": _citation(
|
||||
provider,
|
||||
"damage-procedure",
|
||||
language="nl-BE",
|
||||
section="Damage",
|
||||
),
|
||||
"rank": 1,
|
||||
# RAGcore uses reciprocal-rank fusion; a genuine rank-one result is about
|
||||
# 1 / (60 + 1), not a normalized 0..1 relevance score.
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
from app.core.config import get_settings
|
||||
from app.main import app
|
||||
|
||||
MCP_ROUTE_ALLOWLIST = {
|
||||
("GET", "/api/v1/integrations/mcp/operations-summary"),
|
||||
("GET", "/api/v1/integrations/mcp/attention-vehicles"),
|
||||
("GET", "/api/v1/integrations/mcp/vehicles/{vehicle_ref}"),
|
||||
("POST", "/api/v1/integrations/mcp/search-knowledge"),
|
||||
}
|
||||
|
||||
|
||||
def _headers(token: str | None = None, client_id: str = "test-mcp-client"):
|
||||
@@ -154,3 +162,26 @@ def test_no_write_endpoints_exist_under_mcp_namespace(client):
|
||||
]:
|
||||
response = getattr(client, method)(path, headers=_headers())
|
||||
assert response.status_code in (404, 405)
|
||||
|
||||
|
||||
def test_mcp_namespace_matches_exact_route_allowlist_and_rejects_lookalikes(client):
|
||||
implemented = {
|
||||
(method.upper(), path)
|
||||
for path, operations in app.openapi()["paths"].items()
|
||||
if path.startswith("/api/v1/integrations/mcp/")
|
||||
for method in operations
|
||||
if method in {"get", "post", "put", "patch", "delete"}
|
||||
}
|
||||
assert implemented == MCP_ROUTE_ALLOWLIST
|
||||
|
||||
lookalikes = [
|
||||
("get", "/api/v1/integrations/mcp-extra/operations-summary"),
|
||||
("get", "/api/v1/integrations/mcp/operations-summary-extra"),
|
||||
("get", "/api/v1/integrations/mcp/prefix/attention-vehicles"),
|
||||
("get", "/api/v1/integrations/mcp/attention-vehicles/suffix"),
|
||||
("post", "/api/v1/integrations/mcp/search-knowledge-extra"),
|
||||
("post", "/api/v1/integrations/mcp/prefix/search-knowledge"),
|
||||
]
|
||||
for method, path in lookalikes:
|
||||
response = client.request(method, path, headers=_headers(), json={})
|
||||
assert response.status_code == 404, path
|
||||
|
||||
@@ -63,6 +63,24 @@ def test_eligible_customer_is_irreversibly_anonymized_without_pii_in_audit(ops_c
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "anonymized"
|
||||
search = ops_client.get("/api/v1/customers", params={"query": "CUS-PRIVACY"})
|
||||
assert search.status_code == 200
|
||||
assert search.json() == []
|
||||
window = {
|
||||
"starts_at": "2090-01-01T10:00:00Z",
|
||||
"ends_at": "2090-01-02T10:00:00Z",
|
||||
}
|
||||
available = ops_client.get("/api/v1/bookings/availability", params=window).json()
|
||||
assert available
|
||||
booking = ops_client.post(
|
||||
"/api/v1/bookings",
|
||||
json={
|
||||
"customer_ref": "CUS-PRIVACY",
|
||||
"vehicle_ref": available[0]["public_ref"],
|
||||
**window,
|
||||
},
|
||||
)
|
||||
assert booking.status_code == 422
|
||||
with SessionLocal() as db:
|
||||
customer = db.scalar(select(Customer).where(Customer.id == customer_id))
|
||||
assert customer is not None
|
||||
|
||||
@@ -128,6 +128,23 @@ def test_seed_scenario_s4_booking_overlap():
|
||||
db.close()
|
||||
|
||||
|
||||
def test_seed_odometer_issues_expose_only_the_regressing_booking_as_correctable():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
reset_and_seed(db)
|
||||
issue = _by_ref(db, DataQualityIssue, "DQ-0007")
|
||||
assert issue is not None
|
||||
assert issue.evidence_json["source_type"] == "return"
|
||||
assert issue.evidence_json["related_refs"] == [
|
||||
"INSP-0057",
|
||||
"BK-H-0007",
|
||||
"INSP-0007",
|
||||
]
|
||||
assert issue.evidence_json["correctable_booking_refs"] == ["BK-H-0007"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_seed_scenario_s5_failed_workflow_run():
|
||||
"""S5: one seeded outbox event is durably 'failed' (terminal, retryable), not merely
|
||||
pending, so the background dispatcher never silently auto-heals it away."""
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = next(
|
||||
parent
|
||||
for parent in Path(__file__).resolve().parents
|
||||
if (parent / "seed" / "generate_seed.py").is_file()
|
||||
)
|
||||
SEED_DIRECTORY = REPOSITORY_ROOT / "seed"
|
||||
|
||||
|
||||
def _rows_by_ref(path: Path) -> dict[str, dict[str, str]]:
|
||||
with path.open(newline="", encoding="utf-8") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
assert all(None not in row for row in rows), f"malformed CSV row in {path.name}"
|
||||
return {row["public_ref"]: row for row in rows}
|
||||
|
||||
|
||||
def test_generator_reproduces_committed_seed_snapshot_byte_for_byte(tmp_path: Path):
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SEED_DIRECTORY / "generate_seed.py"),
|
||||
"--anchor",
|
||||
"2026-08-01",
|
||||
"--seed",
|
||||
"20260801",
|
||||
"--out",
|
||||
str(tmp_path),
|
||||
],
|
||||
cwd=REPOSITORY_ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
committed_files = sorted(SEED_DIRECTORY.glob("*.csv"))
|
||||
generated_files = sorted(tmp_path.glob("*.csv"))
|
||||
assert [path.name for path in generated_files] == [path.name for path in committed_files]
|
||||
for committed in committed_files:
|
||||
generated = tmp_path / committed.name
|
||||
assert generated.read_bytes() == committed.read_bytes(), (
|
||||
f"{committed.name} differs from the deterministic generated snapshot"
|
||||
)
|
||||
|
||||
bookings = _rows_by_ref(tmp_path / "bookings.csv")
|
||||
inspections = _rows_by_ref(tmp_path / "inspections.csv")
|
||||
issues = _rows_by_ref(tmp_path / "data_quality_issues.csv")
|
||||
|
||||
assert int(bookings["BK-H-0007"]["end_odometer_km"]) < int(
|
||||
bookings["BK-H-0057"]["end_odometer_km"]
|
||||
)
|
||||
assert int(bookings["BK-H-0010"]["end_odometer_km"]) < int(
|
||||
bookings["BK-H-0060"]["end_odometer_km"]
|
||||
)
|
||||
assert inspections["INSP-0007"]["odometer_km"] == bookings["BK-H-0007"]["end_odometer_km"]
|
||||
assert inspections["INSP-0010"]["odometer_km"] == bookings["BK-H-0010"]["end_odometer_km"]
|
||||
assert inspections["INSP-0001"]["completed_at"] == bookings["BK-H-0001"]["ends_at"]
|
||||
|
||||
expected_issue_refs = {f"DQ-{index:04d}" for index in range(5, 22)}
|
||||
assert expected_issue_refs <= issues.keys()
|
||||
assert all(
|
||||
issues[public_ref]["evidence"] != "Synthetic deterministic seed issue"
|
||||
for public_ref in expected_issue_refs
|
||||
)
|
||||
assert "INSP-0007" in issues["DQ-0007"]["evidence"]
|
||||
assert "INSP-0057" in issues["DQ-0007"]["evidence"]
|
||||
assert "INSP-0010" in issues["DQ-0010"]["evidence"]
|
||||
assert "INSP-0060" in issues["DQ-0010"]["evidence"]
|
||||
@@ -23,3 +23,22 @@ def test_manager_can_create_and_update_user(ops_client):
|
||||
|
||||
def test_employee_cannot_manage_users(employee_client):
|
||||
assert employee_client.get("/api/v1/users").status_code == 403
|
||||
assert (
|
||||
employee_client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"email": "unauthorised@example.test",
|
||||
"display_name": "Unauthorised User",
|
||||
"role": "rental_employee",
|
||||
"password": "a-secure-demo-password",
|
||||
},
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
assert (
|
||||
employee_client.patch(
|
||||
"/api/v1/users/USR-OPS",
|
||||
json={"display_name": "Unauthorised Change"},
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
|
||||
@@ -46,6 +46,12 @@ def test_vehicle_detail_includes_related_records(ops_client):
|
||||
assert len(body["quality_issues"]) >= 1
|
||||
|
||||
|
||||
def test_employee_vehicle_detail_never_exposes_quality_evidence(employee_client):
|
||||
response = employee_client.get("/api/v1/vehicles/MO-016")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["quality_issues"] == []
|
||||
|
||||
|
||||
def test_vehicle_detail_404_for_unknown_ref(ops_client):
|
||||
response = ops_client.get("/api/v1/vehicles/MO-999")
|
||||
assert response.status_code == 404
|
||||
@@ -78,6 +84,53 @@ def test_manager_can_record_maintenance_and_release_vehicle(ops_client):
|
||||
assert released.json()["operational_status"] == "available"
|
||||
|
||||
|
||||
def test_low_maintenance_reading_keeps_canonical_and_opens_quality_issue(ops_client):
|
||||
existing_issues = ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
unavailable_refs = {issue["entity_ref"] for issue in existing_issues}
|
||||
vehicles = ops_client.get("/api/v1/vehicles", params={"status": "available"}).json()
|
||||
vehicle = next(
|
||||
candidate
|
||||
for candidate in vehicles
|
||||
if candidate["odometer_km"] > 0 and candidate["public_ref"] not in unavailable_refs
|
||||
)
|
||||
|
||||
response = ops_client.post(
|
||||
f"/api/v1/vehicles/{vehicle['public_ref']}/maintenance",
|
||||
json={
|
||||
"occurred_at": "2053-01-15T12:00:00Z",
|
||||
"odometer_km": vehicle["odometer_km"] - 1,
|
||||
"category": "inspection",
|
||||
"summary": "Imported workshop reading requires verification.",
|
||||
"mark_maintenance": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
detail = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
|
||||
assert detail["odometer_km"] == vehicle["odometer_km"]
|
||||
issues = ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"status": "open", "rule_type": "odometer_regression"},
|
||||
).json()
|
||||
issue = next(item for item in issues if item["entity_ref"] == vehicle["public_ref"])
|
||||
assert issue["evidence"]["source_type"] == "maintenance"
|
||||
assert issue["evidence"]["correctable_booking_refs"] == []
|
||||
issue_detail = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
|
||||
assert any(
|
||||
snapshot["entity_type"] == "maintenance"
|
||||
and snapshot["public_ref"] == response.json()["public_ref"]
|
||||
for snapshot in issue_detail["related_snapshots"]
|
||||
)
|
||||
retained = ops_client.post(
|
||||
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
|
||||
json={"decision": "retain_canonical"},
|
||||
)
|
||||
assert retained.status_code == 200
|
||||
|
||||
|
||||
def test_employee_cannot_record_maintenance(employee_client):
|
||||
response = employee_client.post(
|
||||
"/api/v1/vehicles/MO-001/maintenance",
|
||||
|
||||
Reference in New Issue
Block a user