M54: harden operations and demo resilience
MobilityOps acceptance / backend (push) Failing after 19s
MobilityOps acceptance / frontend (push) Successful in 25s
MobilityOps acceptance / e2e (push) Skipped

This commit is contained in:
NuklearRabbit
2026-08-24 03:31:03 +02:00
parent b0706989db
commit 81e3fd63bd
101 changed files with 5641 additions and 828 deletions
+105 -2
View File
@@ -1,3 +1,15 @@
import threading
import pytest
from sqlalchemy import delete, func, select
from app.core.db import SessionLocal, engine
from app.models.audit import AuditEvent
from app.models.vehicle import Vehicle
from app.seed_loader import reset_and_seed
from app.services.audit import record_audit_event
def test_unauthenticated_dashboard_is_rejected(client):
response = client.get("/api/v1/dashboard")
assert response.status_code == 401
@@ -101,15 +113,106 @@ def test_demo_reset_rejects_concurrent_rebuild(ops_client):
assert response.status_code == 409
def test_demo_reset_database_lock_blocks_another_api_replica(ops_client):
import app.api.routers.demo as demo_router
# Use a raw connection: an ordinary Session intentionally participates in the
# shared mutation barrier and would make an exclusive reset wait before it can test
# this separate non-blocking replica lock.
with engine.connect() as lock_connection:
lock_connection.scalar(select(func.pg_advisory_lock(demo_router._RESET_ADVISORY_LOCK_ID)))
try:
response = ops_client.post("/api/v1/demo/reset")
finally:
lock_connection.scalar(
select(func.pg_advisory_unlock(demo_router._RESET_ADVISORY_LOCK_ID))
)
assert response.status_code == 409
def test_demo_reset_waits_for_active_mutation_and_then_replaces_it_atomically():
started = threading.Event()
completed = threading.Event()
errors: list[BaseException] = []
def reset_in_other_session() -> None:
started.set()
try:
with SessionLocal() as reset_db:
reset_and_seed(reset_db)
except BaseException as exc: # pragma: no cover - assertion reports thread failures
errors.append(exc)
finally:
completed.set()
with SessionLocal() as mutation_db:
vehicle = mutation_db.scalar(
select(Vehicle).where(Vehicle.public_ref == "MO-001").with_for_update()
)
vehicle.location = "Concurrent mutation marker"
mutation_db.flush()
worker = threading.Thread(target=reset_in_other_session, daemon=True)
worker.start()
assert started.wait(timeout=1)
assert not completed.wait(timeout=0.2)
mutation_db.commit()
worker.join(timeout=10)
assert not worker.is_alive(), "reset deadlocked behind the active mutation"
assert errors == []
with SessionLocal() as db:
restored = db.scalar(select(Vehicle).where(Vehicle.public_ref == "MO-001"))
assert restored.location != "Concurrent mutation marker"
def test_demo_reset_cooldown_returns_retry_after(ops_client, monkeypatch):
import app.api.routers.demo as demo_router
monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 60)
monkeypatch.setattr(demo_router, "_last_reset_monotonic", demo_router.time.monotonic())
with SessionLocal() as db:
cooldown_event = record_audit_event(
db,
actor_type="user",
actor_label="Cooldown test",
action="demo_reset",
entity_type="system",
)
db.commit()
cooldown_event_id = cooldown_event.id
response = ops_client.post("/api/v1/demo/reset")
assert response.status_code == 429
assert int(response.headers["retry-after"]) >= 1
monkeypatch.setattr(demo_router, "_last_reset_monotonic", 0.0)
with SessionLocal() as db:
db.execute(delete(AuditEvent).where(AuditEvent.id == cooldown_event_id))
db.commit()
def test_demo_reset_rolls_back_every_change_when_integrity_check_fails(ops_client, monkeypatch):
import app.api.routers.demo as demo_router
monkeypatch.setattr(demo_router.settings, "demo_reset_cooldown_seconds", 0)
with SessionLocal() as db:
probe = record_audit_event(
db,
actor_type="system",
actor_label="Atomic reset test",
action="reset_atomicity_probe",
entity_type="system",
)
db.commit()
probe_id = probe.id
def fail_integrity_check(_db):
raise RuntimeError("Injected integrity-check failure")
monkeypatch.setattr(demo_router, "scenario_integrity_report", fail_integrity_check)
with pytest.raises(RuntimeError, match="Injected integrity-check failure"):
ops_client.post("/api/v1/demo/reset")
with SessionLocal() as db:
assert db.scalar(select(AuditEvent).where(AuditEvent.id == probe_id)) is not None
db.execute(delete(AuditEvent).where(AuditEvent.id == probe_id))
db.commit()
def test_reset_is_rejected_when_demo_allow_reset_is_disabled(ops_client, monkeypatch):
+127 -1
View File
@@ -95,6 +95,16 @@ def test_booking_requirements_are_explicit_and_audited(ops_client):
)
assert checkout.status_code == 409
whitespace_only = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": " "},
)
assert whitespace_only.status_code == 422
assert (
ops_client.get(f"/api/v1/bookings/{booking['public_ref']}").json()["requirements_complete"]
is False
)
confirmed = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
@@ -106,6 +116,29 @@ def test_booking_requirements_are_explicit_and_audited(ops_client):
assert any(item["entity_ref"] == booking["public_ref"] for item in audit.json())
def test_booking_creation_cannot_preconfirm_requirements(ops_client):
window = {"starts_at": "2033-09-01T10:00:00Z", "ends_at": "2033-09-02T12:00:00Z"}
vehicle = ops_client.get("/api/v1/bookings/availability", params=window).json()[0]
created = ops_client.post(
"/api/v1/bookings",
json={
"customer_ref": "CUS-0001",
"vehicle_ref": vehicle["public_ref"],
"requirements_complete": True,
**window,
},
)
assert created.status_code == 201
assert created.json()["requirements_complete"] is False
audit = ops_client.get(
"/api/v1/audit",
params={"action": "booking_requirements_completed"},
)
assert all(item["entity_ref"] != created.json()["public_ref"] for item in audit.json())
def test_customer_search_returns_canonical_customers(ops_client):
response = ops_client.get("/api/v1/customers", params={"query": "CUS-"})
assert response.status_code == 200
@@ -221,10 +254,14 @@ def test_checkout_records_inspection_and_activates_safe_booking(ops_client):
json={
"customer_ref": "CUS-0001",
"vehicle_ref": vehicle["public_ref"],
"requirements_complete": True,
**window,
},
).json()
confirmed = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
)
assert confirmed.status_code == 200
response = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/checkout",
json={
@@ -241,6 +278,95 @@ def test_checkout_records_inspection_and_activates_safe_booking(ops_client):
updated_vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
assert updated_vehicle["operational_status"] == "rented"
assert any(item["type"] == "checkout" for item in updated_vehicle["inspections"])
checkout_audit = ops_client.get(
"/api/v1/audit",
params={"action": "booking_checkout_recorded"},
).json()
booking_event = next(
item for item in checkout_audit if item["entity_ref"] == booking["public_ref"]
)
vehicle_audit = ops_client.get(
"/api/v1/audit",
params={
"action": "vehicle_status_changed",
"correlation_id": booking_event["correlation_id"],
},
).json()
assert len(vehicle_audit) == 1
assert vehicle_audit[0]["entity_ref"] == vehicle["public_ref"]
assert vehicle_audit[0]["before"]["operational_status"] == "available"
assert vehicle_audit[0]["after"]["operational_status"] == "rented"
def test_checkout_odometer_regression_keeps_canonical_and_opens_bounded_quality_issue(
ops_client,
):
window = {"starts_at": "2052-09-01T10:00:00Z", "ends_at": "2052-09-02T12:00:00Z"}
existing_issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
unavailable_refs = {issue["entity_ref"] for issue in existing_issues}
available = ops_client.get("/api/v1/bookings/availability", params=window).json()
vehicle_option = next(
item
for item in available
if item["operational_status"] == "available" and item["public_ref"] not in unavailable_refs
)
vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle_option['public_ref']}").json()
assert vehicle["odometer_km"] > 0
booking = ops_client.post(
"/api/v1/bookings",
json={
"customer_ref": "CUS-0003",
"vehicle_ref": vehicle["public_ref"],
**window,
},
).json()
ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
)
checkout = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/checkout",
json={
"start_odometer_km": vehicle["odometer_km"] - 1,
"fuel_level_percent": 90,
"cleanliness_ok": True,
"damage_reported": False,
"technical_warning": False,
},
)
assert checkout.status_code == 200
assert checkout.json()["booking_status"] == "blocked"
updated_vehicle = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
assert updated_vehicle["odometer_km"] == vehicle["odometer_km"]
issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
issue = next(item for item in issues if item["entity_ref"] == vehicle["public_ref"])
assert issue["evidence"]["source_type"] == "checkout"
assert issue["evidence"]["correctable_booking_refs"] == []
detail = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
assert any(snapshot["entity_type"] == "inspection" for snapshot in detail["related_snapshots"])
invalid_correction = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": booking["public_ref"],
"corrected_odometer_km": vehicle["odometer_km"],
},
)
assert invalid_correction.status_code == 422
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={"decision": "retain_canonical"},
)
assert retained.status_code == 200
assert retained.json()["status"] == "resolved"
def test_get_booking_detail(ops_client):
+43
View File
@@ -1,3 +1,9 @@
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.vehicle import Vehicle
def test_dashboard_metrics_are_persisted_counts(ops_client):
response = ops_client.get("/api/v1/dashboard")
assert response.status_code == 200
@@ -15,6 +21,43 @@ def test_dashboard_metrics_are_persisted_counts(ops_client):
assert metrics["pending_or_failed_workflows"] >= 1
def test_dashboard_metrics_change_when_persisted_fleet_data_changes(ops_client):
before = ops_client.get("/api/v1/dashboard").json()["metrics"]
db = SessionLocal()
probe = Vehicle(
public_ref="MO-METRIC-PROBE",
make="Synthetic",
model="Metric probe",
model_year=2026,
registration_number="TEST-METRIC-PROBE",
location="Test fixture",
operational_status="blocked",
odometer_km=0,
next_service_km=1,
active=True,
version=1,
)
try:
db.add(probe)
db.commit()
after_response = ops_client.get("/api/v1/dashboard")
assert after_response.status_code == 200
after = after_response.json()["metrics"]
assert after["blocked"] == before["blocked"] + 1
fleet_statuses = ("available", "rented", "cleaning", "maintenance", "blocked")
assert (
sum(after[status] for status in fleet_statuses)
== sum(before[status] for status in fleet_statuses) + 1
)
finally:
persisted_probe = db.scalar(select(Vehicle).where(Vehicle.public_ref == "MO-METRIC-PROBE"))
if persisted_probe is not None:
db.delete(persisted_probe)
db.commit()
db.close()
def test_dashboard_attention_items_link_to_records(ops_client):
response = ops_client.get("/api/v1/dashboard")
body = response.json()
+818 -4
View File
@@ -1,8 +1,25 @@
from sqlalchemy import select
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from threading import Barrier
from sqlalchemy import delete, select, text
from app.core.db import SessionLocal
from app.models.audit import AuditEvent
from app.models.booking import Booking
from app.models.customer import Customer
from app.models.data_quality import DataQualityIssue
from app.models.idempotency import IdempotencyRecord
from app.models.inspection import Inspection
from app.models.maintenance import MaintenanceRecord
from app.models.outbox import OutboxEvent
from app.models.vehicle import Vehicle
from app.schemas import CurrentUser, RegisterReturnRequest, ResolveOdometerRegressionRequest
from app.services.data_quality import (
open_odometer_regression_issue,
resolve_odometer_regression,
)
from app.services.returns import register_vehicle_return
def _activate_booking(vehicle_ref: str, start_odometer_km: int) -> str:
@@ -21,6 +38,30 @@ def _activate_booking(vehicle_ref: str, start_odometer_km: int) -> str:
db.close()
def _cleanup_odometer_scenario(vehicle_id, customer_id) -> None:
"""Remove one isolated DQ-03 scenario in foreign-key-safe order."""
with SessionLocal() as db:
booking_ids = db.scalars(select(Booking.id).where(Booking.vehicle_id == vehicle_id)).all()
issue_ids = db.scalars(
select(DataQualityIssue.id).where(DataQualityIssue.entity_id == vehicle_id)
).all()
audit_entity_ids = [vehicle_id, *booking_ids, *issue_ids]
if audit_entity_ids:
db.execute(delete(AuditEvent).where(AuditEvent.entity_id.in_(audit_entity_ids)))
if booking_ids:
db.execute(
delete(IdempotencyRecord).where(IdempotencyRecord.booking_id.in_(booking_ids))
)
db.execute(delete(OutboxEvent).where(OutboxEvent.aggregate_id.in_(booking_ids)))
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.execute(delete(Customer).where(Customer.id == customer_id))
db.commit()
def test_list_includes_all_five_rule_types(ops_client):
response = ops_client.get("/api/v1/data-quality/issues")
assert response.status_code == 200
@@ -43,6 +84,84 @@ def test_scan_is_idempotent_once_seeded(ops_client):
assert response.json()["created"] == {}
def test_scan_detects_imported_maintenance_odometer_regression(ops_client):
with SessionLocal() as db:
vehicle = Vehicle(
public_ref="MO-DQ-SCAN",
make="Synthetic",
model="Scanner",
model_year=2026,
registration_number="DQ-SCAN-01",
location="Brussels",
operational_status="available",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add(vehicle)
db.flush()
vehicle_id = vehicle.id
db.add_all(
[
MaintenanceRecord(
public_ref="MAINT-DQ-SCAN-A",
vehicle_id=vehicle.id,
occurred_at=datetime(2045, 1, 1, tzinfo=UTC),
odometer_km=20_000,
category="inspection",
summary="Synthetic scan baseline",
),
MaintenanceRecord(
public_ref="MAINT-DQ-SCAN-B",
vehicle_id=vehicle.id,
occurred_at=datetime(2045, 2, 1, tzinfo=UTC),
odometer_km=19_000,
category="inspection",
summary="Synthetic imported regression",
),
]
)
db.commit()
try:
response = ops_client.post("/api/v1/data-quality/scan")
assert response.status_code == 200
assert response.json()["created"]["odometer_regression"] >= 1
issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
issue = next(item for item in issues if item["entity_ref"] == "MO-DQ-SCAN")
assert issue["evidence"]["source_type"] == "maintenance"
assert issue["evidence"]["correctable_booking_refs"] == []
# A later imported regression must enrich the existing open issue instead of
# being silently dropped by the generic check-then-return scanner path.
with SessionLocal() as db:
db.add(
MaintenanceRecord(
public_ref="MAINT-DQ-SCAN-C",
vehicle_id=vehicle_id,
occurred_at=datetime(2045, 3, 1, tzinfo=UTC),
odometer_km=18_000,
category="inspection",
summary="Second synthetic imported regression",
)
)
db.commit()
rescanned = ops_client.post("/api/v1/data-quality/scan")
assert rescanned.status_code == 200
enriched = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
later_refs = {signal["params"]["later_ref"] for signal in enriched["evidence"]["signals"]}
assert {"MAINT-DQ-SCAN-B", "MAINT-DQ-SCAN-C"}.issubset(later_refs)
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.commit()
def test_scan_requires_operations_manager(employee_client):
response = employee_client.post("/api/v1/data-quality/scan")
assert response.status_code == 403
@@ -440,9 +559,8 @@ def test_resolve_odometer_regression_correction_below_canonical_is_rejected_then
def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_client):
# The seeded odometer_regression issues carry no related booking (CSV-only rows).
# Create a fresh one with a real related booking via a live regression return, so
# the "correct_reading" path has an actual booking_ref to target.
# Create a live return regression to prove the complete correction chain updates
# both persisted representations of that reading (booking + return inspection).
booking_ref = _activate_booking("MO-018", start_odometer_km=12000)
vehicle_before = ops_client.get("/api/v1/vehicles/MO-018").json()
low_reading = vehicle_before["odometer_km"] - 200
@@ -459,6 +577,7 @@ def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_clien
)
assert returned.status_code == 201
issue_ref = returned.json()["quality_issue_ref"]
inspection_ref = returned.json()["inspection_ref"]
assert issue_ref is not None
corrected = vehicle_before["odometer_km"] + 500
@@ -477,6 +596,701 @@ def test_resolve_odometer_regression_correct_reading_updates_canonical(ops_clien
assert vehicle["odometer_km"] == corrected
booking = ops_client.get(f"/api/v1/bookings/{booking_ref}").json()
assert booking["end_odometer_km"] == corrected
with SessionLocal() as db:
inspection = db.scalar(select(Inspection).where(Inspection.public_ref == inspection_ref))
assert inspection is not None
assert inspection.odometer_km == corrected
def test_correcting_one_of_two_regressions_keeps_the_other_open(ops_client):
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-MULTI",
first_name="Synthetic",
last_name="Multi",
email="dq-multi@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-MULTI",
make="Synthetic",
model="Multi",
model_year=2026,
registration_number="DQ-MULTI",
location="Brussels",
operational_status="available",
odometer_km=1_000,
next_service_km=2_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
bookings: list[Booking] = []
inspections: list[Inspection] = []
for index, reading in enumerate((900, 800), start=1):
booking = Booking(
public_ref=f"BK-DQ-MULTI-{index}",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=datetime(2046, index, 1, tzinfo=UTC),
ends_at=datetime(2046, index, 2, tzinfo=UTC),
status="returned",
start_odometer_km=reading - 10,
end_odometer_km=reading,
requirements_complete=True,
)
db.add(booking)
db.flush()
inspection = Inspection(
public_ref=f"INSP-DQ-M-{index}",
booking_id=booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=reading,
completed_at=booking.ends_at,
)
db.add(inspection)
db.flush()
open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=inspection.public_ref,
reading_km=reading,
canonical_km=vehicle.odometer_km,
source_type="return",
related_refs=[booking.public_ref, inspection.public_ref],
correctable_booking_refs=[booking.public_ref],
public_ref="DQ-MULTI-SOURCE" if index == 1 else None,
)
bookings.append(booking)
inspections.append(inspection)
booking_refs = [booking.public_ref for booking in bookings]
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
first = ops_client.post(
"/api/v1/data-quality/issues/DQ-MULTI-SOURCE/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": booking_refs[0],
"corrected_odometer_km": 1_100,
},
)
assert first.status_code == 200, first.text
assert first.json()["status"] == "open"
assert first.json()["evidence"]["correctable_booking_refs"] == [booking_refs[1]]
second = ops_client.post(
"/api/v1/data-quality/issues/DQ-MULTI-SOURCE/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": booking_refs[1],
"corrected_odometer_km": 1_200,
},
)
assert second.status_code == 200, second.text
assert second.json()["status"] == "resolved"
with SessionLocal() as db:
persisted = db.scalars(
select(Inspection)
.where(Inspection.vehicle_id == vehicle_id)
.order_by(Inspection.public_ref)
).all()
assert [item.odometer_km for item in persisted] == [1_100, 1_200]
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.execute(delete(Customer).where(Customer.id == customer_id))
db.commit()
def test_retained_regression_is_suppressed_until_source_reading_changes(ops_client):
with SessionLocal() as db:
vehicle = Vehicle(
public_ref="MO-DQ-RETAIN",
make="Synthetic",
model="Retain",
model_year=2026,
registration_number="DQ-RETAIN",
location="Brussels",
operational_status="available",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add(vehicle)
db.flush()
vehicle_id = vehicle.id
db.add_all(
[
MaintenanceRecord(
public_ref="MAINT-DQ-RET-A",
vehicle_id=vehicle.id,
occurred_at=datetime(2047, 1, 1, tzinfo=UTC),
odometer_km=20_000,
category="inspection",
summary="Retain baseline",
),
MaintenanceRecord(
public_ref="MAINT-DQ-RET-B",
vehicle_id=vehicle.id,
occurred_at=datetime(2047, 2, 1, tzinfo=UTC),
odometer_km=19_000,
category="inspection",
summary="Retained source reading",
),
]
)
db.commit()
try:
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
issue = next(
item
for item in ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
if item["entity_ref"] == "MO-DQ-RETAIN"
)
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={"decision": "retain_canonical", "note": "Verified source entry"},
)
assert retained.status_code == 200
assert retained.json()["status"] == "resolved"
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
with SessionLocal() as db:
assert (
db.scalar(
select(DataQualityIssue).where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.status == "open",
)
)
is None
)
changed = db.scalar(
select(MaintenanceRecord).where(MaintenanceRecord.public_ref == "MAINT-DQ-RET-B")
)
changed.odometer_km = 18_999
db.commit()
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
reopened = next(
item
for item in ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
if item["entity_ref"] == "MO-DQ-RETAIN"
)
assert reopened["evidence"]["reopened_from"] == issue["public_ref"]
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(MaintenanceRecord).where(MaintenanceRecord.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.commit()
def test_live_return_retained_as_canonical_is_not_reopened_by_scan(ops_client):
now = datetime.now(UTC)
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-LIVE-RET",
first_name="Synthetic",
last_name="Retained return",
email="dq-live-retain@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-LIVE-RET",
make="Synthetic",
model="Retained return",
model_year=2026,
registration_number="DQ-LIVE-RET",
location="Brussels",
operational_status="rented",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
booking = Booking(
public_ref="BK-DQ-LIVE-RET",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=2),
ends_at=now + timedelta(days=1),
status="active",
start_odometer_km=19_500,
end_odometer_km=None,
requirements_complete=True,
)
db.add_all(
[
booking,
MaintenanceRecord(
public_ref="MNT-DQ-LIVE-RET",
vehicle_id=vehicle.id,
occurred_at=now - timedelta(days=3),
odometer_km=20_000,
category="inspection",
summary="Synthetic canonical baseline",
),
]
)
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
returned = ops_client.post(
"/api/v1/bookings/BK-DQ-LIVE-RET/return",
json={
"end_odometer_km": 19_000,
"fuel_level_percent": 50,
"cleanliness_ok": True,
"damage_reported": False,
"technical_warning": False,
},
headers={"Idempotency-Key": "test-dq-live-retain-001"},
)
assert returned.status_code == 201, returned.text
issue_ref = returned.json()["quality_issue_ref"]
inspection_ref = returned.json()["inspection_ref"]
assert issue_ref is not None
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue_ref}/resolve-odometer-regression",
json={"decision": "retain_canonical", "note": "Source reading verified as wrong."},
)
assert retained.status_code == 200, retained.text
assert retained.json()["status"] == "resolved"
assert retained.json()["evidence"]["retained_odometer_fingerprints"] == [
{
"source_type": "return",
"later_ref": inspection_ref,
"later_km": 19_000,
}
]
scanned = ops_client.post("/api/v1/data-quality/scan")
assert scanned.status_code == 200, scanned.text
with SessionLocal() as db:
issues = db.scalars(
select(DataQualityIssue)
.where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.rule_type == "odometer_regression",
)
.order_by(DataQualityIssue.detected_at)
).all()
assert [(issue.public_ref, issue.status) for issue in issues] == [
(issue_ref, "resolved")
]
finally:
_cleanup_odometer_scenario(vehicle_id, customer_id)
def test_scanned_return_regression_can_correct_booking_all_inspections_and_vehicle(ops_client):
now = datetime.now(UTC)
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-SCAN-RET",
first_name="Synthetic",
last_name="Scanned return",
email="dq-scan-return@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-SCAN-RET",
make="Synthetic",
model="Scanned return",
model_year=2026,
registration_number="DQ-SCAN-RET",
location="Brussels",
operational_status="available",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
booking = Booking(
public_ref="BK-DQ-SCAN-RET",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=3),
ends_at=now - timedelta(hours=12),
status="returned",
start_odometer_km=19_500,
end_odometer_km=18_500,
requirements_complete=True,
)
db.add(booking)
db.flush()
db.add_all(
[
MaintenanceRecord(
public_ref="MNT-DQ-SCAN-RET",
vehicle_id=vehicle.id,
occurred_at=now - timedelta(days=4),
odometer_km=20_000,
category="inspection",
summary="Synthetic canonical baseline",
),
Inspection(
public_ref="INSP-DQ-SCAN-R1",
booking_id=booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=19_000,
completed_at=now - timedelta(days=1),
),
Inspection(
public_ref="INSP-DQ-SCAN-R2",
booking_id=booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=18_500,
completed_at=now - timedelta(hours=12),
),
]
)
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
scanned = ops_client.post("/api/v1/data-quality/scan")
assert scanned.status_code == 200, scanned.text
issue = next(
item
for item in ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
if item["entity_ref"] == "MO-DQ-SCAN-RET"
)
assert issue["evidence"]["correctable_booking_refs"] == ["BK-DQ-SCAN-RET"]
later_refs = {signal["params"]["later_ref"] for signal in issue["evidence"]["signals"]}
assert later_refs == {"INSP-DQ-SCAN-R1", "INSP-DQ-SCAN-R2"}
corrected = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={
"decision": "correct_reading",
"booking_ref": "BK-DQ-SCAN-RET",
"corrected_odometer_km": 20_500,
},
)
assert corrected.status_code == 200, corrected.text
assert corrected.json()["status"] == "resolved"
with SessionLocal() as db:
persisted_vehicle = db.get(Vehicle, vehicle_id)
persisted_booking = db.scalar(
select(Booking).where(Booking.public_ref == "BK-DQ-SCAN-RET")
)
assert persisted_booking is not None
persisted_inspections = db.scalars(
select(Inspection)
.where(Inspection.booking_id == persisted_booking.id, Inspection.type == "return")
.order_by(Inspection.public_ref)
).all()
assert persisted_vehicle is not None
assert persisted_vehicle.odometer_km == 20_500
assert persisted_booking.end_odometer_km == 20_500
assert [inspection.odometer_km for inspection in persisted_inspections] == [
20_500,
20_500,
]
finally:
_cleanup_odometer_scenario(vehicle_id, customer_id)
def test_resolver_and_concurrent_return_finish_without_deadlock_or_lost_evidence():
now = datetime.now(UTC)
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-RACE",
first_name="Synthetic",
last_name="Race",
email="dq-race@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-RACE",
make="Synthetic",
model="Race",
model_year=2026,
registration_number="DQ-RACE",
location="Brussels",
operational_status="rented",
odometer_km=20_000,
next_service_km=30_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
corrected_booking = Booking(
public_ref="BK-DQ-RACE-OLD",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=4),
ends_at=now - timedelta(days=3),
status="returned",
start_odometer_km=19_500,
end_odometer_km=19_000,
requirements_complete=True,
)
concurrent_booking = Booking(
public_ref="BK-DQ-RACE-NEW",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=now - timedelta(days=1),
ends_at=now + timedelta(days=1),
status="active",
start_odometer_km=19_000,
end_odometer_km=None,
requirements_complete=True,
)
db.add_all([corrected_booking, concurrent_booking])
db.flush()
old_inspection = Inspection(
public_ref="INSP-DQ-RACE-OLD",
booking_id=corrected_booking.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=19_000,
completed_at=corrected_booking.ends_at,
)
db.add(old_inspection)
db.flush()
issue = open_odometer_regression_issue(
db,
vehicle=vehicle,
reading_ref=old_inspection.public_ref,
reading_km=19_000,
canonical_km=20_000,
source_type="return",
related_refs=[corrected_booking.public_ref, old_inspection.public_ref],
correctable_booking_refs=[corrected_booking.public_ref],
public_ref="DQ-ODO-RACE",
)
assert issue is not None
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
actor = CurrentUser(
public_ref="USR-DQ-RACE",
display_name="DQ Race Manager",
role="operations_manager",
)
start = Barrier(2)
def correct_existing_reading() -> str:
with SessionLocal() as db:
db.execute(text("SET LOCAL lock_timeout = '5s'"))
start.wait(timeout=5)
resolved = resolve_odometer_regression(
db,
"DQ-ODO-RACE",
ResolveOdometerRegressionRequest(
decision="correct_reading",
booking_ref="BK-DQ-RACE-OLD",
corrected_odometer_km=20_500,
),
actor,
)
return resolved.status
def return_other_booking() -> dict:
with SessionLocal() as db:
db.execute(text("SET LOCAL lock_timeout = '5s'"))
start.wait(timeout=5)
_status, response = register_vehicle_return(
db,
"BK-DQ-RACE-NEW",
RegisterReturnRequest(
end_odometer_km=18_000,
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
),
"test-dq-resolve-return-race-001",
actor,
)
return response
try:
with ThreadPoolExecutor(max_workers=2) as executor:
correction_future = executor.submit(correct_existing_reading)
return_future = executor.submit(return_other_booking)
correction_status = correction_future.result(timeout=15)
return_result = return_future.result(timeout=15)
assert correction_status in {"open", "resolved"}
assert return_result["quality_issue_ref"] is not None
with SessionLocal() as db:
persisted_vehicle = db.get(Vehicle, vehicle_id)
bookings = {
booking.public_ref: booking
for booking in db.scalars(
select(Booking).where(Booking.vehicle_id == vehicle_id)
).all()
}
old_inspections = db.scalars(
select(Inspection).where(Inspection.booking_id == bookings["BK-DQ-RACE-OLD"].id)
).all()
open_issues = db.scalars(
select(DataQualityIssue).where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.rule_type == "odometer_regression",
DataQualityIssue.status == "open",
)
).all()
assert persisted_vehicle is not None
assert persisted_vehicle.odometer_km == 20_500
assert bookings["BK-DQ-RACE-OLD"].end_odometer_km == 20_500
assert [inspection.odometer_km for inspection in old_inspections] == [20_500]
assert bookings["BK-DQ-RACE-NEW"].status == "returned"
assert bookings["BK-DQ-RACE-NEW"].end_odometer_km == 18_000
assert len(open_issues) == 1
assert open_issues[0].evidence_json["correctable_booking_refs"] == ["BK-DQ-RACE-NEW"]
later_refs = {
signal["params"]["later_ref"] for signal in open_issues[0].evidence_json["signals"]
}
assert later_refs == {return_result["inspection_ref"]}
finally:
_cleanup_odometer_scenario(vehicle_id, customer_id)
def test_early_return_uses_inspection_time_without_duplicate_booking_regression(ops_client):
with SessionLocal() as db:
customer = Customer(
public_ref="CUS-DQ-EARLY",
first_name="Synthetic",
last_name="Early",
email="dq-early@example.test",
)
vehicle = Vehicle(
public_ref="MO-DQ-EARLY",
make="Synthetic",
model="Early",
model_year=2026,
registration_number="DQ-EARLY",
location="Brussels",
operational_status="available",
odometer_km=200,
next_service_km=10_000,
active=True,
version=1,
)
db.add_all([customer, vehicle])
db.flush()
early = Booking(
public_ref="BK-DQ-EARLY-A",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=datetime(2048, 1, 1, tzinfo=UTC),
ends_at=datetime(2048, 3, 1, tzinfo=UTC),
status="returned",
start_odometer_km=90,
end_odometer_km=100,
requirements_complete=True,
)
later = Booking(
public_ref="BK-DQ-EARLY-B",
customer_id=customer.id,
vehicle_id=vehicle.id,
starts_at=datetime(2048, 1, 10, tzinfo=UTC),
ends_at=datetime(2048, 2, 1, tzinfo=UTC),
status="returned",
start_odometer_km=100,
end_odometer_km=200,
requirements_complete=True,
)
db.add_all([early, later])
db.flush()
db.add_all(
[
Inspection(
public_ref="INSP-DQ-EARLY-A",
booking_id=early.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=100,
completed_at=datetime(2048, 1, 2, tzinfo=UTC),
),
Inspection(
public_ref="INSP-DQ-EARLY-B",
booking_id=later.id,
vehicle_id=vehicle.id,
type="return",
fuel_level_percent=50,
cleanliness_ok=True,
damage_reported=False,
technical_warning=False,
odometer_km=200,
completed_at=datetime(2048, 2, 1, tzinfo=UTC),
),
]
)
vehicle_id = vehicle.id
customer_id = customer.id
db.commit()
try:
assert ops_client.post("/api/v1/data-quality/scan").status_code == 200
with SessionLocal() as db:
assert (
db.scalar(
select(DataQualityIssue).where(
DataQualityIssue.entity_id == vehicle_id,
DataQualityIssue.rule_type == "odometer_regression",
)
)
is None
)
finally:
with SessionLocal() as db:
db.execute(delete(DataQualityIssue).where(DataQualityIssue.entity_id == vehicle_id))
db.execute(delete(Inspection).where(Inspection.vehicle_id == vehicle_id))
db.execute(delete(Booking).where(Booking.vehicle_id == vehicle_id))
db.execute(delete(Vehicle).where(Vehicle.id == vehicle_id))
db.execute(delete(Customer).where(Customer.id == customer_id))
db.commit()
def test_manual_scan_records_audit_event(ops_client):
+104
View File
@@ -1,5 +1,10 @@
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.outbox import OutboxEvent
from app.seed_loader import reset_and_seed
from app.services import demo_manifest
from app.services.knowledge import KnowledgeHealth
def test_demo_manifest_is_public(client):
@@ -52,3 +57,102 @@ def test_demo_manifest_ragcore_labelled_as_demo_mode_not_live(client):
body = client.get("/api/v1/demo/manifest").json()
ragcore = next(i for i in body["integrations"] if i["key"] == "ragcore")
assert ragcore["status_code"] == "demoMode"
def test_automation_scenario_requires_the_exact_prepared_failure():
db = SessionLocal()
try:
reset_and_seed(db)
event = db.scalar(
select(OutboxEvent).where(OutboxEvent.event_id == demo_manifest._FAILED_DEMO_EVENT_ID)
)
assert event is not None
event.last_error_code = "connectionError"
db.flush()
scenario = next(
item for item in demo_manifest._scenarios(db) if item.id == "automation-retry"
)
assert scenario.ready is False
finally:
db.rollback()
db.close()
def test_knowledge_scenario_rejects_available_provider_with_verified_empty_corpus(
client, monkeypatch
):
class EmptyKnowledgeProvider:
def health(self, language="en-GB"):
return KnowledgeHealth(
provider="ragcore",
available=True,
detail="Ready, but no indexed documents.",
tenant="fleet-ops",
workspace="mobilityops",
collection="procedures",
document_count=0,
source_document_count=4,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="verified",
)
monkeypatch.setattr(demo_manifest, "get_knowledge_provider", EmptyKnowledgeProvider)
body = client.get("/api/v1/demo/manifest").json()
scenario = next(item for item in body["scenarios"] if item["id"] == "knowledge-question")
assert scenario["ready"] is False
assert scenario["blocked_reason_code"] == "knowledgeUnavailable"
ragcore = next(item for item in body["integrations"] if item["key"] == "ragcore")
assert ragcore["status_code"] == "unavailable"
def test_knowledge_scenario_rejects_local_sources_when_index_count_is_unverified(
client, monkeypatch
):
class SourceAwareKnowledgeProvider:
def health(self, language="en-GB"):
return KnowledgeHealth(
provider="ragcore",
available=True,
detail="Ready; index count endpoint is unavailable.",
tenant="fleet-ops",
workspace="mobilityops",
collection="procedures",
document_count=None,
source_document_count=4,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="not_reported",
)
monkeypatch.setattr(demo_manifest, "get_knowledge_provider", SourceAwareKnowledgeProvider)
body = client.get("/api/v1/demo/manifest").json()
scenario = next(item for item in body["scenarios"] if item["id"] == "knowledge-question")
assert scenario["ready"] is False
assert scenario["blocked_reason_code"] == "knowledgeUnavailable"
ragcore = next(item for item in body["integrations"] if item["key"] == "ragcore")
assert ragcore["status_code"] == "unavailable"
def test_knowledge_scenario_requires_provider_availability_even_with_documents():
health = KnowledgeHealth(
provider="ragcore",
available=False,
detail="Provider is unreachable.",
tenant="fleet-ops",
workspace="mobilityops",
collection="procedures",
document_count=4,
source_document_count=4,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="verified",
)
assert demo_manifest._knowledge_scenario_ready(health) is False
+120 -5
View File
@@ -4,6 +4,7 @@ import uuid
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
import pytest
from sqlalchemy import select
from app.core.config import get_settings
@@ -65,12 +66,17 @@ def test_claim_marks_events_delivering():
def test_deliver_one_success(monkeypatch):
event_id = _make_pending_event("MO-002")
execution_id = "n8n-execution-123"
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": execution_id},
},
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
@@ -79,7 +85,7 @@ def test_deliver_one_success(monkeypatch):
event = _get_event(event_id)
assert event.delivery_status == "succeeded"
assert event.attempts == 1
assert event.external_run_id == str(event_id)
assert event.external_run_id == execution_id
assert event.last_error is None
assert event.last_error_code is None
@@ -129,6 +135,106 @@ def test_deliver_one_treats_empty_2xx_body_as_failure(monkeypatch):
assert event.last_error_code == "malformedResponse"
def test_deliver_one_rejects_json_object_without_explicit_success_ack(monkeypatch):
event_id = _make_pending_event("MO-010")
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "pending"
assert event.attempts == 1
assert event.last_error_code == "malformedResponse"
assert event.external_run_id is None
@pytest.mark.parametrize("result", [{}, {"execution_id": 123}, {"execution_id": " "}])
def test_deliver_one_rejects_ack_without_valid_execution_id(monkeypatch, result):
event_id = _make_pending_event("MO-EXEC-ID")
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": result},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "pending"
assert event.last_error_code == "malformedResponse"
assert event.external_run_id is None
def test_deliver_one_rejects_success_ack_for_a_different_event(monkeypatch):
event_id = _make_pending_event("MO-011")
dispatcher._claim_due_events()
other_event_id = uuid.uuid4()
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(other_event_id)},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "pending"
assert event.attempts == 1
assert event.last_error_code == "mismatchedEventId"
assert event.external_run_id is None
def test_late_delivery_outcome_cannot_overwrite_a_newer_lease(monkeypatch):
event_id = _make_pending_event("MO-LEASE-RACE")
dispatcher._claim_due_events()
def fake_post(url, json, headers, timeout):
# Simulate lease expiry/reclaim while the original worker is still in network
# I/O. The later claimant owns a different token and its state must win.
with SessionLocal() as db:
event = db.scalar(
select(OutboxEvent).where(OutboxEvent.event_id == event_id).with_for_update()
)
event.payload_json = {
**event.payload_json,
"_delivery_claim_token": "newer-worker-token",
}
event.next_attempt_at = datetime.now(UTC) + timedelta(minutes=1)
db.commit()
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": "stale-worker-execution"},
},
status_code=200,
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
assert event.delivery_status == "delivering"
assert event.attempts == 0
assert event.external_run_id is None
assert event.payload_json["_delivery_claim_token"] == "newer-worker-token"
def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
event_id = _make_pending_event("MO-004")
settings = get_settings()
@@ -139,7 +245,6 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
for _ in range(settings.n8n_max_attempts):
dispatcher._claim_due_events()
db = SessionLocal()
try:
event = db.scalar(select(OutboxEvent).where(OutboxEvent.event_id == event_id))
@@ -147,6 +252,8 @@ def test_deliver_one_exhausts_attempts_to_failed(monkeypatch):
db.commit()
finally:
db.close()
claimed = dispatcher._claim_due_events(batch_size=1000)
assert event_id in claimed
dispatcher._deliver_one(event_id)
event = _get_event(event_id)
@@ -259,7 +366,11 @@ def test_run_dispatch_cycle_recovers_a_stale_lease_before_claiming(monkeypatch):
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": "n8n-recovered-execution"},
},
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
@@ -275,7 +386,11 @@ def test_run_dispatch_cycle_end_to_end(monkeypatch):
def fake_post(url, json, headers, timeout):
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"ok": True, "event_id": str(event_id), "result": {}},
json=lambda: {
"ok": True,
"event_id": str(event_id),
"result": {"execution_id": "n8n-cycle-execution"},
},
)
monkeypatch.setattr(dispatcher.httpx, "post", fake_post)
+8 -3
View File
@@ -174,11 +174,12 @@ def test_return_idempotency_key_rejects_different_body(ops_client):
def test_return_callback_rejects_malformed_correlation_id(client):
event_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning", "correlation_id": "nope"},
json={"event_id": event_id, "follow_up": "cleaning", "correlation_id": "nope"},
headers={
"Idempotency-Key": str(uuid.uuid4()),
"Idempotency-Key": event_id,
"X-Service-Token": get_settings().n8n_callback_token,
},
)
@@ -195,10 +196,14 @@ def test_blocked_checkout_booking_can_be_cancelled(ops_client):
json={
"customer_ref": "CUS-0002",
"vehicle_ref": vehicle["public_ref"],
"requirements_complete": True,
**window,
},
).json()
confirmed = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/complete-requirements",
json={"confirmation": "Licence and rental conditions checked"},
)
assert confirmed.status_code == 200
checkout = ops_client.post(
f"/api/v1/bookings/{booking['public_ref']}/checkout",
json={
+102 -1
View File
@@ -1,8 +1,14 @@
from sqlalchemy import select
import uuid
from datetime import UTC, datetime
import pytest
from sqlalchemy import delete, select
from app.core.db import SessionLocal
from app.models.audit import AuditEvent
from app.models.outbox import OutboxEvent
from app.seed_loader import reset_and_seed
from app.services import integration_status
def _reseed() -> None:
@@ -108,6 +114,101 @@ def test_integration_status_is_operational_once_all_failed_events_resolved(ops_c
assert body["state"] == "operational"
@pytest.mark.parametrize("delivery_status", ["pending", "delivering"])
def test_queued_work_without_any_success_is_not_reported_operational(delivery_status):
_reseed()
db = SessionLocal()
try:
for event in db.scalars(select(OutboxEvent)).all():
event.delivery_status = delivery_status
event.last_error = None
event.last_error_code = None
db.execute(
delete(AuditEvent).where(
AuditEvent.action.in_(
{
"data_quality_scan_run",
"n8n_procedures_synced",
"n8n_workflow_failure_registered",
"n8n_workflow_heartbeat",
}
)
)
)
db.flush()
status = integration_status.derive_n8n_status(db)
assert status.succeeded == 0
assert getattr(status, delivery_status) > 0
assert status.state == "no_evidence"
finally:
db.rollback()
db.close()
def test_historical_success_is_not_green_when_webhook_is_unconfigured(monkeypatch):
_reseed()
with SessionLocal() as db:
monkeypatch.setattr(integration_status.settings, "n8n_dispatch_enabled", True)
monkeypatch.setattr(integration_status.settings, "n8n_webhook_url", "")
status = integration_status.derive_n8n_status(db)
assert status.configured is False
assert status.succeeded > 0
assert status.state == "unavailable"
def test_null_coded_real_failure_sets_latest_failure_timestamp():
_reseed()
with SessionLocal() as db:
event = db.scalar(
select(OutboxEvent).where(OutboxEvent.delivery_status == "succeeded").limit(1)
)
event.delivery_status = "failed"
event.last_error_code = None
db.flush()
status = integration_status.derive_n8n_status(db)
assert status.unexpected_failed == 1
assert status.latest_failure_at is not None
db.rollback()
def test_unreachable_hub_invalidates_historical_operational_claim(monkeypatch):
db = SessionLocal()
try:
db.add(
AuditEvent(
actor_type="service",
actor_id=None,
actor_label="itworx-mcp-hub",
action="mcp_tool_request",
entity_type="integration",
entity_id=None,
correlation_id=uuid.uuid4(),
before_json=None,
after_json=None,
metadata_json={"tool": "operations_summary"},
occurred_at=datetime.now(UTC),
)
)
db.flush()
monkeypatch.setattr(integration_status.settings, "mcp_hub_registration_enabled", True)
monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: False)
unreachable = integration_status.derive_mcp_hub_status(db)
assert unreachable.total_calls > 0
assert unreachable.hub_reachable is False
assert unreachable.state == "no_evidence"
monkeypatch.setattr(integration_status, "_check_hub_reachable", lambda: True)
reachable = integration_status.derive_mcp_hub_status(db)
assert reachable.state == "operational"
finally:
db.rollback()
db.close()
def test_integration_status_lists_all_four_canonical_workflows(ops_client):
body = ops_client.get("/api/v1/integrations/status").json()["n8n"]
assert body["expected_workflow_count"] == 4
+158 -7
View File
@@ -1,4 +1,5 @@
import uuid
from concurrent.futures import ThreadPoolExecutor
from app.core.config import get_settings
@@ -12,19 +13,29 @@ def _callback_headers(event_id: str, token: str | None = None):
def test_callback_rejects_wrong_service_token(client):
event_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning"},
headers=_callback_headers(str(uuid.uuid4()), token="wrong-token"),
json={
"event_id": event_id,
"correlation_id": str(uuid.uuid4()),
"follow_up": "cleaning",
},
headers=_callback_headers(event_id, token="wrong-token"),
)
assert response.status_code == 401
def test_callback_unknown_event_returns_404(client):
event_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning"},
headers=_callback_headers(str(uuid.uuid4())),
json={
"event_id": event_id,
"correlation_id": str(uuid.uuid4()),
"follow_up": "cleaning",
},
headers=_callback_headers(event_id),
)
assert response.status_code == 404
@@ -39,13 +50,14 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
db = SessionLocal()
try:
booking = db.scalar(select(Booking).limit(1))
correlation_id = uuid.uuid4()
event = OutboxEvent(
event_id=uuid.uuid4(),
event_type="vehicle.returned.v1",
aggregate_type="booking",
aggregate_id=booking.id,
payload_json={
"correlation_id": str(uuid.uuid4()),
"correlation_id": str(correlation_id),
"aggregate": {
"type": "booking",
"id": str(booking.id),
@@ -66,12 +78,22 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
first = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning", "summary": "test"},
json={
"event_id": event_id,
"correlation_id": str(correlation_id),
"follow_up": "cleaning",
"summary": "test",
},
headers=_callback_headers(event_id),
)
second = client.post(
"/api/v1/integrations/n8n/return-callback",
json={"follow_up": "cleaning", "summary": "test"},
json={
"event_id": event_id,
"correlation_id": str(correlation_id),
"follow_up": "cleaning",
"summary": "test",
},
headers=_callback_headers(event_id),
)
assert first.status_code == 200
@@ -82,6 +104,79 @@ def test_callback_is_idempotent_by_event_id(client, ops_client):
).json()
matching = [e for e in audit_events if e["metadata"]["event_id"] == event_id]
assert len(matching) == 1
assert matching[0]["correlation_id"] == str(correlation_id)
def test_callback_rejects_crossed_event_or_correlation(client):
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.outbox import OutboxEvent
with SessionLocal() as db:
event = db.scalar(select(OutboxEvent).limit(1))
event_id = str(event.event_id)
correlation_id = event.payload_json["correlation_id"]
crossed_event = client.post(
"/api/v1/integrations/n8n/return-callback",
json={
"event_id": str(uuid.uuid4()),
"correlation_id": correlation_id,
"follow_up": "cleaning",
},
headers=_callback_headers(event_id),
)
assert crossed_event.status_code == 409
assert crossed_event.json()["error"]["code"] == "CALLBACK_EVENT_MISMATCH"
crossed_correlation = client.post(
"/api/v1/integrations/n8n/return-callback",
json={
"event_id": event_id,
"correlation_id": str(uuid.uuid4()),
"follow_up": "cleaning",
},
headers=_callback_headers(event_id),
)
assert crossed_correlation.status_code == 409
assert crossed_correlation.json()["error"]["code"] == "CALLBACK_CORRELATION_MISMATCH"
def test_callback_concurrent_retries_record_one_audit(client, ops_client):
from sqlalchemy import select
from app.core.db import SessionLocal
from app.models.outbox import OutboxEvent
with SessionLocal() as db:
event = db.scalar(select(OutboxEvent).limit(1))
event_id = str(event.event_id)
correlation_id = event.payload_json["correlation_id"]
body = {
"event_id": event_id,
"correlation_id": correlation_id,
"follow_up": "cleaning",
}
def post_callback(_index: int):
return client.post(
"/api/v1/integrations/n8n/return-callback",
json=body,
headers=_callback_headers(event_id),
)
with ThreadPoolExecutor(max_workers=2) as pool:
responses = list(pool.map(post_callback, range(2)))
assert [response.status_code for response in responses] == [200, 200]
matching = [
event
for event in ops_client.get(
"/api/v1/audit", params={"action": "n8n_return_followup_recorded"}
).json()
if event["metadata"]["event_id"] == event_id
]
assert len(matching) == 1
def test_scheduled_scan_rejects_wrong_service_token(client):
@@ -183,6 +278,42 @@ def test_workflow_error_registers_and_is_idempotent_by_execution_id(client, ops_
assert matching[0]["after"]["retry_action"] == "n8n will retry automatically"
def test_workflow_error_preserves_correlation_and_rejects_malformed(client, ops_client):
settings = get_settings()
headers = {"X-Service-Token": settings.n8n_callback_token}
execution_id = str(uuid.uuid4())
correlation_id = str(uuid.uuid4())
accepted = client.post(
"/api/v1/integrations/n8n/workflow-error",
json=_workflow_error_body(execution_id, correlation_id=correlation_id),
headers=headers,
)
assert accepted.status_code == 200
matching = [
event
for event in ops_client.get(
"/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
).json()
if event["metadata"]["execution_id"] == execution_id
]
assert len(matching) == 1
assert matching[0]["correlation_id"] == correlation_id
malformed_execution = str(uuid.uuid4())
malformed = client.post(
"/api/v1/integrations/n8n/workflow-error",
json=_workflow_error_body(malformed_execution, correlation_id="not-a-uuid"),
headers=headers,
)
assert malformed.status_code == 422
assert all(
event["metadata"]["execution_id"] != malformed_execution
for event in ops_client.get(
"/api/v1/audit", params={"action": "n8n_workflow_failure_registered"}
).json()
)
def test_workflow_error_bounds_summary_length(client):
settings = get_settings()
response = client.post(
@@ -266,6 +397,26 @@ def test_procedures_sync_result_registers_and_is_idempotent(client, ops_client):
assert ragcore_sync["last_seen_at"] is not None
def test_failed_or_partial_procedure_sync_is_never_reported_healthy(client, ops_client):
settings = get_settings()
headers = {"X-Service-Token": settings.n8n_callback_token}
for synced, failed in ((0, 33), (32, 1)):
execution_id = str(uuid.uuid4())
response = client.post(
"/api/v1/integrations/n8n/procedures-sync-result",
json={"execution_id": execution_id, "synced": synced, "failed": failed},
headers=headers,
)
assert response.status_code == 200
status = ops_client.get("/api/v1/integrations/status").json()["n8n"]
workflow = next(w for w in status["workflows"] if "RAGcore" in w["name"])
assert workflow["state"] == "failed"
assert workflow["last_status"] == "failed"
assert workflow["last_execution_id"] == execution_id
assert status["state"] == "degraded"
def test_workflow_heartbeat_is_idempotent_and_drives_live_status(client, ops_client):
settings = get_settings()
execution_id = str(uuid.uuid4())
+409 -70
View File
@@ -1,9 +1,12 @@
from __future__ import annotations
import hashlib
import re
import uuid
from pathlib import Path
import httpx
import pytest
from app.api.routers import knowledge as knowledge_router
from app.core.config import get_settings
@@ -244,11 +247,11 @@ def test_ragcore_status_preserves_stronger_verified_index_evidence(client, ops_c
class _FakeResponse:
def __init__(self, status_code: int, body: dict):
def __init__(self, status_code: int, body: object):
self.status_code = status_code
self._body = body
def json(self) -> dict:
def json(self) -> object:
return self._body
@@ -260,6 +263,7 @@ class _FakeClient:
post_responses=None,
raise_on=None,
get_handler=None,
requests=None,
):
self._get_response = get_response
self._post_response = post_response
@@ -270,6 +274,7 @@ class _FakeClient:
self._post_responses = post_responses or {}
self._raise_on = raise_on
self._get_handler = get_handler
self.requests = requests if requests is not None else []
def __enter__(self):
return self
@@ -289,6 +294,7 @@ class _FakeClient:
def post(self, path, json=None):
if self._raise_on == "post":
raise httpx.ConnectError("no ragcore in this environment")
self.requests.append((path, json))
return self._post_responses.get(path, self._post_response)
@@ -393,22 +399,107 @@ def test_ragcore_provider_health_degrades_on_connection_error(monkeypatch):
assert "unavailable" in health.detail.lower()
def _answers_body(**overrides) -> dict:
_EXCERPTS = {
("en-GB", "damage-procedure"): (
"When a vehicle returns with visible or reported damage, mark damage in the "
"return inspection, add a concise factual description and keep the vehicle "
"blocked."
),
("en-GB", "vehicle-return-procedure"): (
"Open the active booking and record the ending odometer, fuel level, cleanliness, "
"visible damage, technical warnings and relevant notes."
),
("en-GB", "cleaning-checklist"): (
"Cleaning completion alone does not make a blocked or maintenance vehicle "
"available. Fleet Ops derives availability from all active restrictions."
),
("en-GB", "maintenance-escalation"): (
"Escalate when a technical warning is reported, the service threshold is reached, "
"a safety-related defect is observed or an existing maintenance block remains "
"unresolved."
),
("nl-BE", "damage-procedure"): (
"Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schade "
"in de retourinspectie, voeg een beknopte feitelijke beschrijving toe en houd het "
"voertuig geblokkeerd."
),
("nl-BE", "vehicle-return-procedure"): (
"Open de actieve boeking en registreer de eindkilometerstand, het brandstofniveau, "
"de netheid, zichtbare schade, technische waarschuwingen en relevante notities."
),
}
def _managed_document(provider: RAGcoreKnowledgeProvider, document_id: str, language: str):
return next(
document
for document in provider._managed_documents_by_source_id.values()
if document.document_id == document_id and document.language == language
)
def _citation(
provider: RAGcoreKnowledgeProvider,
document_id: str,
*,
language: str = "en-GB",
section: str = "Procedure",
seed: str = "1",
) -> dict:
document = _managed_document(provider, document_id, language)
excerpt = _EXCERPTS[(language, document_id)]
return {
"id": str(uuid.uuid5(uuid.NAMESPACE_URL, f"citation:{language}:{document_id}:{seed}")),
# These are deliberately opaque RAGcore-owned UUIDs, not Fleet Ops' stable
# human-readable procedure id.
"document_id": str(
uuid.uuid5(uuid.NAMESPACE_URL, f"ragcore-document:{language}:{document_id}")
),
"document_version_id": str(
uuid.uuid5(
uuid.NAMESPACE_URL,
f"ragcore-document-version:{language}:{document_id}:{seed}",
)
),
"title": "untrusted-provider-title.md",
"section": section,
"excerpt": excerpt,
"excerpt_sha256": hashlib.sha256(excerpt.encode("utf-8")).hexdigest(),
"source_uri": f"ragcore://source/{document.source_id}",
"source_id": document.source_id,
"locator": f"{document.document_id}.md",
}
def _answers_body(provider: RAGcoreKnowledgeProvider, **overrides) -> dict:
citation = _citation(provider, "damage-procedure", section="Detection")
answer = "Report damage and route the vehicle to maintenance."
body = {
"answer": "Report damage and route the vehicle to maintenance.",
"answer_id": str(uuid.uuid4()),
"retrieval_run_id": str(uuid.uuid4()),
"answer": answer,
"answerability": "answerable",
"citations": [
{
"id": "cite-1",
"document_id": "doc-1",
"document_version_id": "version-1",
"title": "Damage handling procedure",
"section": "Detection",
"excerpt": "Inspect the vehicle for visible damage.",
}
],
"citations": [citation],
"claims": [{"text": answer, "citation_ids": [citation["id"]]}],
}
body.update(overrides)
if "claims" not in overrides:
effective_answer = body.get("answer")
effective_citations = body.get("citations")
body["claims"] = (
[
{
"text": effective_answer,
"citation_ids": [item["id"] for item in effective_citations],
}
]
if isinstance(effective_answer, str)
and effective_answer
and isinstance(effective_citations, list)
and effective_citations
and all(isinstance(item, dict) and "id" in item for item in effective_citations)
else []
)
return body
@@ -418,47 +509,178 @@ def test_ragcore_provider_grounded_answer_maps_citations_to_sources(monkeypatch)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body())),
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(provider))),
)
answer = provider.ask("What must I do about damage?", "test-correlation-grounded")
assert answer.evidence_state == "grounded"
assert answer.answer
assert len(answer.sources) == 1
source = answer.sources[0]
assert source.document_id == "doc-1"
assert source.document_id == "damage-procedure"
assert source.title == "Damage handling procedure"
assert source.version == "version-1"
assert source.version == "1.3"
assert source.section == "Detection"
assert source.excerpt
@pytest.mark.parametrize(
"mutate",
[
lambda body: body.pop("claims"),
lambda body: body.update(claims=[]),
lambda body: body["claims"][0].update(citation_ids=[str(uuid.uuid4())]),
lambda body: body.pop("answer_id"),
lambda body: body.update(retrieval_run_id="not-a-uuid"),
],
)
def test_ragcore_answer_requires_valid_claim_citation_contract(monkeypatch, mutate):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
body = _answers_body(provider)
mutate(body)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, body)),
)
answer = provider.ask("What must I do about damage?", "test-invalid-claim-contract")
assert answer.evidence_state != "grounded"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_requests_are_scoped_to_managed_sources_for_requested_language(
monkeypatch,
):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
requests = []
fake_client = _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
},
requests=requests,
)
monkeypatch.setattr(provider, "_client", lambda: fake_client)
answer = provider.ask(
"Wat is de procedure voor een voertuigretour?",
"language-filter",
"nl-BE",
)
assert answer.evidence_state == "grounded"
assert [path for path, _payload in requests] == ["/v1/answers", "/v1/search"]
expected_source_ids = provider._managed_source_ids("nl-BE")
assert len(expected_source_ids) == 11
assert all(uuid.UUID(source_id) for source_id in expected_source_ids)
for _path, payload in requests:
assert payload["filters"] == {"source_ids": expected_source_ids}
assert payload["requested_space_ids"] == ["space-1"]
def test_ragcore_sources_deduplicate_reuploaded_versions_and_cap_cards(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
citations = []
for index in range(5):
document_ids = [
"damage-procedure",
"damage-procedure",
"vehicle-return-procedure",
"cleaning-checklist",
"maintenance-escalation",
]
for index, document_id in enumerate(document_ids):
citations.append(
{
"id": f"cite-{index}",
"document_id": f"doc-{index}",
"document_version_id": f"version-{index}",
"title": "Damage procedure" if index < 2 else f"Procedure {index}",
"section": "Return",
"excerpt": (
f"Record visible damage before release, chunk {index}."
if index < 2
else f"Unique procedure evidence {index}."
),
}
_citation(
provider,
document_id,
section="Return",
seed=str(index),
)
)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(citations=citations))),
lambda: _FakeClient(
post_response=_FakeResponse(200, _answers_body(provider, citations=citations))
),
)
answer = provider.ask("What must I do about vehicle damage?", "dedupe-test")
assert len(answer.sources) == 3
assert sum(source.title == "Damage procedure" for source in answer.sources) == 1
assert sum(source.title == "Damage handling procedure" for source in answer.sources) == 1
def test_ragcore_answer_rejects_unknown_document_citation(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200,
_answers_body(
provider,
answer="A plausible-looking but unmanaged answer.",
citations=[
{
**_citation(provider, "damage-procedure"),
"source_id": str(uuid.uuid4()),
}
],
),
)
),
)
answer = provider.ask("What must I do about damage?", "unknown-answer-citation")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
@pytest.mark.parametrize(
("field_name", "bad_value"),
[
("id", "not-a-uuid"),
("document_id", "not-a-uuid"),
("document_version_id", "00000000-0000-0000-0000-000000000000"),
("source_id", "00000000-0000-4000-8000-000000000099"),
("source_uri", "ragcore://source/00000000-0000-4000-8000-000000000099"),
("locator", "another-procedure.md"),
("excerpt_sha256", "0" * 64),
("title", None),
("section", {"not": "a string"}),
],
)
def test_ragcore_rejects_broken_managed_source_provenance(field_name, bad_value):
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure")
citation[field_name] = bad_value
assert provider._managed_source(citation, "en-GB") is None
def test_ragcore_rejects_fabricated_excerpt_even_with_matching_hash():
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure")
fabricated = "Invent a repair price and promise it to the customer."
citation["excerpt"] = fabricated
citation["excerpt_sha256"] = hashlib.sha256(fabricated.encode("utf-8")).hexdigest()
assert provider._managed_source(citation, "en-GB") is None
def test_ragcore_never_relabels_an_english_source_as_dutch():
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure", language="en-GB")
assert provider._managed_source(citation, "nl-BE") is None
def test_knowledge_feedback_is_audited_and_can_be_changed(ops_client):
@@ -503,6 +725,7 @@ def test_ragcore_provider_not_answerable_is_insufficient_and_never_fabricates(mo
post_response=_FakeResponse(
200,
_answers_body(
provider,
answer="This should never be shown.",
answerability="not_answerable",
citations=[],
@@ -524,7 +747,8 @@ def test_ragcore_provider_answerable_without_citations_is_insufficient(monkeypat
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200, _answers_body(answerability="answerable", citations=[])
200,
_answers_body(provider, answerability="answerable", citations=[]),
)
),
)
@@ -565,19 +789,66 @@ def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch):
assert answer.evidence_state == "unavailable"
def _search_body(**overrides) -> dict:
def test_ragcore_provider_malformed_top_level_bodies_do_not_escape(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(200, ["not", "an", "object"]),
"/v1/search": _FakeResponse(200, None),
}
),
)
answer = provider.ask("Anything?", "test-correlation-malformed-top-level")
assert answer.evidence_state == "unavailable"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_answer_with_malformed_citation_is_insufficient(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200,
_answers_body(provider, citations=[None]),
)
),
)
answer = provider.ask("What must I do about damage?", "malformed-answer-citation")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def _search_body(
provider: RAGcoreKnowledgeProvider,
language: str = "en-GB",
**overrides,
) -> dict:
body = {
"retrieval_run_id": str(uuid.uuid4()),
"effective_space_ids": ["00000000-0000-4000-8000-000000000001"],
"results": [
{
"chunk_id": "chunk-1",
"citation": {
"id": "cite-1",
"document_id": "doc-1",
"document_version_id": "version-1",
"title": "Vehicle return procedure",
"section": "Return",
"excerpt": "Register the return odometer reading before releasing the vehicle.",
},
"chunk_id": str(uuid.uuid4()),
"text": _EXCERPTS[(language, "vehicle-return-procedure")],
"citation": _citation(
provider,
"vehicle-return-procedure",
language=language,
section="Return",
),
"rank": 1,
"scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None},
}
@@ -588,6 +859,31 @@ def _search_body(**overrides) -> dict:
return body
def test_ragcore_search_ignores_malformed_result_and_citation_bodies(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
malformed_search = _search_body(
provider,
results=[None, {"citation": ["not-an-object"], "scores": "not-an-object"}],
)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, malformed_search),
}
),
)
answer = provider.ask("What is the vehicle return procedure?", "malformed-search-items")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch):
"""/v1/answers itself failing (a real RAGcore-side outage in its generation step,
not a real 'insufficient evidence' classification) must not silently degrade
@@ -601,26 +897,24 @@ def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypa
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider)),
}
),
)
answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback")
assert answer.evidence_state == "grounded"
assert "Register the return odometer reading" in answer.answer
assert "Open the active booking and record the ending odometer" in answer.answer
assert "Vehicle return procedure" in answer.answer
assert len(answer.sources) == 1
assert answer.sources[0].title == "Vehicle return procedure"
assert answer.sources[0].excerpt == (
"Register the return odometer reading before releasing the vehicle."
)
assert answer.sources[0].excerpt == _EXCERPTS[("en-GB", "vehicle-return-procedure")]
def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0)
search_response = _FakeResponse(200, _search_body())
search_response = _FakeResponse(200, _search_body(provider))
monkeypatch.setattr(
provider,
"_client",
@@ -632,8 +926,7 @@ def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
),
)
assert (
provider.ask("What is the vehicle return procedure?", "first").evidence_state
== "grounded"
provider.ask("What is the vehicle return procedure?", "first").evidence_state == "grounded"
)
monkeypatch.setattr(
@@ -654,7 +947,7 @@ def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
}
),
)
@@ -676,7 +969,7 @@ def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkey
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body(results=[])),
"/v1/search": _FakeResponse(200, _search_body(provider, results=[])),
}
),
)
@@ -695,7 +988,7 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider)),
}
),
)
@@ -707,19 +1000,66 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
assert answer.answer == ""
def test_ragcore_search_fallback_rejects_unknown_document_citation(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body(provider)
search["results"][0]["citation"]["source_id"] = str(uuid.uuid4())
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, search),
}
),
)
answer = provider.ask(
"What is the vehicle return procedure?",
"unknown-search-citation",
)
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_search_fallback_rejects_vehicle_colour_despite_high_score(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body(provider)
search["results"][0]["scores"]["fused"] = 1.0
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, search),
}
),
)
answer = provider.ask("What colour is this vehicle?", "vehicle-colour")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body()
search = _search_body(provider, "nl-BE", results=[])
search["results"].append(
{
"citation": {
"document_id": "damage-procedure",
"document_version_id": "version-2",
"title": "damage-procedure.md",
"section": "Damage",
"excerpt": "Record damage and keep the vehicle blocked.",
},
"citation": _citation(
provider,
"damage-procedure",
language="nl-BE",
section="Damage",
),
"rank": 2,
"scores": {"fused": 0.01},
}
@@ -743,16 +1083,15 @@ def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
def test_ragcore_search_fallback_accepts_top_ranked_ragcore_damage_evidence(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body()
search = _search_body(provider, "nl-BE", results=[])
search["results"].append(
{
"citation": {
"document_id": "damage-procedure",
"document_version_id": "version-2",
"title": "damage-procedure.md",
"section": "Damage",
"excerpt": "Record damage and keep the vehicle blocked.",
},
"citation": _citation(
provider,
"damage-procedure",
language="nl-BE",
section="Damage",
),
"rank": 1,
# RAGcore uses reciprocal-rank fusion; a genuine rank-one result is about
# 1 / (60 + 1), not a normalized 0..1 relevance score.
+31
View File
@@ -1,4 +1,12 @@
from app.core.config import get_settings
from app.main import app
MCP_ROUTE_ALLOWLIST = {
("GET", "/api/v1/integrations/mcp/operations-summary"),
("GET", "/api/v1/integrations/mcp/attention-vehicles"),
("GET", "/api/v1/integrations/mcp/vehicles/{vehicle_ref}"),
("POST", "/api/v1/integrations/mcp/search-knowledge"),
}
def _headers(token: str | None = None, client_id: str = "test-mcp-client"):
@@ -154,3 +162,26 @@ def test_no_write_endpoints_exist_under_mcp_namespace(client):
]:
response = getattr(client, method)(path, headers=_headers())
assert response.status_code in (404, 405)
def test_mcp_namespace_matches_exact_route_allowlist_and_rejects_lookalikes(client):
implemented = {
(method.upper(), path)
for path, operations in app.openapi()["paths"].items()
if path.startswith("/api/v1/integrations/mcp/")
for method in operations
if method in {"get", "post", "put", "patch", "delete"}
}
assert implemented == MCP_ROUTE_ALLOWLIST
lookalikes = [
("get", "/api/v1/integrations/mcp-extra/operations-summary"),
("get", "/api/v1/integrations/mcp/operations-summary-extra"),
("get", "/api/v1/integrations/mcp/prefix/attention-vehicles"),
("get", "/api/v1/integrations/mcp/attention-vehicles/suffix"),
("post", "/api/v1/integrations/mcp/search-knowledge-extra"),
("post", "/api/v1/integrations/mcp/prefix/search-knowledge"),
]
for method, path in lookalikes:
response = client.request(method, path, headers=_headers(), json={})
assert response.status_code == 404, path
+18
View File
@@ -63,6 +63,24 @@ def test_eligible_customer_is_irreversibly_anonymized_without_pii_in_audit(ops_c
)
assert response.status_code == 200
assert response.json()["status"] == "anonymized"
search = ops_client.get("/api/v1/customers", params={"query": "CUS-PRIVACY"})
assert search.status_code == 200
assert search.json() == []
window = {
"starts_at": "2090-01-01T10:00:00Z",
"ends_at": "2090-01-02T10:00:00Z",
}
available = ops_client.get("/api/v1/bookings/availability", params=window).json()
assert available
booking = ops_client.post(
"/api/v1/bookings",
json={
"customer_ref": "CUS-PRIVACY",
"vehicle_ref": available[0]["public_ref"],
**window,
},
)
assert booking.status_code == 422
with SessionLocal() as db:
customer = db.scalar(select(Customer).where(Customer.id == customer_id))
assert customer is not None
+17
View File
@@ -128,6 +128,23 @@ def test_seed_scenario_s4_booking_overlap():
db.close()
def test_seed_odometer_issues_expose_only_the_regressing_booking_as_correctable():
db = SessionLocal()
try:
reset_and_seed(db)
issue = _by_ref(db, DataQualityIssue, "DQ-0007")
assert issue is not None
assert issue.evidence_json["source_type"] == "return"
assert issue.evidence_json["related_refs"] == [
"INSP-0057",
"BK-H-0007",
"INSP-0007",
]
assert issue.evidence_json["correctable_booking_refs"] == ["BK-H-0007"]
finally:
db.close()
def test_seed_scenario_s5_failed_workflow_run():
"""S5: one seeded outbox event is durably 'failed' (terminal, retryable), not merely
pending, so the background dispatcher never silently auto-heals it away."""
+73
View File
@@ -0,0 +1,73 @@
from __future__ import annotations
import csv
import subprocess
import sys
from pathlib import Path
REPOSITORY_ROOT = next(
parent
for parent in Path(__file__).resolve().parents
if (parent / "seed" / "generate_seed.py").is_file()
)
SEED_DIRECTORY = REPOSITORY_ROOT / "seed"
def _rows_by_ref(path: Path) -> dict[str, dict[str, str]]:
with path.open(newline="", encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
assert all(None not in row for row in rows), f"malformed CSV row in {path.name}"
return {row["public_ref"]: row for row in rows}
def test_generator_reproduces_committed_seed_snapshot_byte_for_byte(tmp_path: Path):
subprocess.run(
[
sys.executable,
str(SEED_DIRECTORY / "generate_seed.py"),
"--anchor",
"2026-08-01",
"--seed",
"20260801",
"--out",
str(tmp_path),
],
cwd=REPOSITORY_ROOT,
check=True,
capture_output=True,
text=True,
)
committed_files = sorted(SEED_DIRECTORY.glob("*.csv"))
generated_files = sorted(tmp_path.glob("*.csv"))
assert [path.name for path in generated_files] == [path.name for path in committed_files]
for committed in committed_files:
generated = tmp_path / committed.name
assert generated.read_bytes() == committed.read_bytes(), (
f"{committed.name} differs from the deterministic generated snapshot"
)
bookings = _rows_by_ref(tmp_path / "bookings.csv")
inspections = _rows_by_ref(tmp_path / "inspections.csv")
issues = _rows_by_ref(tmp_path / "data_quality_issues.csv")
assert int(bookings["BK-H-0007"]["end_odometer_km"]) < int(
bookings["BK-H-0057"]["end_odometer_km"]
)
assert int(bookings["BK-H-0010"]["end_odometer_km"]) < int(
bookings["BK-H-0060"]["end_odometer_km"]
)
assert inspections["INSP-0007"]["odometer_km"] == bookings["BK-H-0007"]["end_odometer_km"]
assert inspections["INSP-0010"]["odometer_km"] == bookings["BK-H-0010"]["end_odometer_km"]
assert inspections["INSP-0001"]["completed_at"] == bookings["BK-H-0001"]["ends_at"]
expected_issue_refs = {f"DQ-{index:04d}" for index in range(5, 22)}
assert expected_issue_refs <= issues.keys()
assert all(
issues[public_ref]["evidence"] != "Synthetic deterministic seed issue"
for public_ref in expected_issue_refs
)
assert "INSP-0007" in issues["DQ-0007"]["evidence"]
assert "INSP-0057" in issues["DQ-0007"]["evidence"]
assert "INSP-0010" in issues["DQ-0010"]["evidence"]
assert "INSP-0060" in issues["DQ-0010"]["evidence"]
+19
View File
@@ -23,3 +23,22 @@ def test_manager_can_create_and_update_user(ops_client):
def test_employee_cannot_manage_users(employee_client):
assert employee_client.get("/api/v1/users").status_code == 403
assert (
employee_client.post(
"/api/v1/users",
json={
"email": "unauthorised@example.test",
"display_name": "Unauthorised User",
"role": "rental_employee",
"password": "a-secure-demo-password",
},
).status_code
== 403
)
assert (
employee_client.patch(
"/api/v1/users/USR-OPS",
json={"display_name": "Unauthorised Change"},
).status_code
== 403
)
+53
View File
@@ -46,6 +46,12 @@ def test_vehicle_detail_includes_related_records(ops_client):
assert len(body["quality_issues"]) >= 1
def test_employee_vehicle_detail_never_exposes_quality_evidence(employee_client):
response = employee_client.get("/api/v1/vehicles/MO-016")
assert response.status_code == 200
assert response.json()["quality_issues"] == []
def test_vehicle_detail_404_for_unknown_ref(ops_client):
response = ops_client.get("/api/v1/vehicles/MO-999")
assert response.status_code == 404
@@ -78,6 +84,53 @@ def test_manager_can_record_maintenance_and_release_vehicle(ops_client):
assert released.json()["operational_status"] == "available"
def test_low_maintenance_reading_keeps_canonical_and_opens_quality_issue(ops_client):
existing_issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
unavailable_refs = {issue["entity_ref"] for issue in existing_issues}
vehicles = ops_client.get("/api/v1/vehicles", params={"status": "available"}).json()
vehicle = next(
candidate
for candidate in vehicles
if candidate["odometer_km"] > 0 and candidate["public_ref"] not in unavailable_refs
)
response = ops_client.post(
f"/api/v1/vehicles/{vehicle['public_ref']}/maintenance",
json={
"occurred_at": "2053-01-15T12:00:00Z",
"odometer_km": vehicle["odometer_km"] - 1,
"category": "inspection",
"summary": "Imported workshop reading requires verification.",
"mark_maintenance": True,
},
)
assert response.status_code == 201
detail = ops_client.get(f"/api/v1/vehicles/{vehicle['public_ref']}").json()
assert detail["odometer_km"] == vehicle["odometer_km"]
issues = ops_client.get(
"/api/v1/data-quality/issues",
params={"status": "open", "rule_type": "odometer_regression"},
).json()
issue = next(item for item in issues if item["entity_ref"] == vehicle["public_ref"])
assert issue["evidence"]["source_type"] == "maintenance"
assert issue["evidence"]["correctable_booking_refs"] == []
issue_detail = ops_client.get(f"/api/v1/data-quality/issues/{issue['public_ref']}").json()
assert any(
snapshot["entity_type"] == "maintenance"
and snapshot["public_ref"] == response.json()["public_ref"]
for snapshot in issue_detail["related_snapshots"]
)
retained = ops_client.post(
f"/api/v1/data-quality/issues/{issue['public_ref']}/resolve-odometer-regression",
json={"decision": "retain_canonical"},
)
assert retained.status_code == 200
def test_employee_cannot_record_maintenance(employee_client):
response = employee_client.post(
"/api/v1/vehicles/MO-001/maintenance",