M3: implement Data Quality Workbench
Five rule scanners (duplicate customers, missing fields, odometer regression, booking overlap, status conflict) run automatically after seed and via an explicit scan endpoint. Issue defer/reject/merge-customers endpoints with transactional customer merge (booking rewiring, tombstone, audit). Data Quality nav + workbench UI with two-column duplicate comparison and inline (non-native) confirm. Dashboard attention items now link to issues. 35 backend tests passing, ruff clean. Fixed a real false-positive bug in odometer-regression detection found through iteration on seed data, and two TS narrowing errors. Verified end-to-end via browser: S2 merge and S4 overlap scenarios.
This commit is contained in:
NuklearRabbit
@@ -86,9 +86,11 @@ def get_dashboard(
|
||||
detail=issue.evidence_json.get("summary", ""),
|
||||
link_type=link_type,
|
||||
link_ref=link_ref,
|
||||
issue_ref=issue.public_ref,
|
||||
)
|
||||
)
|
||||
attention_items.sort(key=lambda item: _SEVERITY_ORDER.get(item.severity, 3))
|
||||
attention_items = attention_items[:8]
|
||||
|
||||
today = _today()
|
||||
bookings = db.scalars(select(Booking)).all()
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.deps import get_current_user, get_db, require_operations_manager
|
||||
from app.models.customer import Customer
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.schemas import (
|
||||
CurrentUser,
|
||||
DataQualityIssueDetailOut,
|
||||
DataQualityIssueOut,
|
||||
MergeCustomersRequest,
|
||||
MergeCustomersResult,
|
||||
ScanResultOut,
|
||||
)
|
||||
from app.services.data_quality import defer_issue, merge_customers, reject_issue, run_scan
|
||||
|
||||
router = APIRouter(prefix="/api/v1/data-quality", tags=["data-quality"])
|
||||
|
||||
|
||||
def _to_out(issue: DataQualityIssue) -> DataQualityIssueOut:
|
||||
return DataQualityIssueOut(
|
||||
public_ref=issue.public_ref,
|
||||
rule_type=issue.rule_type,
|
||||
entity_type=issue.entity_type,
|
||||
entity_ref=issue.evidence_json.get("entity_ref", ""),
|
||||
severity=issue.severity,
|
||||
status=issue.status,
|
||||
evidence=issue.evidence_json,
|
||||
detected_at=issue.detected_at,
|
||||
resolved_at=issue.resolved_at,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/issues", response_model=list[DataQualityIssueOut])
|
||||
def list_issues(
|
||||
status: str | None = Query(default=None),
|
||||
rule_type: str | None = Query(default=None),
|
||||
severity: str | None = Query(default=None),
|
||||
db: Session = Depends(get_db),
|
||||
_user: CurrentUser = Depends(get_current_user),
|
||||
) -> list[DataQualityIssueOut]:
|
||||
stmt = select(DataQualityIssue).order_by(DataQualityIssue.detected_at.desc())
|
||||
if status:
|
||||
stmt = stmt.where(DataQualityIssue.status == status)
|
||||
if rule_type:
|
||||
stmt = stmt.where(DataQualityIssue.rule_type == rule_type)
|
||||
if severity:
|
||||
stmt = stmt.where(DataQualityIssue.severity == severity)
|
||||
issues = db.scalars(stmt).all()
|
||||
return [_to_out(i) for i in issues]
|
||||
|
||||
|
||||
def _snapshot(entity_type: str, ref: str, db: Session) -> dict | None:
|
||||
if entity_type == "customer":
|
||||
obj = db.scalar(select(Customer).where(Customer.public_ref == ref))
|
||||
if obj is None:
|
||||
return None
|
||||
return {
|
||||
"public_ref": obj.public_ref,
|
||||
"first_name": obj.first_name,
|
||||
"last_name": obj.last_name,
|
||||
"email": obj.email,
|
||||
"phone": obj.phone,
|
||||
"postal_code": obj.postal_code,
|
||||
"city": obj.city,
|
||||
}
|
||||
obj = db.scalar(select(Vehicle).where(Vehicle.public_ref == ref))
|
||||
if obj is None:
|
||||
return None
|
||||
return {
|
||||
"public_ref": obj.public_ref,
|
||||
"make": obj.make,
|
||||
"model": obj.model,
|
||||
"location": obj.location,
|
||||
"operational_status": obj.operational_status,
|
||||
"odometer_km": obj.odometer_km,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/issues/{public_ref}", response_model=DataQualityIssueDetailOut)
|
||||
def get_issue(
|
||||
public_ref: str,
|
||||
db: Session = Depends(get_db),
|
||||
_user: CurrentUser = Depends(get_current_user),
|
||||
) -> DataQualityIssueDetailOut:
|
||||
issue = db.scalar(select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref))
|
||||
if issue is None:
|
||||
raise HTTPException(status_code=404, detail="Data quality issue not found")
|
||||
base = _to_out(issue)
|
||||
related_refs = issue.evidence_json.get("related_refs", [])
|
||||
related_entity_type = (
|
||||
"customer" if issue.rule_type == "possible_duplicate_customer" else "vehicle"
|
||||
)
|
||||
return DataQualityIssueDetailOut(
|
||||
**base.model_dump(),
|
||||
entity_snapshot=_snapshot(issue.entity_type, base.entity_ref, db),
|
||||
related_snapshots=[
|
||||
snap
|
||||
for ref in related_refs
|
||||
if (snap := _snapshot(related_entity_type, ref, db)) is not None
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@router.post("/issues/{public_ref}/defer", response_model=DataQualityIssueOut)
|
||||
def defer(
|
||||
public_ref: str,
|
||||
db: Session = Depends(get_db),
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> DataQualityIssueOut:
|
||||
issue = defer_issue(db, public_ref, user)
|
||||
return _to_out(issue)
|
||||
|
||||
|
||||
@router.post("/issues/{public_ref}/reject", response_model=DataQualityIssueOut)
|
||||
def reject(
|
||||
public_ref: str,
|
||||
db: Session = Depends(get_db),
|
||||
user: CurrentUser = Depends(get_current_user),
|
||||
) -> DataQualityIssueOut:
|
||||
issue = reject_issue(db, public_ref, user)
|
||||
return _to_out(issue)
|
||||
|
||||
|
||||
@router.post("/issues/{public_ref}/merge-customers", response_model=MergeCustomersResult)
|
||||
def merge(
|
||||
public_ref: str,
|
||||
body: MergeCustomersRequest,
|
||||
db: Session = Depends(get_db),
|
||||
user: CurrentUser = Depends(require_operations_manager),
|
||||
) -> MergeCustomersResult:
|
||||
result = merge_customers(db, public_ref, body.survivor_ref, body.field_overrides, user)
|
||||
return MergeCustomersResult(**result)
|
||||
|
||||
|
||||
@router.post("/scan", response_model=ScanResultOut)
|
||||
def scan(
|
||||
db: Session = Depends(get_db),
|
||||
_user: CurrentUser = Depends(require_operations_manager),
|
||||
) -> ScanResultOut:
|
||||
result = run_scan(db)
|
||||
return ScanResultOut(created=result.created)
|
||||
+2
-1
@@ -4,7 +4,7 @@ from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api.routers import audit, bookings, dashboard, demo, vehicles
|
||||
from app.api.routers import audit, bookings, dashboard, data_quality, demo, vehicles
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import AppError, error_body
|
||||
|
||||
@@ -61,3 +61,4 @@ app.include_router(dashboard.router)
|
||||
app.include_router(vehicles.router)
|
||||
app.include_router(bookings.router)
|
||||
app.include_router(audit.router)
|
||||
app.include_router(data_quality.router)
|
||||
|
||||
@@ -106,6 +106,27 @@ class DataQualityIssueOut(BaseModel):
|
||||
resolved_at: datetime | None = None
|
||||
|
||||
|
||||
class DataQualityIssueDetailOut(DataQualityIssueOut):
|
||||
entity_snapshot: dict[str, Any] | None = None
|
||||
related_snapshots: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class MergeCustomersRequest(BaseModel):
|
||||
survivor_ref: str
|
||||
field_overrides: dict[str, str] | None = None
|
||||
|
||||
|
||||
class MergeCustomersResult(BaseModel):
|
||||
issue_ref: str
|
||||
survivor_ref: str
|
||||
loser_ref: str
|
||||
rewired_bookings: int
|
||||
|
||||
|
||||
class ScanResultOut(BaseModel):
|
||||
created: dict[str, int]
|
||||
|
||||
|
||||
class VehicleDetailOut(VehicleOut):
|
||||
bookings: list[BookingSummaryOut] = Field(default_factory=list)
|
||||
inspections: list[InspectionOut] = Field(default_factory=list)
|
||||
@@ -130,6 +151,7 @@ class AttentionItem(BaseModel):
|
||||
detail: str
|
||||
link_type: Literal["vehicle", "booking", "customer"]
|
||||
link_ref: str
|
||||
issue_ref: str | None = None
|
||||
|
||||
|
||||
class TodayItem(BaseModel):
|
||||
|
||||
@@ -260,7 +260,11 @@ def load_seed(db: Session) -> SeedResult:
|
||||
|
||||
|
||||
def reset_and_seed(db: Session) -> SeedResult:
|
||||
from app.services.data_quality import run_scan
|
||||
|
||||
clear_all(db)
|
||||
result = load_seed(db)
|
||||
db.commit()
|
||||
scan = run_scan(db)
|
||||
result.counts["data_quality_issues"] += sum(scan.created.values())
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models.booking import Booking
|
||||
from app.models.customer import Customer
|
||||
from app.models.data_quality import DataQualityIssue
|
||||
from app.models.vehicle import Vehicle
|
||||
from app.schemas import CurrentUser
|
||||
from app.services.audit import record_audit_event
|
||||
|
||||
REQUIRED_CUSTOMER_FIELDS = ("first_name", "last_name")
|
||||
REQUIRED_VEHICLE_FIELDS = ("registration_number", "make", "model", "location")
|
||||
DUPLICATE_THRESHOLD = 70
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
created: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def bump(self, rule_type: str) -> None:
|
||||
self.created[rule_type] = self.created.get(rule_type, 0) + 1
|
||||
|
||||
|
||||
def _normalize(value: str | None) -> str:
|
||||
return (value or "").strip().lower()
|
||||
|
||||
|
||||
def _has_open_issue(db: Session, rule_type: str, entity_type: str, entity_id: uuid.UUID) -> bool:
|
||||
return (
|
||||
db.scalar(
|
||||
select(DataQualityIssue.id).where(
|
||||
DataQualityIssue.rule_type == rule_type,
|
||||
DataQualityIssue.entity_type == entity_type,
|
||||
DataQualityIssue.entity_id == entity_id,
|
||||
DataQualityIssue.status == "open",
|
||||
)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def _next_public_ref(db: Session, prefix: str) -> str:
|
||||
existing = db.execute(select(DataQualityIssue.public_ref)).scalars().all()
|
||||
numbers = [
|
||||
int(ref.rsplit("-", 1)[-1])
|
||||
for ref in existing
|
||||
if ref.startswith(f"{prefix}-") and ref.rsplit("-", 1)[-1].isdigit()
|
||||
]
|
||||
next_number = (max(numbers) + 1) if numbers else 1
|
||||
return f"{prefix}-{next_number:04d}"
|
||||
|
||||
|
||||
def _open_issue(
|
||||
db: Session,
|
||||
scan: ScanResult,
|
||||
*,
|
||||
rule_type: str,
|
||||
entity_type: str,
|
||||
entity_id: uuid.UUID,
|
||||
severity: str,
|
||||
summary: str,
|
||||
entity_ref: str,
|
||||
related_refs: list[str],
|
||||
) -> None:
|
||||
if _has_open_issue(db, rule_type, entity_type, entity_id):
|
||||
return
|
||||
now = datetime.now(UTC)
|
||||
issue = DataQualityIssue(
|
||||
public_ref=_next_public_ref(db, "DQ-SCAN"),
|
||||
rule_type=rule_type,
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
severity=severity,
|
||||
status="open",
|
||||
evidence_json={
|
||||
"summary": summary,
|
||||
"entity_ref": entity_ref,
|
||||
"related_refs": related_refs,
|
||||
},
|
||||
proposed_action_json={},
|
||||
detected_at=now,
|
||||
)
|
||||
db.add(issue)
|
||||
db.flush()
|
||||
scan.bump(rule_type)
|
||||
|
||||
|
||||
def _scan_duplicate_customers(db: Session, scan: ScanResult) -> None:
|
||||
customers = db.scalars(
|
||||
select(Customer).where(Customer.merged_into_customer_id.is_(None))
|
||||
).all()
|
||||
customers.sort(key=lambda c: c.public_ref)
|
||||
|
||||
for i, a in enumerate(customers):
|
||||
for b in customers[i + 1 :]:
|
||||
score = 0
|
||||
signals = []
|
||||
if _normalize(a.email) and _normalize(a.email) == _normalize(b.email):
|
||||
score += 60
|
||||
signals.append("exact email")
|
||||
if _normalize(a.phone) and _normalize(a.phone) == _normalize(b.phone):
|
||||
score += 50
|
||||
signals.append("exact phone")
|
||||
if _normalize(a.postal_code) and _normalize(a.postal_code) == _normalize(b.postal_code):
|
||||
score += 10
|
||||
signals.append("exact postal code")
|
||||
name_a = f"{_normalize(a.first_name)} {_normalize(a.last_name)}"
|
||||
name_b = f"{_normalize(b.first_name)} {_normalize(b.last_name)}"
|
||||
ratio = SequenceMatcher(None, name_a, name_b).ratio()
|
||||
if ratio >= 0.5:
|
||||
score += round(ratio * 30)
|
||||
signals.append("similar name")
|
||||
|
||||
if score >= DUPLICATE_THRESHOLD:
|
||||
_open_issue(
|
||||
db,
|
||||
scan,
|
||||
rule_type="possible_duplicate_customer",
|
||||
entity_type="customer",
|
||||
entity_id=a.id,
|
||||
severity="high",
|
||||
summary="; ".join(signals) + f" (score {score})",
|
||||
entity_ref=a.public_ref,
|
||||
related_refs=[b.public_ref],
|
||||
)
|
||||
|
||||
|
||||
def _scan_missing_required_fields(db: Session, scan: ScanResult) -> None:
|
||||
for customer in db.scalars(
|
||||
select(Customer).where(Customer.merged_into_customer_id.is_(None))
|
||||
).all():
|
||||
missing = [f for f in REQUIRED_CUSTOMER_FIELDS if not getattr(customer, f)]
|
||||
if not customer.email and not customer.phone:
|
||||
missing.append("email_or_phone")
|
||||
if missing:
|
||||
_open_issue(
|
||||
db,
|
||||
scan,
|
||||
rule_type="missing_required_field",
|
||||
entity_type="customer",
|
||||
entity_id=customer.id,
|
||||
severity="low",
|
||||
summary=f"Missing: {', '.join(missing)}",
|
||||
entity_ref=customer.public_ref,
|
||||
related_refs=[],
|
||||
)
|
||||
|
||||
for vehicle in db.scalars(select(Vehicle).where(Vehicle.active.is_(True))).all():
|
||||
missing = [f for f in REQUIRED_VEHICLE_FIELDS if not getattr(vehicle, f)]
|
||||
if missing:
|
||||
_open_issue(
|
||||
db,
|
||||
scan,
|
||||
rule_type="missing_required_field",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
severity="low",
|
||||
summary=f"Missing: {', '.join(missing)}",
|
||||
entity_ref=vehicle.public_ref,
|
||||
related_refs=[],
|
||||
)
|
||||
|
||||
|
||||
def _scan_booking_overlaps(db: Session, scan: ScanResult) -> None:
|
||||
vehicles = db.scalars(select(Vehicle)).all()
|
||||
bookings_by_vehicle: dict[uuid.UUID, list[Booking]] = {}
|
||||
for booking in db.scalars(
|
||||
select(Booking).where(Booking.status.in_(["reserved", "active"]))
|
||||
).all():
|
||||
bookings_by_vehicle.setdefault(booking.vehicle_id, []).append(booking)
|
||||
|
||||
vehicle_by_id = {v.id: v for v in vehicles}
|
||||
for vehicle_id, bookings in bookings_by_vehicle.items():
|
||||
bookings.sort(key=lambda b: b.starts_at)
|
||||
for i, first in enumerate(bookings):
|
||||
for second in bookings[i + 1 :]:
|
||||
if second.starts_at < first.ends_at and first.starts_at < second.ends_at:
|
||||
vehicle = vehicle_by_id[vehicle_id]
|
||||
_open_issue(
|
||||
db,
|
||||
scan,
|
||||
rule_type="booking_overlap",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle_id,
|
||||
severity="high",
|
||||
summary=f"Overlapping bookings {first.public_ref} and {second.public_ref}",
|
||||
entity_ref=vehicle.public_ref,
|
||||
related_refs=[first.public_ref, second.public_ref],
|
||||
)
|
||||
|
||||
|
||||
def _scan_vehicle_status_conflicts(db: Session, scan: ScanResult) -> None:
|
||||
vehicles = db.scalars(select(Vehicle)).all()
|
||||
active_by_vehicle: dict[uuid.UUID, list[Booking]] = {}
|
||||
for booking in db.scalars(select(Booking).where(Booking.status == "active")).all():
|
||||
active_by_vehicle.setdefault(booking.vehicle_id, []).append(booking)
|
||||
|
||||
open_high_by_vehicle = {
|
||||
row[0]
|
||||
for row in db.execute(
|
||||
select(DataQualityIssue.entity_id).where(
|
||||
DataQualityIssue.entity_type == "vehicle",
|
||||
DataQualityIssue.status == "open",
|
||||
DataQualityIssue.severity == "high",
|
||||
)
|
||||
).all()
|
||||
}
|
||||
|
||||
for vehicle in vehicles:
|
||||
has_active_booking = vehicle.id in active_by_vehicle
|
||||
reason = None
|
||||
if vehicle.operational_status == "available" and has_active_booking:
|
||||
reason = "marked available while an active booking exists"
|
||||
elif vehicle.operational_status == "rented" and not has_active_booking:
|
||||
reason = "marked rented without an active booking"
|
||||
elif vehicle.operational_status == "available" and vehicle.id in open_high_by_vehicle:
|
||||
reason = "marked available while a high-severity quality issue is open"
|
||||
elif vehicle.operational_status == "maintenance" and has_active_booking:
|
||||
reason = "marked maintenance while an active booking exists"
|
||||
|
||||
if reason:
|
||||
_open_issue(
|
||||
db,
|
||||
scan,
|
||||
rule_type="vehicle_status_conflict",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle.id,
|
||||
severity="high",
|
||||
summary=f"Vehicle {reason}",
|
||||
entity_ref=vehicle.public_ref,
|
||||
related_refs=[],
|
||||
)
|
||||
|
||||
|
||||
def _scan_odometer_regressions(db: Session, scan: ScanResult) -> None:
|
||||
# The seed dataset's vehicle.odometer_km is generated independently of booking
|
||||
# history, so comparing every historical booking against it produces near-universal
|
||||
# false positives. Instead check the booking sequence's own internal consistency:
|
||||
# each vehicle's completed bookings should show a non-decreasing odometer reading.
|
||||
vehicles = {v.id: v for v in db.scalars(select(Vehicle)).all()}
|
||||
bookings_by_vehicle: dict[uuid.UUID, list[Booking]] = {}
|
||||
for booking in db.scalars(
|
||||
select(Booking).where(
|
||||
Booking.status == "returned", Booking.end_odometer_km.is_not(None)
|
||||
)
|
||||
).all():
|
||||
bookings_by_vehicle.setdefault(booking.vehicle_id, []).append(booking)
|
||||
|
||||
for vehicle_id, bookings in bookings_by_vehicle.items():
|
||||
bookings.sort(key=lambda b: b.ends_at)
|
||||
for earlier, later in zip(bookings, bookings[1:], strict=False):
|
||||
if later.end_odometer_km < earlier.end_odometer_km:
|
||||
vehicle = vehicles[vehicle_id]
|
||||
_open_issue(
|
||||
db,
|
||||
scan,
|
||||
rule_type="odometer_regression",
|
||||
entity_type="vehicle",
|
||||
entity_id=vehicle_id,
|
||||
severity="medium",
|
||||
summary=(
|
||||
f"Booking {later.public_ref} recorded {later.end_odometer_km} km, "
|
||||
f"below the {earlier.end_odometer_km} km recorded by earlier "
|
||||
f"booking {earlier.public_ref}."
|
||||
),
|
||||
entity_ref=vehicle.public_ref,
|
||||
related_refs=[earlier.public_ref, later.public_ref],
|
||||
)
|
||||
break
|
||||
|
||||
|
||||
def run_scan(db: Session) -> ScanResult:
|
||||
scan = ScanResult()
|
||||
_scan_duplicate_customers(db, scan)
|
||||
_scan_missing_required_fields(db, scan)
|
||||
_scan_odometer_regressions(db, scan)
|
||||
_scan_booking_overlaps(db, scan)
|
||||
_scan_vehicle_status_conflicts(db, scan)
|
||||
db.commit()
|
||||
return scan
|
||||
|
||||
|
||||
def _load_open_issue(db: Session, public_ref: str) -> DataQualityIssue:
|
||||
issue = db.scalar(
|
||||
select(DataQualityIssue).where(DataQualityIssue.public_ref == public_ref)
|
||||
)
|
||||
if issue is None:
|
||||
raise AppError("ISSUE_NOT_FOUND", "Data quality issue not found.", status_code=404)
|
||||
if issue.status != "open":
|
||||
raise AppError(
|
||||
"ISSUE_NOT_OPEN",
|
||||
f"Issue is '{issue.status}', not 'open'.",
|
||||
status_code=409,
|
||||
)
|
||||
return issue
|
||||
|
||||
|
||||
def defer_issue(db: Session, public_ref: str, actor: CurrentUser) -> DataQualityIssue:
|
||||
issue = _load_open_issue(db, public_ref)
|
||||
issue.status = "deferred"
|
||||
issue.resolved_at = datetime.now(UTC)
|
||||
issue.resolved_by = actor.display_name
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_issue_deferred",
|
||||
entity_type="data_quality_issue",
|
||||
entity_id=issue.id,
|
||||
before={"status": "open"},
|
||||
after={"status": "deferred"},
|
||||
)
|
||||
db.commit()
|
||||
return issue
|
||||
|
||||
|
||||
def reject_issue(db: Session, public_ref: str, actor: CurrentUser) -> DataQualityIssue:
|
||||
issue = _load_open_issue(db, public_ref)
|
||||
issue.status = "rejected"
|
||||
issue.resolved_at = datetime.now(UTC)
|
||||
issue.resolved_by = actor.display_name
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="data_quality_issue_rejected",
|
||||
entity_type="data_quality_issue",
|
||||
entity_id=issue.id,
|
||||
before={"status": "open"},
|
||||
after={"status": "rejected"},
|
||||
)
|
||||
db.commit()
|
||||
return issue
|
||||
|
||||
|
||||
MERGEABLE_FIELDS = ("first_name", "last_name", "email", "phone", "postal_code", "city")
|
||||
|
||||
|
||||
def merge_customers(
|
||||
db: Session,
|
||||
public_ref: str,
|
||||
survivor_ref: str,
|
||||
field_overrides: dict[str, str] | None,
|
||||
actor: CurrentUser,
|
||||
) -> dict:
|
||||
issue = _load_open_issue(db, public_ref)
|
||||
if issue.rule_type != "possible_duplicate_customer":
|
||||
raise AppError(
|
||||
"NOT_A_DUPLICATE_ISSUE",
|
||||
"This issue is not a possible-duplicate-customer issue.",
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
entity_ref = issue.evidence_json.get("entity_ref")
|
||||
related_refs = issue.evidence_json.get("related_refs", [])
|
||||
candidate_refs = {entity_ref, *related_refs}
|
||||
if survivor_ref not in candidate_refs:
|
||||
raise AppError(
|
||||
"INVALID_SURVIVOR",
|
||||
"The survivor reference must be one of the two customers in this issue.",
|
||||
status_code=422,
|
||||
details={"candidates": sorted(candidate_refs)},
|
||||
)
|
||||
loser_ref = next(ref for ref in candidate_refs if ref != survivor_ref)
|
||||
|
||||
survivor = db.scalar(select(Customer).where(Customer.public_ref == survivor_ref))
|
||||
loser = db.scalar(select(Customer).where(Customer.public_ref == loser_ref))
|
||||
if survivor is None or loser is None:
|
||||
raise AppError(
|
||||
"CUSTOMER_NOT_FOUND", "One of the customers could not be found.", status_code=404
|
||||
)
|
||||
|
||||
before = {
|
||||
"survivor": {f: getattr(survivor, f) for f in MERGEABLE_FIELDS},
|
||||
"loser": {f: getattr(loser, f) for f in MERGEABLE_FIELDS},
|
||||
}
|
||||
|
||||
for field_name, value in (field_overrides or {}).items():
|
||||
if field_name not in MERGEABLE_FIELDS:
|
||||
raise AppError(
|
||||
"INVALID_FIELD_OVERRIDE", f"Field '{field_name}' cannot be merged.", status_code=422
|
||||
)
|
||||
setattr(survivor, field_name, value)
|
||||
|
||||
rewired = db.execute(
|
||||
Booking.__table__.update()
|
||||
.where(Booking.customer_id == loser.id)
|
||||
.values(customer_id=survivor.id)
|
||||
)
|
||||
loser.merged_into_customer_id = survivor.id
|
||||
|
||||
issue.status = "resolved"
|
||||
issue.resolved_at = datetime.now(UTC)
|
||||
issue.resolved_by = actor.display_name
|
||||
|
||||
record_audit_event(
|
||||
db,
|
||||
actor_type="user",
|
||||
actor_label=actor.display_name,
|
||||
action="customer_merged",
|
||||
entity_type="customer",
|
||||
entity_id=survivor.id,
|
||||
before=before,
|
||||
after={"survivor": {f: getattr(survivor, f) for f in MERGEABLE_FIELDS}},
|
||||
metadata={
|
||||
"loser_ref": loser_ref,
|
||||
"survivor_ref": survivor_ref,
|
||||
"rewired_bookings": rewired.rowcount,
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
|
||||
return {
|
||||
"issue_ref": issue.public_ref,
|
||||
"survivor_ref": survivor_ref,
|
||||
"loser_ref": loser_ref,
|
||||
"rewired_bookings": rewired.rowcount,
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
def test_list_includes_all_five_rule_types(ops_client):
|
||||
response = ops_client.get("/api/v1/data-quality/issues")
|
||||
assert response.status_code == 200
|
||||
issues = response.json()
|
||||
rule_types = {i["rule_type"] for i in issues}
|
||||
assert rule_types == {
|
||||
"possible_duplicate_customer",
|
||||
"missing_required_field",
|
||||
"odometer_regression",
|
||||
"booking_overlap",
|
||||
"vehicle_status_conflict",
|
||||
}
|
||||
|
||||
|
||||
def test_scan_is_idempotent_once_seeded(ops_client):
|
||||
# The session fixture already ran a scan as part of seeding; running again
|
||||
# must not create duplicate open issues for the same (rule_type, entity).
|
||||
response = ops_client.post("/api/v1/data-quality/scan")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["created"] == {}
|
||||
|
||||
|
||||
def test_scan_requires_operations_manager(employee_client):
|
||||
response = employee_client.post("/api/v1/data-quality/scan")
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_s2_duplicate_customer_issue_detail(ops_client):
|
||||
response = ops_client.get("/api/v1/data-quality/issues/DQ-DEMO-DUPLICATE")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["entity_ref"] == "CUS-0012"
|
||||
assert body["entity_snapshot"]["public_ref"] == "CUS-0012"
|
||||
assert [s["public_ref"] for s in body["related_snapshots"]] == ["CUS-0178"]
|
||||
|
||||
|
||||
def test_s4_booking_overlap_issue_detail(ops_client):
|
||||
response = ops_client.get("/api/v1/data-quality/issues/DQ-DEMO-OVERLAP")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["entity_ref"] == "MO-016"
|
||||
assert set(body["evidence"]["related_refs"]) == {"BK-DEMO-OVERLAP-A", "BK-DEMO-OVERLAP-B"}
|
||||
|
||||
|
||||
def test_defer_then_reject_are_rejected_on_closed_issue(ops_client):
|
||||
issues = ops_client.get(
|
||||
"/api/v1/data-quality/issues",
|
||||
params={"rule_type": "missing_required_field", "status": "open"},
|
||||
).json()
|
||||
target = issues[0]["public_ref"]
|
||||
|
||||
deferred = ops_client.post(f"/api/v1/data-quality/issues/{target}/defer")
|
||||
assert deferred.status_code == 200
|
||||
assert deferred.json()["status"] == "deferred"
|
||||
|
||||
again = ops_client.post(f"/api/v1/data-quality/issues/{target}/reject")
|
||||
assert again.status_code == 409
|
||||
assert again.json()["error"]["code"] == "ISSUE_NOT_OPEN"
|
||||
|
||||
|
||||
def test_merge_customers_requires_operations_manager(employee_client):
|
||||
response = employee_client.post(
|
||||
"/api/v1/data-quality/issues/DQ-DEMO-DUPLICATE/merge-customers",
|
||||
json={"survivor_ref": "CUS-0012"},
|
||||
)
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_merge_customers_rejects_unrelated_survivor(ops_client):
|
||||
response = ops_client.post(
|
||||
"/api/v1/data-quality/issues/DQ-DEMO-DUPLICATE/merge-customers",
|
||||
json={"survivor_ref": "CUS-0099"},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["error"]["code"] == "INVALID_SURVIVOR"
|
||||
|
||||
|
||||
def test_merge_customers_s2_scenario_rewires_and_audits(ops_client):
|
||||
before_bookings = ops_client.get(
|
||||
"/api/v1/bookings", params={"vehicle_ref": "MO-001"}
|
||||
) # warm the client session; irrelevant vehicle, just a cheap authenticated call
|
||||
assert before_bookings.status_code == 200
|
||||
|
||||
response = ops_client.post(
|
||||
"/api/v1/data-quality/issues/DQ-DEMO-DUPLICATE/merge-customers",
|
||||
json={"survivor_ref": "CUS-0012", "field_overrides": {"city": "Turnhout"}},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["survivor_ref"] == "CUS-0012"
|
||||
assert body["loser_ref"] == "CUS-0178"
|
||||
|
||||
issue = ops_client.get("/api/v1/data-quality/issues/DQ-DEMO-DUPLICATE").json()
|
||||
assert issue["status"] == "resolved"
|
||||
|
||||
audit_events = ops_client.get(
|
||||
"/api/v1/audit", params={"action": "customer_merged"}
|
||||
).json()
|
||||
assert len(audit_events) >= 1
|
||||
|
||||
# Already-resolved issue cannot be merged again.
|
||||
replay = ops_client.post(
|
||||
"/api/v1/data-quality/issues/DQ-DEMO-DUPLICATE/merge-customers",
|
||||
json={"survivor_ref": "CUS-0012"},
|
||||
)
|
||||
assert replay.status_code == 409
|
||||
@@ -20,7 +20,8 @@ def test_seed_counts_match_deterministic_dataset():
|
||||
assert db.scalar(select(func.count()).select_from(Vehicle)) == 50
|
||||
assert db.scalar(select(func.count()).select_from(Customer)) == 180
|
||||
assert db.scalar(select(func.count()).select_from(Booking)) == 246
|
||||
assert db.scalar(select(func.count()).select_from(DataQualityIssue)) == 15
|
||||
# 15 from the CSV plus a deterministic set discovered by the post-seed scan.
|
||||
assert db.scalar(select(func.count()).select_from(DataQualityIssue)) == 26
|
||||
assert db.scalar(select(func.count()).select_from(OutboxEvent)) == 20
|
||||
assert db.scalar(select(func.count()).select_from(User)) == 2
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user