109 lines
3.7 KiB
Python
109 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
import uuid
|
|
from difflib import SequenceMatcher
|
|
from typing import Protocol, TypeVar
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.models.customer import Customer
|
|
|
|
DUPLICATE_THRESHOLD = 70
|
|
|
|
|
|
class ScanAccumulator(Protocol):
|
|
def bump(self, rule_type: str) -> None: ...
|
|
|
|
|
|
ScanTypeContra = TypeVar("ScanTypeContra", bound=ScanAccumulator, contravariant=True)
|
|
|
|
|
|
class OpenIssue(Protocol[ScanTypeContra]):
|
|
def __call__(
|
|
self,
|
|
db: Session,
|
|
scan: ScanTypeContra,
|
|
*,
|
|
rule_type: str,
|
|
entity_type: str,
|
|
entity_id: uuid.UUID,
|
|
severity: str,
|
|
summary: str,
|
|
entity_ref: str,
|
|
related_refs: list[str],
|
|
signals: list[dict] | None = None,
|
|
) -> None: ...
|
|
|
|
|
|
def _normalize(value: str | None) -> str:
|
|
return (value or "").strip().lower()
|
|
|
|
|
|
def scan_duplicate_customers[ScanType: ScanAccumulator](
|
|
db: Session, scan: ScanType, open_issue: OpenIssue[ScanType]
|
|
) -> None:
|
|
customers = list(
|
|
db.scalars(
|
|
select(Customer).where(
|
|
Customer.merged_into_customer_id.is_(None),
|
|
Customer.anonymized_at.is_(None),
|
|
)
|
|
).all()
|
|
)
|
|
customers.sort(key=lambda customer: customer.public_ref)
|
|
|
|
# The threshold cannot be reached without an exact email (60 points) or phone
|
|
# (50 points). Block on normalized identifiers so this remains linear for the
|
|
# overwhelmingly common case and only scores plausible pairs.
|
|
candidate_pairs: set[tuple[int, int]] = set()
|
|
for attribute in ("email", "phone"):
|
|
blocks: dict[str, list[int]] = {}
|
|
for index, customer in enumerate(customers):
|
|
key = _normalize(getattr(customer, attribute))
|
|
if key:
|
|
blocks.setdefault(key, []).append(index)
|
|
for indices in blocks.values():
|
|
for offset, left in enumerate(indices):
|
|
candidate_pairs.update((left, right) for right in indices[offset + 1 :])
|
|
|
|
for left, right in sorted(candidate_pairs):
|
|
a = customers[left]
|
|
b = customers[right]
|
|
score = 0
|
|
signals: list[dict] = []
|
|
summary_parts: list[str] = []
|
|
if _normalize(a.email) and _normalize(a.email) == _normalize(b.email):
|
|
score += 60
|
|
signals.append({"code": "duplicate.exact_email"})
|
|
summary_parts.append("exact email")
|
|
if _normalize(a.phone) and _normalize(a.phone) == _normalize(b.phone):
|
|
score += 50
|
|
signals.append({"code": "duplicate.exact_phone"})
|
|
summary_parts.append("exact phone")
|
|
if _normalize(a.postal_code) and _normalize(a.postal_code) == _normalize(b.postal_code):
|
|
score += 10
|
|
signals.append({"code": "duplicate.same_postal_code"})
|
|
summary_parts.append("exact postal code")
|
|
name_a = f"{_normalize(a.first_name)} {_normalize(a.last_name)}"
|
|
name_b = f"{_normalize(b.first_name)} {_normalize(b.last_name)}"
|
|
ratio = SequenceMatcher(None, name_a, name_b).ratio()
|
|
if ratio >= 0.5:
|
|
score += round(ratio * 30)
|
|
signals.append({"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}})
|
|
summary_parts.append("similar name")
|
|
|
|
if score >= DUPLICATE_THRESHOLD:
|
|
open_issue(
|
|
db,
|
|
scan,
|
|
rule_type="possible_duplicate_customer",
|
|
entity_type="customer",
|
|
entity_id=a.id,
|
|
severity="high",
|
|
summary="; ".join(summary_parts) + f" (score {score})",
|
|
entity_ref=a.public_ref,
|
|
related_refs=[b.public_ref],
|
|
signals=signals,
|
|
)
|