M48: harden demo operations and offsite recovery

backend/app/services/data_quality_duplicate_scan.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import uuid
+from difflib import SequenceMatcher
+from typing import Protocol, TypeVar
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models.customer import Customer
+
+DUPLICATE_THRESHOLD = 70
+
+
+class ScanAccumulator(Protocol):
+    def bump(self, rule_type: str) -> None: ...
+
+
+ScanTypeContra = TypeVar("ScanTypeContra", bound=ScanAccumulator, contravariant=True)
+
+
+class OpenIssue(Protocol[ScanTypeContra]):
+    def __call__(
+        self,
+        db: Session,
+        scan: ScanTypeContra,
+        *,
+        rule_type: str,
+        entity_type: str,
+        entity_id: uuid.UUID,
+        severity: str,
+        summary: str,
+        entity_ref: str,
+        related_refs: list[str],
+        signals: list[dict] | None = None,
+    ) -> None: ...
+
+
+def _normalize(value: str | None) -> str:
+    return (value or "").strip().lower()
+
+
+def scan_duplicate_customers[ScanType: ScanAccumulator](
+    db: Session, scan: ScanType, open_issue: OpenIssue[ScanType]
+) -> None:
+    customers = list(
+        db.scalars(
+            select(Customer).where(
+                Customer.merged_into_customer_id.is_(None),
+                Customer.anonymized_at.is_(None),
+            )
+        ).all()
+    )
+    customers.sort(key=lambda customer: customer.public_ref)
+
+    # The threshold cannot be reached without an exact email (60 points) or phone
+    # (50 points). Block on normalized identifiers so this remains linear for the
+    # overwhelmingly common case and only scores plausible pairs.
+    candidate_pairs: set[tuple[int, int]] = set()
+    for attribute in ("email", "phone"):
+        blocks: dict[str, list[int]] = {}
+        for index, customer in enumerate(customers):
+            key = _normalize(getattr(customer, attribute))
+            if key:
+                blocks.setdefault(key, []).append(index)
+        for indices in blocks.values():
+            for offset, left in enumerate(indices):
+                candidate_pairs.update((left, right) for right in indices[offset + 1 :])
+
+    for left, right in sorted(candidate_pairs):
+        a = customers[left]
+        b = customers[right]
+        score = 0
+        signals: list[dict] = []
+        summary_parts: list[str] = []
+        if _normalize(a.email) and _normalize(a.email) == _normalize(b.email):
+            score += 60
+            signals.append({"code": "duplicate.exact_email"})
+            summary_parts.append("exact email")
+        if _normalize(a.phone) and _normalize(a.phone) == _normalize(b.phone):
+            score += 50
+            signals.append({"code": "duplicate.exact_phone"})
+            summary_parts.append("exact phone")
+        if _normalize(a.postal_code) and _normalize(a.postal_code) == _normalize(b.postal_code):
+            score += 10
+            signals.append({"code": "duplicate.same_postal_code"})
+            summary_parts.append("exact postal code")
+        name_a = f"{_normalize(a.first_name)} {_normalize(a.last_name)}"
+        name_b = f"{_normalize(b.first_name)} {_normalize(b.last_name)}"
+        ratio = SequenceMatcher(None, name_a, name_b).ratio()
+        if ratio >= 0.5:
+            score += round(ratio * 30)
+            signals.append(
+                {"code": "duplicate.similar_name", "params": {"score": round(ratio, 2)}}
+            )
+            summary_parts.append("similar name")
+
+        if score >= DUPLICATE_THRESHOLD:
+            open_issue(
+                db,
+                scan,
+                rule_type="possible_duplicate_customer",
+                entity_type="customer",
+                entity_id=a.id,
+                severity="high",
+                summary="; ".join(summary_parts) + f" (score {score})",
+                entity_ref=a.public_ref,
+                related_refs=[b.public_ref],
+                signals=signals,
+            )