bound the QA evidence overlay and fetch only what it draws
evidence_geojson emitted one feature per false positive, one per false negative and two per match, with no limit. A regional check of 40k detections against 45k reference footprints produced well over a hundred thousand features in a single response, plus one warning string per unresolvable identifier. The endpoint the entire review workflow depends on therefore failed exactly where review matters most. What to draw is now decided before any geometry is fetched, so the query work is proportional to the result rather than to the size of the check — previously 130k geometries were resolved through an IN clause holding every identifier in the check, to then discard most of them. The budget is split between misses and false positives in proportion to their populations with at least one of each, rather than by strict priority, which would mean a check with 50.000 misses and three false positives never showed one. Confirmations fill what remains, and a match is kept or dropped as a pair because half a match is not reviewable evidence. limit_evidence and evidence_role_counts are removed: plan_evidence supersedes them, and helpers kept alive only by their own tests read like a contract. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
@@ -12,84 +13,235 @@ from app.core.errors import AppError
|
||||
from app.models import Detection, DetectionReview, QualityCheck, Segmentation, VectorFeature
|
||||
|
||||
|
||||
class QualityEvidenceService:
|
||||
@staticmethod
|
||||
def evidence_geojson(db: Session, *, project_id: UUID, quality_check_id: UUID) -> dict[str, Any]:
|
||||
quality_check = db.get(QualityCheck, quality_check_id)
|
||||
if not quality_check or quality_check.project_id != project_id:
|
||||
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
|
||||
@dataclass(frozen=True)
|
||||
class EvidenceItem:
|
||||
role: str
|
||||
candidate_feature_id: str | None
|
||||
reference_feature_id: str | None
|
||||
evidence: dict[str, Any]
|
||||
|
||||
findings = quality_check.findings_json or {}
|
||||
features: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
candidate_ids, reference_ids = QualityEvidenceService._evidence_identifiers(findings)
|
||||
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, candidate_ids)
|
||||
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, reference_ids)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvidencePlan:
|
||||
"""Which evidence to draw, decided before any geometry is fetched."""
|
||||
|
||||
items: list[EvidenceItem]
|
||||
candidate_ids: set[str]
|
||||
reference_ids: set[str]
|
||||
total_feature_count: int
|
||||
role_counts: dict[str, int]
|
||||
truncated: bool
|
||||
|
||||
|
||||
class QualityEvidenceService:
|
||||
DEFAULT_EVIDENCE_LIMIT = 5_000
|
||||
|
||||
@staticmethod
|
||||
def plan_evidence(findings: dict[str, Any], *, limit: int) -> EvidencePlan:
|
||||
"""Decide what to draw before resolving a single geometry.
|
||||
|
||||
Building every feature and then discarding most of them meant fetching
|
||||
130k geometries to draw 5k, with an ``IN`` clause holding every
|
||||
identifier in the check. Planning first makes the work proportional to
|
||||
what is returned.
|
||||
|
||||
A match contributes two features and is kept or dropped as a pair; half
|
||||
a match is not reviewable evidence.
|
||||
"""
|
||||
|
||||
matches: list[EvidenceItem] = []
|
||||
false_positives: list[EvidenceItem] = []
|
||||
false_negatives: list[EvidenceItem] = []
|
||||
role_counts: dict[str, int] = {}
|
||||
|
||||
def count(role: str) -> None:
|
||||
role_counts[role] = role_counts.get(role, 0) + 1
|
||||
|
||||
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
|
||||
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
|
||||
iou = evidence.get("iou")
|
||||
if not candidate_id and not reference_id:
|
||||
continue
|
||||
if candidate_id:
|
||||
row = candidate_index.get(candidate_id)
|
||||
if row is not None:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="match_candidate",
|
||||
quality_check=quality_check,
|
||||
evidence=evidence,
|
||||
)
|
||||
)
|
||||
else:
|
||||
warnings.append(f"Candidate evidence feature not found: {candidate_id}")
|
||||
count("match_candidate")
|
||||
if reference_id:
|
||||
row = reference_index.get(reference_id)
|
||||
if row is not None:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="match_reference",
|
||||
quality_check=quality_check,
|
||||
evidence={"candidate_feature_id": candidate_id, "reference_feature_id": reference_id, "iou": iou},
|
||||
)
|
||||
)
|
||||
else:
|
||||
warnings.append(f"Reference evidence feature not found: {reference_id}")
|
||||
count("match_reference")
|
||||
matches.append(EvidenceItem("match", candidate_id, reference_id, evidence))
|
||||
|
||||
for evidence in QualityEvidenceService._evidence_items(findings.get("false_positive_evidence")):
|
||||
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||
if not candidate_id:
|
||||
continue
|
||||
row = candidate_index.get(candidate_id)
|
||||
if row is not None:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="false_positive",
|
||||
quality_check=quality_check,
|
||||
evidence=evidence,
|
||||
)
|
||||
)
|
||||
else:
|
||||
warnings.append(f"False-positive evidence feature not found: {candidate_id}")
|
||||
count("false_positive")
|
||||
false_positives.append(EvidenceItem("false_positive", candidate_id, None, evidence))
|
||||
|
||||
for evidence in QualityEvidenceService._evidence_items(findings.get("false_negative_evidence")):
|
||||
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
|
||||
if not reference_id:
|
||||
continue
|
||||
row = reference_index.get(reference_id)
|
||||
if row is not None:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="false_negative",
|
||||
quality_check=quality_check,
|
||||
evidence=evidence,
|
||||
)
|
||||
)
|
||||
count("false_negative")
|
||||
false_negatives.append(EvidenceItem("false_negative", None, reference_id, evidence))
|
||||
|
||||
total_feature_count = sum(role_counts.values())
|
||||
|
||||
# Both error classes must be visible. Strict priority would mean a check
|
||||
# with 50.000 misses and three false positives never shows one, so the
|
||||
# budget is split between them in proportion to their populations, with
|
||||
# at least one of each. Confirmations fill whatever is left.
|
||||
selected: list[EvidenceItem] = []
|
||||
truncated = False
|
||||
|
||||
if limit <= 0:
|
||||
selected = false_negatives + false_positives + matches
|
||||
else:
|
||||
actionable = len(false_negatives) + len(false_positives)
|
||||
if actionable <= limit:
|
||||
miss_budget, false_positive_budget = len(false_negatives), len(false_positives)
|
||||
else:
|
||||
warnings.append(f"False-negative evidence feature not found: {reference_id}")
|
||||
miss_budget = round(limit * len(false_negatives) / actionable)
|
||||
miss_budget = min(len(false_negatives), max(1 if false_negatives else 0, miss_budget))
|
||||
false_positive_budget = min(len(false_positives), limit - miss_budget)
|
||||
if false_positives and false_positive_budget == 0:
|
||||
false_positive_budget = 1
|
||||
miss_budget = min(miss_budget, limit - 1)
|
||||
|
||||
selected = false_negatives[:miss_budget] + false_positives[:false_positive_budget]
|
||||
truncated = miss_budget < len(false_negatives) or false_positive_budget < len(false_positives)
|
||||
|
||||
drawn = len(selected)
|
||||
for item in matches:
|
||||
cost = sum(1 for value in (item.candidate_feature_id, item.reference_feature_id) if value)
|
||||
if drawn + cost > limit:
|
||||
truncated = True
|
||||
continue
|
||||
selected.append(item)
|
||||
drawn += cost
|
||||
|
||||
return EvidencePlan(
|
||||
items=selected,
|
||||
candidate_ids={item.candidate_feature_id for item in selected if item.candidate_feature_id},
|
||||
reference_ids={item.reference_feature_id for item in selected if item.reference_feature_id},
|
||||
total_feature_count=total_feature_count,
|
||||
role_counts=role_counts,
|
||||
truncated=truncated,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def summarize_missing(*, candidate_ids: list[str], reference_ids: list[str]) -> list[str]:
|
||||
"""One statement instead of one warning per unresolvable identifier.
|
||||
|
||||
A run whose detection rows were removed produced tens of thousands of
|
||||
identically shaped strings, which buries every other warning.
|
||||
"""
|
||||
|
||||
if not candidate_ids and not reference_ids:
|
||||
return []
|
||||
return [
|
||||
f"{len(candidate_ids)} kandidaat- en {len(reference_ids)} referentieobjecten uit dit bewijs zijn niet "
|
||||
"meer als geometrie terug te vinden; ze staan wel in de bewaarde telling."
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def evidence_geojson(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
quality_check = db.get(QualityCheck, quality_check_id)
|
||||
if not quality_check or quality_check.project_id != project_id:
|
||||
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
|
||||
|
||||
findings = quality_check.findings_json or {}
|
||||
warnings: list[str] = []
|
||||
missing_candidates: list[str] = []
|
||||
missing_references: list[str] = []
|
||||
|
||||
resolved_limit = (
|
||||
QualityEvidenceService.DEFAULT_EVIDENCE_LIMIT if limit is None else max(0, int(limit))
|
||||
)
|
||||
# Decide what to draw first, then fetch only those geometries.
|
||||
plan = QualityEvidenceService.plan_evidence(findings, limit=resolved_limit)
|
||||
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, plan.candidate_ids)
|
||||
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, plan.reference_ids)
|
||||
|
||||
features: list[dict[str, Any]] = []
|
||||
for item in plan.items:
|
||||
if item.role == "match":
|
||||
if item.candidate_feature_id:
|
||||
row = candidate_index.get(item.candidate_feature_id)
|
||||
if row is None:
|
||||
missing_candidates.append(item.candidate_feature_id)
|
||||
else:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="match_candidate",
|
||||
quality_check=quality_check,
|
||||
evidence=item.evidence,
|
||||
)
|
||||
)
|
||||
if item.reference_feature_id:
|
||||
row = reference_index.get(item.reference_feature_id)
|
||||
if row is None:
|
||||
missing_references.append(item.reference_feature_id)
|
||||
else:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="match_reference",
|
||||
quality_check=quality_check,
|
||||
evidence={
|
||||
"candidate_feature_id": item.candidate_feature_id,
|
||||
"reference_feature_id": item.reference_feature_id,
|
||||
"iou": item.evidence.get("iou"),
|
||||
},
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if item.role == "false_positive" and item.candidate_feature_id:
|
||||
row = candidate_index.get(item.candidate_feature_id)
|
||||
if row is None:
|
||||
missing_candidates.append(item.candidate_feature_id)
|
||||
else:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="false_positive",
|
||||
quality_check=quality_check,
|
||||
evidence=item.evidence,
|
||||
)
|
||||
)
|
||||
elif item.role == "false_negative" and item.reference_feature_id:
|
||||
row = reference_index.get(item.reference_feature_id)
|
||||
if row is None:
|
||||
missing_references.append(item.reference_feature_id)
|
||||
else:
|
||||
features.append(
|
||||
QualityEvidenceService._row_to_feature(
|
||||
row,
|
||||
role="false_negative",
|
||||
quality_check=quality_check,
|
||||
evidence=item.evidence,
|
||||
)
|
||||
)
|
||||
|
||||
warnings.extend(
|
||||
QualityEvidenceService.summarize_missing(
|
||||
candidate_ids=missing_candidates,
|
||||
reference_ids=missing_references,
|
||||
)
|
||||
)
|
||||
role_counts = plan.role_counts
|
||||
total_feature_count = plan.total_feature_count
|
||||
truncated = plan.truncated
|
||||
if truncated:
|
||||
warnings.append(
|
||||
f"Dit overzicht toont {len(features)} van {total_feature_count} bewijsobjecten: gemiste en "
|
||||
"onterecht gevonden objecten eerst. De tellingen in de kwaliteitscontrole blijven volledig."
|
||||
)
|
||||
|
||||
QualityEvidenceService._annotate_reviews(db, quality_check, features)
|
||||
|
||||
@@ -100,6 +252,10 @@ class QualityEvidenceService:
|
||||
"reference_dataset_id": str(quality_check.reference_dataset_id),
|
||||
"analysis_run_id": str(quality_check.analysis_run_id) if quality_check.analysis_run_id else None,
|
||||
"feature_count": len(features),
|
||||
"total_feature_count": total_feature_count,
|
||||
"role_counts": role_counts,
|
||||
"truncated": truncated,
|
||||
"limit": resolved_limit,
|
||||
"warnings": warnings,
|
||||
"geojson": {
|
||||
"type": "FeatureCollection",
|
||||
|
||||
Reference in New Issue
Block a user