Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bound the QA evidence overlay and fetch only what it draws

Browse Source 
evidence_geojson emitted one feature per false positive, one per false negative
and two per match, with no limit. A regional check of 40k detections against 45k
reference footprints produced well over a hundred thousand features in a single
response, plus one warning string per unresolvable identifier. The endpoint the
entire review workflow depends on therefore failed exactly where review matters
most.

What to draw is now decided before any geometry is fetched, so the query work is
proportional to the result rather than to the size of the check — previously
130k geometries were resolved through an IN clause holding every identifier in
the check, to then discard most of them.

The budget is split between misses and false positives in proportion to their
populations with at least one of each, rather than by strict priority, which
would mean a check with 50.000 misses and three false positives never showed
one. Confirmations fill what remains, and a match is kept or dropped as a pair
because half a match is not reviewable evidence.

limit_evidence and evidence_role_counts are removed: plan_evidence supersedes
them, and helpers kept alive only by their own tests read like a contract.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 15:14:26 +02:00
co-authored by Claude Opus 5
parent 52c2bfd120
commit 5278fcd361
7 changed files with 412 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/routes/quality_checks.py
+14 -1
View File
@@ -40,9 +40,22 @@ def list_quality_checks(
def get_quality_check_evidence_geojson(
project_id: UUID,
quality_check_id: UUID,
limit: int = Query(
default=QualityEvidenceService.DEFAULT_EVIDENCE_LIMIT,
ge=0,
le=100_000,
description="Maximum evidence features to draw; 0 returns everything. Misses and false positives first.",
),
db: Session = Depends(get_db),
) -> dict:
return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id))
return envelope(
QualityEvidenceService.evidence_geojson(
db,
project_id=project_id,
quality_check_id=quality_check_id,
limit=limit,
)
)
@router.get(
backend/app/schemas/qa.py
+6
View File
@@ -89,7 +89,13 @@ class QualityEvidenceResponse(BaseModel):
candidate_dataset_id: UUID | None = None
reference_dataset_id: UUID
analysis_run_id: UUID | None = None
# The overlay is capped so a regional check stays reviewable; the counts in
# the quality check itself are always complete.
feature_count: int
total_feature_count: int | None = None
role_counts: dict[str, int] = Field(default_factory=dict)
truncated: bool = False
limit: int | None = None
warnings: list[str] = Field(default_factory=list)
geojson: GeoJsonFeatureCollection
backend/app/services/quality_evidence_service.py
+216 -60
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from uuid import UUID
@@ -12,84 +13,235 @@ from app.core.errors import AppError
from app.models import Detection, DetectionReview, QualityCheck, Segmentation, VectorFeature
class QualityEvidenceService:
@staticmethod
def evidence_geojson(db: Session, *, project_id: UUID, quality_check_id: UUID) -> dict[str, Any]:
quality_check = db.get(QualityCheck, quality_check_id)
if not quality_check or quality_check.project_id != project_id:
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
@dataclass(frozen=True)
class EvidenceItem:
role: str
candidate_feature_id: str | None
reference_feature_id: str | None
evidence: dict[str, Any]
findings = quality_check.findings_json or {}
features: list[dict[str, Any]] = []
warnings: list[str] = []
candidate_ids, reference_ids = QualityEvidenceService._evidence_identifiers(findings)
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, candidate_ids)
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, reference_ids)
@dataclass(frozen=True)
class EvidencePlan:
"""Which evidence to draw, decided before any geometry is fetched."""
items: list[EvidenceItem]
candidate_ids: set[str]
reference_ids: set[str]
total_feature_count: int
role_counts: dict[str, int]
truncated: bool
class QualityEvidenceService:
DEFAULT_EVIDENCE_LIMIT = 5_000
@staticmethod
def plan_evidence(findings: dict[str, Any], *, limit: int) -> EvidencePlan:
"""Decide what to draw before resolving a single geometry.
Building every feature and then discarding most of them meant fetching
130k geometries to draw 5k, with an ``IN`` clause holding every
identifier in the check. Planning first makes the work proportional to
what is returned.
A match contributes two features and is kept or dropped as a pair; half
a match is not reviewable evidence.
"""
matches: list[EvidenceItem] = []
false_positives: list[EvidenceItem] = []
false_negatives: list[EvidenceItem] = []
role_counts: dict[str, int] = {}
def count(role: str) -> None:
role_counts[role] = role_counts.get(role, 0) + 1
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
iou = evidence.get("iou")
if not candidate_id and not reference_id:
continue
if candidate_id:
row = candidate_index.get(candidate_id)
if row is not None:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="match_candidate",
quality_check=quality_check,
evidence=evidence,
)
)
else:
warnings.append(f"Candidate evidence feature not found: {candidate_id}")
count("match_candidate")
if reference_id:
row = reference_index.get(reference_id)
if row is not None:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="match_reference",
quality_check=quality_check,
evidence={"candidate_feature_id": candidate_id, "reference_feature_id": reference_id, "iou": iou},
)
)
else:
warnings.append(f"Reference evidence feature not found: {reference_id}")
count("match_reference")
matches.append(EvidenceItem("match", candidate_id, reference_id, evidence))
for evidence in QualityEvidenceService._evidence_items(findings.get("false_positive_evidence")):
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
if not candidate_id:
continue
row = candidate_index.get(candidate_id)
if row is not None:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="false_positive",
quality_check=quality_check,
evidence=evidence,
)
)
else:
warnings.append(f"False-positive evidence feature not found: {candidate_id}")
count("false_positive")
false_positives.append(EvidenceItem("false_positive", candidate_id, None, evidence))
for evidence in QualityEvidenceService._evidence_items(findings.get("false_negative_evidence")):
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
if not reference_id:
continue
row = reference_index.get(reference_id)
if row is not None:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="false_negative",
quality_check=quality_check,
evidence=evidence,
)
)
count("false_negative")
false_negatives.append(EvidenceItem("false_negative", None, reference_id, evidence))
total_feature_count = sum(role_counts.values())
# Both error classes must be visible. Strict priority would mean a check
# with 50.000 misses and three false positives never shows one, so the
# budget is split between them in proportion to their populations, with
# at least one of each. Confirmations fill whatever is left.
selected: list[EvidenceItem] = []
truncated = False
if limit <= 0:
selected = false_negatives + false_positives + matches
else:
actionable = len(false_negatives) + len(false_positives)
if actionable <= limit:
miss_budget, false_positive_budget = len(false_negatives), len(false_positives)
else:
warnings.append(f"False-negative evidence feature not found: {reference_id}")
miss_budget = round(limit * len(false_negatives) / actionable)
miss_budget = min(len(false_negatives), max(1 if false_negatives else 0, miss_budget))
false_positive_budget = min(len(false_positives), limit - miss_budget)
if false_positives and false_positive_budget == 0:
false_positive_budget = 1
miss_budget = min(miss_budget, limit - 1)
selected = false_negatives[:miss_budget] + false_positives[:false_positive_budget]
truncated = miss_budget < len(false_negatives) or false_positive_budget < len(false_positives)
drawn = len(selected)
for item in matches:
cost = sum(1 for value in (item.candidate_feature_id, item.reference_feature_id) if value)
if drawn + cost > limit:
truncated = True
continue
selected.append(item)
drawn += cost
return EvidencePlan(
items=selected,
candidate_ids={item.candidate_feature_id for item in selected if item.candidate_feature_id},
reference_ids={item.reference_feature_id for item in selected if item.reference_feature_id},
total_feature_count=total_feature_count,
role_counts=role_counts,
truncated=truncated,
)
@staticmethod
def summarize_missing(*, candidate_ids: list[str], reference_ids: list[str]) -> list[str]:
"""One statement instead of one warning per unresolvable identifier.
A run whose detection rows were removed produced tens of thousands of
identically shaped strings, which buries every other warning.
"""
if not candidate_ids and not reference_ids:
return []
return [
f"{len(candidate_ids)} kandidaat- en {len(reference_ids)} referentieobjecten uit dit bewijs zijn niet "
"meer als geometrie terug te vinden; ze staan wel in de bewaarde telling."
]
@staticmethod
def evidence_geojson(
db: Session,
*,
project_id: UUID,
quality_check_id: UUID,
limit: int | None = None,
) -> dict[str, Any]:
quality_check = db.get(QualityCheck, quality_check_id)
if not quality_check or quality_check.project_id != project_id:
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
findings = quality_check.findings_json or {}
warnings: list[str] = []
missing_candidates: list[str] = []
missing_references: list[str] = []
resolved_limit = (
QualityEvidenceService.DEFAULT_EVIDENCE_LIMIT if limit is None else max(0, int(limit))
)
# Decide what to draw first, then fetch only those geometries.
plan = QualityEvidenceService.plan_evidence(findings, limit=resolved_limit)
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, plan.candidate_ids)
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, plan.reference_ids)
features: list[dict[str, Any]] = []
for item in plan.items:
if item.role == "match":
if item.candidate_feature_id:
row = candidate_index.get(item.candidate_feature_id)
if row is None:
missing_candidates.append(item.candidate_feature_id)
else:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="match_candidate",
quality_check=quality_check,
evidence=item.evidence,
)
)
if item.reference_feature_id:
row = reference_index.get(item.reference_feature_id)
if row is None:
missing_references.append(item.reference_feature_id)
else:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="match_reference",
quality_check=quality_check,
evidence={
"candidate_feature_id": item.candidate_feature_id,
"reference_feature_id": item.reference_feature_id,
"iou": item.evidence.get("iou"),
},
)
)
continue
if item.role == "false_positive" and item.candidate_feature_id:
row = candidate_index.get(item.candidate_feature_id)
if row is None:
missing_candidates.append(item.candidate_feature_id)
else:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="false_positive",
quality_check=quality_check,
evidence=item.evidence,
)
)
elif item.role == "false_negative" and item.reference_feature_id:
row = reference_index.get(item.reference_feature_id)
if row is None:
missing_references.append(item.reference_feature_id)
else:
features.append(
QualityEvidenceService._row_to_feature(
row,
role="false_negative",
quality_check=quality_check,
evidence=item.evidence,
)
)
warnings.extend(
QualityEvidenceService.summarize_missing(
candidate_ids=missing_candidates,
reference_ids=missing_references,
)
)
role_counts = plan.role_counts
total_feature_count = plan.total_feature_count
truncated = plan.truncated
if truncated:
warnings.append(
f"Dit overzicht toont {len(features)} van {total_feature_count} bewijsobjecten: gemiste en "
"onterecht gevonden objecten eerst. De tellingen in de kwaliteitscontrole blijven volledig."
)
QualityEvidenceService._annotate_reviews(db, quality_check, features)
@@ -100,6 +252,10 @@ class QualityEvidenceService:
"reference_dataset_id": str(quality_check.reference_dataset_id),
"analysis_run_id": str(quality_check.analysis_run_id) if quality_check.analysis_run_id else None,
"feature_count": len(features),
"total_feature_count": total_feature_count,
"role_counts": role_counts,
"truncated": truncated,
"limit": resolved_limit,
"warnings": warnings,
"geojson": {
"type": "FeatureCollection",