feat: add measured detection review loop
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-15 03:00:08 +02:00
parent 94ecd377b7
commit d22abe8e7b
27 changed files with 1578 additions and 29 deletions
+23
View File
@@ -7,6 +7,29 @@
# Changelog # Changelog
## Sprint 197 Measured detection accuracy and durable review (2026-07-15)
- Re-ran the active local building model at confidence thresholds `0.10` and
`0.15` over independent Mol holdouts in Achterbos, Gompel, Donk and Postel,
plus the pure-empty Postel forest control. Threshold `0.15` retained the best
F1 in every positive zone and both thresholds produced zero forest-control
detections, so no speculative threshold or model promotion was made.
- Aligned the map-driven building QA flow with the documented operational
footprint match IoU `0.25`. The map now reports candidate count, matches,
precision, recall, F1, false positives and false negatives instead of
presenting every model box as a recognized building.
- Added first-class `detection_reviews` persistence and canonical project QA
endpoints for paginated false-positive/false-negative review decisions.
- Added a focused frontend review queue with role/status filters, notes,
pagination and map handoff. Unreviewed or alignment-mismatch evidence cannot
silently become hard-negative training data.
- Bounded QA evidence resolution to the persisted evidence identifiers instead
of loading complete regional reference datasets into application memory.
- Added regression coverage for migration alignment, decision validation, API
envelopes, bounded evidence access and the map QA threshold.
- Passed the complete readiness gate with 592 backend tests, 88 documented API
routes, one Alembic head and a green frontend typecheck/production build.
## Sprint 196 Map-driven official orthophoto analysis (2026-07-15) ## Sprint 196 Map-driven official orthophoto analysis (2026-07-15)
- Added an explicit bounded endpoint for the official Digitaal Vlaanderen - Added an explicit bounded endpoint for the official Digitaal Vlaanderen
+17
View File
@@ -708,6 +708,23 @@ curl http://localhost:1202/api/v1/projects/{project_id}/quality-checks
The frontend QA/QC Results panel uses this endpoint after loading the demo The frontend QA/QC Results panel uses this endpoint after loading the demo
workflow or running QA. workflow or running QA.
Detection QA evidence can be reviewed without changing its persisted metrics:
```bash
curl "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews?reviewed=false&limit=50"
curl -X POST "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews" \
-H "Content-Type: application/json" \
-d '{"evidence_role":"false_positive","evidence_feature_id":"DETECTION_UUID","decision":"qa_alignment_mismatch","notes":"Box and footprint represent the same building."}'
```
The list is derived from persisted quality-check evidence and paginates at a
maximum of 200 rows. The upsert verifies project ownership, quality-check type,
role-specific decisions and persisted Detection/VectorFeature ownership.
`detection_reviews` never mutates model output, reference geometry or canonical
Metric rows. Evidence GeoJSON queries only stored evidence ids instead of a
complete regional GRB dataset.
### Export foundation ### Export foundation
Persisted exports can be created from the existing workbench state: Persisted exports can be created from the existing workbench state:
@@ -0,0 +1,64 @@
"""Add durable operator review decisions for detection QA evidence."""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision = "202607150001"
down_revision = "202607140001"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"detection_reviews",
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("quality_check_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("analysis_run_id", postgresql.UUID(as_uuid=True), nullable=True),
sa.Column("evidence_role", sa.String(length=32), nullable=False),
sa.Column("evidence_feature_id", sa.String(length=255), nullable=False),
sa.Column("detection_id", postgresql.UUID(as_uuid=True), nullable=True),
sa.Column("reference_feature_id", postgresql.UUID(as_uuid=True), nullable=True),
sa.Column("decision", sa.String(length=64), server_default="unreviewed", nullable=False),
sa.Column("notes", sa.Text(), nullable=True),
sa.Column("reviewed_by", sa.String(length=120), server_default="operator", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.CheckConstraint(
"evidence_role IN ('false_positive', 'false_negative')",
name="ck_detection_reviews_evidence_role",
),
sa.CheckConstraint(
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
"'reference_gap_or_change', 'qa_alignment_mismatch', "
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
name="ck_detection_reviews_decision",
),
sa.ForeignKeyConstraint(["analysis_run_id"], ["analysis_runs.id"], ondelete="SET NULL"),
sa.ForeignKeyConstraint(["detection_id"], ["detections.id"], ondelete="SET NULL"),
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["quality_check_id"], ["quality_checks.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["reference_feature_id"], ["vector_features.id"], ondelete="SET NULL"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"quality_check_id",
"evidence_role",
"evidence_feature_id",
name="uq_detection_reviews_evidence",
),
)
op.create_index("ix_detection_reviews_project_id", "detection_reviews", ["project_id"])
op.create_index("ix_detection_reviews_quality_check_id", "detection_reviews", ["quality_check_id"])
op.create_index("ix_detection_reviews_analysis_run_id", "detection_reviews", ["analysis_run_id"])
op.create_index("ix_detection_reviews_decision", "detection_reviews", ["decision"])
def downgrade() -> None:
op.drop_index("ix_detection_reviews_decision", table_name="detection_reviews")
op.drop_index("ix_detection_reviews_analysis_run_id", table_name="detection_reviews")
op.drop_index("ix_detection_reviews_quality_check_id", table_name="detection_reviews")
op.drop_index("ix_detection_reviews_project_id", table_name="detection_reviews")
op.drop_table("detection_reviews")
+44
View File
@@ -6,7 +6,9 @@ from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.db.session import get_db from app.db.session import get_db
from app.schemas.detection_review import DetectionReviewUpsert
from app.schemas.qa import QualityCheckList from app.schemas.qa import QualityCheckList
from app.services.detection_review_service import DetectionReviewService
from app.services.quality_evidence_service import QualityEvidenceService from app.services.quality_evidence_service import QualityEvidenceService
from app.services.quality_check_service import QualityCheckService from app.services.quality_check_service import QualityCheckService
from app.utils.response import envelope from app.utils.response import envelope
@@ -37,3 +39,45 @@ def get_quality_check_evidence_geojson(
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id)) return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id))
@router.get("/quality-checks/{quality_check_id}/reviews", response_model=dict)
def list_detection_reviews(
project_id: UUID,
quality_check_id: UUID,
evidence_role: str | None = Query(default=None, pattern="^(false_positive|false_negative)$"),
decision: str | None = Query(default=None, max_length=64),
reviewed: bool | None = Query(default=None),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
return envelope(
DetectionReviewService.list_reviews(
db,
project_id=project_id,
quality_check_id=quality_check_id,
evidence_role=evidence_role,
decision=decision,
reviewed=reviewed,
limit=limit,
offset=offset,
).model_dump()
)
@router.post("/quality-checks/{quality_check_id}/reviews", response_model=dict)
def upsert_detection_review(
project_id: UUID,
quality_check_id: UUID,
payload: DetectionReviewUpsert,
db: Session = Depends(get_db),
) -> dict:
return envelope(
DetectionReviewService.upsert_review(
db,
project_id=project_id,
quality_check_id=quality_check_id,
payload=payload,
).model_dump()
)
+2 -1
View File
@@ -1,4 +1,4 @@
from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
__all__ = [ __all__ = [
"AnalysisRun", "AnalysisRun",
@@ -6,6 +6,7 @@ __all__ = [
"Dataset", "Dataset",
"DatasetVersion", "DatasetVersion",
"Detection", "Detection",
"DetectionReview",
"Export", "Export",
"Job", "Job",
"Metric", "Metric",
+57 -1
View File
@@ -4,7 +4,7 @@ import uuid
from datetime import datetime from datetime import datetime
from geoalchemy2 import Geometry from geoalchemy2 import Geometry
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, func from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, UniqueConstraint, func
from sqlalchemy.sql.sqltypes import Integer from sqlalchemy.sql.sqltypes import Integer
from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -272,6 +272,62 @@ class Metric(Base):
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class DetectionReview(Base):
__tablename__ = "detection_reviews"
__table_args__ = (
CheckConstraint(
"evidence_role IN ('false_positive', 'false_negative')",
name="ck_detection_reviews_evidence_role",
),
CheckConstraint(
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
"'reference_gap_or_change', 'qa_alignment_mismatch', "
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
name="ck_detection_reviews_decision",
),
UniqueConstraint(
"quality_check_id",
"evidence_role",
"evidence_feature_id",
name="uq_detection_reviews_evidence",
),
Index("ix_detection_reviews_project_id", "project_id"),
Index("ix_detection_reviews_quality_check_id", "quality_check_id"),
Index("ix_detection_reviews_analysis_run_id", "analysis_run_id"),
Index("ix_detection_reviews_decision", "decision"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
quality_check_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("quality_checks.id", ondelete="CASCADE"),
nullable=False,
)
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("analysis_runs.id", ondelete="SET NULL"),
nullable=True,
)
evidence_role: Mapped[str] = mapped_column(String(32), nullable=False)
evidence_feature_id: Mapped[str] = mapped_column(String(255), nullable=False)
detection_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("detections.id", ondelete="SET NULL"),
nullable=True,
)
reference_feature_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("vector_features.id", ondelete="SET NULL"),
nullable=True,
)
decision: Mapped[str] = mapped_column(String(64), nullable=False, default="unreviewed", server_default="unreviewed")
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
reviewed_by: Mapped[str] = mapped_column(String(120), nullable=False, default="operator", server_default="operator")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class Export(Base): class Export(Base):
__tablename__ = "exports" __tablename__ = "exports"
+5
View File
@@ -18,6 +18,7 @@ from .detection import (
ModelAssetListResponse, ModelAssetListResponse,
ModelAssetRead, ModelAssetRead,
) )
from .detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewSummary, DetectionReviewUpsert
from .segmentation import ( from .segmentation import (
SegmentationListResponse, SegmentationListResponse,
SegmentationModelCapability, SegmentationModelCapability,
@@ -111,6 +112,10 @@ __all__ = [
"DetectionRunResponse", "DetectionRunResponse",
"ModelAssetListResponse", "ModelAssetListResponse",
"ModelAssetRead", "ModelAssetRead",
"DetectionReviewList",
"DetectionReviewRead",
"DetectionReviewSummary",
"DetectionReviewUpsert",
"SegmentationListResponse", "SegmentationListResponse",
"SegmentationModelCapability", "SegmentationModelCapability",
"SegmentationModelsResponse", "SegmentationModelsResponse",
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field
DetectionEvidenceRole = Literal["false_positive", "false_negative"]
DetectionReviewDecision = Literal[
"confirmed_model_false_positive",
"confirmed_model_false_negative",
"reference_gap_or_change",
"qa_alignment_mismatch",
"imagery_obscured_or_uncertain",
"uncertain",
"unreviewed",
]
class DetectionReviewUpsert(BaseModel):
evidence_role: DetectionEvidenceRole
evidence_feature_id: str = Field(min_length=1, max_length=255)
decision: DetectionReviewDecision
notes: str | None = Field(default=None, max_length=2000)
reviewed_by: str = Field(default="operator", min_length=1, max_length=120)
class DetectionReviewRead(BaseModel):
id: UUID | None = None
project_id: UUID
quality_check_id: UUID
analysis_run_id: UUID | None = None
evidence_role: DetectionEvidenceRole
evidence_feature_id: str
detection_id: UUID | None = None
reference_feature_id: UUID | None = None
decision: DetectionReviewDecision = "unreviewed"
notes: str | None = None
reviewed_by: str | None = None
confidence: float | None = None
class_name: str | None = None
source_tile_path: str | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
class DetectionReviewSummary(BaseModel):
total: int
reviewed: int
remaining: int
false_positive_total: int
false_negative_total: int
decision_counts: dict[str, int]
class DetectionReviewList(BaseModel):
items: list[DetectionReviewRead]
total: int
limit: int
offset: int
summary: DetectionReviewSummary
@@ -0,0 +1,252 @@
from __future__ import annotations
from collections import Counter
from typing import Any
from uuid import UUID
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
from app.schemas.detection_review import (
DetectionReviewList,
DetectionReviewRead,
DetectionReviewSummary,
DetectionReviewUpsert,
)
class DetectionReviewService:
ALLOWED_DECISIONS = {
"false_positive": {
"confirmed_model_false_positive",
"reference_gap_or_change",
"qa_alignment_mismatch",
"uncertain",
"unreviewed",
},
"false_negative": {
"confirmed_model_false_negative",
"reference_gap_or_change",
"qa_alignment_mismatch",
"imagery_obscured_or_uncertain",
"uncertain",
"unreviewed",
},
}
@staticmethod
def _quality_check(db: Session, project_id: UUID, quality_check_id: UUID) -> QualityCheck:
quality_check = db.get(QualityCheck, quality_check_id)
if not quality_check or quality_check.project_id != project_id:
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
if quality_check.check_type != "detections_vs_reference":
raise AppError(
code="DETECTION_REVIEW_UNSUPPORTED",
message="Only persisted detection-versus-reference quality checks can be reviewed",
status_code=422,
)
return quality_check
@staticmethod
def _evidence_items(quality_check: QualityCheck) -> list[dict[str, str]]:
findings = quality_check.findings_json or {}
items: list[dict[str, str]] = []
for role, key, id_key in (
("false_positive", "false_positive_evidence", "candidate_feature_id"),
("false_negative", "false_negative_evidence", "reference_feature_id"),
):
evidence_rows = findings.get(key)
if not isinstance(evidence_rows, list):
continue
for evidence in evidence_rows:
if not isinstance(evidence, dict):
continue
value = str(evidence.get(id_key) or "").strip()
if value:
items.append({"evidence_role": role, "evidence_feature_id": value})
return items
@staticmethod
def _uuid(value: str) -> UUID | None:
try:
return UUID(value)
except (TypeError, ValueError):
return None
@staticmethod
def _review_index(db: Session, quality_check_id: UUID) -> dict[tuple[str, str], DetectionReview]:
rows = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check_id).all()
return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
@staticmethod
def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary:
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
decisions = Counter(
reviews[key].decision if key in reviews else "unreviewed"
for key in evidence_keys
)
reviewed = sum(count for decision, count in decisions.items() if decision != "unreviewed")
false_positive_total = sum(1 for item in evidence if item["evidence_role"] == "false_positive")
false_negative_total = sum(1 for item in evidence if item["evidence_role"] == "false_negative")
return DetectionReviewSummary(
total=len(evidence),
reviewed=reviewed,
remaining=max(len(evidence) - reviewed, 0),
false_positive_total=false_positive_total,
false_negative_total=false_negative_total,
decision_counts=dict(sorted(decisions.items())),
)
@staticmethod
def _read_item(
db: Session,
quality_check: QualityCheck,
evidence: dict[str, str],
review: DetectionReview | None,
) -> DetectionReviewRead:
role = evidence["evidence_role"]
feature_id = evidence["evidence_feature_id"]
feature_uuid = DetectionReviewService._uuid(feature_id)
detection = db.get(Detection, feature_uuid) if role == "false_positive" and feature_uuid else None
reference = db.get(VectorFeature, feature_uuid) if role == "false_negative" and feature_uuid else None
return DetectionReviewRead(
id=review.id if review else None,
project_id=quality_check.project_id,
quality_check_id=quality_check.id,
analysis_run_id=quality_check.analysis_run_id,
evidence_role=role,
evidence_feature_id=feature_id,
detection_id=detection.id if detection else review.detection_id if review else None,
reference_feature_id=reference.id if reference else review.reference_feature_id if review else None,
decision=review.decision if review else "unreviewed",
notes=review.notes if review else None,
reviewed_by=review.reviewed_by if review else None,
confidence=detection.confidence if detection else None,
class_name=(detection.class_name if detection else reference.feature_class if reference else None),
source_tile_path=detection.source_tile_path if detection else None,
created_at=review.created_at if review else None,
updated_at=review.updated_at if review else None,
)
@staticmethod
def list_reviews(
db: Session,
*,
project_id: UUID,
quality_check_id: UUID,
evidence_role: str | None = None,
decision: str | None = None,
reviewed: bool | None = None,
limit: int = 50,
offset: int = 0,
) -> DetectionReviewList:
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
evidence = DetectionReviewService._evidence_items(quality_check)
reviews = DetectionReviewService._review_index(db, quality_check_id)
filtered = [item for item in evidence if evidence_role is None or item["evidence_role"] == evidence_role]
if decision is not None:
filtered = [
item
for item in filtered
if (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
else "unreviewed")
== decision
]
if reviewed is not None:
filtered = [
item
for item in filtered
if (
(reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
else "unreviewed")
!= "unreviewed"
)
== reviewed
]
page = filtered[offset : offset + limit]
return DetectionReviewList(
items=[
DetectionReviewService._read_item(
db,
quality_check,
item,
reviews.get((item["evidence_role"], item["evidence_feature_id"])),
)
for item in page
],
total=len(filtered),
limit=limit,
offset=offset,
summary=DetectionReviewService._summary(evidence, reviews),
)
@staticmethod
def upsert_review(
db: Session,
*,
project_id: UUID,
quality_check_id: UUID,
payload: DetectionReviewUpsert,
) -> DetectionReviewRead:
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
if payload.decision not in DetectionReviewService.ALLOWED_DECISIONS[payload.evidence_role]:
raise AppError(
code="INVALID_DETECTION_REVIEW_DECISION",
message="The review decision is not valid for this evidence role",
details={"evidence_role": payload.evidence_role, "decision": payload.decision},
status_code=422,
)
evidence = DetectionReviewService._evidence_items(quality_check)
evidence_key = (payload.evidence_role, payload.evidence_feature_id)
if evidence_key not in {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}:
raise AppError(
code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND",
message="The evidence feature does not belong to this quality check",
status_code=404,
)
feature_uuid = DetectionReviewService._uuid(payload.evidence_feature_id)
detection = db.get(Detection, feature_uuid) if payload.evidence_role == "false_positive" and feature_uuid else None
reference = db.get(VectorFeature, feature_uuid) if payload.evidence_role == "false_negative" and feature_uuid else None
if payload.evidence_role == "false_positive" and (not detection or detection.analysis_run_id != quality_check.analysis_run_id):
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted detection evidence was not found", status_code=404)
if payload.evidence_role == "false_negative" and (not reference or reference.dataset_id != quality_check.reference_dataset_id):
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted reference evidence was not found", status_code=404)
review = (
db.query(DetectionReview)
.filter(
DetectionReview.quality_check_id == quality_check_id,
DetectionReview.evidence_role == payload.evidence_role,
DetectionReview.evidence_feature_id == payload.evidence_feature_id,
)
.first()
)
if review is None:
review = DetectionReview(
project_id=project_id,
quality_check_id=quality_check_id,
analysis_run_id=quality_check.analysis_run_id,
evidence_role=payload.evidence_role,
evidence_feature_id=payload.evidence_feature_id,
detection_id=detection.id if detection else None,
reference_feature_id=reference.id if reference else None,
decision=payload.decision,
notes=payload.notes.strip() if payload.notes and payload.notes.strip() else None,
reviewed_by=payload.reviewed_by.strip(),
)
else:
review.decision = payload.decision
review.notes = payload.notes.strip() if payload.notes and payload.notes.strip() else None
review.reviewed_by = payload.reviewed_by.strip()
db.add(review)
db.commit()
db.refresh(review)
return DetectionReviewService._read_item(
db,
quality_check,
{"evidence_role": payload.evidence_role, "evidence_feature_id": payload.evidence_feature_id},
review,
)
+116 -17
View File
@@ -5,10 +5,11 @@ from uuid import UUID
from geoalchemy2.shape import to_shape from geoalchemy2.shape import to_shape
from shapely.geometry import mapping from shapely.geometry import mapping
from sqlalchemy import or_
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.errors import AppError from app.core.errors import AppError
from app.models import Detection, QualityCheck, Segmentation, VectorFeature from app.models import Detection, DetectionReview, QualityCheck, Segmentation, VectorFeature
class QualityEvidenceService: class QualityEvidenceService:
@@ -21,8 +22,9 @@ class QualityEvidenceService:
findings = quality_check.findings_json or {} findings = quality_check.findings_json or {}
features: list[dict[str, Any]] = [] features: list[dict[str, Any]] = []
warnings: list[str] = [] warnings: list[str] = []
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check) candidate_ids, reference_ids = QualityEvidenceService._evidence_identifiers(findings)
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check) candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, candidate_ids)
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, reference_ids)
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")): for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id")) candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
@@ -89,6 +91,8 @@ class QualityEvidenceService:
else: else:
warnings.append(f"False-negative evidence feature not found: {reference_id}") warnings.append(f"False-negative evidence feature not found: {reference_id}")
QualityEvidenceService._annotate_reviews(db, quality_check, features)
return { return {
"quality_check_id": str(quality_check.id), "quality_check_id": str(quality_check.id),
"project_id": str(quality_check.project_id), "project_id": str(quality_check.project_id),
@@ -117,34 +121,129 @@ class QualityEvidenceService:
return text or None return text or None
@staticmethod @staticmethod
def _candidate_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]: def _evidence_identifiers(findings: dict[str, Any]) -> tuple[set[str], set[str]]:
candidate_ids: set[str] = set()
reference_ids: set[str] = set()
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
if candidate_id:
candidate_ids.add(candidate_id)
if reference_id:
reference_ids.add(reference_id)
for evidence in QualityEvidenceService._evidence_items(findings.get("false_positive_evidence")):
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
if candidate_id:
candidate_ids.add(candidate_id)
for evidence in QualityEvidenceService._evidence_items(findings.get("false_negative_evidence")):
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
if reference_id:
reference_ids.add(reference_id)
return candidate_ids, reference_ids
@staticmethod
def _uuid_identifiers(identifiers: set[str]) -> list[UUID]:
values: list[UUID] = []
for identifier in identifiers:
try:
values.append(UUID(identifier))
except (TypeError, ValueError):
continue
return values
@staticmethod
def _vector_feature_rows(db: Session, dataset_id: UUID, identifiers: set[str]) -> list[VectorFeature]:
if not identifiers:
return []
conditions = [VectorFeature.source_feature_id.in_(identifiers)]
uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
if uuid_identifiers:
conditions.append(VectorFeature.id.in_(uuid_identifiers))
return (
db.query(VectorFeature)
.filter(VectorFeature.dataset_id == dataset_id, or_(*conditions))
.all()
)
@staticmethod
def _candidate_feature_index(
db: Session,
quality_check: QualityCheck,
identifiers: set[str],
) -> dict[str, Any]:
index: dict[str, Any] = {} index: dict[str, Any] = {}
if not identifiers:
return index
uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
if quality_check.candidate_dataset_id: if quality_check.candidate_dataset_id:
for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.candidate_dataset_id).all(): for row in QualityEvidenceService._vector_feature_rows(db, quality_check.candidate_dataset_id, identifiers):
QualityEvidenceService._add_index_keys(index, row) QualityEvidenceService._add_index_keys(index, row)
for row in db.query(Detection).filter(Detection.dataset_id == quality_check.candidate_dataset_id).all(): if uuid_identifiers:
QualityEvidenceService._add_index_keys(index, row) for row in db.query(Detection).filter(
for row in db.query(Segmentation).filter(Segmentation.dataset_id == quality_check.candidate_dataset_id).all(): Detection.dataset_id == quality_check.candidate_dataset_id,
QualityEvidenceService._add_index_keys(index, row) Detection.id.in_(uuid_identifiers),
if quality_check.analysis_run_id: ).all():
for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all():
QualityEvidenceService._add_index_keys(index, row) QualityEvidenceService._add_index_keys(index, row)
for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all(): for row in db.query(Segmentation).filter(
Segmentation.dataset_id == quality_check.candidate_dataset_id,
Segmentation.id.in_(uuid_identifiers),
).all():
QualityEvidenceService._add_index_keys(index, row) QualityEvidenceService._add_index_keys(index, row)
elif quality_check.analysis_run_id: if quality_check.analysis_run_id and uuid_identifiers:
for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all(): for row in db.query(Detection).filter(
Detection.analysis_run_id == quality_check.analysis_run_id,
Detection.id.in_(uuid_identifiers),
).all():
QualityEvidenceService._add_index_keys(index, row) QualityEvidenceService._add_index_keys(index, row)
for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all(): for row in db.query(Segmentation).filter(
Segmentation.analysis_run_id == quality_check.analysis_run_id,
Segmentation.id.in_(uuid_identifiers),
).all():
QualityEvidenceService._add_index_keys(index, row) QualityEvidenceService._add_index_keys(index, row)
return index return index
@staticmethod @staticmethod
def _reference_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]: def _reference_feature_index(
db: Session,
quality_check: QualityCheck,
identifiers: set[str],
) -> dict[str, Any]:
index: dict[str, Any] = {} index: dict[str, Any] = {}
for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.reference_dataset_id).all(): for row in QualityEvidenceService._vector_feature_rows(db, quality_check.reference_dataset_id, identifiers):
QualityEvidenceService._add_index_keys(index, row) QualityEvidenceService._add_index_keys(index, row)
return index return index
@staticmethod
def _annotate_reviews(
db: Session,
quality_check: QualityCheck,
features: list[dict[str, Any]],
) -> None:
if quality_check.check_type != "detections_vs_reference":
return
reviews = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check.id).all()
review_index = {(row.evidence_role, row.evidence_feature_id): row for row in reviews}
for feature in features:
properties = feature.get("properties")
if not isinstance(properties, dict):
continue
role = QualityEvidenceService._string_value(properties.get("qa_evidence_role"))
if role == "false_positive":
evidence_id = QualityEvidenceService._string_value(properties.get("candidate_feature_id"))
elif role == "false_negative":
evidence_id = QualityEvidenceService._string_value(properties.get("reference_feature_id"))
else:
continue
review = review_index.get((role, evidence_id or ""))
properties.update(
{
"review_decision": review.decision if review else "unreviewed",
"review_notes": review.notes if review else None,
"reviewed_by": review.reviewed_by if review else None,
"reviewed_at": review.updated_at.isoformat() if review and review.updated_at else None,
}
)
@staticmethod @staticmethod
def _add_index_keys(index: dict[str, Any], row: Any) -> None: def _add_index_keys(index: dict[str, Any], row: Any) -> None:
for key in QualityEvidenceService._row_identifiers(row): for key in QualityEvidenceService._row_identifiers(row):
@@ -317,7 +317,8 @@ def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() ->
assert "datasetsApi.acquireOrthophoto" in hook_source assert "datasetsApi.acquireOrthophoto" in hook_source
assert "prepareAndRunDetection(datasetId)" in hook_source assert "prepareAndRunDetection(datasetId)" in hook_source
assert "compareDetectionRunWithReference" in hook_source assert "compareDetectionRunWithReference" in hook_source
assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false)" in app_source assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold)" in app_source
assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook_source
assert "vervoerregio|operationele grens" in app_source assert "vervoerregio|operationele grens" in app_source
assert "selectionBbox: mapSelectionBbox" in app_source assert "selectionBbox: mapSelectionBbox" in app_source
assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source
@@ -0,0 +1,253 @@
from __future__ import annotations
from pathlib import Path
from uuid import UUID, uuid4
import pytest
from fastapi.testclient import TestClient
from geoalchemy2.shape import from_shape
from shapely.geometry import box
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
from app.schemas.detection_review import (
DetectionReviewList,
DetectionReviewRead,
DetectionReviewSummary,
DetectionReviewUpsert,
)
from app.services.detection_review_service import DetectionReviewService
ROOT = Path(__file__).resolve().parents[2]
class FakeQuery:
def __init__(self, rows):
self.rows = list(rows)
def filter(self, *criteria):
for criterion in criteria:
left = getattr(criterion, "left", None)
right = getattr(criterion, "right", None)
operator = getattr(criterion, "operator", None)
name = getattr(left, "name", None)
value = getattr(right, "value", right)
if name and operator and operator.__name__ == "eq":
self.rows = [row for row in self.rows if getattr(row, name) == value]
return self
def all(self):
return list(self.rows)
def first(self):
return self.rows[0] if self.rows else None
class FakeSession:
def __init__(self, objects=None, query_rows=None) -> None:
self.objects = objects or {}
self.query_rows = query_rows or {}
def get(self, model, item_id):
return self.objects.get((model, item_id))
def query(self, model):
return FakeQuery(self.query_rows.setdefault(model, []))
def add(self, row):
rows = self.query_rows.setdefault(type(row), [])
if row not in rows:
rows.append(row)
self.objects[(type(row), row.id)] = row
def commit(self):
return None
def refresh(self, _row):
return None
def _review_context() -> tuple[FakeSession, UUID, UUID, Detection, VectorFeature]:
project_id = uuid4()
quality_check_id = uuid4()
analysis_run_id = uuid4()
candidate_dataset_id = uuid4()
reference_dataset_id = uuid4()
detection = Detection(
id=uuid4(),
project_id=project_id,
dataset_id=candidate_dataset_id,
analysis_run_id=analysis_run_id,
model_name="yolo-configured",
class_name="building",
confidence=0.62,
geometry=from_shape(box(5.0, 51.0, 5.001, 51.001), srid=4326),
)
reference = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
source_feature_id="grb-missed",
feature_class="building",
properties_json={},
geometry=from_shape(box(5.002, 51.002, 5.003, 51.003), srid=4326),
)
quality_check = QualityCheck(
id=quality_check_id,
project_id=project_id,
analysis_run_id=analysis_run_id,
candidate_dataset_id=candidate_dataset_id,
reference_dataset_id=reference_dataset_id,
check_type="detections_vs_reference",
status="ok",
findings_json={
"false_positive_evidence": [{"candidate_feature_id": str(detection.id)}],
"false_negative_evidence": [{"reference_feature_id": str(reference.id)}],
},
)
db = FakeSession(
objects={
(QualityCheck, quality_check_id): quality_check,
(Detection, detection.id): detection,
(VectorFeature, reference.id): reference,
},
query_rows={DetectionReview: []},
)
return db, project_id, quality_check_id, detection, reference
def test_detection_review_model_and_migration_are_aligned() -> None:
migration = (ROOT / "backend" / "alembic" / "versions" / "202607150001_detection_reviews.py").read_text(encoding="utf-8")
columns = DetectionReview.__table__.columns
for name in (
"project_id",
"quality_check_id",
"analysis_run_id",
"evidence_role",
"evidence_feature_id",
"detection_id",
"reference_feature_id",
"decision",
"notes",
"reviewed_by",
"created_at",
"updated_at",
):
assert name in columns
assert f'"{name}"' in migration
assert 'op.create_table(\n "detection_reviews"' in migration
assert 'down_revision = "202607140001"' in migration
def test_detection_review_queue_persists_only_valid_operator_decisions() -> None:
db, project_id, quality_check_id, detection, _reference = _review_context()
initial = DetectionReviewService.list_reviews(
db,
project_id=project_id,
quality_check_id=quality_check_id,
)
assert initial.summary.total == 2
assert initial.summary.reviewed == 0
assert initial.summary.decision_counts == {"unreviewed": 2}
saved = DetectionReviewService.upsert_review(
db,
project_id=project_id,
quality_check_id=quality_check_id,
payload=DetectionReviewUpsert(
evidence_role="false_positive",
evidence_feature_id=str(detection.id),
decision="qa_alignment_mismatch",
notes="Box overlaps the official footprint but is not a training negative.",
),
)
assert saved.decision == "qa_alignment_mismatch"
assert saved.detection_id == detection.id
reviewed = DetectionReviewService.list_reviews(
db,
project_id=project_id,
quality_check_id=quality_check_id,
reviewed=True,
)
assert reviewed.total == 1
assert reviewed.summary.reviewed == 1
assert reviewed.summary.remaining == 1
with pytest.raises(AppError) as exc:
DetectionReviewService.upsert_review(
db,
project_id=project_id,
quality_check_id=quality_check_id,
payload=DetectionReviewUpsert(
evidence_role="false_positive",
evidence_feature_id=str(detection.id),
decision="confirmed_model_false_negative",
),
)
assert exc.value.code == "INVALID_DETECTION_REVIEW_DECISION"
def test_detection_review_endpoints_use_canonical_envelopes(monkeypatch) -> None:
project_id = uuid4()
quality_check_id = uuid4()
item = DetectionReviewRead(
project_id=project_id,
quality_check_id=quality_check_id,
evidence_role="false_positive",
evidence_feature_id=str(uuid4()),
decision="unreviewed",
)
result = DetectionReviewList(
items=[item],
total=1,
limit=50,
offset=0,
summary=DetectionReviewSummary(
total=1,
reviewed=0,
remaining=1,
false_positive_total=1,
false_negative_total=0,
decision_counts={"unreviewed": 1},
),
)
monkeypatch.setattr(DetectionReviewService, "list_reviews", lambda *_args, **_kwargs: result)
monkeypatch.setattr(DetectionReviewService, "upsert_review", lambda *_args, **_kwargs: item)
app.dependency_overrides[get_db] = lambda: FakeSession()
try:
listed = TestClient(app).get(f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews")
saved = TestClient(app).post(
f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews",
json={
"evidence_role": "false_positive",
"evidence_feature_id": item.evidence_feature_id,
"decision": "unreviewed",
},
)
finally:
app.dependency_overrides.pop(get_db, None)
assert listed.status_code == 200
assert set(listed.json()) == {"data"}
assert listed.json()["data"]["summary"]["remaining"] == 1
assert saved.status_code == 200
assert saved.json() == {"data": item.model_dump(mode="json")}
def test_map_detection_qa_uses_documented_footprint_threshold_and_honest_labels() -> None:
hook = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8")
app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
evidence_service = (ROOT / "backend" / "app" / "services" / "quality_evidence_service.py").read_text(encoding="utf-8")
assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook
assert "kandidaten" in hook
assert "precision" in hook.lower()
assert "false, iouThreshold" in app_source
assert "VectorFeature.id.in_(uuid_identifiers)" in evidence_service
assert "VectorFeature.source_feature_id.in_(identifiers)" in evidence_service
assert "for row in db.query(VectorFeature).filter(VectorFeature.dataset_id" not in evidence_service
+15
View File
@@ -427,6 +427,21 @@ remains available as a higher-precision legacy `0.15` choice. The older
default-promotion blocker. Every production-like run still requires persisted default-promotion blocker. Every production-like run still requires persisted
QA/QC against suitable reference data. QA/QC against suitable reference data.
The map-driven building workflow uses canonical footprint IoU `0.25`, matching
the promotion evidence above. A July 2026 Mol-only holdout audit compared
confidence `0.10` and `0.15` over Achterbos, Gompel, Donk and Postel. Confidence
`0.15` produced the better F1 in all four positive zones; both thresholds
produced zero detections in the pure-empty Postel forest control. The active
confidence therefore remains `0.15`. This result does not claim production
perfection and does not justify another model-training run by itself.
False-positive and false-negative evidence from persisted detection QA can be
classified through `detection_reviews`. The queue derives from quality-check
evidence ids and resolves persisted Detection and reference VectorFeature rows.
`qa_alignment_mismatch`, `reference_gap_or_change`, uncertain imagery and
unreviewed items must never be exported as hard-negative or missed-positive
training labels. Canonical QA metrics remain unchanged after review.
The persisted seven-AOI evidence for this profile contains 5,568 false The persisted seven-AOI evidence for this profile contains 5,568 false
positives among 13,613 candidate detections. The read-only audit command in positives among 13,613 candidate detections. The read-only audit command in
`scripts/README.md` reports the largest review volumes in Turnhout, Herentals `scripts/README.md` reports the largest review volumes in Turnhout, Herentals
+65
View File
@@ -1300,6 +1300,71 @@ candidate evidence exposes the equivalent persisted model/source fields plus
`segmentation_id`, `mask_path` and `area_m2`. These are additive GeoJSON `segmentation_id`, `mask_path` and `area_m2`. These are additive GeoJSON
properties; the canonical envelope and endpoint path are unchanged. properties; the canonical envelope and endpoint path are unchanged.
For detection QA, false-positive and false-negative evidence properties also
include `review_decision`, `review_notes`, `reviewed_by` and `reviewed_at`.
Missing review rows are represented as `review_decision=unreviewed`. Evidence
resolution is bounded to identifiers stored by the selected quality check.
### GET `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`
Returns the paginated operator review queue for a persisted
`detections_vs_reference` quality check. Optional query parameters are
`evidence_role=false_positive|false_negative`, `decision`, `reviewed=true|false`,
`limit` (1-200) and `offset`. Items derive only from persisted QA evidence.
```json
{
"data": {
"items": [{
"id": null,
"project_id": "uuid",
"quality_check_id": "uuid",
"analysis_run_id": "uuid",
"evidence_role": "false_positive",
"evidence_feature_id": "detection-uuid",
"detection_id": "detection-uuid",
"decision": "unreviewed",
"confidence": 0.62,
"class_name": "building"
}],
"total": 1,
"limit": 50,
"offset": 0,
"summary": {
"total": 73,
"reviewed": 0,
"remaining": 73,
"false_positive_total": 17,
"false_negative_total": 56,
"decision_counts": {"unreviewed": 73}
}
}
}
```
### POST `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`
Creates or updates one durable operator decision. The evidence id must belong
to the quality check and resolve to the persisted Detection or reference
VectorFeature. False-positive and false-negative roles accept only their
role-specific decisions.
```json
{
"evidence_role": "false_positive",
"evidence_feature_id": "detection-uuid",
"decision": "qa_alignment_mismatch",
"notes": "The detection box overlaps the irregular GRB footprint.",
"reviewed_by": "operator"
}
```
Allowed decisions are `confirmed_model_false_positive`,
`confirmed_model_false_negative`, `reference_gap_or_change`,
`qa_alignment_mismatch`, `imagery_obscured_or_uncertain`, `uncertain` and
`unreviewed`. Invalid role/decision combinations return
`INVALID_DETECTION_REVIEW_DECISION`.
## Exports ## Exports
### POST `/api/v1/exports/geojson` ### POST `/api/v1/exports/geojson`
+37
View File
@@ -8135,3 +8135,40 @@ Next:
- Add a review workflow for accepted/rejected detections and use those audited - Add a review workflow for accepted/rejected detections and use those audited
labels as the gate for model calibration or retraining. Do not present the labels as the gate for model calibration or retraining. Do not present the
current F1 score as production-grade accuracy. current F1 score as production-grade accuracy.
## Sprint 197 - Measured detection accuracy and durable review (2026-07-15)
Implemented:
- Added first-class `detection_reviews` persistence linked to Project,
QualityCheck, AnalysisRun, Detection and reference VectorFeature, with
role-constrained decisions and one durable row per evidence item.
- Added canonical paginated GET/POST review endpoints and a frontend review
queue with role/status filters, notes, summary counts and map handoff.
- Changed map-driven detection QA from generic UI IoU `0.50` to the documented
box-versus-footprint operational IoU `0.25`.
- Reworded map output as candidates and exposed persisted matches, precision,
recall, F1, false positives and false negatives.
- Bounded evidence lookup to persisted evidence ids; complete regional GRB
layers are no longer materialized for a small review overlay.
Live model evidence before deployment:
- Re-ran the active model at confidence `0.10` and `0.15` on Mol Achterbos,
Gompel, Donk and Postel with QA IoU `0.25`.
- Confidence `0.15` won F1 in all four positive holdouts: `0.6694`, `0.6564`,
`0.5894` and `0.4749`. Confidence `0.10` measured `0.6287`, `0.6348`,
`0.5636` and `0.4435` respectively.
- Both thresholds produced zero detections on the pure-empty Postel forest
control. The active confidence remains `0.15`; no model or asset was trained,
downloaded or promoted.
Validation before deployment:
- The complete readiness gate passed 592 backend tests, the API/document audit
for 88 implemented routes, backend compilation, one Alembic head, offline
migration SQL generation, frontend typecheck/build and shell smoke checks.
- Focused review/evidence regressions passed and the production bundle retained
separate React, application and MapLibre chunks.
Next:
- Deploy the migration and UI, verify one live review queue and map QA result,
then complete representative manual decisions before constructing any new
model-training corpus.
+22
View File
@@ -196,6 +196,28 @@ diagnostic reference-envelope comparison are persisted in the existing
`quality_checks.findings_json`; `parameters_json.coverage_policy` records the `quality_checks.findings_json`; `parameters_json.coverage_policy` records the
evaluation policy used for reproducibility. evaluation policy used for reproducibility.
### detection_reviews
- `id uuid primary key`
- `project_id uuid references projects(id) on delete cascade`
- `quality_check_id uuid references quality_checks(id) on delete cascade`
- `analysis_run_id uuid nullable references analysis_runs(id) on delete set null`
- `evidence_role text not null` (`false_positive` or `false_negative`)
- `evidence_feature_id text not null`
- `detection_id uuid nullable references detections(id) on delete set null`
- `reference_feature_id uuid nullable references vector_features(id) on delete set null`
- `decision text not null default 'unreviewed'`
- `notes text nullable`
- `reviewed_by text not null default 'operator'`
- `created_at timestamptz`
- `updated_at timestamptz`
The unique key is `(quality_check_id, evidence_role, evidence_feature_id)`.
Indexes cover project, quality check, analysis run and decision. Reviews
classify persisted QA evidence only; they do not replace or modify Detection,
VectorFeature, QualityCheck or Metric records. Unreviewed, reference-gap,
imagery-uncertain and QA-alignment cases are not training labels.
### exports ### exports
- `id uuid primary key` - `id uuid primary key`
+4
View File
@@ -46,6 +46,10 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training. - [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training.
- [x] Add a coverage-aware Mol multi-zone benchmark report with explicit positive-zone, per-zone collapse, reference-coverage and pure-empty background gates. - [x] Add a coverage-aware Mol multi-zone benchmark report with explicit positive-zone, per-zone collapse, reference-coverage and pure-empty background gates.
- [x] Execute the refreshed coverage-aware Mol operational benchmark against the active local model and record the resulting retain/review decision. - [x] Execute the refreshed coverage-aware Mol operational benchmark against the active local model and record the resulting retain/review decision.
- [x] Align map-driven building QA with footprint IoU `0.25` and show measured candidate/match/error metrics instead of calling every detection a building.
- [x] Persist paginated false-positive/false-negative operator decisions and expose them in the Quality workspace.
- [x] Bound QA evidence resolution to persisted evidence ids instead of loading complete regional reference datasets.
- [x] Compare confidence `0.10` and `0.15` over independent Mol holdouts; retain `0.15` because it wins F1 in every positive zone while both pass the empty-background control.
- [ ] Complete manual decisions for the generated 48 false-negative and 48 false-positive review cards before constructing any new training corpus. - [ ] Complete manual decisions for the generated 48 false-negative and 48 false-positive review cards before constructing any new training corpus.
- [x] Backend FastAPI foundation, health endpoint and service structure. - [x] Backend FastAPI foundation, health endpoint and service structure.
- [x] React/TypeScript frontend foundation and MapLibre workbench. - [x] React/TypeScript frontend foundation and MapLibre workbench.
+7
View File
@@ -8,6 +8,12 @@ The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanal
Detection defaults to the configured local YOLO asset and automatically selects an available raster and active model asset where possible. The model registry and preflight remain honest when PyTorch, Ultralytics, a local model file or a tile manifest is unavailable. The active building profile is operational but remains review-required: its recorded coverage-aware benchmark is approximately precision 0.590, recall 0.577 and F1 0.582 over seven positive AOIs, with zero detections in the empty-background control. Another training pass is intentionally blocked until the generated false-positive and false-negative review decisions are completed. Detection defaults to the configured local YOLO asset and automatically selects an available raster and active model asset where possible. The model registry and preflight remain honest when PyTorch, Ultralytics, a local model file or a tile manifest is unavailable. The active building profile is operational but remains review-required: its recorded coverage-aware benchmark is approximately precision 0.590, recall 0.577 and F1 0.582 over seven positive AOIs, with zero detections in the empty-background control. Another training pass is intentionally blocked until the generated false-positive and false-negative review decisions are completed.
Map-driven building analysis uses the documented footprint-IoU `0.25` and
distinguishes model candidates from verified buildings. It shows persisted
matches, precision, recall, F1, false positives and false negatives. The
Quality workspace includes a paginated review queue for persisted detection-QA
evidence. Reviews do not rewrite detections, GRB geometry or QA metrics.
The map-first explorer has two deliberate modes. `Latest state` selects the The map-first explorer has two deliberate modes. `Latest state` selects the
latest explicitly dated source snapshot without claiming an old edition is latest explicitly dated source snapshot without claiming an old edition is
current, while `Evolution` lets the operator compare an earlier and later current, while `Evolution` lets the operator compare an earlier and later
@@ -373,6 +379,7 @@ Before creating tiles, the guided action inspects raster dimensions and estimate
- The QA/QC workspace includes a selected-check evidence drilldown with candidate/reference provenance, false-positive/negative metric evidence, map handoff context and parameters/findings JSON. - The QA/QC workspace includes a selected-check evidence drilldown with candidate/reference provenance, false-positive/negative metric evidence, map handoff context and parameters/findings JSON.
- QA/QC findings now persist feature-level evidence in `findings_json`: matched candidate/reference feature ids with IoU, false-positive candidate feature ids and false-negative reference feature ids. The QA/QC drilldown renders these as compact evidence lists before the raw JSON. - QA/QC findings now persist feature-level evidence in `findings_json`: matched candidate/reference feature ids with IoU, false-positive candidate feature ids and false-negative reference feature ids. The QA/QC drilldown renders these as compact evidence lists before the raw JSON.
- Persisted QA/QC checks can be rendered as a Map workspace evidence overlay. The QA/QC panel calls `GET /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/evidence/geojson`, then MapLibre draws matched candidate/reference geometries, false positives and false negatives with distinct styling and a compact legend. - Persisted QA/QC checks can be rendered as a Map workspace evidence overlay. The QA/QC panel calls `GET /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/evidence/geojson`, then MapLibre draws matched candidate/reference geometries, false positives and false negatives with distinct styling and a compact legend.
- Detection QA checks expose a filtered, paginated operator review queue through `GET/POST /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`. The UI keeps confirmed model errors separate from reference gaps and box-to-footprint alignment mismatches.
## Raster dependency visibility ## Raster dependency visibility
+4 -2
View File
@@ -477,8 +477,8 @@ function App(): JSX.Element {
datasets, datasets,
loadProjectData, loadProjectData,
prepareAndRunDetection: (datasetId) => prepareAndRunDetection(datasetId, 'yolo-configured'), prepareAndRunDetection: (datasetId) => prepareAndRunDetection(datasetId, 'yolo-configured'),
compareDetectionRunWithReference: (analysisRunId, referenceDatasetId) => compareDetectionRunWithReference: (analysisRunId, referenceDatasetId, iouThreshold) =>
compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false), compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold),
onAnalysisReady: () => { onAnalysisReady: () => {
setMapContentMode('analysis') setMapContentMode('analysis')
setMapLayerVisible(true) setMapLayerVisible(true)
@@ -1011,6 +1011,8 @@ function App(): JSX.Element {
orthophotoAnalysisStatus={mapOrthophotoAnalysis.status} orthophotoAnalysisStatus={mapOrthophotoAnalysis.status}
orthophotoAnalysisError={mapOrthophotoAnalysis.error} orthophotoAnalysisError={mapOrthophotoAnalysis.error}
orthophotoAnalysisRunning={mapOrthophotoAnalysis.running} orthophotoAnalysisRunning={mapOrthophotoAnalysis.running}
orthophotoAnalysisQuality={mapOrthophotoAnalysis.lastQuality}
orthophotoAnalysisDetectionCount={mapOrthophotoAnalysis.lastDetectionCount}
availableMapDatasets={availableMapDatasets} availableMapDatasets={availableMapDatasets}
selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''} selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''}
selectedFeature={selectedMapFeature} selectedFeature={selectedMapFeature}
+21 -1
View File
@@ -1,6 +1,6 @@
import { useEffect, useMemo, useState } from 'react' import { useEffect, useMemo, useState } from 'react'
import GeoMap from '../GeoMap' import GeoMap from '../GeoMap'
import type { AreaRead, DatasetCreateResponse, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types' import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
import { featureCollectionBounds } from '../../lib/geojsonBounds' import { featureCollectionBounds } from '../../lib/geojsonBounds'
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights' import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
import { useTemporalComparison } from '../../hooks/useTemporalComparison' import { useTemporalComparison } from '../../hooks/useTemporalComparison'
@@ -327,6 +327,12 @@ function formatBboxLabel(bbox: VectorSelectionBBox | null): string {
return `${formatCoordinate(bbox.min_x)}, ${formatCoordinate(bbox.min_y)} -> ${formatCoordinate(bbox.max_x)}, ${formatCoordinate(bbox.max_y)}` return `${formatCoordinate(bbox.min_x)}, ${formatCoordinate(bbox.min_y)} -> ${formatCoordinate(bbox.max_x)}, ${formatCoordinate(bbox.max_y)}`
} }
function formatPercentage(value: number | null | undefined): string {
return typeof value === 'number' && Number.isFinite(value)
? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`
: 'n.v.t.'
}
function bboxToInputState(bbox: VectorSelectionBBox | null) { function bboxToInputState(bbox: VectorSelectionBBox | null) {
return { return {
min_x: bbox ? String(bbox.min_x) : '', min_x: bbox ? String(bbox.min_x) : '',
@@ -445,6 +451,8 @@ interface MapWorkspaceProps {
orthophotoAnalysisStatus: string orthophotoAnalysisStatus: string
orthophotoAnalysisError: string | null orthophotoAnalysisError: string | null
orthophotoAnalysisRunning: boolean orthophotoAnalysisRunning: boolean
orthophotoAnalysisQuality: DetectionQaResult | null
orthophotoAnalysisDetectionCount: number | null
availableMapDatasets: DatasetCreateResponse[] availableMapDatasets: DatasetCreateResponse[]
selectedMapDatasetId: string selectedMapDatasetId: string
onSelectMapArea: (areaId: string) => void onSelectMapArea: (areaId: string) => void
@@ -518,6 +526,8 @@ export function MapWorkspace({
orthophotoAnalysisStatus, orthophotoAnalysisStatus,
orthophotoAnalysisError, orthophotoAnalysisError,
orthophotoAnalysisRunning, orthophotoAnalysisRunning,
orthophotoAnalysisQuality,
orthophotoAnalysisDetectionCount,
availableMapDatasets, availableMapDatasets,
selectedMapDatasetId, selectedMapDatasetId,
onSelectMapArea, onSelectMapArea,
@@ -1194,6 +1204,16 @@ export function MapWorkspace({
</button> </button>
{orthophotoAnalysisStatus ? <p role="status">{orthophotoAnalysisStatus}</p> : null} {orthophotoAnalysisStatus ? <p role="status">{orthophotoAnalysisStatus}</p> : null}
{orthophotoAnalysisError ? <p className="error" role="alert">{orthophotoAnalysisError}</p> : null} {orthophotoAnalysisError ? <p className="error" role="alert">{orthophotoAnalysisError}</p> : null}
{orthophotoAnalysisQuality ? (
<div className="geo-image-quality-metrics" aria-label="Gemeten kwaliteit van de beeldanalyse">
<div><span>Kandidaten</span><strong>{orthophotoAnalysisDetectionCount?.toLocaleString('nl-BE') ?? 'n.v.t.'}</strong></div>
<div><span>Precision</span><strong>{formatPercentage(orthophotoAnalysisQuality.precision)}</strong></div>
<div><span>Recall</span><strong>{formatPercentage(orthophotoAnalysisQuality.recall)}</strong></div>
<div><span>F1</span><strong>{formatPercentage(orthophotoAnalysisQuality.f1_score)}</strong></div>
<div><span>Fout</span><strong>{orthophotoAnalysisQuality.false_positives.toLocaleString('nl-BE')}</strong></div>
<div><span>Gemist</span><strong>{orthophotoAnalysisQuality.false_negatives.toLocaleString('nl-BE')}</strong></div>
</div>
) : null}
</div> </div>
) : null} ) : null}
@@ -0,0 +1,244 @@
import { useEffect, useState } from 'react'
import { qaApi } from '../../services/api'
import type {
DetectionEvidenceRole,
DetectionReviewDecision,
DetectionReviewList,
DetectionReviewRead,
} from '../../types'
import { formatError } from '../../lib/formatError'
interface DetectionReviewPanelProps {
projectId: string
qualityCheckId: string
onOpenEvidenceMap?: (qualityCheckId: string) => void
}
const ROLE_LABELS: Record<DetectionEvidenceRole, string> = {
false_positive: 'Onterecht gevonden',
false_negative: 'Gemist gebouw',
}
const DECISION_LABELS: Record<DetectionReviewDecision, string> = {
confirmed_model_false_positive: 'Bevestigde foutdetectie',
confirmed_model_false_negative: 'Bevestigd gemist gebouw',
reference_gap_or_change: 'Referentie ontbreekt of is verouderd',
qa_alignment_mismatch: 'Vormvergelijking is te streng',
imagery_obscured_or_uncertain: 'Luchtbeeld is onduidelijk',
uncertain: 'Verder onderzoek nodig',
unreviewed: 'Nog niet beoordeeld',
}
const ROLE_DECISIONS: Record<DetectionEvidenceRole, DetectionReviewDecision[]> = {
false_positive: [
'unreviewed',
'confirmed_model_false_positive',
'reference_gap_or_change',
'qa_alignment_mismatch',
'uncertain',
],
false_negative: [
'unreviewed',
'confirmed_model_false_negative',
'reference_gap_or_change',
'qa_alignment_mismatch',
'imagery_obscured_or_uncertain',
'uncertain',
],
}
function shortId(value: string): string {
return value.length > 18 ? `${value.slice(0, 8)}...${value.slice(-6)}` : value
}
export function DetectionReviewPanel({
projectId,
qualityCheckId,
onOpenEvidenceMap,
}: DetectionReviewPanelProps): JSX.Element {
const [queue, setQueue] = useState<DetectionReviewList | null>(null)
const [loading, setLoading] = useState(false)
const [savingKey, setSavingKey] = useState<string | null>(null)
const [error, setError] = useState<string | null>(null)
const [roleFilter, setRoleFilter] = useState<'all' | DetectionEvidenceRole>('all')
const [statusFilter, setStatusFilter] = useState<'all' | 'reviewed' | 'unreviewed'>('unreviewed')
const [offset, setOffset] = useState(0)
const [draftDecisions, setDraftDecisions] = useState<Record<string, DetectionReviewDecision>>({})
const [draftNotes, setDraftNotes] = useState<Record<string, string>>({})
const load = async () => {
setLoading(true)
setError(null)
try {
const result = await qaApi.listDetectionReviews(projectId, qualityCheckId, {
evidenceRole: roleFilter === 'all' ? undefined : roleFilter,
reviewed: statusFilter === 'all' ? undefined : statusFilter === 'reviewed',
limit: 50,
offset,
})
setQueue(result)
setDraftDecisions(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.decision])))
setDraftNotes(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.notes ?? ''])))
} catch (caught) {
setError(formatError(caught, 'De controlelijst kon niet worden geladen'))
} finally {
setLoading(false)
}
}
useEffect(() => {
void load()
}, [projectId, qualityCheckId, roleFilter, statusFilter, offset])
const save = async (item: DetectionReviewRead) => {
const key = reviewKey(item)
setSavingKey(key)
setError(null)
try {
await qaApi.upsertDetectionReview(projectId, qualityCheckId, {
evidence_role: item.evidence_role,
evidence_feature_id: item.evidence_feature_id,
decision: draftDecisions[key] ?? item.decision,
notes: draftNotes[key]?.trim() || null,
reviewed_by: 'operator',
})
await load()
} catch (caught) {
setError(formatError(caught, 'De beoordeling kon niet worden bewaard'))
} finally {
setSavingKey(null)
}
}
return (
<section className="detection-review-panel" aria-label="Handmatige controle van beeldanalyse">
<div className="panel-title-row">
<div>
<h3>Fouten controleren</h3>
<p className="muted">Beoordeel alleen twijfelgevallen. Bevestigde fouten kunnen later veilig als trainingsfeedback worden gebruikt.</p>
</div>
<button type="button" className="secondary-action" onClick={() => onOpenEvidenceMap?.(qualityCheckId)}>
Op kaart bekijken
</button>
</div>
{queue ? (
<div className="detection-review-summary">
<div><span>Te beoordelen</span><strong>{queue.summary.total}</strong></div>
<div><span>Afgerond</span><strong>{queue.summary.reviewed}</strong></div>
<div><span>Resterend</span><strong>{queue.summary.remaining}</strong></div>
<div><span>Fout gevonden</span><strong>{queue.summary.false_positive_total}</strong></div>
<div><span>Gemist</span><strong>{queue.summary.false_negative_total}</strong></div>
</div>
) : null}
<div className="detection-review-filters">
<label>
Soort
<select value={roleFilter} onChange={(event) => {
setRoleFilter(event.target.value as typeof roleFilter)
setOffset(0)
}}>
<option value="all">Alles</option>
<option value="false_positive">Onterecht gevonden</option>
<option value="false_negative">Gemist gebouw</option>
</select>
</label>
<label>
Status
<select value={statusFilter} onChange={(event) => {
setStatusFilter(event.target.value as typeof statusFilter)
setOffset(0)
}}>
<option value="unreviewed">Nog te beoordelen</option>
<option value="reviewed">Beoordeeld</option>
<option value="all">Alles</option>
</select>
</label>
<button type="button" className="secondary-action" disabled={loading} onClick={() => void load()}>
{loading ? 'Laden...' : 'Vernieuwen'}
</button>
</div>
{error ? <p className="error" role="alert">{error}</p> : null}
{!loading && queue && queue.items.length === 0 ? (
<div className="result-state result-state-empty">
<strong>Geen objecten in deze selectie</strong>
<p>Pas de filters aan of open de bewijslaag op de kaart.</p>
</div>
) : null}
<ol className="detection-review-list">
{queue?.items.map((item) => {
const key = reviewKey(item)
const decision = draftDecisions[key] ?? item.decision
return (
<li key={key} className="detection-review-item">
<div className="detection-review-item-heading">
<div>
<span>{ROLE_LABELS[item.evidence_role]}</span>
<strong>{item.class_name ?? 'gebouw'} / {shortId(item.evidence_feature_id)}</strong>
</div>
{typeof item.confidence === 'number' ? <span className="count-pill">{Math.round(item.confidence * 100)}% vertrouwen</span> : null}
</div>
<div className="detection-review-editor">
<label>
Beoordeling
<select
value={decision}
onChange={(event) => setDraftDecisions((current) => ({
...current,
[key]: event.target.value as DetectionReviewDecision,
}))}
>
{ROLE_DECISIONS[item.evidence_role].map((option) => (
<option key={option} value={option}>{DECISION_LABELS[option]}</option>
))}
</select>
</label>
<label>
Notitie
<input
type="text"
maxLength={2000}
placeholder="Waarom is dit correct, fout of onzeker?"
value={draftNotes[key] ?? ''}
onChange={(event) => setDraftNotes((current) => ({ ...current, [key]: event.target.value }))}
/>
</label>
<button type="button" className="primary-action" disabled={savingKey === key} onClick={() => void save(item)}>
{savingKey === key ? 'Bewaren...' : 'Beoordeling bewaren'}
</button>
</div>
</li>
)
})}
</ol>
{queue && queue.total > queue.limit ? (
<div className="detection-review-pagination" aria-label="Pagina's van de controlelijst">
<button
type="button"
className="secondary-action"
disabled={offset === 0 || loading}
onClick={() => setOffset((current) => Math.max(current - queue.limit, 0))}
>
Vorige
</button>
<span>{offset + 1}-{Math.min(offset + queue.limit, queue.total)} van {queue.total}</span>
<button
type="button"
className="secondary-action"
disabled={offset + queue.limit >= queue.total || loading}
onClick={() => setOffset((current) => current + queue.limit)}
>
Volgende
</button>
</div>
) : null}
</section>
)
}
function reviewKey(item: Pick<DetectionReviewRead, 'evidence_role' | 'evidence_feature_id'>): string {
return `${item.evidence_role}:${item.evidence_feature_id}`
}
@@ -1,5 +1,6 @@
import { useMemo, useState } from 'react' import { useMemo, useState } from 'react'
import type { DatasetCreateResponse, MetricRead, QualityCheckRead } from '../../types' import type { DatasetCreateResponse, MetricRead, QualityCheckRead } from '../../types'
import { DetectionReviewPanel } from './DetectionReviewPanel'
const CORE_METRIC_ORDER = [ const CORE_METRIC_ORDER = [
'precision', 'precision',
@@ -304,6 +305,13 @@ export function QualityResultsPanel({
</button> </button>
</div> </div>
</div> </div>
{selectedQualityCheck.check_type === 'detections_vs_reference' && selectedProjectId ? (
<DetectionReviewPanel
projectId={selectedProjectId}
qualityCheckId={selectedQualityCheck.id}
onOpenEvidenceMap={onOpenEvidenceMap}
/>
) : null}
<div className="quality-feature-evidence-grid" aria-label="Feature-level QA/QC evidence"> <div className="quality-feature-evidence-grid" aria-label="Feature-level QA/QC evidence">
<div> <div>
<span>Overeenkomende object-ID's</span> <span>Overeenkomende object-ID's</span>
+2 -1
View File
@@ -405,6 +405,7 @@ export function useDetectionWorkflow({
analysisRunId: string, analysisRunId: string,
referenceDatasetId: string, referenceDatasetId: string,
useCurrentFilters = true, useCurrentFilters = true,
iouThresholdOverride?: number,
): Promise<DetectionQaResult | null> => { ): Promise<DetectionQaResult | null> => {
if (!analysisRunId) { if (!analysisRunId) {
setDetectionQaError('Select a detection run') setDetectionQaError('Select a detection run')
@@ -422,7 +423,7 @@ export function useDetectionWorkflow({
try { try {
const result = await detectionApi.compareWithReference(analysisRunId, { const result = await detectionApi.compareWithReference(analysisRunId, {
reference_dataset_id: referenceDatasetId, reference_dataset_id: referenceDatasetId,
iou_threshold: qaIouThreshold, iou_threshold: iouThresholdOverride ?? qaIouThreshold,
class_name: useCurrentFilters ? detectionClassFilter || null : null, class_name: useCurrentFilters ? detectionClassFilter || null : null,
min_confidence: useCurrentFilters && detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null, min_confidence: useCurrentFilters && detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null,
}) })
+32 -4
View File
@@ -17,6 +17,8 @@ export type MapOrthophotoAnalysisStage =
| 'complete' | 'complete'
| 'failed' | 'failed'
export const MAP_BUILDING_QA_IOU_THRESHOLD = 0.25
interface MapOrthophotoAnalysisOptions { interface MapOrthophotoAnalysisOptions {
selectedProjectId: string | null selectedProjectId: string | null
selectedAreaId: string selectedAreaId: string
@@ -27,6 +29,7 @@ interface MapOrthophotoAnalysisOptions {
compareDetectionRunWithReference: ( compareDetectionRunWithReference: (
analysisRunId: string, analysisRunId: string,
referenceDatasetId: string, referenceDatasetId: string,
iouThreshold: number,
) => Promise<DetectionQaResult | null> ) => Promise<DetectionQaResult | null>
onAnalysisReady: () => void onAnalysisReady: () => void
} }
@@ -61,6 +64,12 @@ function formatOrthophotoError(caught: unknown): string {
return formatError(caught, 'De kaartgestuurde beeldanalyse is mislukt') return formatError(caught, 'De kaartgestuurde beeldanalyse is mislukt')
} }
function formatQualityPercentage(value: number | null | undefined): string {
return typeof value === 'number' && Number.isFinite(value)
? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`
: 'n.v.t.'
}
export function useMapOrthophotoAnalysis({ export function useMapOrthophotoAnalysis({
selectedProjectId, selectedProjectId,
selectedAreaId, selectedAreaId,
@@ -75,12 +84,18 @@ export function useMapOrthophotoAnalysis({
const [status, setStatus] = useState('') const [status, setStatus] = useState('')
const [error, setError] = useState<string | null>(null) const [error, setError] = useState<string | null>(null)
const [lastResult, setLastResult] = useState<OrthophotoAcquisitionResult | null>(null) const [lastResult, setLastResult] = useState<OrthophotoAcquisitionResult | null>(null)
const [lastQuality, setLastQuality] = useState<DetectionQaResult | null>(null)
const [lastDetectionCount, setLastDetectionCount] = useState<number | null>(null)
const [lastAnalysisRunId, setLastAnalysisRunId] = useState<string | null>(null)
useEffect(() => { useEffect(() => {
setStage('idle') setStage('idle')
setStatus('') setStatus('')
setError(null) setError(null)
setLastResult(null) setLastResult(null)
setLastQuality(null)
setLastDetectionCount(null)
setLastAnalysisRunId(null)
}, [selectionBbox?.min_x, selectionBbox?.min_y, selectionBbox?.max_x, selectionBbox?.max_y]) }, [selectionBbox?.min_x, selectionBbox?.min_y, selectionBbox?.max_x, selectionBbox?.max_y])
const run = async (bbox: VectorSelectionBBox): Promise<boolean> => { const run = async (bbox: VectorSelectionBBox): Promise<boolean> => {
@@ -91,6 +106,9 @@ export function useMapOrthophotoAnalysis({
} }
setError(null) setError(null)
setLastResult(null) setLastResult(null)
setLastQuality(null)
setLastDetectionCount(null)
setLastAnalysisRunId(null)
setStage('acquiring') setStage('acquiring')
setStatus('1/3 Officieel luchtbeeld voor de rechthoek ophalen...') setStatus('1/3 Officieel luchtbeeld voor de rechthoek ophalen...')
try { try {
@@ -112,20 +130,27 @@ export function useMapOrthophotoAnalysis({
if (!detection) { if (!detection) {
throw new Error('De beeldanalyse stopte. Open Beeldanalyse voor de technische oorzaak.') throw new Error('De beeldanalyse stopte. Open Beeldanalyse voor de technische oorzaak.')
} }
setLastDetectionCount(detection.detection_count)
setLastAnalysisRunId(detection.analysis_run_id)
const reference = findBuildingReference(datasets) const reference = findBuildingReference(datasets)
if (reference) { if (reference) {
setStage('validating') setStage('validating')
setStatus('3/3 Resultaat vergelijken met officiële GRB-gebouwen...') setStatus('3/3 Resultaat vergelijken met officiële GRB-gebouwen...')
const quality = await compareDetectionRunWithReference(detection.analysis_run_id, reference.id) const quality = await compareDetectionRunWithReference(
detection.analysis_run_id,
reference.id,
MAP_BUILDING_QA_IOU_THRESHOLD,
)
setLastQuality(quality)
setStatus( setStatus(
quality quality
? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend en gecontroleerd.` ? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten, ${quality.matches.toLocaleString('nl-BE')} gekoppeld aan GRB. Precision ${formatQualityPercentage(quality.precision)}, recall ${formatQualityPercentage(quality.recall)}.`
: `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend; kwaliteitscontrole kon niet afronden.`, : `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten; kwaliteitscontrole kon niet afronden.`,
) )
} else { } else {
setStatus( setStatus(
`Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend. De GRB-referentielaag ontbreekt voor automatische controle.`, `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaatvormen. De GRB-referentielaag ontbreekt voor automatische controle.`,
) )
} }
setStage('complete') setStage('complete')
@@ -144,6 +169,9 @@ export function useMapOrthophotoAnalysis({
status, status,
error, error,
lastResult, lastResult,
lastQuality,
lastDetectionCount,
lastAnalysisRunId,
running: stage === 'acquiring' || stage === 'detecting' || stage === 'validating', running: stage === 'acquiring' || stage === 'detecting' || stage === 'validating',
run, run,
} }
+28 -1
View File
@@ -1,5 +1,13 @@
import { apiGet, apiPost } from './client' import { apiGet, apiPost } from './client'
import type { QaComparisonRequest, JobRead, QualityCheckListResponse, QualityEvidenceGeoJsonResponse } from '../../types' import type {
DetectionReviewList,
DetectionReviewRead,
DetectionReviewUpsert,
JobRead,
QaComparisonRequest,
QualityCheckListResponse,
QualityEvidenceGeoJsonResponse,
} from '../../types'
export const qaApi = { export const qaApi = {
runQa: (payload: QaComparisonRequest): Promise<JobRead> => runQa: (payload: QaComparisonRequest): Promise<JobRead> =>
@@ -8,4 +16,23 @@ export const qaApi = {
apiGet<QualityCheckListResponse>(`/api/v1/projects/${projectId}/quality-checks`), apiGet<QualityCheckListResponse>(`/api/v1/projects/${projectId}/quality-checks`),
getQualityEvidenceGeoJson: (projectId: string, qualityCheckId: string): Promise<QualityEvidenceGeoJsonResponse> => getQualityEvidenceGeoJson: (projectId: string, qualityCheckId: string): Promise<QualityEvidenceGeoJsonResponse> =>
apiGet<QualityEvidenceGeoJsonResponse>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/evidence/geojson`), apiGet<QualityEvidenceGeoJsonResponse>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/evidence/geojson`),
listDetectionReviews: (
projectId: string,
qualityCheckId: string,
options: { evidenceRole?: string; reviewed?: boolean; limit?: number; offset?: number } = {},
): Promise<DetectionReviewList> => {
const query = new URLSearchParams({
limit: String(options.limit ?? 50),
offset: String(options.offset ?? 0),
})
if (options.evidenceRole) query.set('evidence_role', options.evidenceRole)
if (typeof options.reviewed === 'boolean') query.set('reviewed', String(options.reviewed))
return apiGet<DetectionReviewList>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews?${query}`)
},
upsertDetectionReview: (
projectId: string,
qualityCheckId: string,
payload: DetectionReviewUpsert,
): Promise<DetectionReviewRead> =>
apiPost<DetectionReviewRead>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews`, payload),
} }
+136
View File
@@ -1862,3 +1862,139 @@ details.ai-lab-model-surface > summary strong {
background: var(--accent-soft); background: var(--accent-soft);
} }
} }
.geo-image-quality-metrics,
.detection-review-summary {
display: grid;
grid-template-columns: repeat(6, minmax(0, 1fr));
gap: 0.5rem;
}
.geo-image-quality-metrics {
grid-column: 1 / -1;
margin-top: 0.35rem;
border-top: 1px solid var(--line);
padding-top: 0.65rem;
}
.geo-image-quality-metrics > div,
.detection-review-summary > div {
display: grid;
min-width: 0;
gap: 0.15rem;
border: 1px solid var(--line);
border-radius: 6px;
padding: 0.55rem;
background: var(--panel-soft);
}
.geo-image-quality-metrics span,
.detection-review-summary span,
.detection-review-item-heading span {
color: var(--muted);
font-size: 0.7rem;
}
.geo-image-quality-metrics strong,
.detection-review-summary strong {
color: var(--text);
font-size: 0.9rem;
}
.detection-review-panel {
display: grid;
gap: 0.75rem;
margin-top: 0.85rem;
border-top: 1px solid var(--line);
padding-top: 0.85rem;
}
.detection-review-summary {
grid-template-columns: repeat(5, minmax(0, 1fr));
}
.detection-review-filters,
.detection-review-editor,
.detection-review-pagination {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr)) auto;
gap: 0.65rem;
align-items: end;
}
.detection-review-filters label,
.detection-review-editor label {
display: grid;
min-width: 0;
gap: 0.3rem;
color: var(--muted);
font-size: 0.72rem;
}
.detection-review-list {
display: grid;
max-height: 34rem;
gap: 0.5rem;
margin: 0;
overflow: auto;
padding: 0;
list-style: none;
}
.detection-review-item {
display: grid;
gap: 0.6rem;
border: 1px solid var(--line);
border-radius: 6px;
padding: 0.7rem;
background: #ffffff;
}
.detection-review-item-heading {
display: flex;
min-width: 0;
gap: 0.75rem;
align-items: center;
justify-content: space-between;
}
.detection-review-item-heading > div {
display: grid;
min-width: 0;
gap: 0.15rem;
}
.detection-review-item-heading strong {
overflow-wrap: anywhere;
font-size: 0.82rem;
}
.detection-review-pagination {
display: flex;
align-items: center;
justify-content: flex-end;
}
.detection-review-pagination span {
color: var(--muted);
font-size: 0.76rem;
}
@media (max-width: 900px) {
.geo-image-quality-metrics,
.detection-review-summary {
grid-template-columns: repeat(3, minmax(0, 1fr));
}
.detection-review-editor,
.detection-review-filters {
grid-template-columns: 1fr;
}
}
@media (max-width: 560px) {
.geo-image-quality-metrics,
.detection-review-summary {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
}
+55
View File
@@ -894,6 +894,61 @@ export interface QualityEvidenceGeoJsonResponse {
geojson: GeoJSON.FeatureCollection geojson: GeoJSON.FeatureCollection
} }
export type DetectionEvidenceRole = 'false_positive' | 'false_negative'
export type DetectionReviewDecision =
| 'confirmed_model_false_positive'
| 'confirmed_model_false_negative'
| 'reference_gap_or_change'
| 'qa_alignment_mismatch'
| 'imagery_obscured_or_uncertain'
| 'uncertain'
| 'unreviewed'
export interface DetectionReviewUpsert {
evidence_role: DetectionEvidenceRole
evidence_feature_id: string
decision: DetectionReviewDecision
notes?: string | null
reviewed_by?: string
}
export interface DetectionReviewRead {
id?: string | null
project_id: string
quality_check_id: string
analysis_run_id?: string | null
evidence_role: DetectionEvidenceRole
evidence_feature_id: string
detection_id?: string | null
reference_feature_id?: string | null
decision: DetectionReviewDecision
notes?: string | null
reviewed_by?: string | null
confidence?: number | null
class_name?: string | null
source_tile_path?: string | null
created_at?: string | null
updated_at?: string | null
}
export interface DetectionReviewSummary {
total: number
reviewed: number
remaining: number
false_positive_total: number
false_negative_total: number
decision_counts: Record<string, number>
}
export interface DetectionReviewList {
items: DetectionReviewRead[]
total: number
limit: number
offset: number
summary: DetectionReviewSummary
}
export type ExportKind = 'dataset' | 'detection_run' | 'segmentation_run' | 'vector_selection' export type ExportKind = 'dataset' | 'detection_run' | 'segmentation_run' | 'vector_selection'
export interface ExportRead { export interface ExportRead {