feat: add measured detection review loop
This commit is contained in:
@@ -7,6 +7,29 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 197 Measured detection accuracy and durable review (2026-07-15)
|
||||
|
||||
- Re-ran the active local building model at confidence thresholds `0.10` and
|
||||
`0.15` over independent Mol holdouts in Achterbos, Gompel, Donk and Postel,
|
||||
plus the pure-empty Postel forest control. Threshold `0.15` retained the best
|
||||
F1 in every positive zone and both thresholds produced zero forest-control
|
||||
detections, so no speculative threshold or model promotion was made.
|
||||
- Aligned the map-driven building QA flow with the documented operational
|
||||
footprint match IoU `0.25`. The map now reports candidate count, matches,
|
||||
precision, recall, F1, false positives and false negatives instead of
|
||||
presenting every model box as a recognized building.
|
||||
- Added first-class `detection_reviews` persistence and canonical project QA
|
||||
endpoints for paginated false-positive/false-negative review decisions.
|
||||
- Added a focused frontend review queue with role/status filters, notes,
|
||||
pagination and map handoff. Unreviewed or alignment-mismatch evidence cannot
|
||||
silently become hard-negative training data.
|
||||
- Bounded QA evidence resolution to the persisted evidence identifiers instead
|
||||
of loading complete regional reference datasets into application memory.
|
||||
- Added regression coverage for migration alignment, decision validation, API
|
||||
envelopes, bounded evidence access and the map QA threshold.
|
||||
- Passed the complete readiness gate with 592 backend tests, 88 documented API
|
||||
routes, one Alembic head and a green frontend typecheck/production build.
|
||||
|
||||
## Sprint 196 Map-driven official orthophoto analysis (2026-07-15)
|
||||
|
||||
- Added an explicit bounded endpoint for the official Digitaal Vlaanderen
|
||||
|
||||
@@ -708,6 +708,23 @@ curl http://localhost:1202/api/v1/projects/{project_id}/quality-checks
|
||||
The frontend QA/QC Results panel uses this endpoint after loading the demo
|
||||
workflow or running QA.
|
||||
|
||||
Detection QA evidence can be reviewed without changing its persisted metrics:
|
||||
|
||||
```bash
|
||||
curl "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews?reviewed=false&limit=50"
|
||||
|
||||
curl -X POST "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"evidence_role":"false_positive","evidence_feature_id":"DETECTION_UUID","decision":"qa_alignment_mismatch","notes":"Box and footprint represent the same building."}'
|
||||
```
|
||||
|
||||
The list is derived from persisted quality-check evidence and paginates at a
|
||||
maximum of 200 rows. The upsert verifies project ownership, quality-check type,
|
||||
role-specific decisions and persisted Detection/VectorFeature ownership.
|
||||
`detection_reviews` never mutates model output, reference geometry or canonical
|
||||
Metric rows. Evidence GeoJSON queries only stored evidence ids instead of a
|
||||
complete regional GRB dataset.
|
||||
|
||||
### Export foundation
|
||||
|
||||
Persisted exports can be created from the existing workbench state:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Add durable operator review decisions for detection QA evidence."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
|
||||
revision = "202607150001"
|
||||
down_revision = "202607140001"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"detection_reviews",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("quality_check_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("analysis_run_id", postgresql.UUID(as_uuid=True), nullable=True),
|
||||
sa.Column("evidence_role", sa.String(length=32), nullable=False),
|
||||
sa.Column("evidence_feature_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("detection_id", postgresql.UUID(as_uuid=True), nullable=True),
|
||||
sa.Column("reference_feature_id", postgresql.UUID(as_uuid=True), nullable=True),
|
||||
sa.Column("decision", sa.String(length=64), server_default="unreviewed", nullable=False),
|
||||
sa.Column("notes", sa.Text(), nullable=True),
|
||||
sa.Column("reviewed_by", sa.String(length=120), server_default="operator", nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||
sa.CheckConstraint(
|
||||
"evidence_role IN ('false_positive', 'false_negative')",
|
||||
name="ck_detection_reviews_evidence_role",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
|
||||
"'reference_gap_or_change', 'qa_alignment_mismatch', "
|
||||
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
|
||||
name="ck_detection_reviews_decision",
|
||||
),
|
||||
sa.ForeignKeyConstraint(["analysis_run_id"], ["analysis_runs.id"], ondelete="SET NULL"),
|
||||
sa.ForeignKeyConstraint(["detection_id"], ["detections.id"], ondelete="SET NULL"),
|
||||
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["quality_check_id"], ["quality_checks.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["reference_feature_id"], ["vector_features.id"], ondelete="SET NULL"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"quality_check_id",
|
||||
"evidence_role",
|
||||
"evidence_feature_id",
|
||||
name="uq_detection_reviews_evidence",
|
||||
),
|
||||
)
|
||||
op.create_index("ix_detection_reviews_project_id", "detection_reviews", ["project_id"])
|
||||
op.create_index("ix_detection_reviews_quality_check_id", "detection_reviews", ["quality_check_id"])
|
||||
op.create_index("ix_detection_reviews_analysis_run_id", "detection_reviews", ["analysis_run_id"])
|
||||
op.create_index("ix_detection_reviews_decision", "detection_reviews", ["decision"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_detection_reviews_decision", table_name="detection_reviews")
|
||||
op.drop_index("ix_detection_reviews_analysis_run_id", table_name="detection_reviews")
|
||||
op.drop_index("ix_detection_reviews_quality_check_id", table_name="detection_reviews")
|
||||
op.drop_index("ix_detection_reviews_project_id", table_name="detection_reviews")
|
||||
op.drop_table("detection_reviews")
|
||||
@@ -6,7 +6,9 @@ from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas.detection_review import DetectionReviewUpsert
|
||||
from app.schemas.qa import QualityCheckList
|
||||
from app.services.detection_review_service import DetectionReviewService
|
||||
from app.services.quality_evidence_service import QualityEvidenceService
|
||||
from app.services.quality_check_service import QualityCheckService
|
||||
from app.utils.response import envelope
|
||||
@@ -37,3 +39,45 @@ def get_quality_check_evidence_geojson(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id))
|
||||
|
||||
|
||||
@router.get("/quality-checks/{quality_check_id}/reviews", response_model=dict)
|
||||
def list_detection_reviews(
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
evidence_role: str | None = Query(default=None, pattern="^(false_positive|false_negative)$"),
|
||||
decision: str | None = Query(default=None, max_length=64),
|
||||
reviewed: bool | None = Query(default=None),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(
|
||||
DetectionReviewService.list_reviews(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
evidence_role=evidence_role,
|
||||
decision=decision,
|
||||
reviewed=reviewed,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.post("/quality-checks/{quality_check_id}/reviews", response_model=dict)
|
||||
def upsert_detection_review(
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
payload: DetectionReviewUpsert,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(
|
||||
DetectionReviewService.upsert_review(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
payload=payload,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
||||
from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
||||
|
||||
__all__ = [
|
||||
"AnalysisRun",
|
||||
@@ -6,6 +6,7 @@ __all__ = [
|
||||
"Dataset",
|
||||
"DatasetVersion",
|
||||
"Detection",
|
||||
"DetectionReview",
|
||||
"Export",
|
||||
"Job",
|
||||
"Metric",
|
||||
|
||||
@@ -4,7 +4,7 @@ import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, func
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.sql.sqltypes import Integer
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
@@ -272,6 +272,62 @@ class Metric(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class DetectionReview(Base):
|
||||
__tablename__ = "detection_reviews"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"evidence_role IN ('false_positive', 'false_negative')",
|
||||
name="ck_detection_reviews_evidence_role",
|
||||
),
|
||||
CheckConstraint(
|
||||
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
|
||||
"'reference_gap_or_change', 'qa_alignment_mismatch', "
|
||||
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
|
||||
name="ck_detection_reviews_decision",
|
||||
),
|
||||
UniqueConstraint(
|
||||
"quality_check_id",
|
||||
"evidence_role",
|
||||
"evidence_feature_id",
|
||||
name="uq_detection_reviews_evidence",
|
||||
),
|
||||
Index("ix_detection_reviews_project_id", "project_id"),
|
||||
Index("ix_detection_reviews_quality_check_id", "quality_check_id"),
|
||||
Index("ix_detection_reviews_analysis_run_id", "analysis_run_id"),
|
||||
Index("ix_detection_reviews_decision", "decision"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
quality_check_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("quality_checks.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("analysis_runs.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
evidence_role: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
evidence_feature_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
detection_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("detections.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
reference_feature_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("vector_features.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
decision: Mapped[str] = mapped_column(String(64), nullable=False, default="unreviewed", server_default="unreviewed")
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
reviewed_by: Mapped[str] = mapped_column(String(120), nullable=False, default="operator", server_default="operator")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class Export(Base):
|
||||
__tablename__ = "exports"
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from .detection import (
|
||||
ModelAssetListResponse,
|
||||
ModelAssetRead,
|
||||
)
|
||||
from .detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewSummary, DetectionReviewUpsert
|
||||
from .segmentation import (
|
||||
SegmentationListResponse,
|
||||
SegmentationModelCapability,
|
||||
@@ -111,6 +112,10 @@ __all__ = [
|
||||
"DetectionRunResponse",
|
||||
"ModelAssetListResponse",
|
||||
"ModelAssetRead",
|
||||
"DetectionReviewList",
|
||||
"DetectionReviewRead",
|
||||
"DetectionReviewSummary",
|
||||
"DetectionReviewUpsert",
|
||||
"SegmentationListResponse",
|
||||
"SegmentationModelCapability",
|
||||
"SegmentationModelsResponse",
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
DetectionEvidenceRole = Literal["false_positive", "false_negative"]
|
||||
DetectionReviewDecision = Literal[
|
||||
"confirmed_model_false_positive",
|
||||
"confirmed_model_false_negative",
|
||||
"reference_gap_or_change",
|
||||
"qa_alignment_mismatch",
|
||||
"imagery_obscured_or_uncertain",
|
||||
"uncertain",
|
||||
"unreviewed",
|
||||
]
|
||||
|
||||
|
||||
class DetectionReviewUpsert(BaseModel):
|
||||
evidence_role: DetectionEvidenceRole
|
||||
evidence_feature_id: str = Field(min_length=1, max_length=255)
|
||||
decision: DetectionReviewDecision
|
||||
notes: str | None = Field(default=None, max_length=2000)
|
||||
reviewed_by: str = Field(default="operator", min_length=1, max_length=120)
|
||||
|
||||
|
||||
class DetectionReviewRead(BaseModel):
|
||||
id: UUID | None = None
|
||||
project_id: UUID
|
||||
quality_check_id: UUID
|
||||
analysis_run_id: UUID | None = None
|
||||
evidence_role: DetectionEvidenceRole
|
||||
evidence_feature_id: str
|
||||
detection_id: UUID | None = None
|
||||
reference_feature_id: UUID | None = None
|
||||
decision: DetectionReviewDecision = "unreviewed"
|
||||
notes: str | None = None
|
||||
reviewed_by: str | None = None
|
||||
confidence: float | None = None
|
||||
class_name: str | None = None
|
||||
source_tile_path: str | None = None
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
|
||||
|
||||
class DetectionReviewSummary(BaseModel):
|
||||
total: int
|
||||
reviewed: int
|
||||
remaining: int
|
||||
false_positive_total: int
|
||||
false_negative_total: int
|
||||
decision_counts: dict[str, int]
|
||||
|
||||
|
||||
class DetectionReviewList(BaseModel):
|
||||
items: list[DetectionReviewRead]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
summary: DetectionReviewSummary
|
||||
@@ -0,0 +1,252 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
|
||||
from app.schemas.detection_review import (
|
||||
DetectionReviewList,
|
||||
DetectionReviewRead,
|
||||
DetectionReviewSummary,
|
||||
DetectionReviewUpsert,
|
||||
)
|
||||
|
||||
|
||||
class DetectionReviewService:
|
||||
ALLOWED_DECISIONS = {
|
||||
"false_positive": {
|
||||
"confirmed_model_false_positive",
|
||||
"reference_gap_or_change",
|
||||
"qa_alignment_mismatch",
|
||||
"uncertain",
|
||||
"unreviewed",
|
||||
},
|
||||
"false_negative": {
|
||||
"confirmed_model_false_negative",
|
||||
"reference_gap_or_change",
|
||||
"qa_alignment_mismatch",
|
||||
"imagery_obscured_or_uncertain",
|
||||
"uncertain",
|
||||
"unreviewed",
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _quality_check(db: Session, project_id: UUID, quality_check_id: UUID) -> QualityCheck:
|
||||
quality_check = db.get(QualityCheck, quality_check_id)
|
||||
if not quality_check or quality_check.project_id != project_id:
|
||||
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
|
||||
if quality_check.check_type != "detections_vs_reference":
|
||||
raise AppError(
|
||||
code="DETECTION_REVIEW_UNSUPPORTED",
|
||||
message="Only persisted detection-versus-reference quality checks can be reviewed",
|
||||
status_code=422,
|
||||
)
|
||||
return quality_check
|
||||
|
||||
@staticmethod
|
||||
def _evidence_items(quality_check: QualityCheck) -> list[dict[str, str]]:
|
||||
findings = quality_check.findings_json or {}
|
||||
items: list[dict[str, str]] = []
|
||||
for role, key, id_key in (
|
||||
("false_positive", "false_positive_evidence", "candidate_feature_id"),
|
||||
("false_negative", "false_negative_evidence", "reference_feature_id"),
|
||||
):
|
||||
evidence_rows = findings.get(key)
|
||||
if not isinstance(evidence_rows, list):
|
||||
continue
|
||||
for evidence in evidence_rows:
|
||||
if not isinstance(evidence, dict):
|
||||
continue
|
||||
value = str(evidence.get(id_key) or "").strip()
|
||||
if value:
|
||||
items.append({"evidence_role": role, "evidence_feature_id": value})
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def _uuid(value: str) -> UUID | None:
|
||||
try:
|
||||
return UUID(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _review_index(db: Session, quality_check_id: UUID) -> dict[tuple[str, str], DetectionReview]:
|
||||
rows = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check_id).all()
|
||||
return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
|
||||
|
||||
@staticmethod
|
||||
def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary:
|
||||
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
|
||||
decisions = Counter(
|
||||
reviews[key].decision if key in reviews else "unreviewed"
|
||||
for key in evidence_keys
|
||||
)
|
||||
reviewed = sum(count for decision, count in decisions.items() if decision != "unreviewed")
|
||||
false_positive_total = sum(1 for item in evidence if item["evidence_role"] == "false_positive")
|
||||
false_negative_total = sum(1 for item in evidence if item["evidence_role"] == "false_negative")
|
||||
return DetectionReviewSummary(
|
||||
total=len(evidence),
|
||||
reviewed=reviewed,
|
||||
remaining=max(len(evidence) - reviewed, 0),
|
||||
false_positive_total=false_positive_total,
|
||||
false_negative_total=false_negative_total,
|
||||
decision_counts=dict(sorted(decisions.items())),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _read_item(
|
||||
db: Session,
|
||||
quality_check: QualityCheck,
|
||||
evidence: dict[str, str],
|
||||
review: DetectionReview | None,
|
||||
) -> DetectionReviewRead:
|
||||
role = evidence["evidence_role"]
|
||||
feature_id = evidence["evidence_feature_id"]
|
||||
feature_uuid = DetectionReviewService._uuid(feature_id)
|
||||
detection = db.get(Detection, feature_uuid) if role == "false_positive" and feature_uuid else None
|
||||
reference = db.get(VectorFeature, feature_uuid) if role == "false_negative" and feature_uuid else None
|
||||
return DetectionReviewRead(
|
||||
id=review.id if review else None,
|
||||
project_id=quality_check.project_id,
|
||||
quality_check_id=quality_check.id,
|
||||
analysis_run_id=quality_check.analysis_run_id,
|
||||
evidence_role=role,
|
||||
evidence_feature_id=feature_id,
|
||||
detection_id=detection.id if detection else review.detection_id if review else None,
|
||||
reference_feature_id=reference.id if reference else review.reference_feature_id if review else None,
|
||||
decision=review.decision if review else "unreviewed",
|
||||
notes=review.notes if review else None,
|
||||
reviewed_by=review.reviewed_by if review else None,
|
||||
confidence=detection.confidence if detection else None,
|
||||
class_name=(detection.class_name if detection else reference.feature_class if reference else None),
|
||||
source_tile_path=detection.source_tile_path if detection else None,
|
||||
created_at=review.created_at if review else None,
|
||||
updated_at=review.updated_at if review else None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def list_reviews(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
evidence_role: str | None = None,
|
||||
decision: str | None = None,
|
||||
reviewed: bool | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> DetectionReviewList:
|
||||
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
|
||||
evidence = DetectionReviewService._evidence_items(quality_check)
|
||||
reviews = DetectionReviewService._review_index(db, quality_check_id)
|
||||
filtered = [item for item in evidence if evidence_role is None or item["evidence_role"] == evidence_role]
|
||||
if decision is not None:
|
||||
filtered = [
|
||||
item
|
||||
for item in filtered
|
||||
if (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
|
||||
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
|
||||
else "unreviewed")
|
||||
== decision
|
||||
]
|
||||
if reviewed is not None:
|
||||
filtered = [
|
||||
item
|
||||
for item in filtered
|
||||
if (
|
||||
(reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
|
||||
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
|
||||
else "unreviewed")
|
||||
!= "unreviewed"
|
||||
)
|
||||
== reviewed
|
||||
]
|
||||
page = filtered[offset : offset + limit]
|
||||
return DetectionReviewList(
|
||||
items=[
|
||||
DetectionReviewService._read_item(
|
||||
db,
|
||||
quality_check,
|
||||
item,
|
||||
reviews.get((item["evidence_role"], item["evidence_feature_id"])),
|
||||
)
|
||||
for item in page
|
||||
],
|
||||
total=len(filtered),
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
summary=DetectionReviewService._summary(evidence, reviews),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def upsert_review(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
payload: DetectionReviewUpsert,
|
||||
) -> DetectionReviewRead:
|
||||
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
|
||||
if payload.decision not in DetectionReviewService.ALLOWED_DECISIONS[payload.evidence_role]:
|
||||
raise AppError(
|
||||
code="INVALID_DETECTION_REVIEW_DECISION",
|
||||
message="The review decision is not valid for this evidence role",
|
||||
details={"evidence_role": payload.evidence_role, "decision": payload.decision},
|
||||
status_code=422,
|
||||
)
|
||||
evidence = DetectionReviewService._evidence_items(quality_check)
|
||||
evidence_key = (payload.evidence_role, payload.evidence_feature_id)
|
||||
if evidence_key not in {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}:
|
||||
raise AppError(
|
||||
code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND",
|
||||
message="The evidence feature does not belong to this quality check",
|
||||
status_code=404,
|
||||
)
|
||||
feature_uuid = DetectionReviewService._uuid(payload.evidence_feature_id)
|
||||
detection = db.get(Detection, feature_uuid) if payload.evidence_role == "false_positive" and feature_uuid else None
|
||||
reference = db.get(VectorFeature, feature_uuid) if payload.evidence_role == "false_negative" and feature_uuid else None
|
||||
if payload.evidence_role == "false_positive" and (not detection or detection.analysis_run_id != quality_check.analysis_run_id):
|
||||
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted detection evidence was not found", status_code=404)
|
||||
if payload.evidence_role == "false_negative" and (not reference or reference.dataset_id != quality_check.reference_dataset_id):
|
||||
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted reference evidence was not found", status_code=404)
|
||||
|
||||
review = (
|
||||
db.query(DetectionReview)
|
||||
.filter(
|
||||
DetectionReview.quality_check_id == quality_check_id,
|
||||
DetectionReview.evidence_role == payload.evidence_role,
|
||||
DetectionReview.evidence_feature_id == payload.evidence_feature_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if review is None:
|
||||
review = DetectionReview(
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
analysis_run_id=quality_check.analysis_run_id,
|
||||
evidence_role=payload.evidence_role,
|
||||
evidence_feature_id=payload.evidence_feature_id,
|
||||
detection_id=detection.id if detection else None,
|
||||
reference_feature_id=reference.id if reference else None,
|
||||
decision=payload.decision,
|
||||
notes=payload.notes.strip() if payload.notes and payload.notes.strip() else None,
|
||||
reviewed_by=payload.reviewed_by.strip(),
|
||||
)
|
||||
else:
|
||||
review.decision = payload.decision
|
||||
review.notes = payload.notes.strip() if payload.notes and payload.notes.strip() else None
|
||||
review.reviewed_by = payload.reviewed_by.strip()
|
||||
db.add(review)
|
||||
db.commit()
|
||||
db.refresh(review)
|
||||
return DetectionReviewService._read_item(
|
||||
db,
|
||||
quality_check,
|
||||
{"evidence_role": payload.evidence_role, "evidence_feature_id": payload.evidence_feature_id},
|
||||
review,
|
||||
)
|
||||
@@ -5,10 +5,11 @@ from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import mapping
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Detection, QualityCheck, Segmentation, VectorFeature
|
||||
from app.models import Detection, DetectionReview, QualityCheck, Segmentation, VectorFeature
|
||||
|
||||
|
||||
class QualityEvidenceService:
|
||||
@@ -21,8 +22,9 @@ class QualityEvidenceService:
|
||||
findings = quality_check.findings_json or {}
|
||||
features: list[dict[str, Any]] = []
|
||||
warnings: list[str] = []
|
||||
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check)
|
||||
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check)
|
||||
candidate_ids, reference_ids = QualityEvidenceService._evidence_identifiers(findings)
|
||||
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, candidate_ids)
|
||||
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, reference_ids)
|
||||
|
||||
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
|
||||
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||
@@ -89,6 +91,8 @@ class QualityEvidenceService:
|
||||
else:
|
||||
warnings.append(f"False-negative evidence feature not found: {reference_id}")
|
||||
|
||||
QualityEvidenceService._annotate_reviews(db, quality_check, features)
|
||||
|
||||
return {
|
||||
"quality_check_id": str(quality_check.id),
|
||||
"project_id": str(quality_check.project_id),
|
||||
@@ -117,34 +121,129 @@ class QualityEvidenceService:
|
||||
return text or None
|
||||
|
||||
@staticmethod
|
||||
def _candidate_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]:
|
||||
def _evidence_identifiers(findings: dict[str, Any]) -> tuple[set[str], set[str]]:
|
||||
candidate_ids: set[str] = set()
|
||||
reference_ids: set[str] = set()
|
||||
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
|
||||
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
|
||||
if candidate_id:
|
||||
candidate_ids.add(candidate_id)
|
||||
if reference_id:
|
||||
reference_ids.add(reference_id)
|
||||
for evidence in QualityEvidenceService._evidence_items(findings.get("false_positive_evidence")):
|
||||
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||
if candidate_id:
|
||||
candidate_ids.add(candidate_id)
|
||||
for evidence in QualityEvidenceService._evidence_items(findings.get("false_negative_evidence")):
|
||||
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
|
||||
if reference_id:
|
||||
reference_ids.add(reference_id)
|
||||
return candidate_ids, reference_ids
|
||||
|
||||
@staticmethod
|
||||
def _uuid_identifiers(identifiers: set[str]) -> list[UUID]:
|
||||
values: list[UUID] = []
|
||||
for identifier in identifiers:
|
||||
try:
|
||||
values.append(UUID(identifier))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return values
|
||||
|
||||
@staticmethod
|
||||
def _vector_feature_rows(db: Session, dataset_id: UUID, identifiers: set[str]) -> list[VectorFeature]:
|
||||
if not identifiers:
|
||||
return []
|
||||
conditions = [VectorFeature.source_feature_id.in_(identifiers)]
|
||||
uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
|
||||
if uuid_identifiers:
|
||||
conditions.append(VectorFeature.id.in_(uuid_identifiers))
|
||||
return (
|
||||
db.query(VectorFeature)
|
||||
.filter(VectorFeature.dataset_id == dataset_id, or_(*conditions))
|
||||
.all()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _candidate_feature_index(
|
||||
db: Session,
|
||||
quality_check: QualityCheck,
|
||||
identifiers: set[str],
|
||||
) -> dict[str, Any]:
|
||||
index: dict[str, Any] = {}
|
||||
if not identifiers:
|
||||
return index
|
||||
uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
|
||||
if quality_check.candidate_dataset_id:
|
||||
for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.candidate_dataset_id).all():
|
||||
for row in QualityEvidenceService._vector_feature_rows(db, quality_check.candidate_dataset_id, identifiers):
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
for row in db.query(Detection).filter(Detection.dataset_id == quality_check.candidate_dataset_id).all():
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
for row in db.query(Segmentation).filter(Segmentation.dataset_id == quality_check.candidate_dataset_id).all():
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
if quality_check.analysis_run_id:
|
||||
for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all():
|
||||
if uuid_identifiers:
|
||||
for row in db.query(Detection).filter(
|
||||
Detection.dataset_id == quality_check.candidate_dataset_id,
|
||||
Detection.id.in_(uuid_identifiers),
|
||||
).all():
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all():
|
||||
for row in db.query(Segmentation).filter(
|
||||
Segmentation.dataset_id == quality_check.candidate_dataset_id,
|
||||
Segmentation.id.in_(uuid_identifiers),
|
||||
).all():
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
elif quality_check.analysis_run_id:
|
||||
for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all():
|
||||
if quality_check.analysis_run_id and uuid_identifiers:
|
||||
for row in db.query(Detection).filter(
|
||||
Detection.analysis_run_id == quality_check.analysis_run_id,
|
||||
Detection.id.in_(uuid_identifiers),
|
||||
).all():
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all():
|
||||
for row in db.query(Segmentation).filter(
|
||||
Segmentation.analysis_run_id == quality_check.analysis_run_id,
|
||||
Segmentation.id.in_(uuid_identifiers),
|
||||
).all():
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
return index
|
||||
|
||||
@staticmethod
|
||||
def _reference_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]:
|
||||
def _reference_feature_index(
|
||||
db: Session,
|
||||
quality_check: QualityCheck,
|
||||
identifiers: set[str],
|
||||
) -> dict[str, Any]:
|
||||
index: dict[str, Any] = {}
|
||||
for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.reference_dataset_id).all():
|
||||
for row in QualityEvidenceService._vector_feature_rows(db, quality_check.reference_dataset_id, identifiers):
|
||||
QualityEvidenceService._add_index_keys(index, row)
|
||||
return index
|
||||
|
||||
@staticmethod
|
||||
def _annotate_reviews(
|
||||
db: Session,
|
||||
quality_check: QualityCheck,
|
||||
features: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if quality_check.check_type != "detections_vs_reference":
|
||||
return
|
||||
reviews = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check.id).all()
|
||||
review_index = {(row.evidence_role, row.evidence_feature_id): row for row in reviews}
|
||||
for feature in features:
|
||||
properties = feature.get("properties")
|
||||
if not isinstance(properties, dict):
|
||||
continue
|
||||
role = QualityEvidenceService._string_value(properties.get("qa_evidence_role"))
|
||||
if role == "false_positive":
|
||||
evidence_id = QualityEvidenceService._string_value(properties.get("candidate_feature_id"))
|
||||
elif role == "false_negative":
|
||||
evidence_id = QualityEvidenceService._string_value(properties.get("reference_feature_id"))
|
||||
else:
|
||||
continue
|
||||
review = review_index.get((role, evidence_id or ""))
|
||||
properties.update(
|
||||
{
|
||||
"review_decision": review.decision if review else "unreviewed",
|
||||
"review_notes": review.notes if review else None,
|
||||
"reviewed_by": review.reviewed_by if review else None,
|
||||
"reviewed_at": review.updated_at.isoformat() if review and review.updated_at else None,
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _add_index_keys(index: dict[str, Any], row: Any) -> None:
|
||||
for key in QualityEvidenceService._row_identifiers(row):
|
||||
|
||||
@@ -317,7 +317,8 @@ def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() ->
|
||||
assert "datasetsApi.acquireOrthophoto" in hook_source
|
||||
assert "prepareAndRunDetection(datasetId)" in hook_source
|
||||
assert "compareDetectionRunWithReference" in hook_source
|
||||
assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false)" in app_source
|
||||
assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold)" in app_source
|
||||
assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook_source
|
||||
assert "vervoerregio|operationele grens" in app_source
|
||||
assert "selectionBbox: mapSelectionBbox" in app_source
|
||||
assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from geoalchemy2.shape import from_shape
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
|
||||
from app.schemas.detection_review import (
|
||||
DetectionReviewList,
|
||||
DetectionReviewRead,
|
||||
DetectionReviewSummary,
|
||||
DetectionReviewUpsert,
|
||||
)
|
||||
from app.services.detection_review_service import DetectionReviewService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, rows):
|
||||
self.rows = list(rows)
|
||||
|
||||
def filter(self, *criteria):
|
||||
for criterion in criteria:
|
||||
left = getattr(criterion, "left", None)
|
||||
right = getattr(criterion, "right", None)
|
||||
operator = getattr(criterion, "operator", None)
|
||||
name = getattr(left, "name", None)
|
||||
value = getattr(right, "value", right)
|
||||
if name and operator and operator.__name__ == "eq":
|
||||
self.rows = [row for row in self.rows if getattr(row, name) == value]
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return list(self.rows)
|
||||
|
||||
def first(self):
|
||||
return self.rows[0] if self.rows else None
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None, query_rows=None) -> None:
|
||||
self.objects = objects or {}
|
||||
self.query_rows = query_rows or {}
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
def query(self, model):
|
||||
return FakeQuery(self.query_rows.setdefault(model, []))
|
||||
|
||||
def add(self, row):
|
||||
rows = self.query_rows.setdefault(type(row), [])
|
||||
if row not in rows:
|
||||
rows.append(row)
|
||||
self.objects[(type(row), row.id)] = row
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def refresh(self, _row):
|
||||
return None
|
||||
|
||||
|
||||
def _review_context() -> tuple[FakeSession, UUID, UUID, Detection, VectorFeature]:
|
||||
project_id = uuid4()
|
||||
quality_check_id = uuid4()
|
||||
analysis_run_id = uuid4()
|
||||
candidate_dataset_id = uuid4()
|
||||
reference_dataset_id = uuid4()
|
||||
detection = Detection(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
dataset_id=candidate_dataset_id,
|
||||
analysis_run_id=analysis_run_id,
|
||||
model_name="yolo-configured",
|
||||
class_name="building",
|
||||
confidence=0.62,
|
||||
geometry=from_shape(box(5.0, 51.0, 5.001, 51.001), srid=4326),
|
||||
)
|
||||
reference = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
source_feature_id="grb-missed",
|
||||
feature_class="building",
|
||||
properties_json={},
|
||||
geometry=from_shape(box(5.002, 51.002, 5.003, 51.003), srid=4326),
|
||||
)
|
||||
quality_check = QualityCheck(
|
||||
id=quality_check_id,
|
||||
project_id=project_id,
|
||||
analysis_run_id=analysis_run_id,
|
||||
candidate_dataset_id=candidate_dataset_id,
|
||||
reference_dataset_id=reference_dataset_id,
|
||||
check_type="detections_vs_reference",
|
||||
status="ok",
|
||||
findings_json={
|
||||
"false_positive_evidence": [{"candidate_feature_id": str(detection.id)}],
|
||||
"false_negative_evidence": [{"reference_feature_id": str(reference.id)}],
|
||||
},
|
||||
)
|
||||
db = FakeSession(
|
||||
objects={
|
||||
(QualityCheck, quality_check_id): quality_check,
|
||||
(Detection, detection.id): detection,
|
||||
(VectorFeature, reference.id): reference,
|
||||
},
|
||||
query_rows={DetectionReview: []},
|
||||
)
|
||||
return db, project_id, quality_check_id, detection, reference
|
||||
|
||||
|
||||
def test_detection_review_model_and_migration_are_aligned() -> None:
|
||||
migration = (ROOT / "backend" / "alembic" / "versions" / "202607150001_detection_reviews.py").read_text(encoding="utf-8")
|
||||
columns = DetectionReview.__table__.columns
|
||||
|
||||
for name in (
|
||||
"project_id",
|
||||
"quality_check_id",
|
||||
"analysis_run_id",
|
||||
"evidence_role",
|
||||
"evidence_feature_id",
|
||||
"detection_id",
|
||||
"reference_feature_id",
|
||||
"decision",
|
||||
"notes",
|
||||
"reviewed_by",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
):
|
||||
assert name in columns
|
||||
assert f'"{name}"' in migration
|
||||
assert 'op.create_table(\n "detection_reviews"' in migration
|
||||
assert 'down_revision = "202607140001"' in migration
|
||||
|
||||
|
||||
def test_detection_review_queue_persists_only_valid_operator_decisions() -> None:
|
||||
db, project_id, quality_check_id, detection, _reference = _review_context()
|
||||
|
||||
initial = DetectionReviewService.list_reviews(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
)
|
||||
assert initial.summary.total == 2
|
||||
assert initial.summary.reviewed == 0
|
||||
assert initial.summary.decision_counts == {"unreviewed": 2}
|
||||
|
||||
saved = DetectionReviewService.upsert_review(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
payload=DetectionReviewUpsert(
|
||||
evidence_role="false_positive",
|
||||
evidence_feature_id=str(detection.id),
|
||||
decision="qa_alignment_mismatch",
|
||||
notes="Box overlaps the official footprint but is not a training negative.",
|
||||
),
|
||||
)
|
||||
assert saved.decision == "qa_alignment_mismatch"
|
||||
assert saved.detection_id == detection.id
|
||||
|
||||
reviewed = DetectionReviewService.list_reviews(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
reviewed=True,
|
||||
)
|
||||
assert reviewed.total == 1
|
||||
assert reviewed.summary.reviewed == 1
|
||||
assert reviewed.summary.remaining == 1
|
||||
|
||||
with pytest.raises(AppError) as exc:
|
||||
DetectionReviewService.upsert_review(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
payload=DetectionReviewUpsert(
|
||||
evidence_role="false_positive",
|
||||
evidence_feature_id=str(detection.id),
|
||||
decision="confirmed_model_false_negative",
|
||||
),
|
||||
)
|
||||
assert exc.value.code == "INVALID_DETECTION_REVIEW_DECISION"
|
||||
|
||||
|
||||
def test_detection_review_endpoints_use_canonical_envelopes(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
quality_check_id = uuid4()
|
||||
item = DetectionReviewRead(
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
evidence_role="false_positive",
|
||||
evidence_feature_id=str(uuid4()),
|
||||
decision="unreviewed",
|
||||
)
|
||||
result = DetectionReviewList(
|
||||
items=[item],
|
||||
total=1,
|
||||
limit=50,
|
||||
offset=0,
|
||||
summary=DetectionReviewSummary(
|
||||
total=1,
|
||||
reviewed=0,
|
||||
remaining=1,
|
||||
false_positive_total=1,
|
||||
false_negative_total=0,
|
||||
decision_counts={"unreviewed": 1},
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(DetectionReviewService, "list_reviews", lambda *_args, **_kwargs: result)
|
||||
monkeypatch.setattr(DetectionReviewService, "upsert_review", lambda *_args, **_kwargs: item)
|
||||
app.dependency_overrides[get_db] = lambda: FakeSession()
|
||||
try:
|
||||
listed = TestClient(app).get(f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews")
|
||||
saved = TestClient(app).post(
|
||||
f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews",
|
||||
json={
|
||||
"evidence_role": "false_positive",
|
||||
"evidence_feature_id": item.evidence_feature_id,
|
||||
"decision": "unreviewed",
|
||||
},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(get_db, None)
|
||||
|
||||
assert listed.status_code == 200
|
||||
assert set(listed.json()) == {"data"}
|
||||
assert listed.json()["data"]["summary"]["remaining"] == 1
|
||||
assert saved.status_code == 200
|
||||
assert saved.json() == {"data": item.model_dump(mode="json")}
|
||||
|
||||
|
||||
def test_map_detection_qa_uses_documented_footprint_threshold_and_honest_labels() -> None:
|
||||
hook = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8")
|
||||
app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
|
||||
evidence_service = (ROOT / "backend" / "app" / "services" / "quality_evidence_service.py").read_text(encoding="utf-8")
|
||||
|
||||
assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook
|
||||
assert "kandidaten" in hook
|
||||
assert "precision" in hook.lower()
|
||||
assert "false, iouThreshold" in app_source
|
||||
assert "VectorFeature.id.in_(uuid_identifiers)" in evidence_service
|
||||
assert "VectorFeature.source_feature_id.in_(identifiers)" in evidence_service
|
||||
assert "for row in db.query(VectorFeature).filter(VectorFeature.dataset_id" not in evidence_service
|
||||
@@ -427,6 +427,21 @@ remains available as a higher-precision legacy `0.15` choice. The older
|
||||
default-promotion blocker. Every production-like run still requires persisted
|
||||
QA/QC against suitable reference data.
|
||||
|
||||
The map-driven building workflow uses canonical footprint IoU `0.25`, matching
|
||||
the promotion evidence above. A July 2026 Mol-only holdout audit compared
|
||||
confidence `0.10` and `0.15` over Achterbos, Gompel, Donk and Postel. Confidence
|
||||
`0.15` produced the better F1 in all four positive zones; both thresholds
|
||||
produced zero detections in the pure-empty Postel forest control. The active
|
||||
confidence therefore remains `0.15`. This result does not claim production
|
||||
perfection and does not justify another model-training run by itself.
|
||||
|
||||
False-positive and false-negative evidence from persisted detection QA can be
|
||||
classified through `detection_reviews`. The queue derives from quality-check
|
||||
evidence ids and resolves persisted Detection and reference VectorFeature rows.
|
||||
`qa_alignment_mismatch`, `reference_gap_or_change`, uncertain imagery and
|
||||
unreviewed items must never be exported as hard-negative or missed-positive
|
||||
training labels. Canonical QA metrics remain unchanged after review.
|
||||
|
||||
The persisted seven-AOI evidence for this profile contains 5,568 false
|
||||
positives among 13,613 candidate detections. The read-only audit command in
|
||||
`scripts/README.md` reports the largest review volumes in Turnhout, Herentals
|
||||
|
||||
@@ -1300,6 +1300,71 @@ candidate evidence exposes the equivalent persisted model/source fields plus
|
||||
`segmentation_id`, `mask_path` and `area_m2`. These are additive GeoJSON
|
||||
properties; the canonical envelope and endpoint path are unchanged.
|
||||
|
||||
For detection QA, false-positive and false-negative evidence properties also
|
||||
include `review_decision`, `review_notes`, `reviewed_by` and `reviewed_at`.
|
||||
Missing review rows are represented as `review_decision=unreviewed`. Evidence
|
||||
resolution is bounded to identifiers stored by the selected quality check.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`
|
||||
|
||||
Returns the paginated operator review queue for a persisted
|
||||
`detections_vs_reference` quality check. Optional query parameters are
|
||||
`evidence_role=false_positive|false_negative`, `decision`, `reviewed=true|false`,
|
||||
`limit` (1-200) and `offset`. Items derive only from persisted QA evidence.
|
||||
|
||||
```json
|
||||
{
|
||||
"data": {
|
||||
"items": [{
|
||||
"id": null,
|
||||
"project_id": "uuid",
|
||||
"quality_check_id": "uuid",
|
||||
"analysis_run_id": "uuid",
|
||||
"evidence_role": "false_positive",
|
||||
"evidence_feature_id": "detection-uuid",
|
||||
"detection_id": "detection-uuid",
|
||||
"decision": "unreviewed",
|
||||
"confidence": 0.62,
|
||||
"class_name": "building"
|
||||
}],
|
||||
"total": 1,
|
||||
"limit": 50,
|
||||
"offset": 0,
|
||||
"summary": {
|
||||
"total": 73,
|
||||
"reviewed": 0,
|
||||
"remaining": 73,
|
||||
"false_positive_total": 17,
|
||||
"false_negative_total": 56,
|
||||
"decision_counts": {"unreviewed": 73}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`
|
||||
|
||||
Creates or updates one durable operator decision. The evidence id must belong
|
||||
to the quality check and resolve to the persisted Detection or reference
|
||||
VectorFeature. False-positive and false-negative roles accept only their
|
||||
role-specific decisions.
|
||||
|
||||
```json
|
||||
{
|
||||
"evidence_role": "false_positive",
|
||||
"evidence_feature_id": "detection-uuid",
|
||||
"decision": "qa_alignment_mismatch",
|
||||
"notes": "The detection box overlaps the irregular GRB footprint.",
|
||||
"reviewed_by": "operator"
|
||||
}
|
||||
```
|
||||
|
||||
Allowed decisions are `confirmed_model_false_positive`,
|
||||
`confirmed_model_false_negative`, `reference_gap_or_change`,
|
||||
`qa_alignment_mismatch`, `imagery_obscured_or_uncertain`, `uncertain` and
|
||||
`unreviewed`. Invalid role/decision combinations return
|
||||
`INVALID_DETECTION_REVIEW_DECISION`.
|
||||
|
||||
## Exports
|
||||
|
||||
### POST `/api/v1/exports/geojson`
|
||||
|
||||
@@ -8135,3 +8135,40 @@ Next:
|
||||
- Add a review workflow for accepted/rejected detections and use those audited
|
||||
labels as the gate for model calibration or retraining. Do not present the
|
||||
current F1 score as production-grade accuracy.
|
||||
|
||||
## Sprint 197 - Measured detection accuracy and durable review (2026-07-15)
|
||||
|
||||
Implemented:
|
||||
- Added first-class `detection_reviews` persistence linked to Project,
|
||||
QualityCheck, AnalysisRun, Detection and reference VectorFeature, with
|
||||
role-constrained decisions and one durable row per evidence item.
|
||||
- Added canonical paginated GET/POST review endpoints and a frontend review
|
||||
queue with role/status filters, notes, summary counts and map handoff.
|
||||
- Changed map-driven detection QA from generic UI IoU `0.50` to the documented
|
||||
box-versus-footprint operational IoU `0.25`.
|
||||
- Reworded map output as candidates and exposed persisted matches, precision,
|
||||
recall, F1, false positives and false negatives.
|
||||
- Bounded evidence lookup to persisted evidence ids; complete regional GRB
|
||||
layers are no longer materialized for a small review overlay.
|
||||
|
||||
Live model evidence before deployment:
|
||||
- Re-ran the active model at confidence `0.10` and `0.15` on Mol Achterbos,
|
||||
Gompel, Donk and Postel with QA IoU `0.25`.
|
||||
- Confidence `0.15` won F1 in all four positive holdouts: `0.6694`, `0.6564`,
|
||||
`0.5894` and `0.4749`. Confidence `0.10` measured `0.6287`, `0.6348`,
|
||||
`0.5636` and `0.4435` respectively.
|
||||
- Both thresholds produced zero detections on the pure-empty Postel forest
|
||||
control. The active confidence remains `0.15`; no model or asset was trained,
|
||||
downloaded or promoted.
|
||||
|
||||
Validation before deployment:
|
||||
- The complete readiness gate passed 592 backend tests, the API/document audit
|
||||
for 88 implemented routes, backend compilation, one Alembic head, offline
|
||||
migration SQL generation, frontend typecheck/build and shell smoke checks.
|
||||
- Focused review/evidence regressions passed and the production bundle retained
|
||||
separate React, application and MapLibre chunks.
|
||||
|
||||
Next:
|
||||
- Deploy the migration and UI, verify one live review queue and map QA result,
|
||||
then complete representative manual decisions before constructing any new
|
||||
model-training corpus.
|
||||
|
||||
@@ -196,6 +196,28 @@ diagnostic reference-envelope comparison are persisted in the existing
|
||||
`quality_checks.findings_json`; `parameters_json.coverage_policy` records the
|
||||
evaluation policy used for reproducibility.
|
||||
|
||||
### detection_reviews
|
||||
|
||||
- `id uuid primary key`
|
||||
- `project_id uuid references projects(id) on delete cascade`
|
||||
- `quality_check_id uuid references quality_checks(id) on delete cascade`
|
||||
- `analysis_run_id uuid nullable references analysis_runs(id) on delete set null`
|
||||
- `evidence_role text not null` (`false_positive` or `false_negative`)
|
||||
- `evidence_feature_id text not null`
|
||||
- `detection_id uuid nullable references detections(id) on delete set null`
|
||||
- `reference_feature_id uuid nullable references vector_features(id) on delete set null`
|
||||
- `decision text not null default 'unreviewed'`
|
||||
- `notes text nullable`
|
||||
- `reviewed_by text not null default 'operator'`
|
||||
- `created_at timestamptz`
|
||||
- `updated_at timestamptz`
|
||||
|
||||
The unique key is `(quality_check_id, evidence_role, evidence_feature_id)`.
|
||||
Indexes cover project, quality check, analysis run and decision. Reviews
|
||||
classify persisted QA evidence only; they do not replace or modify Detection,
|
||||
VectorFeature, QualityCheck or Metric records. Unreviewed, reference-gap,
|
||||
imagery-uncertain and QA-alignment cases are not training labels.
|
||||
|
||||
### exports
|
||||
|
||||
- `id uuid primary key`
|
||||
|
||||
@@ -46,6 +46,10 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training.
|
||||
- [x] Add a coverage-aware Mol multi-zone benchmark report with explicit positive-zone, per-zone collapse, reference-coverage and pure-empty background gates.
|
||||
- [x] Execute the refreshed coverage-aware Mol operational benchmark against the active local model and record the resulting retain/review decision.
|
||||
- [x] Align map-driven building QA with footprint IoU `0.25` and show measured candidate/match/error metrics instead of calling every detection a building.
|
||||
- [x] Persist paginated false-positive/false-negative operator decisions and expose them in the Quality workspace.
|
||||
- [x] Bound QA evidence resolution to persisted evidence ids instead of loading complete regional reference datasets.
|
||||
- [x] Compare confidence `0.10` and `0.15` over independent Mol holdouts; retain `0.15` because it wins F1 in every positive zone while both pass the empty-background control.
|
||||
- [ ] Complete manual decisions for the generated 48 false-negative and 48 false-positive review cards before constructing any new training corpus.
|
||||
- [x] Backend FastAPI foundation, health endpoint and service structure.
|
||||
- [x] React/TypeScript frontend foundation and MapLibre workbench.
|
||||
|
||||
@@ -8,6 +8,12 @@ The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanal
|
||||
|
||||
Detection defaults to the configured local YOLO asset and automatically selects an available raster and active model asset where possible. The model registry and preflight remain honest when PyTorch, Ultralytics, a local model file or a tile manifest is unavailable. The active building profile is operational but remains review-required: its recorded coverage-aware benchmark is approximately precision 0.590, recall 0.577 and F1 0.582 over seven positive AOIs, with zero detections in the empty-background control. Another training pass is intentionally blocked until the generated false-positive and false-negative review decisions are completed.
|
||||
|
||||
Map-driven building analysis uses the documented footprint-IoU `0.25` and
|
||||
distinguishes model candidates from verified buildings. It shows persisted
|
||||
matches, precision, recall, F1, false positives and false negatives. The
|
||||
Quality workspace includes a paginated review queue for persisted detection-QA
|
||||
evidence. Reviews do not rewrite detections, GRB geometry or QA metrics.
|
||||
|
||||
The map-first explorer has two deliberate modes. `Latest state` selects the
|
||||
latest explicitly dated source snapshot without claiming an old edition is
|
||||
current, while `Evolution` lets the operator compare an earlier and later
|
||||
@@ -373,6 +379,7 @@ Before creating tiles, the guided action inspects raster dimensions and estimate
|
||||
- The QA/QC workspace includes a selected-check evidence drilldown with candidate/reference provenance, false-positive/negative metric evidence, map handoff context and parameters/findings JSON.
|
||||
- QA/QC findings now persist feature-level evidence in `findings_json`: matched candidate/reference feature ids with IoU, false-positive candidate feature ids and false-negative reference feature ids. The QA/QC drilldown renders these as compact evidence lists before the raw JSON.
|
||||
- Persisted QA/QC checks can be rendered as a Map workspace evidence overlay. The QA/QC panel calls `GET /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/evidence/geojson`, then MapLibre draws matched candidate/reference geometries, false positives and false negatives with distinct styling and a compact legend.
|
||||
- Detection QA checks expose a filtered, paginated operator review queue through `GET/POST /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`. The UI keeps confirmed model errors separate from reference gaps and box-to-footprint alignment mismatches.
|
||||
|
||||
## Raster dependency visibility
|
||||
|
||||
|
||||
@@ -477,8 +477,8 @@ function App(): JSX.Element {
|
||||
datasets,
|
||||
loadProjectData,
|
||||
prepareAndRunDetection: (datasetId) => prepareAndRunDetection(datasetId, 'yolo-configured'),
|
||||
compareDetectionRunWithReference: (analysisRunId, referenceDatasetId) =>
|
||||
compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false),
|
||||
compareDetectionRunWithReference: (analysisRunId, referenceDatasetId, iouThreshold) =>
|
||||
compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold),
|
||||
onAnalysisReady: () => {
|
||||
setMapContentMode('analysis')
|
||||
setMapLayerVisible(true)
|
||||
@@ -1011,6 +1011,8 @@ function App(): JSX.Element {
|
||||
orthophotoAnalysisStatus={mapOrthophotoAnalysis.status}
|
||||
orthophotoAnalysisError={mapOrthophotoAnalysis.error}
|
||||
orthophotoAnalysisRunning={mapOrthophotoAnalysis.running}
|
||||
orthophotoAnalysisQuality={mapOrthophotoAnalysis.lastQuality}
|
||||
orthophotoAnalysisDetectionCount={mapOrthophotoAnalysis.lastDetectionCount}
|
||||
availableMapDatasets={availableMapDatasets}
|
||||
selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''}
|
||||
selectedFeature={selectedMapFeature}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import GeoMap from '../GeoMap'
|
||||
import type { AreaRead, DatasetCreateResponse, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||
import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
||||
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
|
||||
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
||||
@@ -327,6 +327,12 @@ function formatBboxLabel(bbox: VectorSelectionBBox | null): string {
|
||||
return `${formatCoordinate(bbox.min_x)}, ${formatCoordinate(bbox.min_y)} -> ${formatCoordinate(bbox.max_x)}, ${formatCoordinate(bbox.max_y)}`
|
||||
}
|
||||
|
||||
function formatPercentage(value: number | null | undefined): string {
|
||||
return typeof value === 'number' && Number.isFinite(value)
|
||||
? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`
|
||||
: 'n.v.t.'
|
||||
}
|
||||
|
||||
function bboxToInputState(bbox: VectorSelectionBBox | null) {
|
||||
return {
|
||||
min_x: bbox ? String(bbox.min_x) : '',
|
||||
@@ -445,6 +451,8 @@ interface MapWorkspaceProps {
|
||||
orthophotoAnalysisStatus: string
|
||||
orthophotoAnalysisError: string | null
|
||||
orthophotoAnalysisRunning: boolean
|
||||
orthophotoAnalysisQuality: DetectionQaResult | null
|
||||
orthophotoAnalysisDetectionCount: number | null
|
||||
availableMapDatasets: DatasetCreateResponse[]
|
||||
selectedMapDatasetId: string
|
||||
onSelectMapArea: (areaId: string) => void
|
||||
@@ -518,6 +526,8 @@ export function MapWorkspace({
|
||||
orthophotoAnalysisStatus,
|
||||
orthophotoAnalysisError,
|
||||
orthophotoAnalysisRunning,
|
||||
orthophotoAnalysisQuality,
|
||||
orthophotoAnalysisDetectionCount,
|
||||
availableMapDatasets,
|
||||
selectedMapDatasetId,
|
||||
onSelectMapArea,
|
||||
@@ -1194,6 +1204,16 @@ export function MapWorkspace({
|
||||
</button>
|
||||
{orthophotoAnalysisStatus ? <p role="status">{orthophotoAnalysisStatus}</p> : null}
|
||||
{orthophotoAnalysisError ? <p className="error" role="alert">{orthophotoAnalysisError}</p> : null}
|
||||
{orthophotoAnalysisQuality ? (
|
||||
<div className="geo-image-quality-metrics" aria-label="Gemeten kwaliteit van de beeldanalyse">
|
||||
<div><span>Kandidaten</span><strong>{orthophotoAnalysisDetectionCount?.toLocaleString('nl-BE') ?? 'n.v.t.'}</strong></div>
|
||||
<div><span>Precision</span><strong>{formatPercentage(orthophotoAnalysisQuality.precision)}</strong></div>
|
||||
<div><span>Recall</span><strong>{formatPercentage(orthophotoAnalysisQuality.recall)}</strong></div>
|
||||
<div><span>F1</span><strong>{formatPercentage(orthophotoAnalysisQuality.f1_score)}</strong></div>
|
||||
<div><span>Fout</span><strong>{orthophotoAnalysisQuality.false_positives.toLocaleString('nl-BE')}</strong></div>
|
||||
<div><span>Gemist</span><strong>{orthophotoAnalysisQuality.false_negatives.toLocaleString('nl-BE')}</strong></div>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { qaApi } from '../../services/api'
|
||||
import type {
|
||||
DetectionEvidenceRole,
|
||||
DetectionReviewDecision,
|
||||
DetectionReviewList,
|
||||
DetectionReviewRead,
|
||||
} from '../../types'
|
||||
import { formatError } from '../../lib/formatError'
|
||||
|
||||
interface DetectionReviewPanelProps {
|
||||
projectId: string
|
||||
qualityCheckId: string
|
||||
onOpenEvidenceMap?: (qualityCheckId: string) => void
|
||||
}
|
||||
|
||||
const ROLE_LABELS: Record<DetectionEvidenceRole, string> = {
|
||||
false_positive: 'Onterecht gevonden',
|
||||
false_negative: 'Gemist gebouw',
|
||||
}
|
||||
|
||||
const DECISION_LABELS: Record<DetectionReviewDecision, string> = {
|
||||
confirmed_model_false_positive: 'Bevestigde foutdetectie',
|
||||
confirmed_model_false_negative: 'Bevestigd gemist gebouw',
|
||||
reference_gap_or_change: 'Referentie ontbreekt of is verouderd',
|
||||
qa_alignment_mismatch: 'Vormvergelijking is te streng',
|
||||
imagery_obscured_or_uncertain: 'Luchtbeeld is onduidelijk',
|
||||
uncertain: 'Verder onderzoek nodig',
|
||||
unreviewed: 'Nog niet beoordeeld',
|
||||
}
|
||||
|
||||
const ROLE_DECISIONS: Record<DetectionEvidenceRole, DetectionReviewDecision[]> = {
|
||||
false_positive: [
|
||||
'unreviewed',
|
||||
'confirmed_model_false_positive',
|
||||
'reference_gap_or_change',
|
||||
'qa_alignment_mismatch',
|
||||
'uncertain',
|
||||
],
|
||||
false_negative: [
|
||||
'unreviewed',
|
||||
'confirmed_model_false_negative',
|
||||
'reference_gap_or_change',
|
||||
'qa_alignment_mismatch',
|
||||
'imagery_obscured_or_uncertain',
|
||||
'uncertain',
|
||||
],
|
||||
}
|
||||
|
||||
function shortId(value: string): string {
|
||||
return value.length > 18 ? `${value.slice(0, 8)}...${value.slice(-6)}` : value
|
||||
}
|
||||
|
||||
export function DetectionReviewPanel({
|
||||
projectId,
|
||||
qualityCheckId,
|
||||
onOpenEvidenceMap,
|
||||
}: DetectionReviewPanelProps): JSX.Element {
|
||||
const [queue, setQueue] = useState<DetectionReviewList | null>(null)
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [savingKey, setSavingKey] = useState<string | null>(null)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [roleFilter, setRoleFilter] = useState<'all' | DetectionEvidenceRole>('all')
|
||||
const [statusFilter, setStatusFilter] = useState<'all' | 'reviewed' | 'unreviewed'>('unreviewed')
|
||||
const [offset, setOffset] = useState(0)
|
||||
const [draftDecisions, setDraftDecisions] = useState<Record<string, DetectionReviewDecision>>({})
|
||||
const [draftNotes, setDraftNotes] = useState<Record<string, string>>({})
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const result = await qaApi.listDetectionReviews(projectId, qualityCheckId, {
|
||||
evidenceRole: roleFilter === 'all' ? undefined : roleFilter,
|
||||
reviewed: statusFilter === 'all' ? undefined : statusFilter === 'reviewed',
|
||||
limit: 50,
|
||||
offset,
|
||||
})
|
||||
setQueue(result)
|
||||
setDraftDecisions(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.decision])))
|
||||
setDraftNotes(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.notes ?? ''])))
|
||||
} catch (caught) {
|
||||
setError(formatError(caught, 'De controlelijst kon niet worden geladen'))
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
void load()
|
||||
}, [projectId, qualityCheckId, roleFilter, statusFilter, offset])
|
||||
|
||||
const save = async (item: DetectionReviewRead) => {
|
||||
const key = reviewKey(item)
|
||||
setSavingKey(key)
|
||||
setError(null)
|
||||
try {
|
||||
await qaApi.upsertDetectionReview(projectId, qualityCheckId, {
|
||||
evidence_role: item.evidence_role,
|
||||
evidence_feature_id: item.evidence_feature_id,
|
||||
decision: draftDecisions[key] ?? item.decision,
|
||||
notes: draftNotes[key]?.trim() || null,
|
||||
reviewed_by: 'operator',
|
||||
})
|
||||
await load()
|
||||
} catch (caught) {
|
||||
setError(formatError(caught, 'De beoordeling kon niet worden bewaard'))
|
||||
} finally {
|
||||
setSavingKey(null)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<section className="detection-review-panel" aria-label="Handmatige controle van beeldanalyse">
|
||||
<div className="panel-title-row">
|
||||
<div>
|
||||
<h3>Fouten controleren</h3>
|
||||
<p className="muted">Beoordeel alleen twijfelgevallen. Bevestigde fouten kunnen later veilig als trainingsfeedback worden gebruikt.</p>
|
||||
</div>
|
||||
<button type="button" className="secondary-action" onClick={() => onOpenEvidenceMap?.(qualityCheckId)}>
|
||||
Op kaart bekijken
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{queue ? (
|
||||
<div className="detection-review-summary">
|
||||
<div><span>Te beoordelen</span><strong>{queue.summary.total}</strong></div>
|
||||
<div><span>Afgerond</span><strong>{queue.summary.reviewed}</strong></div>
|
||||
<div><span>Resterend</span><strong>{queue.summary.remaining}</strong></div>
|
||||
<div><span>Fout gevonden</span><strong>{queue.summary.false_positive_total}</strong></div>
|
||||
<div><span>Gemist</span><strong>{queue.summary.false_negative_total}</strong></div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="detection-review-filters">
|
||||
<label>
|
||||
Soort
|
||||
<select value={roleFilter} onChange={(event) => {
|
||||
setRoleFilter(event.target.value as typeof roleFilter)
|
||||
setOffset(0)
|
||||
}}>
|
||||
<option value="all">Alles</option>
|
||||
<option value="false_positive">Onterecht gevonden</option>
|
||||
<option value="false_negative">Gemist gebouw</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Status
|
||||
<select value={statusFilter} onChange={(event) => {
|
||||
setStatusFilter(event.target.value as typeof statusFilter)
|
||||
setOffset(0)
|
||||
}}>
|
||||
<option value="unreviewed">Nog te beoordelen</option>
|
||||
<option value="reviewed">Beoordeeld</option>
|
||||
<option value="all">Alles</option>
|
||||
</select>
|
||||
</label>
|
||||
<button type="button" className="secondary-action" disabled={loading} onClick={() => void load()}>
|
||||
{loading ? 'Laden...' : 'Vernieuwen'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error ? <p className="error" role="alert">{error}</p> : null}
|
||||
{!loading && queue && queue.items.length === 0 ? (
|
||||
<div className="result-state result-state-empty">
|
||||
<strong>Geen objecten in deze selectie</strong>
|
||||
<p>Pas de filters aan of open de bewijslaag op de kaart.</p>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<ol className="detection-review-list">
|
||||
{queue?.items.map((item) => {
|
||||
const key = reviewKey(item)
|
||||
const decision = draftDecisions[key] ?? item.decision
|
||||
return (
|
||||
<li key={key} className="detection-review-item">
|
||||
<div className="detection-review-item-heading">
|
||||
<div>
|
||||
<span>{ROLE_LABELS[item.evidence_role]}</span>
|
||||
<strong>{item.class_name ?? 'gebouw'} / {shortId(item.evidence_feature_id)}</strong>
|
||||
</div>
|
||||
{typeof item.confidence === 'number' ? <span className="count-pill">{Math.round(item.confidence * 100)}% vertrouwen</span> : null}
|
||||
</div>
|
||||
<div className="detection-review-editor">
|
||||
<label>
|
||||
Beoordeling
|
||||
<select
|
||||
value={decision}
|
||||
onChange={(event) => setDraftDecisions((current) => ({
|
||||
...current,
|
||||
[key]: event.target.value as DetectionReviewDecision,
|
||||
}))}
|
||||
>
|
||||
{ROLE_DECISIONS[item.evidence_role].map((option) => (
|
||||
<option key={option} value={option}>{DECISION_LABELS[option]}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Notitie
|
||||
<input
|
||||
type="text"
|
||||
maxLength={2000}
|
||||
placeholder="Waarom is dit correct, fout of onzeker?"
|
||||
value={draftNotes[key] ?? ''}
|
||||
onChange={(event) => setDraftNotes((current) => ({ ...current, [key]: event.target.value }))}
|
||||
/>
|
||||
</label>
|
||||
<button type="button" className="primary-action" disabled={savingKey === key} onClick={() => void save(item)}>
|
||||
{savingKey === key ? 'Bewaren...' : 'Beoordeling bewaren'}
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ol>
|
||||
{queue && queue.total > queue.limit ? (
|
||||
<div className="detection-review-pagination" aria-label="Pagina's van de controlelijst">
|
||||
<button
|
||||
type="button"
|
||||
className="secondary-action"
|
||||
disabled={offset === 0 || loading}
|
||||
onClick={() => setOffset((current) => Math.max(current - queue.limit, 0))}
|
||||
>
|
||||
Vorige
|
||||
</button>
|
||||
<span>{offset + 1}-{Math.min(offset + queue.limit, queue.total)} van {queue.total}</span>
|
||||
<button
|
||||
type="button"
|
||||
className="secondary-action"
|
||||
disabled={offset + queue.limit >= queue.total || loading}
|
||||
onClick={() => setOffset((current) => current + queue.limit)}
|
||||
>
|
||||
Volgende
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function reviewKey(item: Pick<DetectionReviewRead, 'evidence_role' | 'evidence_feature_id'>): string {
|
||||
return `${item.evidence_role}:${item.evidence_feature_id}`
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
import type { DatasetCreateResponse, MetricRead, QualityCheckRead } from '../../types'
|
||||
import { DetectionReviewPanel } from './DetectionReviewPanel'
|
||||
|
||||
const CORE_METRIC_ORDER = [
|
||||
'precision',
|
||||
@@ -304,6 +305,13 @@ export function QualityResultsPanel({
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{selectedQualityCheck.check_type === 'detections_vs_reference' && selectedProjectId ? (
|
||||
<DetectionReviewPanel
|
||||
projectId={selectedProjectId}
|
||||
qualityCheckId={selectedQualityCheck.id}
|
||||
onOpenEvidenceMap={onOpenEvidenceMap}
|
||||
/>
|
||||
) : null}
|
||||
<div className="quality-feature-evidence-grid" aria-label="Feature-level QA/QC evidence">
|
||||
<div>
|
||||
<span>Overeenkomende object-ID's</span>
|
||||
|
||||
@@ -405,6 +405,7 @@ export function useDetectionWorkflow({
|
||||
analysisRunId: string,
|
||||
referenceDatasetId: string,
|
||||
useCurrentFilters = true,
|
||||
iouThresholdOverride?: number,
|
||||
): Promise<DetectionQaResult | null> => {
|
||||
if (!analysisRunId) {
|
||||
setDetectionQaError('Select a detection run')
|
||||
@@ -422,7 +423,7 @@ export function useDetectionWorkflow({
|
||||
try {
|
||||
const result = await detectionApi.compareWithReference(analysisRunId, {
|
||||
reference_dataset_id: referenceDatasetId,
|
||||
iou_threshold: qaIouThreshold,
|
||||
iou_threshold: iouThresholdOverride ?? qaIouThreshold,
|
||||
class_name: useCurrentFilters ? detectionClassFilter || null : null,
|
||||
min_confidence: useCurrentFilters && detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null,
|
||||
})
|
||||
|
||||
@@ -17,6 +17,8 @@ export type MapOrthophotoAnalysisStage =
|
||||
| 'complete'
|
||||
| 'failed'
|
||||
|
||||
export const MAP_BUILDING_QA_IOU_THRESHOLD = 0.25
|
||||
|
||||
interface MapOrthophotoAnalysisOptions {
|
||||
selectedProjectId: string | null
|
||||
selectedAreaId: string
|
||||
@@ -27,6 +29,7 @@ interface MapOrthophotoAnalysisOptions {
|
||||
compareDetectionRunWithReference: (
|
||||
analysisRunId: string,
|
||||
referenceDatasetId: string,
|
||||
iouThreshold: number,
|
||||
) => Promise<DetectionQaResult | null>
|
||||
onAnalysisReady: () => void
|
||||
}
|
||||
@@ -61,6 +64,12 @@ function formatOrthophotoError(caught: unknown): string {
|
||||
return formatError(caught, 'De kaartgestuurde beeldanalyse is mislukt')
|
||||
}
|
||||
|
||||
function formatQualityPercentage(value: number | null | undefined): string {
|
||||
return typeof value === 'number' && Number.isFinite(value)
|
||||
? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`
|
||||
: 'n.v.t.'
|
||||
}
|
||||
|
||||
export function useMapOrthophotoAnalysis({
|
||||
selectedProjectId,
|
||||
selectedAreaId,
|
||||
@@ -75,12 +84,18 @@ export function useMapOrthophotoAnalysis({
|
||||
const [status, setStatus] = useState('')
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [lastResult, setLastResult] = useState<OrthophotoAcquisitionResult | null>(null)
|
||||
const [lastQuality, setLastQuality] = useState<DetectionQaResult | null>(null)
|
||||
const [lastDetectionCount, setLastDetectionCount] = useState<number | null>(null)
|
||||
const [lastAnalysisRunId, setLastAnalysisRunId] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
setStage('idle')
|
||||
setStatus('')
|
||||
setError(null)
|
||||
setLastResult(null)
|
||||
setLastQuality(null)
|
||||
setLastDetectionCount(null)
|
||||
setLastAnalysisRunId(null)
|
||||
}, [selectionBbox?.min_x, selectionBbox?.min_y, selectionBbox?.max_x, selectionBbox?.max_y])
|
||||
|
||||
const run = async (bbox: VectorSelectionBBox): Promise<boolean> => {
|
||||
@@ -91,6 +106,9 @@ export function useMapOrthophotoAnalysis({
|
||||
}
|
||||
setError(null)
|
||||
setLastResult(null)
|
||||
setLastQuality(null)
|
||||
setLastDetectionCount(null)
|
||||
setLastAnalysisRunId(null)
|
||||
setStage('acquiring')
|
||||
setStatus('1/3 Officieel luchtbeeld voor de rechthoek ophalen...')
|
||||
try {
|
||||
@@ -112,20 +130,27 @@ export function useMapOrthophotoAnalysis({
|
||||
if (!detection) {
|
||||
throw new Error('De beeldanalyse stopte. Open Beeldanalyse voor de technische oorzaak.')
|
||||
}
|
||||
setLastDetectionCount(detection.detection_count)
|
||||
setLastAnalysisRunId(detection.analysis_run_id)
|
||||
|
||||
const reference = findBuildingReference(datasets)
|
||||
if (reference) {
|
||||
setStage('validating')
|
||||
setStatus('3/3 Resultaat vergelijken met officiële GRB-gebouwen...')
|
||||
const quality = await compareDetectionRunWithReference(detection.analysis_run_id, reference.id)
|
||||
const quality = await compareDetectionRunWithReference(
|
||||
detection.analysis_run_id,
|
||||
reference.id,
|
||||
MAP_BUILDING_QA_IOU_THRESHOLD,
|
||||
)
|
||||
setLastQuality(quality)
|
||||
setStatus(
|
||||
quality
|
||||
? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend en gecontroleerd.`
|
||||
: `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend; kwaliteitscontrole kon niet afronden.`,
|
||||
? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten, ${quality.matches.toLocaleString('nl-BE')} gekoppeld aan GRB. Precision ${formatQualityPercentage(quality.precision)}, recall ${formatQualityPercentage(quality.recall)}.`
|
||||
: `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten; kwaliteitscontrole kon niet afronden.`,
|
||||
)
|
||||
} else {
|
||||
setStatus(
|
||||
`Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend. De GRB-referentielaag ontbreekt voor automatische controle.`,
|
||||
`Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaatvormen. De GRB-referentielaag ontbreekt voor automatische controle.`,
|
||||
)
|
||||
}
|
||||
setStage('complete')
|
||||
@@ -144,6 +169,9 @@ export function useMapOrthophotoAnalysis({
|
||||
status,
|
||||
error,
|
||||
lastResult,
|
||||
lastQuality,
|
||||
lastDetectionCount,
|
||||
lastAnalysisRunId,
|
||||
running: stage === 'acquiring' || stage === 'detecting' || stage === 'validating',
|
||||
run,
|
||||
}
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
import { apiGet, apiPost } from './client'
|
||||
import type { QaComparisonRequest, JobRead, QualityCheckListResponse, QualityEvidenceGeoJsonResponse } from '../../types'
|
||||
import type {
|
||||
DetectionReviewList,
|
||||
DetectionReviewRead,
|
||||
DetectionReviewUpsert,
|
||||
JobRead,
|
||||
QaComparisonRequest,
|
||||
QualityCheckListResponse,
|
||||
QualityEvidenceGeoJsonResponse,
|
||||
} from '../../types'
|
||||
|
||||
export const qaApi = {
|
||||
runQa: (payload: QaComparisonRequest): Promise<JobRead> =>
|
||||
@@ -8,4 +16,23 @@ export const qaApi = {
|
||||
apiGet<QualityCheckListResponse>(`/api/v1/projects/${projectId}/quality-checks`),
|
||||
getQualityEvidenceGeoJson: (projectId: string, qualityCheckId: string): Promise<QualityEvidenceGeoJsonResponse> =>
|
||||
apiGet<QualityEvidenceGeoJsonResponse>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/evidence/geojson`),
|
||||
listDetectionReviews: (
|
||||
projectId: string,
|
||||
qualityCheckId: string,
|
||||
options: { evidenceRole?: string; reviewed?: boolean; limit?: number; offset?: number } = {},
|
||||
): Promise<DetectionReviewList> => {
|
||||
const query = new URLSearchParams({
|
||||
limit: String(options.limit ?? 50),
|
||||
offset: String(options.offset ?? 0),
|
||||
})
|
||||
if (options.evidenceRole) query.set('evidence_role', options.evidenceRole)
|
||||
if (typeof options.reviewed === 'boolean') query.set('reviewed', String(options.reviewed))
|
||||
return apiGet<DetectionReviewList>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews?${query}`)
|
||||
},
|
||||
upsertDetectionReview: (
|
||||
projectId: string,
|
||||
qualityCheckId: string,
|
||||
payload: DetectionReviewUpsert,
|
||||
): Promise<DetectionReviewRead> =>
|
||||
apiPost<DetectionReviewRead>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews`, payload),
|
||||
}
|
||||
|
||||
@@ -1862,3 +1862,139 @@ details.ai-lab-model-surface > summary strong {
|
||||
background: var(--accent-soft);
|
||||
}
|
||||
}
|
||||
|
||||
.geo-image-quality-metrics,
|
||||
.detection-review-summary {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(6, minmax(0, 1fr));
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.geo-image-quality-metrics {
|
||||
grid-column: 1 / -1;
|
||||
margin-top: 0.35rem;
|
||||
border-top: 1px solid var(--line);
|
||||
padding-top: 0.65rem;
|
||||
}
|
||||
|
||||
.geo-image-quality-metrics > div,
|
||||
.detection-review-summary > div {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.15rem;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 6px;
|
||||
padding: 0.55rem;
|
||||
background: var(--panel-soft);
|
||||
}
|
||||
|
||||
.geo-image-quality-metrics span,
|
||||
.detection-review-summary span,
|
||||
.detection-review-item-heading span {
|
||||
color: var(--muted);
|
||||
font-size: 0.7rem;
|
||||
}
|
||||
|
||||
.geo-image-quality-metrics strong,
|
||||
.detection-review-summary strong {
|
||||
color: var(--text);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.detection-review-panel {
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
margin-top: 0.85rem;
|
||||
border-top: 1px solid var(--line);
|
||||
padding-top: 0.85rem;
|
||||
}
|
||||
|
||||
.detection-review-summary {
|
||||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.detection-review-filters,
|
||||
.detection-review-editor,
|
||||
.detection-review-pagination {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr)) auto;
|
||||
gap: 0.65rem;
|
||||
align-items: end;
|
||||
}
|
||||
|
||||
.detection-review-filters label,
|
||||
.detection-review-editor label {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.3rem;
|
||||
color: var(--muted);
|
||||
font-size: 0.72rem;
|
||||
}
|
||||
|
||||
.detection-review-list {
|
||||
display: grid;
|
||||
max-height: 34rem;
|
||||
gap: 0.5rem;
|
||||
margin: 0;
|
||||
overflow: auto;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.detection-review-item {
|
||||
display: grid;
|
||||
gap: 0.6rem;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 6px;
|
||||
padding: 0.7rem;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
.detection-review-item-heading {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
gap: 0.75rem;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.detection-review-item-heading > div {
|
||||
display: grid;
|
||||
min-width: 0;
|
||||
gap: 0.15rem;
|
||||
}
|
||||
|
||||
.detection-review-item-heading strong {
|
||||
overflow-wrap: anywhere;
|
||||
font-size: 0.82rem;
|
||||
}
|
||||
|
||||
.detection-review-pagination {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
.detection-review-pagination span {
|
||||
color: var(--muted);
|
||||
font-size: 0.76rem;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.geo-image-quality-metrics,
|
||||
.detection-review-summary {
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.detection-review-editor,
|
||||
.detection-review-filters {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 560px) {
|
||||
.geo-image-quality-metrics,
|
||||
.detection-review-summary {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -894,6 +894,61 @@ export interface QualityEvidenceGeoJsonResponse {
|
||||
geojson: GeoJSON.FeatureCollection
|
||||
}
|
||||
|
||||
export type DetectionEvidenceRole = 'false_positive' | 'false_negative'
|
||||
|
||||
export type DetectionReviewDecision =
|
||||
| 'confirmed_model_false_positive'
|
||||
| 'confirmed_model_false_negative'
|
||||
| 'reference_gap_or_change'
|
||||
| 'qa_alignment_mismatch'
|
||||
| 'imagery_obscured_or_uncertain'
|
||||
| 'uncertain'
|
||||
| 'unreviewed'
|
||||
|
||||
export interface DetectionReviewUpsert {
|
||||
evidence_role: DetectionEvidenceRole
|
||||
evidence_feature_id: string
|
||||
decision: DetectionReviewDecision
|
||||
notes?: string | null
|
||||
reviewed_by?: string
|
||||
}
|
||||
|
||||
export interface DetectionReviewRead {
|
||||
id?: string | null
|
||||
project_id: string
|
||||
quality_check_id: string
|
||||
analysis_run_id?: string | null
|
||||
evidence_role: DetectionEvidenceRole
|
||||
evidence_feature_id: string
|
||||
detection_id?: string | null
|
||||
reference_feature_id?: string | null
|
||||
decision: DetectionReviewDecision
|
||||
notes?: string | null
|
||||
reviewed_by?: string | null
|
||||
confidence?: number | null
|
||||
class_name?: string | null
|
||||
source_tile_path?: string | null
|
||||
created_at?: string | null
|
||||
updated_at?: string | null
|
||||
}
|
||||
|
||||
export interface DetectionReviewSummary {
|
||||
total: number
|
||||
reviewed: number
|
||||
remaining: number
|
||||
false_positive_total: number
|
||||
false_negative_total: number
|
||||
decision_counts: Record<string, number>
|
||||
}
|
||||
|
||||
export interface DetectionReviewList {
|
||||
items: DetectionReviewRead[]
|
||||
total: number
|
||||
limit: number
|
||||
offset: number
|
||||
summary: DetectionReviewSummary
|
||||
}
|
||||
|
||||
export type ExportKind = 'dataset' | 'detection_run' | 'segmentation_run' | 'vector_selection'
|
||||
|
||||
export interface ExportRead {
|
||||
|
||||
Reference in New Issue
Block a user