feat: add measured detection review loop
This commit is contained in:
@@ -7,6 +7,29 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 197 Measured detection accuracy and durable review (2026-07-15)
|
||||||
|
|
||||||
|
- Re-ran the active local building model at confidence thresholds `0.10` and
|
||||||
|
`0.15` over independent Mol holdouts in Achterbos, Gompel, Donk and Postel,
|
||||||
|
plus the pure-empty Postel forest control. Threshold `0.15` retained the best
|
||||||
|
F1 in every positive zone and both thresholds produced zero forest-control
|
||||||
|
detections, so no speculative threshold or model promotion was made.
|
||||||
|
- Aligned the map-driven building QA flow with the documented operational
|
||||||
|
footprint match IoU `0.25`. The map now reports candidate count, matches,
|
||||||
|
precision, recall, F1, false positives and false negatives instead of
|
||||||
|
presenting every model box as a recognized building.
|
||||||
|
- Added first-class `detection_reviews` persistence and canonical project QA
|
||||||
|
endpoints for paginated false-positive/false-negative review decisions.
|
||||||
|
- Added a focused frontend review queue with role/status filters, notes,
|
||||||
|
pagination and map handoff. Unreviewed or alignment-mismatch evidence cannot
|
||||||
|
silently become hard-negative training data.
|
||||||
|
- Bounded QA evidence resolution to the persisted evidence identifiers instead
|
||||||
|
of loading complete regional reference datasets into application memory.
|
||||||
|
- Added regression coverage for migration alignment, decision validation, API
|
||||||
|
envelopes, bounded evidence access and the map QA threshold.
|
||||||
|
- Passed the complete readiness gate with 592 backend tests, 88 documented API
|
||||||
|
routes, one Alembic head and a green frontend typecheck/production build.
|
||||||
|
|
||||||
## Sprint 196 Map-driven official orthophoto analysis (2026-07-15)
|
## Sprint 196 Map-driven official orthophoto analysis (2026-07-15)
|
||||||
|
|
||||||
- Added an explicit bounded endpoint for the official Digitaal Vlaanderen
|
- Added an explicit bounded endpoint for the official Digitaal Vlaanderen
|
||||||
|
|||||||
@@ -708,6 +708,23 @@ curl http://localhost:1202/api/v1/projects/{project_id}/quality-checks
|
|||||||
The frontend QA/QC Results panel uses this endpoint after loading the demo
|
The frontend QA/QC Results panel uses this endpoint after loading the demo
|
||||||
workflow or running QA.
|
workflow or running QA.
|
||||||
|
|
||||||
|
Detection QA evidence can be reviewed without changing its persisted metrics:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews?reviewed=false&limit=50"
|
||||||
|
|
||||||
|
curl -X POST "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"evidence_role":"false_positive","evidence_feature_id":"DETECTION_UUID","decision":"qa_alignment_mismatch","notes":"Box and footprint represent the same building."}'
|
||||||
|
```
|
||||||
|
|
||||||
|
The list is derived from persisted quality-check evidence and paginates at a
|
||||||
|
maximum of 200 rows. The upsert verifies project ownership, quality-check type,
|
||||||
|
role-specific decisions and persisted Detection/VectorFeature ownership.
|
||||||
|
`detection_reviews` never mutates model output, reference geometry or canonical
|
||||||
|
Metric rows. Evidence GeoJSON queries only stored evidence ids instead of a
|
||||||
|
complete regional GRB dataset.
|
||||||
|
|
||||||
### Export foundation
|
### Export foundation
|
||||||
|
|
||||||
Persisted exports can be created from the existing workbench state:
|
Persisted exports can be created from the existing workbench state:
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""Add durable operator review decisions for detection QA evidence."""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
|
||||||
|
revision = "202607150001"
|
||||||
|
down_revision = "202607140001"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"detection_reviews",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||||
|
sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||||
|
sa.Column("quality_check_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||||
|
sa.Column("analysis_run_id", postgresql.UUID(as_uuid=True), nullable=True),
|
||||||
|
sa.Column("evidence_role", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("evidence_feature_id", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("detection_id", postgresql.UUID(as_uuid=True), nullable=True),
|
||||||
|
sa.Column("reference_feature_id", postgresql.UUID(as_uuid=True), nullable=True),
|
||||||
|
sa.Column("decision", sa.String(length=64), server_default="unreviewed", nullable=False),
|
||||||
|
sa.Column("notes", sa.Text(), nullable=True),
|
||||||
|
sa.Column("reviewed_by", sa.String(length=120), server_default="operator", nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"evidence_role IN ('false_positive', 'false_negative')",
|
||||||
|
name="ck_detection_reviews_evidence_role",
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
|
||||||
|
"'reference_gap_or_change', 'qa_alignment_mismatch', "
|
||||||
|
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
|
||||||
|
name="ck_detection_reviews_decision",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(["analysis_run_id"], ["analysis_runs.id"], ondelete="SET NULL"),
|
||||||
|
sa.ForeignKeyConstraint(["detection_id"], ["detections.id"], ondelete="SET NULL"),
|
||||||
|
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"),
|
||||||
|
sa.ForeignKeyConstraint(["quality_check_id"], ["quality_checks.id"], ondelete="CASCADE"),
|
||||||
|
sa.ForeignKeyConstraint(["reference_feature_id"], ["vector_features.id"], ondelete="SET NULL"),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"quality_check_id",
|
||||||
|
"evidence_role",
|
||||||
|
"evidence_feature_id",
|
||||||
|
name="uq_detection_reviews_evidence",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_detection_reviews_project_id", "detection_reviews", ["project_id"])
|
||||||
|
op.create_index("ix_detection_reviews_quality_check_id", "detection_reviews", ["quality_check_id"])
|
||||||
|
op.create_index("ix_detection_reviews_analysis_run_id", "detection_reviews", ["analysis_run_id"])
|
||||||
|
op.create_index("ix_detection_reviews_decision", "detection_reviews", ["decision"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_detection_reviews_decision", table_name="detection_reviews")
|
||||||
|
op.drop_index("ix_detection_reviews_analysis_run_id", table_name="detection_reviews")
|
||||||
|
op.drop_index("ix_detection_reviews_quality_check_id", table_name="detection_reviews")
|
||||||
|
op.drop_index("ix_detection_reviews_project_id", table_name="detection_reviews")
|
||||||
|
op.drop_table("detection_reviews")
|
||||||
@@ -6,7 +6,9 @@ from fastapi import APIRouter, Depends, Query
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
|
from app.schemas.detection_review import DetectionReviewUpsert
|
||||||
from app.schemas.qa import QualityCheckList
|
from app.schemas.qa import QualityCheckList
|
||||||
|
from app.services.detection_review_service import DetectionReviewService
|
||||||
from app.services.quality_evidence_service import QualityEvidenceService
|
from app.services.quality_evidence_service import QualityEvidenceService
|
||||||
from app.services.quality_check_service import QualityCheckService
|
from app.services.quality_check_service import QualityCheckService
|
||||||
from app.utils.response import envelope
|
from app.utils.response import envelope
|
||||||
@@ -37,3 +39,45 @@ def get_quality_check_evidence_geojson(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id))
|
return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/quality-checks/{quality_check_id}/reviews", response_model=dict)
|
||||||
|
def list_detection_reviews(
|
||||||
|
project_id: UUID,
|
||||||
|
quality_check_id: UUID,
|
||||||
|
evidence_role: str | None = Query(default=None, pattern="^(false_positive|false_negative)$"),
|
||||||
|
decision: str | None = Query(default=None, max_length=64),
|
||||||
|
reviewed: bool | None = Query(default=None),
|
||||||
|
limit: int = Query(default=50, ge=1, le=200),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
return envelope(
|
||||||
|
DetectionReviewService.list_reviews(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
evidence_role=evidence_role,
|
||||||
|
decision=decision,
|
||||||
|
reviewed=reviewed,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
).model_dump()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/quality-checks/{quality_check_id}/reviews", response_model=dict)
|
||||||
|
def upsert_detection_review(
|
||||||
|
project_id: UUID,
|
||||||
|
quality_check_id: UUID,
|
||||||
|
payload: DetectionReviewUpsert,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
return envelope(
|
||||||
|
DetectionReviewService.upsert_review(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
payload=payload,
|
||||||
|
).model_dump()
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"AnalysisRun",
|
"AnalysisRun",
|
||||||
@@ -6,6 +6,7 @@ __all__ = [
|
|||||||
"Dataset",
|
"Dataset",
|
||||||
"DatasetVersion",
|
"DatasetVersion",
|
||||||
"Detection",
|
"Detection",
|
||||||
|
"DetectionReview",
|
||||||
"Export",
|
"Export",
|
||||||
"Job",
|
"Job",
|
||||||
"Metric",
|
"Metric",
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import uuid
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from geoalchemy2 import Geometry
|
from geoalchemy2 import Geometry
|
||||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, func
|
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, UniqueConstraint, func
|
||||||
from sqlalchemy.sql.sqltypes import Integer
|
from sqlalchemy.sql.sqltypes import Integer
|
||||||
from sqlalchemy.dialects.postgresql import UUID
|
from sqlalchemy.dialects.postgresql import UUID
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
@@ -272,6 +272,62 @@ class Metric(Base):
|
|||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionReview(Base):
|
||||||
|
__tablename__ = "detection_reviews"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"evidence_role IN ('false_positive', 'false_negative')",
|
||||||
|
name="ck_detection_reviews_evidence_role",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
|
||||||
|
"'reference_gap_or_change', 'qa_alignment_mismatch', "
|
||||||
|
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
|
||||||
|
name="ck_detection_reviews_decision",
|
||||||
|
),
|
||||||
|
UniqueConstraint(
|
||||||
|
"quality_check_id",
|
||||||
|
"evidence_role",
|
||||||
|
"evidence_feature_id",
|
||||||
|
name="uq_detection_reviews_evidence",
|
||||||
|
),
|
||||||
|
Index("ix_detection_reviews_project_id", "project_id"),
|
||||||
|
Index("ix_detection_reviews_quality_check_id", "quality_check_id"),
|
||||||
|
Index("ix_detection_reviews_analysis_run_id", "analysis_run_id"),
|
||||||
|
Index("ix_detection_reviews_decision", "decision"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||||
|
quality_check_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("quality_checks.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("analysis_runs.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
evidence_role: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
evidence_feature_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
detection_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("detections.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
reference_feature_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("vector_features.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
decision: Mapped[str] = mapped_column(String(64), nullable=False, default="unreviewed", server_default="unreviewed")
|
||||||
|
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
reviewed_by: Mapped[str] = mapped_column(String(120), nullable=False, default="operator", server_default="operator")
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
|
||||||
class Export(Base):
|
class Export(Base):
|
||||||
__tablename__ = "exports"
|
__tablename__ = "exports"
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from .detection import (
|
|||||||
ModelAssetListResponse,
|
ModelAssetListResponse,
|
||||||
ModelAssetRead,
|
ModelAssetRead,
|
||||||
)
|
)
|
||||||
|
from .detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewSummary, DetectionReviewUpsert
|
||||||
from .segmentation import (
|
from .segmentation import (
|
||||||
SegmentationListResponse,
|
SegmentationListResponse,
|
||||||
SegmentationModelCapability,
|
SegmentationModelCapability,
|
||||||
@@ -111,6 +112,10 @@ __all__ = [
|
|||||||
"DetectionRunResponse",
|
"DetectionRunResponse",
|
||||||
"ModelAssetListResponse",
|
"ModelAssetListResponse",
|
||||||
"ModelAssetRead",
|
"ModelAssetRead",
|
||||||
|
"DetectionReviewList",
|
||||||
|
"DetectionReviewRead",
|
||||||
|
"DetectionReviewSummary",
|
||||||
|
"DetectionReviewUpsert",
|
||||||
"SegmentationListResponse",
|
"SegmentationListResponse",
|
||||||
"SegmentationModelCapability",
|
"SegmentationModelCapability",
|
||||||
"SegmentationModelsResponse",
|
"SegmentationModelsResponse",
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Literal
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
DetectionEvidenceRole = Literal["false_positive", "false_negative"]
|
||||||
|
DetectionReviewDecision = Literal[
|
||||||
|
"confirmed_model_false_positive",
|
||||||
|
"confirmed_model_false_negative",
|
||||||
|
"reference_gap_or_change",
|
||||||
|
"qa_alignment_mismatch",
|
||||||
|
"imagery_obscured_or_uncertain",
|
||||||
|
"uncertain",
|
||||||
|
"unreviewed",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionReviewUpsert(BaseModel):
|
||||||
|
evidence_role: DetectionEvidenceRole
|
||||||
|
evidence_feature_id: str = Field(min_length=1, max_length=255)
|
||||||
|
decision: DetectionReviewDecision
|
||||||
|
notes: str | None = Field(default=None, max_length=2000)
|
||||||
|
reviewed_by: str = Field(default="operator", min_length=1, max_length=120)
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionReviewRead(BaseModel):
|
||||||
|
id: UUID | None = None
|
||||||
|
project_id: UUID
|
||||||
|
quality_check_id: UUID
|
||||||
|
analysis_run_id: UUID | None = None
|
||||||
|
evidence_role: DetectionEvidenceRole
|
||||||
|
evidence_feature_id: str
|
||||||
|
detection_id: UUID | None = None
|
||||||
|
reference_feature_id: UUID | None = None
|
||||||
|
decision: DetectionReviewDecision = "unreviewed"
|
||||||
|
notes: str | None = None
|
||||||
|
reviewed_by: str | None = None
|
||||||
|
confidence: float | None = None
|
||||||
|
class_name: str | None = None
|
||||||
|
source_tile_path: str | None = None
|
||||||
|
created_at: datetime | None = None
|
||||||
|
updated_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionReviewSummary(BaseModel):
|
||||||
|
total: int
|
||||||
|
reviewed: int
|
||||||
|
remaining: int
|
||||||
|
false_positive_total: int
|
||||||
|
false_negative_total: int
|
||||||
|
decision_counts: dict[str, int]
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionReviewList(BaseModel):
|
||||||
|
items: list[DetectionReviewRead]
|
||||||
|
total: int
|
||||||
|
limit: int
|
||||||
|
offset: int
|
||||||
|
summary: DetectionReviewSummary
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
|
||||||
|
from app.schemas.detection_review import (
|
||||||
|
DetectionReviewList,
|
||||||
|
DetectionReviewRead,
|
||||||
|
DetectionReviewSummary,
|
||||||
|
DetectionReviewUpsert,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionReviewService:
|
||||||
|
ALLOWED_DECISIONS = {
|
||||||
|
"false_positive": {
|
||||||
|
"confirmed_model_false_positive",
|
||||||
|
"reference_gap_or_change",
|
||||||
|
"qa_alignment_mismatch",
|
||||||
|
"uncertain",
|
||||||
|
"unreviewed",
|
||||||
|
},
|
||||||
|
"false_negative": {
|
||||||
|
"confirmed_model_false_negative",
|
||||||
|
"reference_gap_or_change",
|
||||||
|
"qa_alignment_mismatch",
|
||||||
|
"imagery_obscured_or_uncertain",
|
||||||
|
"uncertain",
|
||||||
|
"unreviewed",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _quality_check(db: Session, project_id: UUID, quality_check_id: UUID) -> QualityCheck:
|
||||||
|
quality_check = db.get(QualityCheck, quality_check_id)
|
||||||
|
if not quality_check or quality_check.project_id != project_id:
|
||||||
|
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
|
||||||
|
if quality_check.check_type != "detections_vs_reference":
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_REVIEW_UNSUPPORTED",
|
||||||
|
message="Only persisted detection-versus-reference quality checks can be reviewed",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return quality_check
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _evidence_items(quality_check: QualityCheck) -> list[dict[str, str]]:
|
||||||
|
findings = quality_check.findings_json or {}
|
||||||
|
items: list[dict[str, str]] = []
|
||||||
|
for role, key, id_key in (
|
||||||
|
("false_positive", "false_positive_evidence", "candidate_feature_id"),
|
||||||
|
("false_negative", "false_negative_evidence", "reference_feature_id"),
|
||||||
|
):
|
||||||
|
evidence_rows = findings.get(key)
|
||||||
|
if not isinstance(evidence_rows, list):
|
||||||
|
continue
|
||||||
|
for evidence in evidence_rows:
|
||||||
|
if not isinstance(evidence, dict):
|
||||||
|
continue
|
||||||
|
value = str(evidence.get(id_key) or "").strip()
|
||||||
|
if value:
|
||||||
|
items.append({"evidence_role": role, "evidence_feature_id": value})
|
||||||
|
return items
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _uuid(value: str) -> UUID | None:
|
||||||
|
try:
|
||||||
|
return UUID(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _review_index(db: Session, quality_check_id: UUID) -> dict[tuple[str, str], DetectionReview]:
|
||||||
|
rows = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check_id).all()
|
||||||
|
return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary:
|
||||||
|
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
|
||||||
|
decisions = Counter(
|
||||||
|
reviews[key].decision if key in reviews else "unreviewed"
|
||||||
|
for key in evidence_keys
|
||||||
|
)
|
||||||
|
reviewed = sum(count for decision, count in decisions.items() if decision != "unreviewed")
|
||||||
|
false_positive_total = sum(1 for item in evidence if item["evidence_role"] == "false_positive")
|
||||||
|
false_negative_total = sum(1 for item in evidence if item["evidence_role"] == "false_negative")
|
||||||
|
return DetectionReviewSummary(
|
||||||
|
total=len(evidence),
|
||||||
|
reviewed=reviewed,
|
||||||
|
remaining=max(len(evidence) - reviewed, 0),
|
||||||
|
false_positive_total=false_positive_total,
|
||||||
|
false_negative_total=false_negative_total,
|
||||||
|
decision_counts=dict(sorted(decisions.items())),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_item(
|
||||||
|
db: Session,
|
||||||
|
quality_check: QualityCheck,
|
||||||
|
evidence: dict[str, str],
|
||||||
|
review: DetectionReview | None,
|
||||||
|
) -> DetectionReviewRead:
|
||||||
|
role = evidence["evidence_role"]
|
||||||
|
feature_id = evidence["evidence_feature_id"]
|
||||||
|
feature_uuid = DetectionReviewService._uuid(feature_id)
|
||||||
|
detection = db.get(Detection, feature_uuid) if role == "false_positive" and feature_uuid else None
|
||||||
|
reference = db.get(VectorFeature, feature_uuid) if role == "false_negative" and feature_uuid else None
|
||||||
|
return DetectionReviewRead(
|
||||||
|
id=review.id if review else None,
|
||||||
|
project_id=quality_check.project_id,
|
||||||
|
quality_check_id=quality_check.id,
|
||||||
|
analysis_run_id=quality_check.analysis_run_id,
|
||||||
|
evidence_role=role,
|
||||||
|
evidence_feature_id=feature_id,
|
||||||
|
detection_id=detection.id if detection else review.detection_id if review else None,
|
||||||
|
reference_feature_id=reference.id if reference else review.reference_feature_id if review else None,
|
||||||
|
decision=review.decision if review else "unreviewed",
|
||||||
|
notes=review.notes if review else None,
|
||||||
|
reviewed_by=review.reviewed_by if review else None,
|
||||||
|
confidence=detection.confidence if detection else None,
|
||||||
|
class_name=(detection.class_name if detection else reference.feature_class if reference else None),
|
||||||
|
source_tile_path=detection.source_tile_path if detection else None,
|
||||||
|
created_at=review.created_at if review else None,
|
||||||
|
updated_at=review.updated_at if review else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def list_reviews(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
project_id: UUID,
|
||||||
|
quality_check_id: UUID,
|
||||||
|
evidence_role: str | None = None,
|
||||||
|
decision: str | None = None,
|
||||||
|
reviewed: bool | None = None,
|
||||||
|
limit: int = 50,
|
||||||
|
offset: int = 0,
|
||||||
|
) -> DetectionReviewList:
|
||||||
|
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
|
||||||
|
evidence = DetectionReviewService._evidence_items(quality_check)
|
||||||
|
reviews = DetectionReviewService._review_index(db, quality_check_id)
|
||||||
|
filtered = [item for item in evidence if evidence_role is None or item["evidence_role"] == evidence_role]
|
||||||
|
if decision is not None:
|
||||||
|
filtered = [
|
||||||
|
item
|
||||||
|
for item in filtered
|
||||||
|
if (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
|
||||||
|
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
|
||||||
|
else "unreviewed")
|
||||||
|
== decision
|
||||||
|
]
|
||||||
|
if reviewed is not None:
|
||||||
|
filtered = [
|
||||||
|
item
|
||||||
|
for item in filtered
|
||||||
|
if (
|
||||||
|
(reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
|
||||||
|
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
|
||||||
|
else "unreviewed")
|
||||||
|
!= "unreviewed"
|
||||||
|
)
|
||||||
|
== reviewed
|
||||||
|
]
|
||||||
|
page = filtered[offset : offset + limit]
|
||||||
|
return DetectionReviewList(
|
||||||
|
items=[
|
||||||
|
DetectionReviewService._read_item(
|
||||||
|
db,
|
||||||
|
quality_check,
|
||||||
|
item,
|
||||||
|
reviews.get((item["evidence_role"], item["evidence_feature_id"])),
|
||||||
|
)
|
||||||
|
for item in page
|
||||||
|
],
|
||||||
|
total=len(filtered),
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
summary=DetectionReviewService._summary(evidence, reviews),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def upsert_review(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
project_id: UUID,
|
||||||
|
quality_check_id: UUID,
|
||||||
|
payload: DetectionReviewUpsert,
|
||||||
|
) -> DetectionReviewRead:
|
||||||
|
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
|
||||||
|
if payload.decision not in DetectionReviewService.ALLOWED_DECISIONS[payload.evidence_role]:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_DETECTION_REVIEW_DECISION",
|
||||||
|
message="The review decision is not valid for this evidence role",
|
||||||
|
details={"evidence_role": payload.evidence_role, "decision": payload.decision},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
evidence = DetectionReviewService._evidence_items(quality_check)
|
||||||
|
evidence_key = (payload.evidence_role, payload.evidence_feature_id)
|
||||||
|
if evidence_key not in {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}:
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND",
|
||||||
|
message="The evidence feature does not belong to this quality check",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
|
feature_uuid = DetectionReviewService._uuid(payload.evidence_feature_id)
|
||||||
|
detection = db.get(Detection, feature_uuid) if payload.evidence_role == "false_positive" and feature_uuid else None
|
||||||
|
reference = db.get(VectorFeature, feature_uuid) if payload.evidence_role == "false_negative" and feature_uuid else None
|
||||||
|
if payload.evidence_role == "false_positive" and (not detection or detection.analysis_run_id != quality_check.analysis_run_id):
|
||||||
|
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted detection evidence was not found", status_code=404)
|
||||||
|
if payload.evidence_role == "false_negative" and (not reference or reference.dataset_id != quality_check.reference_dataset_id):
|
||||||
|
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted reference evidence was not found", status_code=404)
|
||||||
|
|
||||||
|
review = (
|
||||||
|
db.query(DetectionReview)
|
||||||
|
.filter(
|
||||||
|
DetectionReview.quality_check_id == quality_check_id,
|
||||||
|
DetectionReview.evidence_role == payload.evidence_role,
|
||||||
|
DetectionReview.evidence_feature_id == payload.evidence_feature_id,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if review is None:
|
||||||
|
review = DetectionReview(
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
analysis_run_id=quality_check.analysis_run_id,
|
||||||
|
evidence_role=payload.evidence_role,
|
||||||
|
evidence_feature_id=payload.evidence_feature_id,
|
||||||
|
detection_id=detection.id if detection else None,
|
||||||
|
reference_feature_id=reference.id if reference else None,
|
||||||
|
decision=payload.decision,
|
||||||
|
notes=payload.notes.strip() if payload.notes and payload.notes.strip() else None,
|
||||||
|
reviewed_by=payload.reviewed_by.strip(),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
review.decision = payload.decision
|
||||||
|
review.notes = payload.notes.strip() if payload.notes and payload.notes.strip() else None
|
||||||
|
review.reviewed_by = payload.reviewed_by.strip()
|
||||||
|
db.add(review)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(review)
|
||||||
|
return DetectionReviewService._read_item(
|
||||||
|
db,
|
||||||
|
quality_check,
|
||||||
|
{"evidence_role": payload.evidence_role, "evidence_feature_id": payload.evidence_feature_id},
|
||||||
|
review,
|
||||||
|
)
|
||||||
@@ -5,10 +5,11 @@ from uuid import UUID
|
|||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
from shapely.geometry import mapping
|
from shapely.geometry import mapping
|
||||||
|
from sqlalchemy import or_
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Detection, QualityCheck, Segmentation, VectorFeature
|
from app.models import Detection, DetectionReview, QualityCheck, Segmentation, VectorFeature
|
||||||
|
|
||||||
|
|
||||||
class QualityEvidenceService:
|
class QualityEvidenceService:
|
||||||
@@ -21,8 +22,9 @@ class QualityEvidenceService:
|
|||||||
findings = quality_check.findings_json or {}
|
findings = quality_check.findings_json or {}
|
||||||
features: list[dict[str, Any]] = []
|
features: list[dict[str, Any]] = []
|
||||||
warnings: list[str] = []
|
warnings: list[str] = []
|
||||||
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check)
|
candidate_ids, reference_ids = QualityEvidenceService._evidence_identifiers(findings)
|
||||||
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check)
|
candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, candidate_ids)
|
||||||
|
reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, reference_ids)
|
||||||
|
|
||||||
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
|
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
|
||||||
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||||
@@ -89,6 +91,8 @@ class QualityEvidenceService:
|
|||||||
else:
|
else:
|
||||||
warnings.append(f"False-negative evidence feature not found: {reference_id}")
|
warnings.append(f"False-negative evidence feature not found: {reference_id}")
|
||||||
|
|
||||||
|
QualityEvidenceService._annotate_reviews(db, quality_check, features)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"quality_check_id": str(quality_check.id),
|
"quality_check_id": str(quality_check.id),
|
||||||
"project_id": str(quality_check.project_id),
|
"project_id": str(quality_check.project_id),
|
||||||
@@ -117,34 +121,129 @@ class QualityEvidenceService:
|
|||||||
return text or None
|
return text or None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _candidate_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]:
|
def _evidence_identifiers(findings: dict[str, Any]) -> tuple[set[str], set[str]]:
|
||||||
|
candidate_ids: set[str] = set()
|
||||||
|
reference_ids: set[str] = set()
|
||||||
|
for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
|
||||||
|
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||||
|
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
|
||||||
|
if candidate_id:
|
||||||
|
candidate_ids.add(candidate_id)
|
||||||
|
if reference_id:
|
||||||
|
reference_ids.add(reference_id)
|
||||||
|
for evidence in QualityEvidenceService._evidence_items(findings.get("false_positive_evidence")):
|
||||||
|
candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
|
||||||
|
if candidate_id:
|
||||||
|
candidate_ids.add(candidate_id)
|
||||||
|
for evidence in QualityEvidenceService._evidence_items(findings.get("false_negative_evidence")):
|
||||||
|
reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
|
||||||
|
if reference_id:
|
||||||
|
reference_ids.add(reference_id)
|
||||||
|
return candidate_ids, reference_ids
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _uuid_identifiers(identifiers: set[str]) -> list[UUID]:
|
||||||
|
values: list[UUID] = []
|
||||||
|
for identifier in identifiers:
|
||||||
|
try:
|
||||||
|
values.append(UUID(identifier))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
return values
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _vector_feature_rows(db: Session, dataset_id: UUID, identifiers: set[str]) -> list[VectorFeature]:
|
||||||
|
if not identifiers:
|
||||||
|
return []
|
||||||
|
conditions = [VectorFeature.source_feature_id.in_(identifiers)]
|
||||||
|
uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
|
||||||
|
if uuid_identifiers:
|
||||||
|
conditions.append(VectorFeature.id.in_(uuid_identifiers))
|
||||||
|
return (
|
||||||
|
db.query(VectorFeature)
|
||||||
|
.filter(VectorFeature.dataset_id == dataset_id, or_(*conditions))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _candidate_feature_index(
|
||||||
|
db: Session,
|
||||||
|
quality_check: QualityCheck,
|
||||||
|
identifiers: set[str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
index: dict[str, Any] = {}
|
index: dict[str, Any] = {}
|
||||||
|
if not identifiers:
|
||||||
|
return index
|
||||||
|
uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
|
||||||
if quality_check.candidate_dataset_id:
|
if quality_check.candidate_dataset_id:
|
||||||
for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.candidate_dataset_id).all():
|
for row in QualityEvidenceService._vector_feature_rows(db, quality_check.candidate_dataset_id, identifiers):
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
QualityEvidenceService._add_index_keys(index, row)
|
||||||
for row in db.query(Detection).filter(Detection.dataset_id == quality_check.candidate_dataset_id).all():
|
if uuid_identifiers:
|
||||||
|
for row in db.query(Detection).filter(
|
||||||
|
Detection.dataset_id == quality_check.candidate_dataset_id,
|
||||||
|
Detection.id.in_(uuid_identifiers),
|
||||||
|
).all():
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
QualityEvidenceService._add_index_keys(index, row)
|
||||||
for row in db.query(Segmentation).filter(Segmentation.dataset_id == quality_check.candidate_dataset_id).all():
|
for row in db.query(Segmentation).filter(
|
||||||
|
Segmentation.dataset_id == quality_check.candidate_dataset_id,
|
||||||
|
Segmentation.id.in_(uuid_identifiers),
|
||||||
|
).all():
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
QualityEvidenceService._add_index_keys(index, row)
|
||||||
if quality_check.analysis_run_id:
|
if quality_check.analysis_run_id and uuid_identifiers:
|
||||||
for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all():
|
for row in db.query(Detection).filter(
|
||||||
|
Detection.analysis_run_id == quality_check.analysis_run_id,
|
||||||
|
Detection.id.in_(uuid_identifiers),
|
||||||
|
).all():
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
QualityEvidenceService._add_index_keys(index, row)
|
||||||
for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all():
|
for row in db.query(Segmentation).filter(
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
Segmentation.analysis_run_id == quality_check.analysis_run_id,
|
||||||
elif quality_check.analysis_run_id:
|
Segmentation.id.in_(uuid_identifiers),
|
||||||
for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all():
|
).all():
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
|
||||||
for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all():
|
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
QualityEvidenceService._add_index_keys(index, row)
|
||||||
return index
|
return index
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _reference_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]:
|
def _reference_feature_index(
|
||||||
|
db: Session,
|
||||||
|
quality_check: QualityCheck,
|
||||||
|
identifiers: set[str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
index: dict[str, Any] = {}
|
index: dict[str, Any] = {}
|
||||||
for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.reference_dataset_id).all():
|
for row in QualityEvidenceService._vector_feature_rows(db, quality_check.reference_dataset_id, identifiers):
|
||||||
QualityEvidenceService._add_index_keys(index, row)
|
QualityEvidenceService._add_index_keys(index, row)
|
||||||
return index
|
return index
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _annotate_reviews(
|
||||||
|
db: Session,
|
||||||
|
quality_check: QualityCheck,
|
||||||
|
features: list[dict[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
if quality_check.check_type != "detections_vs_reference":
|
||||||
|
return
|
||||||
|
reviews = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check.id).all()
|
||||||
|
review_index = {(row.evidence_role, row.evidence_feature_id): row for row in reviews}
|
||||||
|
for feature in features:
|
||||||
|
properties = feature.get("properties")
|
||||||
|
if not isinstance(properties, dict):
|
||||||
|
continue
|
||||||
|
role = QualityEvidenceService._string_value(properties.get("qa_evidence_role"))
|
||||||
|
if role == "false_positive":
|
||||||
|
evidence_id = QualityEvidenceService._string_value(properties.get("candidate_feature_id"))
|
||||||
|
elif role == "false_negative":
|
||||||
|
evidence_id = QualityEvidenceService._string_value(properties.get("reference_feature_id"))
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
review = review_index.get((role, evidence_id or ""))
|
||||||
|
properties.update(
|
||||||
|
{
|
||||||
|
"review_decision": review.decision if review else "unreviewed",
|
||||||
|
"review_notes": review.notes if review else None,
|
||||||
|
"reviewed_by": review.reviewed_by if review else None,
|
||||||
|
"reviewed_at": review.updated_at.isoformat() if review and review.updated_at else None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _add_index_keys(index: dict[str, Any], row: Any) -> None:
|
def _add_index_keys(index: dict[str, Any], row: Any) -> None:
|
||||||
for key in QualityEvidenceService._row_identifiers(row):
|
for key in QualityEvidenceService._row_identifiers(row):
|
||||||
|
|||||||
@@ -317,7 +317,8 @@ def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() ->
|
|||||||
assert "datasetsApi.acquireOrthophoto" in hook_source
|
assert "datasetsApi.acquireOrthophoto" in hook_source
|
||||||
assert "prepareAndRunDetection(datasetId)" in hook_source
|
assert "prepareAndRunDetection(datasetId)" in hook_source
|
||||||
assert "compareDetectionRunWithReference" in hook_source
|
assert "compareDetectionRunWithReference" in hook_source
|
||||||
assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false)" in app_source
|
assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold)" in app_source
|
||||||
|
assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook_source
|
||||||
assert "vervoerregio|operationele grens" in app_source
|
assert "vervoerregio|operationele grens" in app_source
|
||||||
assert "selectionBbox: mapSelectionBbox" in app_source
|
assert "selectionBbox: mapSelectionBbox" in app_source
|
||||||
assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source
|
assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source
|
||||||
|
|||||||
@@ -0,0 +1,253 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from geoalchemy2.shape import from_shape
|
||||||
|
from shapely.geometry import box
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.db.session import get_db
|
||||||
|
from app.main import app
|
||||||
|
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
|
||||||
|
from app.schemas.detection_review import (
|
||||||
|
DetectionReviewList,
|
||||||
|
DetectionReviewRead,
|
||||||
|
DetectionReviewSummary,
|
||||||
|
DetectionReviewUpsert,
|
||||||
|
)
|
||||||
|
from app.services.detection_review_service import DetectionReviewService
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
class FakeQuery:
|
||||||
|
def __init__(self, rows):
|
||||||
|
self.rows = list(rows)
|
||||||
|
|
||||||
|
def filter(self, *criteria):
|
||||||
|
for criterion in criteria:
|
||||||
|
left = getattr(criterion, "left", None)
|
||||||
|
right = getattr(criterion, "right", None)
|
||||||
|
operator = getattr(criterion, "operator", None)
|
||||||
|
name = getattr(left, "name", None)
|
||||||
|
value = getattr(right, "value", right)
|
||||||
|
if name and operator and operator.__name__ == "eq":
|
||||||
|
self.rows = [row for row in self.rows if getattr(row, name) == value]
|
||||||
|
return self
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return list(self.rows)
|
||||||
|
|
||||||
|
def first(self):
|
||||||
|
return self.rows[0] if self.rows else None
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSession:
|
||||||
|
def __init__(self, objects=None, query_rows=None) -> None:
|
||||||
|
self.objects = objects or {}
|
||||||
|
self.query_rows = query_rows or {}
|
||||||
|
|
||||||
|
def get(self, model, item_id):
|
||||||
|
return self.objects.get((model, item_id))
|
||||||
|
|
||||||
|
def query(self, model):
|
||||||
|
return FakeQuery(self.query_rows.setdefault(model, []))
|
||||||
|
|
||||||
|
def add(self, row):
|
||||||
|
rows = self.query_rows.setdefault(type(row), [])
|
||||||
|
if row not in rows:
|
||||||
|
rows.append(row)
|
||||||
|
self.objects[(type(row), row.id)] = row
|
||||||
|
|
||||||
|
def commit(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def refresh(self, _row):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _review_context() -> tuple[FakeSession, UUID, UUID, Detection, VectorFeature]:
|
||||||
|
project_id = uuid4()
|
||||||
|
quality_check_id = uuid4()
|
||||||
|
analysis_run_id = uuid4()
|
||||||
|
candidate_dataset_id = uuid4()
|
||||||
|
reference_dataset_id = uuid4()
|
||||||
|
detection = Detection(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=candidate_dataset_id,
|
||||||
|
analysis_run_id=analysis_run_id,
|
||||||
|
model_name="yolo-configured",
|
||||||
|
class_name="building",
|
||||||
|
confidence=0.62,
|
||||||
|
geometry=from_shape(box(5.0, 51.0, 5.001, 51.001), srid=4326),
|
||||||
|
)
|
||||||
|
reference = VectorFeature(
|
||||||
|
id=uuid4(),
|
||||||
|
dataset_id=reference_dataset_id,
|
||||||
|
source_feature_id="grb-missed",
|
||||||
|
feature_class="building",
|
||||||
|
properties_json={},
|
||||||
|
geometry=from_shape(box(5.002, 51.002, 5.003, 51.003), srid=4326),
|
||||||
|
)
|
||||||
|
quality_check = QualityCheck(
|
||||||
|
id=quality_check_id,
|
||||||
|
project_id=project_id,
|
||||||
|
analysis_run_id=analysis_run_id,
|
||||||
|
candidate_dataset_id=candidate_dataset_id,
|
||||||
|
reference_dataset_id=reference_dataset_id,
|
||||||
|
check_type="detections_vs_reference",
|
||||||
|
status="ok",
|
||||||
|
findings_json={
|
||||||
|
"false_positive_evidence": [{"candidate_feature_id": str(detection.id)}],
|
||||||
|
"false_negative_evidence": [{"reference_feature_id": str(reference.id)}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
db = FakeSession(
|
||||||
|
objects={
|
||||||
|
(QualityCheck, quality_check_id): quality_check,
|
||||||
|
(Detection, detection.id): detection,
|
||||||
|
(VectorFeature, reference.id): reference,
|
||||||
|
},
|
||||||
|
query_rows={DetectionReview: []},
|
||||||
|
)
|
||||||
|
return db, project_id, quality_check_id, detection, reference
|
||||||
|
|
||||||
|
|
||||||
|
def test_detection_review_model_and_migration_are_aligned() -> None:
|
||||||
|
migration = (ROOT / "backend" / "alembic" / "versions" / "202607150001_detection_reviews.py").read_text(encoding="utf-8")
|
||||||
|
columns = DetectionReview.__table__.columns
|
||||||
|
|
||||||
|
for name in (
|
||||||
|
"project_id",
|
||||||
|
"quality_check_id",
|
||||||
|
"analysis_run_id",
|
||||||
|
"evidence_role",
|
||||||
|
"evidence_feature_id",
|
||||||
|
"detection_id",
|
||||||
|
"reference_feature_id",
|
||||||
|
"decision",
|
||||||
|
"notes",
|
||||||
|
"reviewed_by",
|
||||||
|
"created_at",
|
||||||
|
"updated_at",
|
||||||
|
):
|
||||||
|
assert name in columns
|
||||||
|
assert f'"{name}"' in migration
|
||||||
|
assert 'op.create_table(\n "detection_reviews"' in migration
|
||||||
|
assert 'down_revision = "202607140001"' in migration
|
||||||
|
|
||||||
|
|
||||||
|
def test_detection_review_queue_persists_only_valid_operator_decisions() -> None:
|
||||||
|
db, project_id, quality_check_id, detection, _reference = _review_context()
|
||||||
|
|
||||||
|
initial = DetectionReviewService.list_reviews(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
)
|
||||||
|
assert initial.summary.total == 2
|
||||||
|
assert initial.summary.reviewed == 0
|
||||||
|
assert initial.summary.decision_counts == {"unreviewed": 2}
|
||||||
|
|
||||||
|
saved = DetectionReviewService.upsert_review(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
payload=DetectionReviewUpsert(
|
||||||
|
evidence_role="false_positive",
|
||||||
|
evidence_feature_id=str(detection.id),
|
||||||
|
decision="qa_alignment_mismatch",
|
||||||
|
notes="Box overlaps the official footprint but is not a training negative.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert saved.decision == "qa_alignment_mismatch"
|
||||||
|
assert saved.detection_id == detection.id
|
||||||
|
|
||||||
|
reviewed = DetectionReviewService.list_reviews(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
reviewed=True,
|
||||||
|
)
|
||||||
|
assert reviewed.total == 1
|
||||||
|
assert reviewed.summary.reviewed == 1
|
||||||
|
assert reviewed.summary.remaining == 1
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc:
|
||||||
|
DetectionReviewService.upsert_review(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
payload=DetectionReviewUpsert(
|
||||||
|
evidence_role="false_positive",
|
||||||
|
evidence_feature_id=str(detection.id),
|
||||||
|
decision="confirmed_model_false_negative",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert exc.value.code == "INVALID_DETECTION_REVIEW_DECISION"
|
||||||
|
|
||||||
|
|
||||||
|
def test_detection_review_endpoints_use_canonical_envelopes(monkeypatch) -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
quality_check_id = uuid4()
|
||||||
|
item = DetectionReviewRead(
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
evidence_role="false_positive",
|
||||||
|
evidence_feature_id=str(uuid4()),
|
||||||
|
decision="unreviewed",
|
||||||
|
)
|
||||||
|
result = DetectionReviewList(
|
||||||
|
items=[item],
|
||||||
|
total=1,
|
||||||
|
limit=50,
|
||||||
|
offset=0,
|
||||||
|
summary=DetectionReviewSummary(
|
||||||
|
total=1,
|
||||||
|
reviewed=0,
|
||||||
|
remaining=1,
|
||||||
|
false_positive_total=1,
|
||||||
|
false_negative_total=0,
|
||||||
|
decision_counts={"unreviewed": 1},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(DetectionReviewService, "list_reviews", lambda *_args, **_kwargs: result)
|
||||||
|
monkeypatch.setattr(DetectionReviewService, "upsert_review", lambda *_args, **_kwargs: item)
|
||||||
|
app.dependency_overrides[get_db] = lambda: FakeSession()
|
||||||
|
try:
|
||||||
|
listed = TestClient(app).get(f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews")
|
||||||
|
saved = TestClient(app).post(
|
||||||
|
f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews",
|
||||||
|
json={
|
||||||
|
"evidence_role": "false_positive",
|
||||||
|
"evidence_feature_id": item.evidence_feature_id,
|
||||||
|
"decision": "unreviewed",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
app.dependency_overrides.pop(get_db, None)
|
||||||
|
|
||||||
|
assert listed.status_code == 200
|
||||||
|
assert set(listed.json()) == {"data"}
|
||||||
|
assert listed.json()["data"]["summary"]["remaining"] == 1
|
||||||
|
assert saved.status_code == 200
|
||||||
|
assert saved.json() == {"data": item.model_dump(mode="json")}
|
||||||
|
|
||||||
|
|
||||||
|
def test_map_detection_qa_uses_documented_footprint_threshold_and_honest_labels() -> None:
|
||||||
|
hook = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8")
|
||||||
|
app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
|
||||||
|
evidence_service = (ROOT / "backend" / "app" / "services" / "quality_evidence_service.py").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook
|
||||||
|
assert "kandidaten" in hook
|
||||||
|
assert "precision" in hook.lower()
|
||||||
|
assert "false, iouThreshold" in app_source
|
||||||
|
assert "VectorFeature.id.in_(uuid_identifiers)" in evidence_service
|
||||||
|
assert "VectorFeature.source_feature_id.in_(identifiers)" in evidence_service
|
||||||
|
assert "for row in db.query(VectorFeature).filter(VectorFeature.dataset_id" not in evidence_service
|
||||||
@@ -427,6 +427,21 @@ remains available as a higher-precision legacy `0.15` choice. The older
|
|||||||
default-promotion blocker. Every production-like run still requires persisted
|
default-promotion blocker. Every production-like run still requires persisted
|
||||||
QA/QC against suitable reference data.
|
QA/QC against suitable reference data.
|
||||||
|
|
||||||
|
The map-driven building workflow uses canonical footprint IoU `0.25`, matching
|
||||||
|
the promotion evidence above. A July 2026 Mol-only holdout audit compared
|
||||||
|
confidence `0.10` and `0.15` over Achterbos, Gompel, Donk and Postel. Confidence
|
||||||
|
`0.15` produced the better F1 in all four positive zones; both thresholds
|
||||||
|
produced zero detections in the pure-empty Postel forest control. The active
|
||||||
|
confidence therefore remains `0.15`. This result does not claim production
|
||||||
|
perfection and does not justify another model-training run by itself.
|
||||||
|
|
||||||
|
False-positive and false-negative evidence from persisted detection QA can be
|
||||||
|
classified through `detection_reviews`. The queue derives from quality-check
|
||||||
|
evidence ids and resolves persisted Detection and reference VectorFeature rows.
|
||||||
|
`qa_alignment_mismatch`, `reference_gap_or_change`, uncertain imagery and
|
||||||
|
unreviewed items must never be exported as hard-negative or missed-positive
|
||||||
|
training labels. Canonical QA metrics remain unchanged after review.
|
||||||
|
|
||||||
The persisted seven-AOI evidence for this profile contains 5,568 false
|
The persisted seven-AOI evidence for this profile contains 5,568 false
|
||||||
positives among 13,613 candidate detections. The read-only audit command in
|
positives among 13,613 candidate detections. The read-only audit command in
|
||||||
`scripts/README.md` reports the largest review volumes in Turnhout, Herentals
|
`scripts/README.md` reports the largest review volumes in Turnhout, Herentals
|
||||||
|
|||||||
@@ -1300,6 +1300,71 @@ candidate evidence exposes the equivalent persisted model/source fields plus
|
|||||||
`segmentation_id`, `mask_path` and `area_m2`. These are additive GeoJSON
|
`segmentation_id`, `mask_path` and `area_m2`. These are additive GeoJSON
|
||||||
properties; the canonical envelope and endpoint path are unchanged.
|
properties; the canonical envelope and endpoint path are unchanged.
|
||||||
|
|
||||||
|
For detection QA, false-positive and false-negative evidence properties also
|
||||||
|
include `review_decision`, `review_notes`, `reviewed_by` and `reviewed_at`.
|
||||||
|
Missing review rows are represented as `review_decision=unreviewed`. Evidence
|
||||||
|
resolution is bounded to identifiers stored by the selected quality check.
|
||||||
|
|
||||||
|
### GET `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`
|
||||||
|
|
||||||
|
Returns the paginated operator review queue for a persisted
|
||||||
|
`detections_vs_reference` quality check. Optional query parameters are
|
||||||
|
`evidence_role=false_positive|false_negative`, `decision`, `reviewed=true|false`,
|
||||||
|
`limit` (1-200) and `offset`. Items derive only from persisted QA evidence.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"items": [{
|
||||||
|
"id": null,
|
||||||
|
"project_id": "uuid",
|
||||||
|
"quality_check_id": "uuid",
|
||||||
|
"analysis_run_id": "uuid",
|
||||||
|
"evidence_role": "false_positive",
|
||||||
|
"evidence_feature_id": "detection-uuid",
|
||||||
|
"detection_id": "detection-uuid",
|
||||||
|
"decision": "unreviewed",
|
||||||
|
"confidence": 0.62,
|
||||||
|
"class_name": "building"
|
||||||
|
}],
|
||||||
|
"total": 1,
|
||||||
|
"limit": 50,
|
||||||
|
"offset": 0,
|
||||||
|
"summary": {
|
||||||
|
"total": 73,
|
||||||
|
"reviewed": 0,
|
||||||
|
"remaining": 73,
|
||||||
|
"false_positive_total": 17,
|
||||||
|
"false_negative_total": 56,
|
||||||
|
"decision_counts": {"unreviewed": 73}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`
|
||||||
|
|
||||||
|
Creates or updates one durable operator decision. The evidence id must belong
|
||||||
|
to the quality check and resolve to the persisted Detection or reference
|
||||||
|
VectorFeature. False-positive and false-negative roles accept only their
|
||||||
|
role-specific decisions.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"evidence_role": "false_positive",
|
||||||
|
"evidence_feature_id": "detection-uuid",
|
||||||
|
"decision": "qa_alignment_mismatch",
|
||||||
|
"notes": "The detection box overlaps the irregular GRB footprint.",
|
||||||
|
"reviewed_by": "operator"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Allowed decisions are `confirmed_model_false_positive`,
|
||||||
|
`confirmed_model_false_negative`, `reference_gap_or_change`,
|
||||||
|
`qa_alignment_mismatch`, `imagery_obscured_or_uncertain`, `uncertain` and
|
||||||
|
`unreviewed`. Invalid role/decision combinations return
|
||||||
|
`INVALID_DETECTION_REVIEW_DECISION`.
|
||||||
|
|
||||||
## Exports
|
## Exports
|
||||||
|
|
||||||
### POST `/api/v1/exports/geojson`
|
### POST `/api/v1/exports/geojson`
|
||||||
|
|||||||
@@ -8135,3 +8135,40 @@ Next:
|
|||||||
- Add a review workflow for accepted/rejected detections and use those audited
|
- Add a review workflow for accepted/rejected detections and use those audited
|
||||||
labels as the gate for model calibration or retraining. Do not present the
|
labels as the gate for model calibration or retraining. Do not present the
|
||||||
current F1 score as production-grade accuracy.
|
current F1 score as production-grade accuracy.
|
||||||
|
|
||||||
|
## Sprint 197 - Measured detection accuracy and durable review (2026-07-15)
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
- Added first-class `detection_reviews` persistence linked to Project,
|
||||||
|
QualityCheck, AnalysisRun, Detection and reference VectorFeature, with
|
||||||
|
role-constrained decisions and one durable row per evidence item.
|
||||||
|
- Added canonical paginated GET/POST review endpoints and a frontend review
|
||||||
|
queue with role/status filters, notes, summary counts and map handoff.
|
||||||
|
- Changed map-driven detection QA from generic UI IoU `0.50` to the documented
|
||||||
|
box-versus-footprint operational IoU `0.25`.
|
||||||
|
- Reworded map output as candidates and exposed persisted matches, precision,
|
||||||
|
recall, F1, false positives and false negatives.
|
||||||
|
- Bounded evidence lookup to persisted evidence ids; complete regional GRB
|
||||||
|
layers are no longer materialized for a small review overlay.
|
||||||
|
|
||||||
|
Live model evidence before deployment:
|
||||||
|
- Re-ran the active model at confidence `0.10` and `0.15` on Mol Achterbos,
|
||||||
|
Gompel, Donk and Postel with QA IoU `0.25`.
|
||||||
|
- Confidence `0.15` won F1 in all four positive holdouts: `0.6694`, `0.6564`,
|
||||||
|
`0.5894` and `0.4749`. Confidence `0.10` measured `0.6287`, `0.6348`,
|
||||||
|
`0.5636` and `0.4435` respectively.
|
||||||
|
- Both thresholds produced zero detections on the pure-empty Postel forest
|
||||||
|
control. The active confidence remains `0.15`; no model or asset was trained,
|
||||||
|
downloaded or promoted.
|
||||||
|
|
||||||
|
Validation before deployment:
|
||||||
|
- The complete readiness gate passed 592 backend tests, the API/document audit
|
||||||
|
for 88 implemented routes, backend compilation, one Alembic head, offline
|
||||||
|
migration SQL generation, frontend typecheck/build and shell smoke checks.
|
||||||
|
- Focused review/evidence regressions passed and the production bundle retained
|
||||||
|
separate React, application and MapLibre chunks.
|
||||||
|
|
||||||
|
Next:
|
||||||
|
- Deploy the migration and UI, verify one live review queue and map QA result,
|
||||||
|
then complete representative manual decisions before constructing any new
|
||||||
|
model-training corpus.
|
||||||
|
|||||||
@@ -196,6 +196,28 @@ diagnostic reference-envelope comparison are persisted in the existing
|
|||||||
`quality_checks.findings_json`; `parameters_json.coverage_policy` records the
|
`quality_checks.findings_json`; `parameters_json.coverage_policy` records the
|
||||||
evaluation policy used for reproducibility.
|
evaluation policy used for reproducibility.
|
||||||
|
|
||||||
|
### detection_reviews
|
||||||
|
|
||||||
|
- `id uuid primary key`
|
||||||
|
- `project_id uuid references projects(id) on delete cascade`
|
||||||
|
- `quality_check_id uuid references quality_checks(id) on delete cascade`
|
||||||
|
- `analysis_run_id uuid nullable references analysis_runs(id) on delete set null`
|
||||||
|
- `evidence_role text not null` (`false_positive` or `false_negative`)
|
||||||
|
- `evidence_feature_id text not null`
|
||||||
|
- `detection_id uuid nullable references detections(id) on delete set null`
|
||||||
|
- `reference_feature_id uuid nullable references vector_features(id) on delete set null`
|
||||||
|
- `decision text not null default 'unreviewed'`
|
||||||
|
- `notes text nullable`
|
||||||
|
- `reviewed_by text not null default 'operator'`
|
||||||
|
- `created_at timestamptz`
|
||||||
|
- `updated_at timestamptz`
|
||||||
|
|
||||||
|
The unique key is `(quality_check_id, evidence_role, evidence_feature_id)`.
|
||||||
|
Indexes cover project, quality check, analysis run and decision. Reviews
|
||||||
|
classify persisted QA evidence only; they do not replace or modify Detection,
|
||||||
|
VectorFeature, QualityCheck or Metric records. Unreviewed, reference-gap,
|
||||||
|
imagery-uncertain and QA-alignment cases are not training labels.
|
||||||
|
|
||||||
### exports
|
### exports
|
||||||
|
|
||||||
- `id uuid primary key`
|
- `id uuid primary key`
|
||||||
|
|||||||
@@ -46,6 +46,10 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training.
|
- [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training.
|
||||||
- [x] Add a coverage-aware Mol multi-zone benchmark report with explicit positive-zone, per-zone collapse, reference-coverage and pure-empty background gates.
|
- [x] Add a coverage-aware Mol multi-zone benchmark report with explicit positive-zone, per-zone collapse, reference-coverage and pure-empty background gates.
|
||||||
- [x] Execute the refreshed coverage-aware Mol operational benchmark against the active local model and record the resulting retain/review decision.
|
- [x] Execute the refreshed coverage-aware Mol operational benchmark against the active local model and record the resulting retain/review decision.
|
||||||
|
- [x] Align map-driven building QA with footprint IoU `0.25` and show measured candidate/match/error metrics instead of calling every detection a building.
|
||||||
|
- [x] Persist paginated false-positive/false-negative operator decisions and expose them in the Quality workspace.
|
||||||
|
- [x] Bound QA evidence resolution to persisted evidence ids instead of loading complete regional reference datasets.
|
||||||
|
- [x] Compare confidence `0.10` and `0.15` over independent Mol holdouts; retain `0.15` because it wins F1 in every positive zone while both pass the empty-background control.
|
||||||
- [ ] Complete manual decisions for the generated 48 false-negative and 48 false-positive review cards before constructing any new training corpus.
|
- [ ] Complete manual decisions for the generated 48 false-negative and 48 false-positive review cards before constructing any new training corpus.
|
||||||
- [x] Backend FastAPI foundation, health endpoint and service structure.
|
- [x] Backend FastAPI foundation, health endpoint and service structure.
|
||||||
- [x] React/TypeScript frontend foundation and MapLibre workbench.
|
- [x] React/TypeScript frontend foundation and MapLibre workbench.
|
||||||
|
|||||||
@@ -8,6 +8,12 @@ The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanal
|
|||||||
|
|
||||||
Detection defaults to the configured local YOLO asset and automatically selects an available raster and active model asset where possible. The model registry and preflight remain honest when PyTorch, Ultralytics, a local model file or a tile manifest is unavailable. The active building profile is operational but remains review-required: its recorded coverage-aware benchmark is approximately precision 0.590, recall 0.577 and F1 0.582 over seven positive AOIs, with zero detections in the empty-background control. Another training pass is intentionally blocked until the generated false-positive and false-negative review decisions are completed.
|
Detection defaults to the configured local YOLO asset and automatically selects an available raster and active model asset where possible. The model registry and preflight remain honest when PyTorch, Ultralytics, a local model file or a tile manifest is unavailable. The active building profile is operational but remains review-required: its recorded coverage-aware benchmark is approximately precision 0.590, recall 0.577 and F1 0.582 over seven positive AOIs, with zero detections in the empty-background control. Another training pass is intentionally blocked until the generated false-positive and false-negative review decisions are completed.
|
||||||
|
|
||||||
|
Map-driven building analysis uses the documented footprint-IoU `0.25` and
|
||||||
|
distinguishes model candidates from verified buildings. It shows persisted
|
||||||
|
matches, precision, recall, F1, false positives and false negatives. The
|
||||||
|
Quality workspace includes a paginated review queue for persisted detection-QA
|
||||||
|
evidence. Reviews do not rewrite detections, GRB geometry or QA metrics.
|
||||||
|
|
||||||
The map-first explorer has two deliberate modes. `Latest state` selects the
|
The map-first explorer has two deliberate modes. `Latest state` selects the
|
||||||
latest explicitly dated source snapshot without claiming an old edition is
|
latest explicitly dated source snapshot without claiming an old edition is
|
||||||
current, while `Evolution` lets the operator compare an earlier and later
|
current, while `Evolution` lets the operator compare an earlier and later
|
||||||
@@ -373,6 +379,7 @@ Before creating tiles, the guided action inspects raster dimensions and estimate
|
|||||||
- The QA/QC workspace includes a selected-check evidence drilldown with candidate/reference provenance, false-positive/negative metric evidence, map handoff context and parameters/findings JSON.
|
- The QA/QC workspace includes a selected-check evidence drilldown with candidate/reference provenance, false-positive/negative metric evidence, map handoff context and parameters/findings JSON.
|
||||||
- QA/QC findings now persist feature-level evidence in `findings_json`: matched candidate/reference feature ids with IoU, false-positive candidate feature ids and false-negative reference feature ids. The QA/QC drilldown renders these as compact evidence lists before the raw JSON.
|
- QA/QC findings now persist feature-level evidence in `findings_json`: matched candidate/reference feature ids with IoU, false-positive candidate feature ids and false-negative reference feature ids. The QA/QC drilldown renders these as compact evidence lists before the raw JSON.
|
||||||
- Persisted QA/QC checks can be rendered as a Map workspace evidence overlay. The QA/QC panel calls `GET /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/evidence/geojson`, then MapLibre draws matched candidate/reference geometries, false positives and false negatives with distinct styling and a compact legend.
|
- Persisted QA/QC checks can be rendered as a Map workspace evidence overlay. The QA/QC panel calls `GET /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/evidence/geojson`, then MapLibre draws matched candidate/reference geometries, false positives and false negatives with distinct styling and a compact legend.
|
||||||
|
- Detection QA checks expose a filtered, paginated operator review queue through `GET/POST /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`. The UI keeps confirmed model errors separate from reference gaps and box-to-footprint alignment mismatches.
|
||||||
|
|
||||||
## Raster dependency visibility
|
## Raster dependency visibility
|
||||||
|
|
||||||
|
|||||||
@@ -477,8 +477,8 @@ function App(): JSX.Element {
|
|||||||
datasets,
|
datasets,
|
||||||
loadProjectData,
|
loadProjectData,
|
||||||
prepareAndRunDetection: (datasetId) => prepareAndRunDetection(datasetId, 'yolo-configured'),
|
prepareAndRunDetection: (datasetId) => prepareAndRunDetection(datasetId, 'yolo-configured'),
|
||||||
compareDetectionRunWithReference: (analysisRunId, referenceDatasetId) =>
|
compareDetectionRunWithReference: (analysisRunId, referenceDatasetId, iouThreshold) =>
|
||||||
compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false),
|
compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold),
|
||||||
onAnalysisReady: () => {
|
onAnalysisReady: () => {
|
||||||
setMapContentMode('analysis')
|
setMapContentMode('analysis')
|
||||||
setMapLayerVisible(true)
|
setMapLayerVisible(true)
|
||||||
@@ -1011,6 +1011,8 @@ function App(): JSX.Element {
|
|||||||
orthophotoAnalysisStatus={mapOrthophotoAnalysis.status}
|
orthophotoAnalysisStatus={mapOrthophotoAnalysis.status}
|
||||||
orthophotoAnalysisError={mapOrthophotoAnalysis.error}
|
orthophotoAnalysisError={mapOrthophotoAnalysis.error}
|
||||||
orthophotoAnalysisRunning={mapOrthophotoAnalysis.running}
|
orthophotoAnalysisRunning={mapOrthophotoAnalysis.running}
|
||||||
|
orthophotoAnalysisQuality={mapOrthophotoAnalysis.lastQuality}
|
||||||
|
orthophotoAnalysisDetectionCount={mapOrthophotoAnalysis.lastDetectionCount}
|
||||||
availableMapDatasets={availableMapDatasets}
|
availableMapDatasets={availableMapDatasets}
|
||||||
selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''}
|
selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''}
|
||||||
selectedFeature={selectedMapFeature}
|
selectedFeature={selectedMapFeature}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useEffect, useMemo, useState } from 'react'
|
import { useEffect, useMemo, useState } from 'react'
|
||||||
import GeoMap from '../GeoMap'
|
import GeoMap from '../GeoMap'
|
||||||
import type { AreaRead, DatasetCreateResponse, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||||
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
||||||
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
|
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
|
||||||
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
||||||
@@ -327,6 +327,12 @@ function formatBboxLabel(bbox: VectorSelectionBBox | null): string {
|
|||||||
return `${formatCoordinate(bbox.min_x)}, ${formatCoordinate(bbox.min_y)} -> ${formatCoordinate(bbox.max_x)}, ${formatCoordinate(bbox.max_y)}`
|
return `${formatCoordinate(bbox.min_x)}, ${formatCoordinate(bbox.min_y)} -> ${formatCoordinate(bbox.max_x)}, ${formatCoordinate(bbox.max_y)}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatPercentage(value: number | null | undefined): string {
|
||||||
|
return typeof value === 'number' && Number.isFinite(value)
|
||||||
|
? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`
|
||||||
|
: 'n.v.t.'
|
||||||
|
}
|
||||||
|
|
||||||
function bboxToInputState(bbox: VectorSelectionBBox | null) {
|
function bboxToInputState(bbox: VectorSelectionBBox | null) {
|
||||||
return {
|
return {
|
||||||
min_x: bbox ? String(bbox.min_x) : '',
|
min_x: bbox ? String(bbox.min_x) : '',
|
||||||
@@ -445,6 +451,8 @@ interface MapWorkspaceProps {
|
|||||||
orthophotoAnalysisStatus: string
|
orthophotoAnalysisStatus: string
|
||||||
orthophotoAnalysisError: string | null
|
orthophotoAnalysisError: string | null
|
||||||
orthophotoAnalysisRunning: boolean
|
orthophotoAnalysisRunning: boolean
|
||||||
|
orthophotoAnalysisQuality: DetectionQaResult | null
|
||||||
|
orthophotoAnalysisDetectionCount: number | null
|
||||||
availableMapDatasets: DatasetCreateResponse[]
|
availableMapDatasets: DatasetCreateResponse[]
|
||||||
selectedMapDatasetId: string
|
selectedMapDatasetId: string
|
||||||
onSelectMapArea: (areaId: string) => void
|
onSelectMapArea: (areaId: string) => void
|
||||||
@@ -518,6 +526,8 @@ export function MapWorkspace({
|
|||||||
orthophotoAnalysisStatus,
|
orthophotoAnalysisStatus,
|
||||||
orthophotoAnalysisError,
|
orthophotoAnalysisError,
|
||||||
orthophotoAnalysisRunning,
|
orthophotoAnalysisRunning,
|
||||||
|
orthophotoAnalysisQuality,
|
||||||
|
orthophotoAnalysisDetectionCount,
|
||||||
availableMapDatasets,
|
availableMapDatasets,
|
||||||
selectedMapDatasetId,
|
selectedMapDatasetId,
|
||||||
onSelectMapArea,
|
onSelectMapArea,
|
||||||
@@ -1194,6 +1204,16 @@ export function MapWorkspace({
|
|||||||
</button>
|
</button>
|
||||||
{orthophotoAnalysisStatus ? <p role="status">{orthophotoAnalysisStatus}</p> : null}
|
{orthophotoAnalysisStatus ? <p role="status">{orthophotoAnalysisStatus}</p> : null}
|
||||||
{orthophotoAnalysisError ? <p className="error" role="alert">{orthophotoAnalysisError}</p> : null}
|
{orthophotoAnalysisError ? <p className="error" role="alert">{orthophotoAnalysisError}</p> : null}
|
||||||
|
{orthophotoAnalysisQuality ? (
|
||||||
|
<div className="geo-image-quality-metrics" aria-label="Gemeten kwaliteit van de beeldanalyse">
|
||||||
|
<div><span>Kandidaten</span><strong>{orthophotoAnalysisDetectionCount?.toLocaleString('nl-BE') ?? 'n.v.t.'}</strong></div>
|
||||||
|
<div><span>Precision</span><strong>{formatPercentage(orthophotoAnalysisQuality.precision)}</strong></div>
|
||||||
|
<div><span>Recall</span><strong>{formatPercentage(orthophotoAnalysisQuality.recall)}</strong></div>
|
||||||
|
<div><span>F1</span><strong>{formatPercentage(orthophotoAnalysisQuality.f1_score)}</strong></div>
|
||||||
|
<div><span>Fout</span><strong>{orthophotoAnalysisQuality.false_positives.toLocaleString('nl-BE')}</strong></div>
|
||||||
|
<div><span>Gemist</span><strong>{orthophotoAnalysisQuality.false_negatives.toLocaleString('nl-BE')}</strong></div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,244 @@
|
|||||||
|
import { useEffect, useState } from 'react'
|
||||||
|
import { qaApi } from '../../services/api'
|
||||||
|
import type {
|
||||||
|
DetectionEvidenceRole,
|
||||||
|
DetectionReviewDecision,
|
||||||
|
DetectionReviewList,
|
||||||
|
DetectionReviewRead,
|
||||||
|
} from '../../types'
|
||||||
|
import { formatError } from '../../lib/formatError'
|
||||||
|
|
||||||
|
interface DetectionReviewPanelProps {
|
||||||
|
projectId: string
|
||||||
|
qualityCheckId: string
|
||||||
|
onOpenEvidenceMap?: (qualityCheckId: string) => void
|
||||||
|
}
|
||||||
|
|
||||||
|
const ROLE_LABELS: Record<DetectionEvidenceRole, string> = {
|
||||||
|
false_positive: 'Onterecht gevonden',
|
||||||
|
false_negative: 'Gemist gebouw',
|
||||||
|
}
|
||||||
|
|
||||||
|
const DECISION_LABELS: Record<DetectionReviewDecision, string> = {
|
||||||
|
confirmed_model_false_positive: 'Bevestigde foutdetectie',
|
||||||
|
confirmed_model_false_negative: 'Bevestigd gemist gebouw',
|
||||||
|
reference_gap_or_change: 'Referentie ontbreekt of is verouderd',
|
||||||
|
qa_alignment_mismatch: 'Vormvergelijking is te streng',
|
||||||
|
imagery_obscured_or_uncertain: 'Luchtbeeld is onduidelijk',
|
||||||
|
uncertain: 'Verder onderzoek nodig',
|
||||||
|
unreviewed: 'Nog niet beoordeeld',
|
||||||
|
}
|
||||||
|
|
||||||
|
const ROLE_DECISIONS: Record<DetectionEvidenceRole, DetectionReviewDecision[]> = {
|
||||||
|
false_positive: [
|
||||||
|
'unreviewed',
|
||||||
|
'confirmed_model_false_positive',
|
||||||
|
'reference_gap_or_change',
|
||||||
|
'qa_alignment_mismatch',
|
||||||
|
'uncertain',
|
||||||
|
],
|
||||||
|
false_negative: [
|
||||||
|
'unreviewed',
|
||||||
|
'confirmed_model_false_negative',
|
||||||
|
'reference_gap_or_change',
|
||||||
|
'qa_alignment_mismatch',
|
||||||
|
'imagery_obscured_or_uncertain',
|
||||||
|
'uncertain',
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
function shortId(value: string): string {
|
||||||
|
return value.length > 18 ? `${value.slice(0, 8)}...${value.slice(-6)}` : value
|
||||||
|
}
|
||||||
|
|
||||||
|
export function DetectionReviewPanel({
|
||||||
|
projectId,
|
||||||
|
qualityCheckId,
|
||||||
|
onOpenEvidenceMap,
|
||||||
|
}: DetectionReviewPanelProps): JSX.Element {
|
||||||
|
const [queue, setQueue] = useState<DetectionReviewList | null>(null)
|
||||||
|
const [loading, setLoading] = useState(false)
|
||||||
|
const [savingKey, setSavingKey] = useState<string | null>(null)
|
||||||
|
const [error, setError] = useState<string | null>(null)
|
||||||
|
const [roleFilter, setRoleFilter] = useState<'all' | DetectionEvidenceRole>('all')
|
||||||
|
const [statusFilter, setStatusFilter] = useState<'all' | 'reviewed' | 'unreviewed'>('unreviewed')
|
||||||
|
const [offset, setOffset] = useState(0)
|
||||||
|
const [draftDecisions, setDraftDecisions] = useState<Record<string, DetectionReviewDecision>>({})
|
||||||
|
const [draftNotes, setDraftNotes] = useState<Record<string, string>>({})
|
||||||
|
|
||||||
|
const load = async () => {
|
||||||
|
setLoading(true)
|
||||||
|
setError(null)
|
||||||
|
try {
|
||||||
|
const result = await qaApi.listDetectionReviews(projectId, qualityCheckId, {
|
||||||
|
evidenceRole: roleFilter === 'all' ? undefined : roleFilter,
|
||||||
|
reviewed: statusFilter === 'all' ? undefined : statusFilter === 'reviewed',
|
||||||
|
limit: 50,
|
||||||
|
offset,
|
||||||
|
})
|
||||||
|
setQueue(result)
|
||||||
|
setDraftDecisions(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.decision])))
|
||||||
|
setDraftNotes(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.notes ?? ''])))
|
||||||
|
} catch (caught) {
|
||||||
|
setError(formatError(caught, 'De controlelijst kon niet worden geladen'))
|
||||||
|
} finally {
|
||||||
|
setLoading(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void load()
|
||||||
|
}, [projectId, qualityCheckId, roleFilter, statusFilter, offset])
|
||||||
|
|
||||||
|
const save = async (item: DetectionReviewRead) => {
|
||||||
|
const key = reviewKey(item)
|
||||||
|
setSavingKey(key)
|
||||||
|
setError(null)
|
||||||
|
try {
|
||||||
|
await qaApi.upsertDetectionReview(projectId, qualityCheckId, {
|
||||||
|
evidence_role: item.evidence_role,
|
||||||
|
evidence_feature_id: item.evidence_feature_id,
|
||||||
|
decision: draftDecisions[key] ?? item.decision,
|
||||||
|
notes: draftNotes[key]?.trim() || null,
|
||||||
|
reviewed_by: 'operator',
|
||||||
|
})
|
||||||
|
await load()
|
||||||
|
} catch (caught) {
|
||||||
|
setError(formatError(caught, 'De beoordeling kon niet worden bewaard'))
|
||||||
|
} finally {
|
||||||
|
setSavingKey(null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="detection-review-panel" aria-label="Handmatige controle van beeldanalyse">
|
||||||
|
<div className="panel-title-row">
|
||||||
|
<div>
|
||||||
|
<h3>Fouten controleren</h3>
|
||||||
|
<p className="muted">Beoordeel alleen twijfelgevallen. Bevestigde fouten kunnen later veilig als trainingsfeedback worden gebruikt.</p>
|
||||||
|
</div>
|
||||||
|
<button type="button" className="secondary-action" onClick={() => onOpenEvidenceMap?.(qualityCheckId)}>
|
||||||
|
Op kaart bekijken
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{queue ? (
|
||||||
|
<div className="detection-review-summary">
|
||||||
|
<div><span>Te beoordelen</span><strong>{queue.summary.total}</strong></div>
|
||||||
|
<div><span>Afgerond</span><strong>{queue.summary.reviewed}</strong></div>
|
||||||
|
<div><span>Resterend</span><strong>{queue.summary.remaining}</strong></div>
|
||||||
|
<div><span>Fout gevonden</span><strong>{queue.summary.false_positive_total}</strong></div>
|
||||||
|
<div><span>Gemist</span><strong>{queue.summary.false_negative_total}</strong></div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
|
<div className="detection-review-filters">
|
||||||
|
<label>
|
||||||
|
Soort
|
||||||
|
<select value={roleFilter} onChange={(event) => {
|
||||||
|
setRoleFilter(event.target.value as typeof roleFilter)
|
||||||
|
setOffset(0)
|
||||||
|
}}>
|
||||||
|
<option value="all">Alles</option>
|
||||||
|
<option value="false_positive">Onterecht gevonden</option>
|
||||||
|
<option value="false_negative">Gemist gebouw</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
Status
|
||||||
|
<select value={statusFilter} onChange={(event) => {
|
||||||
|
setStatusFilter(event.target.value as typeof statusFilter)
|
||||||
|
setOffset(0)
|
||||||
|
}}>
|
||||||
|
<option value="unreviewed">Nog te beoordelen</option>
|
||||||
|
<option value="reviewed">Beoordeeld</option>
|
||||||
|
<option value="all">Alles</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<button type="button" className="secondary-action" disabled={loading} onClick={() => void load()}>
|
||||||
|
{loading ? 'Laden...' : 'Vernieuwen'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{error ? <p className="error" role="alert">{error}</p> : null}
|
||||||
|
{!loading && queue && queue.items.length === 0 ? (
|
||||||
|
<div className="result-state result-state-empty">
|
||||||
|
<strong>Geen objecten in deze selectie</strong>
|
||||||
|
<p>Pas de filters aan of open de bewijslaag op de kaart.</p>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
|
<ol className="detection-review-list">
|
||||||
|
{queue?.items.map((item) => {
|
||||||
|
const key = reviewKey(item)
|
||||||
|
const decision = draftDecisions[key] ?? item.decision
|
||||||
|
return (
|
||||||
|
<li key={key} className="detection-review-item">
|
||||||
|
<div className="detection-review-item-heading">
|
||||||
|
<div>
|
||||||
|
<span>{ROLE_LABELS[item.evidence_role]}</span>
|
||||||
|
<strong>{item.class_name ?? 'gebouw'} / {shortId(item.evidence_feature_id)}</strong>
|
||||||
|
</div>
|
||||||
|
{typeof item.confidence === 'number' ? <span className="count-pill">{Math.round(item.confidence * 100)}% vertrouwen</span> : null}
|
||||||
|
</div>
|
||||||
|
<div className="detection-review-editor">
|
||||||
|
<label>
|
||||||
|
Beoordeling
|
||||||
|
<select
|
||||||
|
value={decision}
|
||||||
|
onChange={(event) => setDraftDecisions((current) => ({
|
||||||
|
...current,
|
||||||
|
[key]: event.target.value as DetectionReviewDecision,
|
||||||
|
}))}
|
||||||
|
>
|
||||||
|
{ROLE_DECISIONS[item.evidence_role].map((option) => (
|
||||||
|
<option key={option} value={option}>{DECISION_LABELS[option]}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
Notitie
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
maxLength={2000}
|
||||||
|
placeholder="Waarom is dit correct, fout of onzeker?"
|
||||||
|
value={draftNotes[key] ?? ''}
|
||||||
|
onChange={(event) => setDraftNotes((current) => ({ ...current, [key]: event.target.value }))}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<button type="button" className="primary-action" disabled={savingKey === key} onClick={() => void save(item)}>
|
||||||
|
{savingKey === key ? 'Bewaren...' : 'Beoordeling bewaren'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</li>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</ol>
|
||||||
|
{queue && queue.total > queue.limit ? (
|
||||||
|
<div className="detection-review-pagination" aria-label="Pagina's van de controlelijst">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="secondary-action"
|
||||||
|
disabled={offset === 0 || loading}
|
||||||
|
onClick={() => setOffset((current) => Math.max(current - queue.limit, 0))}
|
||||||
|
>
|
||||||
|
Vorige
|
||||||
|
</button>
|
||||||
|
<span>{offset + 1}-{Math.min(offset + queue.limit, queue.total)} van {queue.total}</span>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="secondary-action"
|
||||||
|
disabled={offset + queue.limit >= queue.total || loading}
|
||||||
|
onClick={() => setOffset((current) => current + queue.limit)}
|
||||||
|
>
|
||||||
|
Volgende
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</section>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function reviewKey(item: Pick<DetectionReviewRead, 'evidence_role' | 'evidence_feature_id'>): string {
|
||||||
|
return `${item.evidence_role}:${item.evidence_feature_id}`
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { useMemo, useState } from 'react'
|
import { useMemo, useState } from 'react'
|
||||||
import type { DatasetCreateResponse, MetricRead, QualityCheckRead } from '../../types'
|
import type { DatasetCreateResponse, MetricRead, QualityCheckRead } from '../../types'
|
||||||
|
import { DetectionReviewPanel } from './DetectionReviewPanel'
|
||||||
|
|
||||||
const CORE_METRIC_ORDER = [
|
const CORE_METRIC_ORDER = [
|
||||||
'precision',
|
'precision',
|
||||||
@@ -304,6 +305,13 @@ export function QualityResultsPanel({
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{selectedQualityCheck.check_type === 'detections_vs_reference' && selectedProjectId ? (
|
||||||
|
<DetectionReviewPanel
|
||||||
|
projectId={selectedProjectId}
|
||||||
|
qualityCheckId={selectedQualityCheck.id}
|
||||||
|
onOpenEvidenceMap={onOpenEvidenceMap}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
<div className="quality-feature-evidence-grid" aria-label="Feature-level QA/QC evidence">
|
<div className="quality-feature-evidence-grid" aria-label="Feature-level QA/QC evidence">
|
||||||
<div>
|
<div>
|
||||||
<span>Overeenkomende object-ID's</span>
|
<span>Overeenkomende object-ID's</span>
|
||||||
|
|||||||
@@ -405,6 +405,7 @@ export function useDetectionWorkflow({
|
|||||||
analysisRunId: string,
|
analysisRunId: string,
|
||||||
referenceDatasetId: string,
|
referenceDatasetId: string,
|
||||||
useCurrentFilters = true,
|
useCurrentFilters = true,
|
||||||
|
iouThresholdOverride?: number,
|
||||||
): Promise<DetectionQaResult | null> => {
|
): Promise<DetectionQaResult | null> => {
|
||||||
if (!analysisRunId) {
|
if (!analysisRunId) {
|
||||||
setDetectionQaError('Select a detection run')
|
setDetectionQaError('Select a detection run')
|
||||||
@@ -422,7 +423,7 @@ export function useDetectionWorkflow({
|
|||||||
try {
|
try {
|
||||||
const result = await detectionApi.compareWithReference(analysisRunId, {
|
const result = await detectionApi.compareWithReference(analysisRunId, {
|
||||||
reference_dataset_id: referenceDatasetId,
|
reference_dataset_id: referenceDatasetId,
|
||||||
iou_threshold: qaIouThreshold,
|
iou_threshold: iouThresholdOverride ?? qaIouThreshold,
|
||||||
class_name: useCurrentFilters ? detectionClassFilter || null : null,
|
class_name: useCurrentFilters ? detectionClassFilter || null : null,
|
||||||
min_confidence: useCurrentFilters && detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null,
|
min_confidence: useCurrentFilters && detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ export type MapOrthophotoAnalysisStage =
|
|||||||
| 'complete'
|
| 'complete'
|
||||||
| 'failed'
|
| 'failed'
|
||||||
|
|
||||||
|
export const MAP_BUILDING_QA_IOU_THRESHOLD = 0.25
|
||||||
|
|
||||||
interface MapOrthophotoAnalysisOptions {
|
interface MapOrthophotoAnalysisOptions {
|
||||||
selectedProjectId: string | null
|
selectedProjectId: string | null
|
||||||
selectedAreaId: string
|
selectedAreaId: string
|
||||||
@@ -27,6 +29,7 @@ interface MapOrthophotoAnalysisOptions {
|
|||||||
compareDetectionRunWithReference: (
|
compareDetectionRunWithReference: (
|
||||||
analysisRunId: string,
|
analysisRunId: string,
|
||||||
referenceDatasetId: string,
|
referenceDatasetId: string,
|
||||||
|
iouThreshold: number,
|
||||||
) => Promise<DetectionQaResult | null>
|
) => Promise<DetectionQaResult | null>
|
||||||
onAnalysisReady: () => void
|
onAnalysisReady: () => void
|
||||||
}
|
}
|
||||||
@@ -61,6 +64,12 @@ function formatOrthophotoError(caught: unknown): string {
|
|||||||
return formatError(caught, 'De kaartgestuurde beeldanalyse is mislukt')
|
return formatError(caught, 'De kaartgestuurde beeldanalyse is mislukt')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatQualityPercentage(value: number | null | undefined): string {
|
||||||
|
return typeof value === 'number' && Number.isFinite(value)
|
||||||
|
? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%`
|
||||||
|
: 'n.v.t.'
|
||||||
|
}
|
||||||
|
|
||||||
export function useMapOrthophotoAnalysis({
|
export function useMapOrthophotoAnalysis({
|
||||||
selectedProjectId,
|
selectedProjectId,
|
||||||
selectedAreaId,
|
selectedAreaId,
|
||||||
@@ -75,12 +84,18 @@ export function useMapOrthophotoAnalysis({
|
|||||||
const [status, setStatus] = useState('')
|
const [status, setStatus] = useState('')
|
||||||
const [error, setError] = useState<string | null>(null)
|
const [error, setError] = useState<string | null>(null)
|
||||||
const [lastResult, setLastResult] = useState<OrthophotoAcquisitionResult | null>(null)
|
const [lastResult, setLastResult] = useState<OrthophotoAcquisitionResult | null>(null)
|
||||||
|
const [lastQuality, setLastQuality] = useState<DetectionQaResult | null>(null)
|
||||||
|
const [lastDetectionCount, setLastDetectionCount] = useState<number | null>(null)
|
||||||
|
const [lastAnalysisRunId, setLastAnalysisRunId] = useState<string | null>(null)
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setStage('idle')
|
setStage('idle')
|
||||||
setStatus('')
|
setStatus('')
|
||||||
setError(null)
|
setError(null)
|
||||||
setLastResult(null)
|
setLastResult(null)
|
||||||
|
setLastQuality(null)
|
||||||
|
setLastDetectionCount(null)
|
||||||
|
setLastAnalysisRunId(null)
|
||||||
}, [selectionBbox?.min_x, selectionBbox?.min_y, selectionBbox?.max_x, selectionBbox?.max_y])
|
}, [selectionBbox?.min_x, selectionBbox?.min_y, selectionBbox?.max_x, selectionBbox?.max_y])
|
||||||
|
|
||||||
const run = async (bbox: VectorSelectionBBox): Promise<boolean> => {
|
const run = async (bbox: VectorSelectionBBox): Promise<boolean> => {
|
||||||
@@ -91,6 +106,9 @@ export function useMapOrthophotoAnalysis({
|
|||||||
}
|
}
|
||||||
setError(null)
|
setError(null)
|
||||||
setLastResult(null)
|
setLastResult(null)
|
||||||
|
setLastQuality(null)
|
||||||
|
setLastDetectionCount(null)
|
||||||
|
setLastAnalysisRunId(null)
|
||||||
setStage('acquiring')
|
setStage('acquiring')
|
||||||
setStatus('1/3 Officieel luchtbeeld voor de rechthoek ophalen...')
|
setStatus('1/3 Officieel luchtbeeld voor de rechthoek ophalen...')
|
||||||
try {
|
try {
|
||||||
@@ -112,20 +130,27 @@ export function useMapOrthophotoAnalysis({
|
|||||||
if (!detection) {
|
if (!detection) {
|
||||||
throw new Error('De beeldanalyse stopte. Open Beeldanalyse voor de technische oorzaak.')
|
throw new Error('De beeldanalyse stopte. Open Beeldanalyse voor de technische oorzaak.')
|
||||||
}
|
}
|
||||||
|
setLastDetectionCount(detection.detection_count)
|
||||||
|
setLastAnalysisRunId(detection.analysis_run_id)
|
||||||
|
|
||||||
const reference = findBuildingReference(datasets)
|
const reference = findBuildingReference(datasets)
|
||||||
if (reference) {
|
if (reference) {
|
||||||
setStage('validating')
|
setStage('validating')
|
||||||
setStatus('3/3 Resultaat vergelijken met officiële GRB-gebouwen...')
|
setStatus('3/3 Resultaat vergelijken met officiële GRB-gebouwen...')
|
||||||
const quality = await compareDetectionRunWithReference(detection.analysis_run_id, reference.id)
|
const quality = await compareDetectionRunWithReference(
|
||||||
|
detection.analysis_run_id,
|
||||||
|
reference.id,
|
||||||
|
MAP_BUILDING_QA_IOU_THRESHOLD,
|
||||||
|
)
|
||||||
|
setLastQuality(quality)
|
||||||
setStatus(
|
setStatus(
|
||||||
quality
|
quality
|
||||||
? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend en gecontroleerd.`
|
? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten, ${quality.matches.toLocaleString('nl-BE')} gekoppeld aan GRB. Precision ${formatQualityPercentage(quality.precision)}, recall ${formatQualityPercentage(quality.recall)}.`
|
||||||
: `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend; kwaliteitscontrole kon niet afronden.`,
|
: `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten; kwaliteitscontrole kon niet afronden.`,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
setStatus(
|
setStatus(
|
||||||
`Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend. De GRB-referentielaag ontbreekt voor automatische controle.`,
|
`Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaatvormen. De GRB-referentielaag ontbreekt voor automatische controle.`,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
setStage('complete')
|
setStage('complete')
|
||||||
@@ -144,6 +169,9 @@ export function useMapOrthophotoAnalysis({
|
|||||||
status,
|
status,
|
||||||
error,
|
error,
|
||||||
lastResult,
|
lastResult,
|
||||||
|
lastQuality,
|
||||||
|
lastDetectionCount,
|
||||||
|
lastAnalysisRunId,
|
||||||
running: stage === 'acquiring' || stage === 'detecting' || stage === 'validating',
|
running: stage === 'acquiring' || stage === 'detecting' || stage === 'validating',
|
||||||
run,
|
run,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,13 @@
|
|||||||
import { apiGet, apiPost } from './client'
|
import { apiGet, apiPost } from './client'
|
||||||
import type { QaComparisonRequest, JobRead, QualityCheckListResponse, QualityEvidenceGeoJsonResponse } from '../../types'
|
import type {
|
||||||
|
DetectionReviewList,
|
||||||
|
DetectionReviewRead,
|
||||||
|
DetectionReviewUpsert,
|
||||||
|
JobRead,
|
||||||
|
QaComparisonRequest,
|
||||||
|
QualityCheckListResponse,
|
||||||
|
QualityEvidenceGeoJsonResponse,
|
||||||
|
} from '../../types'
|
||||||
|
|
||||||
export const qaApi = {
|
export const qaApi = {
|
||||||
runQa: (payload: QaComparisonRequest): Promise<JobRead> =>
|
runQa: (payload: QaComparisonRequest): Promise<JobRead> =>
|
||||||
@@ -8,4 +16,23 @@ export const qaApi = {
|
|||||||
apiGet<QualityCheckListResponse>(`/api/v1/projects/${projectId}/quality-checks`),
|
apiGet<QualityCheckListResponse>(`/api/v1/projects/${projectId}/quality-checks`),
|
||||||
getQualityEvidenceGeoJson: (projectId: string, qualityCheckId: string): Promise<QualityEvidenceGeoJsonResponse> =>
|
getQualityEvidenceGeoJson: (projectId: string, qualityCheckId: string): Promise<QualityEvidenceGeoJsonResponse> =>
|
||||||
apiGet<QualityEvidenceGeoJsonResponse>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/evidence/geojson`),
|
apiGet<QualityEvidenceGeoJsonResponse>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/evidence/geojson`),
|
||||||
|
listDetectionReviews: (
|
||||||
|
projectId: string,
|
||||||
|
qualityCheckId: string,
|
||||||
|
options: { evidenceRole?: string; reviewed?: boolean; limit?: number; offset?: number } = {},
|
||||||
|
): Promise<DetectionReviewList> => {
|
||||||
|
const query = new URLSearchParams({
|
||||||
|
limit: String(options.limit ?? 50),
|
||||||
|
offset: String(options.offset ?? 0),
|
||||||
|
})
|
||||||
|
if (options.evidenceRole) query.set('evidence_role', options.evidenceRole)
|
||||||
|
if (typeof options.reviewed === 'boolean') query.set('reviewed', String(options.reviewed))
|
||||||
|
return apiGet<DetectionReviewList>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews?${query}`)
|
||||||
|
},
|
||||||
|
upsertDetectionReview: (
|
||||||
|
projectId: string,
|
||||||
|
qualityCheckId: string,
|
||||||
|
payload: DetectionReviewUpsert,
|
||||||
|
): Promise<DetectionReviewRead> =>
|
||||||
|
apiPost<DetectionReviewRead>(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews`, payload),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1862,3 +1862,139 @@ details.ai-lab-model-surface > summary strong {
|
|||||||
background: var(--accent-soft);
|
background: var(--accent-soft);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.geo-image-quality-metrics,
|
||||||
|
.detection-review-summary {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(6, minmax(0, 1fr));
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.geo-image-quality-metrics {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
margin-top: 0.35rem;
|
||||||
|
border-top: 1px solid var(--line);
|
||||||
|
padding-top: 0.65rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.geo-image-quality-metrics > div,
|
||||||
|
.detection-review-summary > div {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.15rem;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 0.55rem;
|
||||||
|
background: var(--panel-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.geo-image-quality-metrics span,
|
||||||
|
.detection-review-summary span,
|
||||||
|
.detection-review-item-heading span {
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 0.7rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.geo-image-quality-metrics strong,
|
||||||
|
.detection-review-summary strong {
|
||||||
|
color: var(--text);
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-panel {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.75rem;
|
||||||
|
margin-top: 0.85rem;
|
||||||
|
border-top: 1px solid var(--line);
|
||||||
|
padding-top: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-summary {
|
||||||
|
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-filters,
|
||||||
|
.detection-review-editor,
|
||||||
|
.detection-review-pagination {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr)) auto;
|
||||||
|
gap: 0.65rem;
|
||||||
|
align-items: end;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-filters label,
|
||||||
|
.detection-review-editor label {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.3rem;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 0.72rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-list {
|
||||||
|
display: grid;
|
||||||
|
max-height: 34rem;
|
||||||
|
gap: 0.5rem;
|
||||||
|
margin: 0;
|
||||||
|
overflow: auto;
|
||||||
|
padding: 0;
|
||||||
|
list-style: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-item {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.6rem;
|
||||||
|
border: 1px solid var(--line);
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 0.7rem;
|
||||||
|
background: #ffffff;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-item-heading {
|
||||||
|
display: flex;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.75rem;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-item-heading > div {
|
||||||
|
display: grid;
|
||||||
|
min-width: 0;
|
||||||
|
gap: 0.15rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-item-heading strong {
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
font-size: 0.82rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-pagination {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: flex-end;
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-pagination span {
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 0.76rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 900px) {
|
||||||
|
.geo-image-quality-metrics,
|
||||||
|
.detection-review-summary {
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
|
.detection-review-editor,
|
||||||
|
.detection-review-filters {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (max-width: 560px) {
|
||||||
|
.geo-image-quality-metrics,
|
||||||
|
.detection-review-summary {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -894,6 +894,61 @@ export interface QualityEvidenceGeoJsonResponse {
|
|||||||
geojson: GeoJSON.FeatureCollection
|
geojson: GeoJSON.FeatureCollection
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type DetectionEvidenceRole = 'false_positive' | 'false_negative'
|
||||||
|
|
||||||
|
export type DetectionReviewDecision =
|
||||||
|
| 'confirmed_model_false_positive'
|
||||||
|
| 'confirmed_model_false_negative'
|
||||||
|
| 'reference_gap_or_change'
|
||||||
|
| 'qa_alignment_mismatch'
|
||||||
|
| 'imagery_obscured_or_uncertain'
|
||||||
|
| 'uncertain'
|
||||||
|
| 'unreviewed'
|
||||||
|
|
||||||
|
export interface DetectionReviewUpsert {
|
||||||
|
evidence_role: DetectionEvidenceRole
|
||||||
|
evidence_feature_id: string
|
||||||
|
decision: DetectionReviewDecision
|
||||||
|
notes?: string | null
|
||||||
|
reviewed_by?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DetectionReviewRead {
|
||||||
|
id?: string | null
|
||||||
|
project_id: string
|
||||||
|
quality_check_id: string
|
||||||
|
analysis_run_id?: string | null
|
||||||
|
evidence_role: DetectionEvidenceRole
|
||||||
|
evidence_feature_id: string
|
||||||
|
detection_id?: string | null
|
||||||
|
reference_feature_id?: string | null
|
||||||
|
decision: DetectionReviewDecision
|
||||||
|
notes?: string | null
|
||||||
|
reviewed_by?: string | null
|
||||||
|
confidence?: number | null
|
||||||
|
class_name?: string | null
|
||||||
|
source_tile_path?: string | null
|
||||||
|
created_at?: string | null
|
||||||
|
updated_at?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DetectionReviewSummary {
|
||||||
|
total: number
|
||||||
|
reviewed: number
|
||||||
|
remaining: number
|
||||||
|
false_positive_total: number
|
||||||
|
false_negative_total: number
|
||||||
|
decision_counts: Record<string, number>
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DetectionReviewList {
|
||||||
|
items: DetectionReviewRead[]
|
||||||
|
total: number
|
||||||
|
limit: number
|
||||||
|
offset: number
|
||||||
|
summary: DetectionReviewSummary
|
||||||
|
}
|
||||||
|
|
||||||
export type ExportKind = 'dataset' | 'detection_run' | 'segmentation_run' | 'vector_selection'
|
export type ExportKind = 'dataset' | 'detection_run' | 'segmentation_run' | 'vector_selection'
|
||||||
|
|
||||||
export interface ExportRead {
|
export interface ExportRead {
|
||||||
|
|||||||
Reference in New Issue
Block a user