diff --git a/CHANGELOG.md b/CHANGELOG.md index d1fd9d44..2d60cb05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,29 @@ # Changelog +## Sprint 197 Measured detection accuracy and durable review (2026-07-15) + +- Re-ran the active local building model at confidence thresholds `0.10` and + `0.15` over independent Mol holdouts in Achterbos, Gompel, Donk and Postel, + plus the pure-empty Postel forest control. Threshold `0.15` retained the best + F1 in every positive zone and both thresholds produced zero forest-control + detections, so no speculative threshold or model promotion was made. +- Aligned the map-driven building QA flow with the documented operational + footprint match IoU `0.25`. The map now reports candidate count, matches, + precision, recall, F1, false positives and false negatives instead of + presenting every model box as a recognized building. +- Added first-class `detection_reviews` persistence and canonical project QA + endpoints for paginated false-positive/false-negative review decisions. +- Added a focused frontend review queue with role/status filters, notes, + pagination and map handoff. Unreviewed or alignment-mismatch evidence cannot + silently become hard-negative training data. +- Bounded QA evidence resolution to the persisted evidence identifiers instead + of loading complete regional reference datasets into application memory. +- Added regression coverage for migration alignment, decision validation, API + envelopes, bounded evidence access and the map QA threshold. +- Passed the complete readiness gate with 592 backend tests, 88 documented API + routes, one Alembic head and a green frontend typecheck/production build. + ## Sprint 196 Map-driven official orthophoto analysis (2026-07-15) - Added an explicit bounded endpoint for the official Digitaal Vlaanderen diff --git a/backend/README.md b/backend/README.md index ab610aaa..ac6372ed 100644 --- a/backend/README.md +++ b/backend/README.md @@ -708,6 +708,23 @@ curl http://localhost:1202/api/v1/projects/{project_id}/quality-checks The frontend QA/QC Results panel uses this endpoint after loading the demo workflow or running QA. +Detection QA evidence can be reviewed without changing its persisted metrics: + +```bash +curl "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews?reviewed=false&limit=50" + +curl -X POST "http://localhost:1202/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews" \ + -H "Content-Type: application/json" \ + -d '{"evidence_role":"false_positive","evidence_feature_id":"DETECTION_UUID","decision":"qa_alignment_mismatch","notes":"Box and footprint represent the same building."}' +``` + +The list is derived from persisted quality-check evidence and paginates at a +maximum of 200 rows. The upsert verifies project ownership, quality-check type, +role-specific decisions and persisted Detection/VectorFeature ownership. +`detection_reviews` never mutates model output, reference geometry or canonical +Metric rows. Evidence GeoJSON queries only stored evidence ids instead of a +complete regional GRB dataset. + ### Export foundation Persisted exports can be created from the existing workbench state: diff --git a/backend/alembic/versions/202607150001_detection_reviews.py b/backend/alembic/versions/202607150001_detection_reviews.py new file mode 100644 index 00000000..5e08b34f --- /dev/null +++ b/backend/alembic/versions/202607150001_detection_reviews.py @@ -0,0 +1,64 @@ +"""Add durable operator review decisions for detection QA evidence.""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision = "202607150001" +down_revision = "202607140001" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "detection_reviews", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("quality_check_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("analysis_run_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("evidence_role", sa.String(length=32), nullable=False), + sa.Column("evidence_feature_id", sa.String(length=255), nullable=False), + sa.Column("detection_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("reference_feature_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("decision", sa.String(length=64), server_default="unreviewed", nullable=False), + sa.Column("notes", sa.Text(), nullable=True), + sa.Column("reviewed_by", sa.String(length=120), server_default="operator", nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.CheckConstraint( + "evidence_role IN ('false_positive', 'false_negative')", + name="ck_detection_reviews_evidence_role", + ), + sa.CheckConstraint( + "decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', " + "'reference_gap_or_change', 'qa_alignment_mismatch', " + "'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')", + name="ck_detection_reviews_decision", + ), + sa.ForeignKeyConstraint(["analysis_run_id"], ["analysis_runs.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["detection_id"], ["detections.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["quality_check_id"], ["quality_checks.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["reference_feature_id"], ["vector_features.id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "quality_check_id", + "evidence_role", + "evidence_feature_id", + name="uq_detection_reviews_evidence", + ), + ) + op.create_index("ix_detection_reviews_project_id", "detection_reviews", ["project_id"]) + op.create_index("ix_detection_reviews_quality_check_id", "detection_reviews", ["quality_check_id"]) + op.create_index("ix_detection_reviews_analysis_run_id", "detection_reviews", ["analysis_run_id"]) + op.create_index("ix_detection_reviews_decision", "detection_reviews", ["decision"]) + + +def downgrade() -> None: + op.drop_index("ix_detection_reviews_decision", table_name="detection_reviews") + op.drop_index("ix_detection_reviews_analysis_run_id", table_name="detection_reviews") + op.drop_index("ix_detection_reviews_quality_check_id", table_name="detection_reviews") + op.drop_index("ix_detection_reviews_project_id", table_name="detection_reviews") + op.drop_table("detection_reviews") diff --git a/backend/app/api/routes/quality_checks.py b/backend/app/api/routes/quality_checks.py index c1b64c71..ac22316c 100644 --- a/backend/app/api/routes/quality_checks.py +++ b/backend/app/api/routes/quality_checks.py @@ -6,7 +6,9 @@ from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session from app.db.session import get_db +from app.schemas.detection_review import DetectionReviewUpsert from app.schemas.qa import QualityCheckList +from app.services.detection_review_service import DetectionReviewService from app.services.quality_evidence_service import QualityEvidenceService from app.services.quality_check_service import QualityCheckService from app.utils.response import envelope @@ -37,3 +39,45 @@ def get_quality_check_evidence_geojson( db: Session = Depends(get_db), ) -> dict: return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id)) + + +@router.get("/quality-checks/{quality_check_id}/reviews", response_model=dict) +def list_detection_reviews( + project_id: UUID, + quality_check_id: UUID, + evidence_role: str | None = Query(default=None, pattern="^(false_positive|false_negative)$"), + decision: str | None = Query(default=None, max_length=64), + reviewed: bool | None = Query(default=None), + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + db: Session = Depends(get_db), +) -> dict: + return envelope( + DetectionReviewService.list_reviews( + db, + project_id=project_id, + quality_check_id=quality_check_id, + evidence_role=evidence_role, + decision=decision, + reviewed=reviewed, + limit=limit, + offset=offset, + ).model_dump() + ) + + +@router.post("/quality-checks/{quality_check_id}/reviews", response_model=dict) +def upsert_detection_review( + project_id: UUID, + quality_check_id: UUID, + payload: DetectionReviewUpsert, + db: Session = Depends(get_db), +) -> dict: + return envelope( + DetectionReviewService.upsert_review( + db, + project_id=project_id, + quality_check_id=quality_check_id, + payload=payload, + ).model_dump() + ) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index aa87ee4d..447ffd0b 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -1,4 +1,4 @@ -from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature +from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature __all__ = [ "AnalysisRun", @@ -6,6 +6,7 @@ __all__ = [ "Dataset", "DatasetVersion", "Detection", + "DetectionReview", "Export", "Job", "Metric", diff --git a/backend/app/models/entities.py b/backend/app/models/entities.py index d283a795..72395040 100644 --- a/backend/app/models/entities.py +++ b/backend/app/models/entities.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime from geoalchemy2 import Geometry -from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, func +from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, UniqueConstraint, func from sqlalchemy.sql.sqltypes import Integer from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -272,6 +272,62 @@ class Metric(Base): created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) +class DetectionReview(Base): + __tablename__ = "detection_reviews" + __table_args__ = ( + CheckConstraint( + "evidence_role IN ('false_positive', 'false_negative')", + name="ck_detection_reviews_evidence_role", + ), + CheckConstraint( + "decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', " + "'reference_gap_or_change', 'qa_alignment_mismatch', " + "'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')", + name="ck_detection_reviews_decision", + ), + UniqueConstraint( + "quality_check_id", + "evidence_role", + "evidence_feature_id", + name="uq_detection_reviews_evidence", + ), + Index("ix_detection_reviews_project_id", "project_id"), + Index("ix_detection_reviews_quality_check_id", "quality_check_id"), + Index("ix_detection_reviews_analysis_run_id", "analysis_run_id"), + Index("ix_detection_reviews_decision", "decision"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) + quality_check_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("quality_checks.id", ondelete="CASCADE"), + nullable=False, + ) + analysis_run_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("analysis_runs.id", ondelete="SET NULL"), + nullable=True, + ) + evidence_role: Mapped[str] = mapped_column(String(32), nullable=False) + evidence_feature_id: Mapped[str] = mapped_column(String(255), nullable=False) + detection_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("detections.id", ondelete="SET NULL"), + nullable=True, + ) + reference_feature_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), + ForeignKey("vector_features.id", ondelete="SET NULL"), + nullable=True, + ) + decision: Mapped[str] = mapped_column(String(64), nullable=False, default="unreviewed", server_default="unreviewed") + notes: Mapped[str | None] = mapped_column(Text, nullable=True) + reviewed_by: Mapped[str] = mapped_column(String(120), nullable=False, default="operator", server_default="operator") + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + class Export(Base): __tablename__ = "exports" diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 3b897979..4c915258 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -18,6 +18,7 @@ from .detection import ( ModelAssetListResponse, ModelAssetRead, ) +from .detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewSummary, DetectionReviewUpsert from .segmentation import ( SegmentationListResponse, SegmentationModelCapability, @@ -111,6 +112,10 @@ __all__ = [ "DetectionRunResponse", "ModelAssetListResponse", "ModelAssetRead", + "DetectionReviewList", + "DetectionReviewRead", + "DetectionReviewSummary", + "DetectionReviewUpsert", "SegmentationListResponse", "SegmentationModelCapability", "SegmentationModelsResponse", diff --git a/backend/app/schemas/detection_review.py b/backend/app/schemas/detection_review.py new file mode 100644 index 00000000..82e4ee7f --- /dev/null +++ b/backend/app/schemas/detection_review.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field + + +DetectionEvidenceRole = Literal["false_positive", "false_negative"] +DetectionReviewDecision = Literal[ + "confirmed_model_false_positive", + "confirmed_model_false_negative", + "reference_gap_or_change", + "qa_alignment_mismatch", + "imagery_obscured_or_uncertain", + "uncertain", + "unreviewed", +] + + +class DetectionReviewUpsert(BaseModel): + evidence_role: DetectionEvidenceRole + evidence_feature_id: str = Field(min_length=1, max_length=255) + decision: DetectionReviewDecision + notes: str | None = Field(default=None, max_length=2000) + reviewed_by: str = Field(default="operator", min_length=1, max_length=120) + + +class DetectionReviewRead(BaseModel): + id: UUID | None = None + project_id: UUID + quality_check_id: UUID + analysis_run_id: UUID | None = None + evidence_role: DetectionEvidenceRole + evidence_feature_id: str + detection_id: UUID | None = None + reference_feature_id: UUID | None = None + decision: DetectionReviewDecision = "unreviewed" + notes: str | None = None + reviewed_by: str | None = None + confidence: float | None = None + class_name: str | None = None + source_tile_path: str | None = None + created_at: datetime | None = None + updated_at: datetime | None = None + + +class DetectionReviewSummary(BaseModel): + total: int + reviewed: int + remaining: int + false_positive_total: int + false_negative_total: int + decision_counts: dict[str, int] + + +class DetectionReviewList(BaseModel): + items: list[DetectionReviewRead] + total: int + limit: int + offset: int + summary: DetectionReviewSummary diff --git a/backend/app/services/detection_review_service.py b/backend/app/services/detection_review_service.py new file mode 100644 index 00000000..8c836323 --- /dev/null +++ b/backend/app/services/detection_review_service.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import Detection, DetectionReview, QualityCheck, VectorFeature +from app.schemas.detection_review import ( + DetectionReviewList, + DetectionReviewRead, + DetectionReviewSummary, + DetectionReviewUpsert, +) + + +class DetectionReviewService: + ALLOWED_DECISIONS = { + "false_positive": { + "confirmed_model_false_positive", + "reference_gap_or_change", + "qa_alignment_mismatch", + "uncertain", + "unreviewed", + }, + "false_negative": { + "confirmed_model_false_negative", + "reference_gap_or_change", + "qa_alignment_mismatch", + "imagery_obscured_or_uncertain", + "uncertain", + "unreviewed", + }, + } + + @staticmethod + def _quality_check(db: Session, project_id: UUID, quality_check_id: UUID) -> QualityCheck: + quality_check = db.get(QualityCheck, quality_check_id) + if not quality_check or quality_check.project_id != project_id: + raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404) + if quality_check.check_type != "detections_vs_reference": + raise AppError( + code="DETECTION_REVIEW_UNSUPPORTED", + message="Only persisted detection-versus-reference quality checks can be reviewed", + status_code=422, + ) + return quality_check + + @staticmethod + def _evidence_items(quality_check: QualityCheck) -> list[dict[str, str]]: + findings = quality_check.findings_json or {} + items: list[dict[str, str]] = [] + for role, key, id_key in ( + ("false_positive", "false_positive_evidence", "candidate_feature_id"), + ("false_negative", "false_negative_evidence", "reference_feature_id"), + ): + evidence_rows = findings.get(key) + if not isinstance(evidence_rows, list): + continue + for evidence in evidence_rows: + if not isinstance(evidence, dict): + continue + value = str(evidence.get(id_key) or "").strip() + if value: + items.append({"evidence_role": role, "evidence_feature_id": value}) + return items + + @staticmethod + def _uuid(value: str) -> UUID | None: + try: + return UUID(value) + except (TypeError, ValueError): + return None + + @staticmethod + def _review_index(db: Session, quality_check_id: UUID) -> dict[tuple[str, str], DetectionReview]: + rows = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check_id).all() + return {(row.evidence_role, row.evidence_feature_id): row for row in rows} + + @staticmethod + def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary: + evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence} + decisions = Counter( + reviews[key].decision if key in reviews else "unreviewed" + for key in evidence_keys + ) + reviewed = sum(count for decision, count in decisions.items() if decision != "unreviewed") + false_positive_total = sum(1 for item in evidence if item["evidence_role"] == "false_positive") + false_negative_total = sum(1 for item in evidence if item["evidence_role"] == "false_negative") + return DetectionReviewSummary( + total=len(evidence), + reviewed=reviewed, + remaining=max(len(evidence) - reviewed, 0), + false_positive_total=false_positive_total, + false_negative_total=false_negative_total, + decision_counts=dict(sorted(decisions.items())), + ) + + @staticmethod + def _read_item( + db: Session, + quality_check: QualityCheck, + evidence: dict[str, str], + review: DetectionReview | None, + ) -> DetectionReviewRead: + role = evidence["evidence_role"] + feature_id = evidence["evidence_feature_id"] + feature_uuid = DetectionReviewService._uuid(feature_id) + detection = db.get(Detection, feature_uuid) if role == "false_positive" and feature_uuid else None + reference = db.get(VectorFeature, feature_uuid) if role == "false_negative" and feature_uuid else None + return DetectionReviewRead( + id=review.id if review else None, + project_id=quality_check.project_id, + quality_check_id=quality_check.id, + analysis_run_id=quality_check.analysis_run_id, + evidence_role=role, + evidence_feature_id=feature_id, + detection_id=detection.id if detection else review.detection_id if review else None, + reference_feature_id=reference.id if reference else review.reference_feature_id if review else None, + decision=review.decision if review else "unreviewed", + notes=review.notes if review else None, + reviewed_by=review.reviewed_by if review else None, + confidence=detection.confidence if detection else None, + class_name=(detection.class_name if detection else reference.feature_class if reference else None), + source_tile_path=detection.source_tile_path if detection else None, + created_at=review.created_at if review else None, + updated_at=review.updated_at if review else None, + ) + + @staticmethod + def list_reviews( + db: Session, + *, + project_id: UUID, + quality_check_id: UUID, + evidence_role: str | None = None, + decision: str | None = None, + reviewed: bool | None = None, + limit: int = 50, + offset: int = 0, + ) -> DetectionReviewList: + quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id) + evidence = DetectionReviewService._evidence_items(quality_check) + reviews = DetectionReviewService._review_index(db, quality_check_id) + filtered = [item for item in evidence if evidence_role is None or item["evidence_role"] == evidence_role] + if decision is not None: + filtered = [ + item + for item in filtered + if (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision + if reviews.get((item["evidence_role"], item["evidence_feature_id"])) + else "unreviewed") + == decision + ] + if reviewed is not None: + filtered = [ + item + for item in filtered + if ( + (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision + if reviews.get((item["evidence_role"], item["evidence_feature_id"])) + else "unreviewed") + != "unreviewed" + ) + == reviewed + ] + page = filtered[offset : offset + limit] + return DetectionReviewList( + items=[ + DetectionReviewService._read_item( + db, + quality_check, + item, + reviews.get((item["evidence_role"], item["evidence_feature_id"])), + ) + for item in page + ], + total=len(filtered), + limit=limit, + offset=offset, + summary=DetectionReviewService._summary(evidence, reviews), + ) + + @staticmethod + def upsert_review( + db: Session, + *, + project_id: UUID, + quality_check_id: UUID, + payload: DetectionReviewUpsert, + ) -> DetectionReviewRead: + quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id) + if payload.decision not in DetectionReviewService.ALLOWED_DECISIONS[payload.evidence_role]: + raise AppError( + code="INVALID_DETECTION_REVIEW_DECISION", + message="The review decision is not valid for this evidence role", + details={"evidence_role": payload.evidence_role, "decision": payload.decision}, + status_code=422, + ) + evidence = DetectionReviewService._evidence_items(quality_check) + evidence_key = (payload.evidence_role, payload.evidence_feature_id) + if evidence_key not in {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}: + raise AppError( + code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", + message="The evidence feature does not belong to this quality check", + status_code=404, + ) + feature_uuid = DetectionReviewService._uuid(payload.evidence_feature_id) + detection = db.get(Detection, feature_uuid) if payload.evidence_role == "false_positive" and feature_uuid else None + reference = db.get(VectorFeature, feature_uuid) if payload.evidence_role == "false_negative" and feature_uuid else None + if payload.evidence_role == "false_positive" and (not detection or detection.analysis_run_id != quality_check.analysis_run_id): + raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted detection evidence was not found", status_code=404) + if payload.evidence_role == "false_negative" and (not reference or reference.dataset_id != quality_check.reference_dataset_id): + raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted reference evidence was not found", status_code=404) + + review = ( + db.query(DetectionReview) + .filter( + DetectionReview.quality_check_id == quality_check_id, + DetectionReview.evidence_role == payload.evidence_role, + DetectionReview.evidence_feature_id == payload.evidence_feature_id, + ) + .first() + ) + if review is None: + review = DetectionReview( + project_id=project_id, + quality_check_id=quality_check_id, + analysis_run_id=quality_check.analysis_run_id, + evidence_role=payload.evidence_role, + evidence_feature_id=payload.evidence_feature_id, + detection_id=detection.id if detection else None, + reference_feature_id=reference.id if reference else None, + decision=payload.decision, + notes=payload.notes.strip() if payload.notes and payload.notes.strip() else None, + reviewed_by=payload.reviewed_by.strip(), + ) + else: + review.decision = payload.decision + review.notes = payload.notes.strip() if payload.notes and payload.notes.strip() else None + review.reviewed_by = payload.reviewed_by.strip() + db.add(review) + db.commit() + db.refresh(review) + return DetectionReviewService._read_item( + db, + quality_check, + {"evidence_role": payload.evidence_role, "evidence_feature_id": payload.evidence_feature_id}, + review, + ) diff --git a/backend/app/services/quality_evidence_service.py b/backend/app/services/quality_evidence_service.py index 9cc5b782..141ea0b8 100644 --- a/backend/app/services/quality_evidence_service.py +++ b/backend/app/services/quality_evidence_service.py @@ -5,10 +5,11 @@ from uuid import UUID from geoalchemy2.shape import to_shape from shapely.geometry import mapping +from sqlalchemy import or_ from sqlalchemy.orm import Session from app.core.errors import AppError -from app.models import Detection, QualityCheck, Segmentation, VectorFeature +from app.models import Detection, DetectionReview, QualityCheck, Segmentation, VectorFeature class QualityEvidenceService: @@ -21,8 +22,9 @@ class QualityEvidenceService: findings = quality_check.findings_json or {} features: list[dict[str, Any]] = [] warnings: list[str] = [] - candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check) - reference_index = QualityEvidenceService._reference_feature_index(db, quality_check) + candidate_ids, reference_ids = QualityEvidenceService._evidence_identifiers(findings) + candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, candidate_ids) + reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, reference_ids) for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")): candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id")) @@ -89,6 +91,8 @@ class QualityEvidenceService: else: warnings.append(f"False-negative evidence feature not found: {reference_id}") + QualityEvidenceService._annotate_reviews(db, quality_check, features) + return { "quality_check_id": str(quality_check.id), "project_id": str(quality_check.project_id), @@ -117,34 +121,129 @@ class QualityEvidenceService: return text or None @staticmethod - def _candidate_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]: + def _evidence_identifiers(findings: dict[str, Any]) -> tuple[set[str], set[str]]: + candidate_ids: set[str] = set() + reference_ids: set[str] = set() + for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")): + candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id")) + reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id")) + if candidate_id: + candidate_ids.add(candidate_id) + if reference_id: + reference_ids.add(reference_id) + for evidence in QualityEvidenceService._evidence_items(findings.get("false_positive_evidence")): + candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id")) + if candidate_id: + candidate_ids.add(candidate_id) + for evidence in QualityEvidenceService._evidence_items(findings.get("false_negative_evidence")): + reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id")) + if reference_id: + reference_ids.add(reference_id) + return candidate_ids, reference_ids + + @staticmethod + def _uuid_identifiers(identifiers: set[str]) -> list[UUID]: + values: list[UUID] = [] + for identifier in identifiers: + try: + values.append(UUID(identifier)) + except (TypeError, ValueError): + continue + return values + + @staticmethod + def _vector_feature_rows(db: Session, dataset_id: UUID, identifiers: set[str]) -> list[VectorFeature]: + if not identifiers: + return [] + conditions = [VectorFeature.source_feature_id.in_(identifiers)] + uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers) + if uuid_identifiers: + conditions.append(VectorFeature.id.in_(uuid_identifiers)) + return ( + db.query(VectorFeature) + .filter(VectorFeature.dataset_id == dataset_id, or_(*conditions)) + .all() + ) + + @staticmethod + def _candidate_feature_index( + db: Session, + quality_check: QualityCheck, + identifiers: set[str], + ) -> dict[str, Any]: index: dict[str, Any] = {} + if not identifiers: + return index + uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers) if quality_check.candidate_dataset_id: - for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.candidate_dataset_id).all(): + for row in QualityEvidenceService._vector_feature_rows(db, quality_check.candidate_dataset_id, identifiers): QualityEvidenceService._add_index_keys(index, row) - for row in db.query(Detection).filter(Detection.dataset_id == quality_check.candidate_dataset_id).all(): - QualityEvidenceService._add_index_keys(index, row) - for row in db.query(Segmentation).filter(Segmentation.dataset_id == quality_check.candidate_dataset_id).all(): - QualityEvidenceService._add_index_keys(index, row) - if quality_check.analysis_run_id: - for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all(): + if uuid_identifiers: + for row in db.query(Detection).filter( + Detection.dataset_id == quality_check.candidate_dataset_id, + Detection.id.in_(uuid_identifiers), + ).all(): QualityEvidenceService._add_index_keys(index, row) - for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all(): + for row in db.query(Segmentation).filter( + Segmentation.dataset_id == quality_check.candidate_dataset_id, + Segmentation.id.in_(uuid_identifiers), + ).all(): QualityEvidenceService._add_index_keys(index, row) - elif quality_check.analysis_run_id: - for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all(): + if quality_check.analysis_run_id and uuid_identifiers: + for row in db.query(Detection).filter( + Detection.analysis_run_id == quality_check.analysis_run_id, + Detection.id.in_(uuid_identifiers), + ).all(): QualityEvidenceService._add_index_keys(index, row) - for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all(): + for row in db.query(Segmentation).filter( + Segmentation.analysis_run_id == quality_check.analysis_run_id, + Segmentation.id.in_(uuid_identifiers), + ).all(): QualityEvidenceService._add_index_keys(index, row) return index @staticmethod - def _reference_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]: + def _reference_feature_index( + db: Session, + quality_check: QualityCheck, + identifiers: set[str], + ) -> dict[str, Any]: index: dict[str, Any] = {} - for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.reference_dataset_id).all(): + for row in QualityEvidenceService._vector_feature_rows(db, quality_check.reference_dataset_id, identifiers): QualityEvidenceService._add_index_keys(index, row) return index + @staticmethod + def _annotate_reviews( + db: Session, + quality_check: QualityCheck, + features: list[dict[str, Any]], + ) -> None: + if quality_check.check_type != "detections_vs_reference": + return + reviews = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check.id).all() + review_index = {(row.evidence_role, row.evidence_feature_id): row for row in reviews} + for feature in features: + properties = feature.get("properties") + if not isinstance(properties, dict): + continue + role = QualityEvidenceService._string_value(properties.get("qa_evidence_role")) + if role == "false_positive": + evidence_id = QualityEvidenceService._string_value(properties.get("candidate_feature_id")) + elif role == "false_negative": + evidence_id = QualityEvidenceService._string_value(properties.get("reference_feature_id")) + else: + continue + review = review_index.get((role, evidence_id or "")) + properties.update( + { + "review_decision": review.decision if review else "unreviewed", + "review_notes": review.notes if review else None, + "reviewed_by": review.reviewed_by if review else None, + "reviewed_at": review.updated_at.isoformat() if review and review.updated_at else None, + } + ) + @staticmethod def _add_index_keys(index: dict[str, Any], row: Any) -> None: for key in QualityEvidenceService._row_identifiers(row): diff --git a/backend/tests/test_sprint196_map_orthophoto_analysis.py b/backend/tests/test_sprint196_map_orthophoto_analysis.py index 778e34ca..4d7033fe 100644 --- a/backend/tests/test_sprint196_map_orthophoto_analysis.py +++ b/backend/tests/test_sprint196_map_orthophoto_analysis.py @@ -317,7 +317,8 @@ def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() -> assert "datasetsApi.acquireOrthophoto" in hook_source assert "prepareAndRunDetection(datasetId)" in hook_source assert "compareDetectionRunWithReference" in hook_source - assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false)" in app_source + assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold)" in app_source + assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook_source assert "vervoerregio|operationele grens" in app_source assert "selectionBbox: mapSelectionBbox" in app_source assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source diff --git a/backend/tests/test_sprint197_accuracy_review_loop.py b/backend/tests/test_sprint197_accuracy_review_loop.py new file mode 100644 index 00000000..43b5e84b --- /dev/null +++ b/backend/tests/test_sprint197_accuracy_review_loop.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from pathlib import Path +from uuid import UUID, uuid4 + +import pytest +from fastapi.testclient import TestClient +from geoalchemy2.shape import from_shape +from shapely.geometry import box + +from app.core.errors import AppError +from app.db.session import get_db +from app.main import app +from app.models import Detection, DetectionReview, QualityCheck, VectorFeature +from app.schemas.detection_review import ( + DetectionReviewList, + DetectionReviewRead, + DetectionReviewSummary, + DetectionReviewUpsert, +) +from app.services.detection_review_service import DetectionReviewService + + +ROOT = Path(__file__).resolve().parents[2] + + +class FakeQuery: + def __init__(self, rows): + self.rows = list(rows) + + def filter(self, *criteria): + for criterion in criteria: + left = getattr(criterion, "left", None) + right = getattr(criterion, "right", None) + operator = getattr(criterion, "operator", None) + name = getattr(left, "name", None) + value = getattr(right, "value", right) + if name and operator and operator.__name__ == "eq": + self.rows = [row for row in self.rows if getattr(row, name) == value] + return self + + def all(self): + return list(self.rows) + + def first(self): + return self.rows[0] if self.rows else None + + +class FakeSession: + def __init__(self, objects=None, query_rows=None) -> None: + self.objects = objects or {} + self.query_rows = query_rows or {} + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def query(self, model): + return FakeQuery(self.query_rows.setdefault(model, [])) + + def add(self, row): + rows = self.query_rows.setdefault(type(row), []) + if row not in rows: + rows.append(row) + self.objects[(type(row), row.id)] = row + + def commit(self): + return None + + def refresh(self, _row): + return None + + +def _review_context() -> tuple[FakeSession, UUID, UUID, Detection, VectorFeature]: + project_id = uuid4() + quality_check_id = uuid4() + analysis_run_id = uuid4() + candidate_dataset_id = uuid4() + reference_dataset_id = uuid4() + detection = Detection( + id=uuid4(), + project_id=project_id, + dataset_id=candidate_dataset_id, + analysis_run_id=analysis_run_id, + model_name="yolo-configured", + class_name="building", + confidence=0.62, + geometry=from_shape(box(5.0, 51.0, 5.001, 51.001), srid=4326), + ) + reference = VectorFeature( + id=uuid4(), + dataset_id=reference_dataset_id, + source_feature_id="grb-missed", + feature_class="building", + properties_json={}, + geometry=from_shape(box(5.002, 51.002, 5.003, 51.003), srid=4326), + ) + quality_check = QualityCheck( + id=quality_check_id, + project_id=project_id, + analysis_run_id=analysis_run_id, + candidate_dataset_id=candidate_dataset_id, + reference_dataset_id=reference_dataset_id, + check_type="detections_vs_reference", + status="ok", + findings_json={ + "false_positive_evidence": [{"candidate_feature_id": str(detection.id)}], + "false_negative_evidence": [{"reference_feature_id": str(reference.id)}], + }, + ) + db = FakeSession( + objects={ + (QualityCheck, quality_check_id): quality_check, + (Detection, detection.id): detection, + (VectorFeature, reference.id): reference, + }, + query_rows={DetectionReview: []}, + ) + return db, project_id, quality_check_id, detection, reference + + +def test_detection_review_model_and_migration_are_aligned() -> None: + migration = (ROOT / "backend" / "alembic" / "versions" / "202607150001_detection_reviews.py").read_text(encoding="utf-8") + columns = DetectionReview.__table__.columns + + for name in ( + "project_id", + "quality_check_id", + "analysis_run_id", + "evidence_role", + "evidence_feature_id", + "detection_id", + "reference_feature_id", + "decision", + "notes", + "reviewed_by", + "created_at", + "updated_at", + ): + assert name in columns + assert f'"{name}"' in migration + assert 'op.create_table(\n "detection_reviews"' in migration + assert 'down_revision = "202607140001"' in migration + + +def test_detection_review_queue_persists_only_valid_operator_decisions() -> None: + db, project_id, quality_check_id, detection, _reference = _review_context() + + initial = DetectionReviewService.list_reviews( + db, + project_id=project_id, + quality_check_id=quality_check_id, + ) + assert initial.summary.total == 2 + assert initial.summary.reviewed == 0 + assert initial.summary.decision_counts == {"unreviewed": 2} + + saved = DetectionReviewService.upsert_review( + db, + project_id=project_id, + quality_check_id=quality_check_id, + payload=DetectionReviewUpsert( + evidence_role="false_positive", + evidence_feature_id=str(detection.id), + decision="qa_alignment_mismatch", + notes="Box overlaps the official footprint but is not a training negative.", + ), + ) + assert saved.decision == "qa_alignment_mismatch" + assert saved.detection_id == detection.id + + reviewed = DetectionReviewService.list_reviews( + db, + project_id=project_id, + quality_check_id=quality_check_id, + reviewed=True, + ) + assert reviewed.total == 1 + assert reviewed.summary.reviewed == 1 + assert reviewed.summary.remaining == 1 + + with pytest.raises(AppError) as exc: + DetectionReviewService.upsert_review( + db, + project_id=project_id, + quality_check_id=quality_check_id, + payload=DetectionReviewUpsert( + evidence_role="false_positive", + evidence_feature_id=str(detection.id), + decision="confirmed_model_false_negative", + ), + ) + assert exc.value.code == "INVALID_DETECTION_REVIEW_DECISION" + + +def test_detection_review_endpoints_use_canonical_envelopes(monkeypatch) -> None: + project_id = uuid4() + quality_check_id = uuid4() + item = DetectionReviewRead( + project_id=project_id, + quality_check_id=quality_check_id, + evidence_role="false_positive", + evidence_feature_id=str(uuid4()), + decision="unreviewed", + ) + result = DetectionReviewList( + items=[item], + total=1, + limit=50, + offset=0, + summary=DetectionReviewSummary( + total=1, + reviewed=0, + remaining=1, + false_positive_total=1, + false_negative_total=0, + decision_counts={"unreviewed": 1}, + ), + ) + monkeypatch.setattr(DetectionReviewService, "list_reviews", lambda *_args, **_kwargs: result) + monkeypatch.setattr(DetectionReviewService, "upsert_review", lambda *_args, **_kwargs: item) + app.dependency_overrides[get_db] = lambda: FakeSession() + try: + listed = TestClient(app).get(f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews") + saved = TestClient(app).post( + f"/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews", + json={ + "evidence_role": "false_positive", + "evidence_feature_id": item.evidence_feature_id, + "decision": "unreviewed", + }, + ) + finally: + app.dependency_overrides.pop(get_db, None) + + assert listed.status_code == 200 + assert set(listed.json()) == {"data"} + assert listed.json()["data"]["summary"]["remaining"] == 1 + assert saved.status_code == 200 + assert saved.json() == {"data": item.model_dump(mode="json")} + + +def test_map_detection_qa_uses_documented_footprint_threshold_and_honest_labels() -> None: + hook = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8") + app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8") + evidence_service = (ROOT / "backend" / "app" / "services" / "quality_evidence_service.py").read_text(encoding="utf-8") + + assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook + assert "kandidaten" in hook + assert "precision" in hook.lower() + assert "false, iouThreshold" in app_source + assert "VectorFeature.id.in_(uuid_identifiers)" in evidence_service + assert "VectorFeature.source_feature_id.in_(identifiers)" in evidence_service + assert "for row in db.query(VectorFeature).filter(VectorFeature.dataset_id" not in evidence_service diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 6668811b..daf87bd8 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -427,6 +427,21 @@ remains available as a higher-precision legacy `0.15` choice. The older default-promotion blocker. Every production-like run still requires persisted QA/QC against suitable reference data. +The map-driven building workflow uses canonical footprint IoU `0.25`, matching +the promotion evidence above. A July 2026 Mol-only holdout audit compared +confidence `0.10` and `0.15` over Achterbos, Gompel, Donk and Postel. Confidence +`0.15` produced the better F1 in all four positive zones; both thresholds +produced zero detections in the pure-empty Postel forest control. The active +confidence therefore remains `0.15`. This result does not claim production +perfection and does not justify another model-training run by itself. + +False-positive and false-negative evidence from persisted detection QA can be +classified through `detection_reviews`. The queue derives from quality-check +evidence ids and resolves persisted Detection and reference VectorFeature rows. +`qa_alignment_mismatch`, `reference_gap_or_change`, uncertain imagery and +unreviewed items must never be exported as hard-negative or missed-positive +training labels. Canonical QA metrics remain unchanged after review. + The persisted seven-AOI evidence for this profile contains 5,568 false positives among 13,613 candidate detections. The read-only audit command in `scripts/README.md` reports the largest review volumes in Turnhout, Herentals diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 4ffc378f..48a6fc99 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1300,6 +1300,71 @@ candidate evidence exposes the equivalent persisted model/source fields plus `segmentation_id`, `mask_path` and `area_m2`. These are additive GeoJSON properties; the canonical envelope and endpoint path are unchanged. +For detection QA, false-positive and false-negative evidence properties also +include `review_decision`, `review_notes`, `reviewed_by` and `reviewed_at`. +Missing review rows are represented as `review_decision=unreviewed`. Evidence +resolution is bounded to identifiers stored by the selected quality check. + +### GET `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews` + +Returns the paginated operator review queue for a persisted +`detections_vs_reference` quality check. Optional query parameters are +`evidence_role=false_positive|false_negative`, `decision`, `reviewed=true|false`, +`limit` (1-200) and `offset`. Items derive only from persisted QA evidence. + +```json +{ + "data": { + "items": [{ + "id": null, + "project_id": "uuid", + "quality_check_id": "uuid", + "analysis_run_id": "uuid", + "evidence_role": "false_positive", + "evidence_feature_id": "detection-uuid", + "detection_id": "detection-uuid", + "decision": "unreviewed", + "confidence": 0.62, + "class_name": "building" + }], + "total": 1, + "limit": 50, + "offset": 0, + "summary": { + "total": 73, + "reviewed": 0, + "remaining": 73, + "false_positive_total": 17, + "false_negative_total": 56, + "decision_counts": {"unreviewed": 73} + } + } +} +``` + +### POST `/api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews` + +Creates or updates one durable operator decision. The evidence id must belong +to the quality check and resolve to the persisted Detection or reference +VectorFeature. False-positive and false-negative roles accept only their +role-specific decisions. + +```json +{ + "evidence_role": "false_positive", + "evidence_feature_id": "detection-uuid", + "decision": "qa_alignment_mismatch", + "notes": "The detection box overlaps the irregular GRB footprint.", + "reviewed_by": "operator" +} +``` + +Allowed decisions are `confirmed_model_false_positive`, +`confirmed_model_false_negative`, `reference_gap_or_change`, +`qa_alignment_mismatch`, `imagery_obscured_or_uncertain`, `uncertain` and +`unreviewed`. Invalid role/decision combinations return +`INVALID_DETECTION_REVIEW_DECISION`. + ## Exports ### POST `/api/v1/exports/geojson` diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index cf409126..f8693060 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8135,3 +8135,40 @@ Next: - Add a review workflow for accepted/rejected detections and use those audited labels as the gate for model calibration or retraining. Do not present the current F1 score as production-grade accuracy. + +## Sprint 197 - Measured detection accuracy and durable review (2026-07-15) + +Implemented: +- Added first-class `detection_reviews` persistence linked to Project, + QualityCheck, AnalysisRun, Detection and reference VectorFeature, with + role-constrained decisions and one durable row per evidence item. +- Added canonical paginated GET/POST review endpoints and a frontend review + queue with role/status filters, notes, summary counts and map handoff. +- Changed map-driven detection QA from generic UI IoU `0.50` to the documented + box-versus-footprint operational IoU `0.25`. +- Reworded map output as candidates and exposed persisted matches, precision, + recall, F1, false positives and false negatives. +- Bounded evidence lookup to persisted evidence ids; complete regional GRB + layers are no longer materialized for a small review overlay. + +Live model evidence before deployment: +- Re-ran the active model at confidence `0.10` and `0.15` on Mol Achterbos, + Gompel, Donk and Postel with QA IoU `0.25`. +- Confidence `0.15` won F1 in all four positive holdouts: `0.6694`, `0.6564`, + `0.5894` and `0.4749`. Confidence `0.10` measured `0.6287`, `0.6348`, + `0.5636` and `0.4435` respectively. +- Both thresholds produced zero detections on the pure-empty Postel forest + control. The active confidence remains `0.15`; no model or asset was trained, + downloaded or promoted. + +Validation before deployment: +- The complete readiness gate passed 592 backend tests, the API/document audit + for 88 implemented routes, backend compilation, one Alembic head, offline + migration SQL generation, frontend typecheck/build and shell smoke checks. +- Focused review/evidence regressions passed and the production bundle retained + separate React, application and MapLibre chunks. + +Next: +- Deploy the migration and UI, verify one live review queue and map QA result, + then complete representative manual decisions before constructing any new + model-training corpus. diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index 0ef3a246..d1e475cc 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -196,6 +196,28 @@ diagnostic reference-envelope comparison are persisted in the existing `quality_checks.findings_json`; `parameters_json.coverage_policy` records the evaluation policy used for reproducibility. +### detection_reviews + +- `id uuid primary key` +- `project_id uuid references projects(id) on delete cascade` +- `quality_check_id uuid references quality_checks(id) on delete cascade` +- `analysis_run_id uuid nullable references analysis_runs(id) on delete set null` +- `evidence_role text not null` (`false_positive` or `false_negative`) +- `evidence_feature_id text not null` +- `detection_id uuid nullable references detections(id) on delete set null` +- `reference_feature_id uuid nullable references vector_features(id) on delete set null` +- `decision text not null default 'unreviewed'` +- `notes text nullable` +- `reviewed_by text not null default 'operator'` +- `created_at timestamptz` +- `updated_at timestamptz` + +The unique key is `(quality_check_id, evidence_role, evidence_feature_id)`. +Indexes cover project, quality check, analysis run and decision. Reviews +classify persisted QA evidence only; they do not replace or modify Detection, +VectorFeature, QualityCheck or Metric records. Unreviewed, reference-gap, +imagery-uncertain and QA-alignment cases are not training labels. + ### exports - `id uuid primary key` diff --git a/docs/TODO.md b/docs/TODO.md index 27fec3eb..0aa9d092 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -46,6 +46,10 @@ This file now starts with the current implementation status. Older preparation/b - [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training. - [x] Add a coverage-aware Mol multi-zone benchmark report with explicit positive-zone, per-zone collapse, reference-coverage and pure-empty background gates. - [x] Execute the refreshed coverage-aware Mol operational benchmark against the active local model and record the resulting retain/review decision. +- [x] Align map-driven building QA with footprint IoU `0.25` and show measured candidate/match/error metrics instead of calling every detection a building. +- [x] Persist paginated false-positive/false-negative operator decisions and expose them in the Quality workspace. +- [x] Bound QA evidence resolution to persisted evidence ids instead of loading complete regional reference datasets. +- [x] Compare confidence `0.10` and `0.15` over independent Mol holdouts; retain `0.15` because it wins F1 in every positive zone while both pass the empty-background control. - [ ] Complete manual decisions for the generated 48 false-negative and 48 false-positive review cards before constructing any new training corpus. - [x] Backend FastAPI foundation, health endpoint and service structure. - [x] React/TypeScript frontend foundation and MapLibre workbench. diff --git a/frontend/README.md b/frontend/README.md index f2fc0a7a..c09862cf 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -8,6 +8,12 @@ The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanal Detection defaults to the configured local YOLO asset and automatically selects an available raster and active model asset where possible. The model registry and preflight remain honest when PyTorch, Ultralytics, a local model file or a tile manifest is unavailable. The active building profile is operational but remains review-required: its recorded coverage-aware benchmark is approximately precision 0.590, recall 0.577 and F1 0.582 over seven positive AOIs, with zero detections in the empty-background control. Another training pass is intentionally blocked until the generated false-positive and false-negative review decisions are completed. +Map-driven building analysis uses the documented footprint-IoU `0.25` and +distinguishes model candidates from verified buildings. It shows persisted +matches, precision, recall, F1, false positives and false negatives. The +Quality workspace includes a paginated review queue for persisted detection-QA +evidence. Reviews do not rewrite detections, GRB geometry or QA metrics. + The map-first explorer has two deliberate modes. `Latest state` selects the latest explicitly dated source snapshot without claiming an old edition is current, while `Evolution` lets the operator compare an earlier and later @@ -373,6 +379,7 @@ Before creating tiles, the guided action inspects raster dimensions and estimate - The QA/QC workspace includes a selected-check evidence drilldown with candidate/reference provenance, false-positive/negative metric evidence, map handoff context and parameters/findings JSON. - QA/QC findings now persist feature-level evidence in `findings_json`: matched candidate/reference feature ids with IoU, false-positive candidate feature ids and false-negative reference feature ids. The QA/QC drilldown renders these as compact evidence lists before the raw JSON. - Persisted QA/QC checks can be rendered as a Map workspace evidence overlay. The QA/QC panel calls `GET /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/evidence/geojson`, then MapLibre draws matched candidate/reference geometries, false positives and false negatives with distinct styling and a compact legend. +- Detection QA checks expose a filtered, paginated operator review queue through `GET/POST /api/v1/projects/{project_id}/quality-checks/{quality_check_id}/reviews`. The UI keeps confirmed model errors separate from reference gaps and box-to-footprint alignment mismatches. ## Raster dependency visibility diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index a26b5af2..2ff66b50 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -477,8 +477,8 @@ function App(): JSX.Element { datasets, loadProjectData, prepareAndRunDetection: (datasetId) => prepareAndRunDetection(datasetId, 'yolo-configured'), - compareDetectionRunWithReference: (analysisRunId, referenceDatasetId) => - compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false), + compareDetectionRunWithReference: (analysisRunId, referenceDatasetId, iouThreshold) => + compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold), onAnalysisReady: () => { setMapContentMode('analysis') setMapLayerVisible(true) @@ -1011,6 +1011,8 @@ function App(): JSX.Element { orthophotoAnalysisStatus={mapOrthophotoAnalysis.status} orthophotoAnalysisError={mapOrthophotoAnalysis.error} orthophotoAnalysisRunning={mapOrthophotoAnalysis.running} + orthophotoAnalysisQuality={mapOrthophotoAnalysis.lastQuality} + orthophotoAnalysisDetectionCount={mapOrthophotoAnalysis.lastDetectionCount} availableMapDatasets={availableMapDatasets} selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''} selectedFeature={selectedMapFeature} diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index b402f3db..0855569b 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from 'react' import GeoMap from '../GeoMap' -import type { AreaRead, DatasetCreateResponse, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types' +import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types' import { featureCollectionBounds } from '../../lib/geojsonBounds' import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights' import { useTemporalComparison } from '../../hooks/useTemporalComparison' @@ -327,6 +327,12 @@ function formatBboxLabel(bbox: VectorSelectionBBox | null): string { return `${formatCoordinate(bbox.min_x)}, ${formatCoordinate(bbox.min_y)} -> ${formatCoordinate(bbox.max_x)}, ${formatCoordinate(bbox.max_y)}` } +function formatPercentage(value: number | null | undefined): string { + return typeof value === 'number' && Number.isFinite(value) + ? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%` + : 'n.v.t.' +} + function bboxToInputState(bbox: VectorSelectionBBox | null) { return { min_x: bbox ? String(bbox.min_x) : '', @@ -445,6 +451,8 @@ interface MapWorkspaceProps { orthophotoAnalysisStatus: string orthophotoAnalysisError: string | null orthophotoAnalysisRunning: boolean + orthophotoAnalysisQuality: DetectionQaResult | null + orthophotoAnalysisDetectionCount: number | null availableMapDatasets: DatasetCreateResponse[] selectedMapDatasetId: string onSelectMapArea: (areaId: string) => void @@ -518,6 +526,8 @@ export function MapWorkspace({ orthophotoAnalysisStatus, orthophotoAnalysisError, orthophotoAnalysisRunning, + orthophotoAnalysisQuality, + orthophotoAnalysisDetectionCount, availableMapDatasets, selectedMapDatasetId, onSelectMapArea, @@ -1194,6 +1204,16 @@ export function MapWorkspace({ {orthophotoAnalysisStatus ?

{orthophotoAnalysisStatus}

: null} {orthophotoAnalysisError ?

{orthophotoAnalysisError}

: null} + {orthophotoAnalysisQuality ? ( +
+
Kandidaten{orthophotoAnalysisDetectionCount?.toLocaleString('nl-BE') ?? 'n.v.t.'}
+
Precision{formatPercentage(orthophotoAnalysisQuality.precision)}
+
Recall{formatPercentage(orthophotoAnalysisQuality.recall)}
+
F1{formatPercentage(orthophotoAnalysisQuality.f1_score)}
+
Fout{orthophotoAnalysisQuality.false_positives.toLocaleString('nl-BE')}
+
Gemist{orthophotoAnalysisQuality.false_negatives.toLocaleString('nl-BE')}
+
+ ) : null} ) : null} diff --git a/frontend/src/components/quality/DetectionReviewPanel.tsx b/frontend/src/components/quality/DetectionReviewPanel.tsx new file mode 100644 index 00000000..f2bf9f4c --- /dev/null +++ b/frontend/src/components/quality/DetectionReviewPanel.tsx @@ -0,0 +1,244 @@ +import { useEffect, useState } from 'react' +import { qaApi } from '../../services/api' +import type { + DetectionEvidenceRole, + DetectionReviewDecision, + DetectionReviewList, + DetectionReviewRead, +} from '../../types' +import { formatError } from '../../lib/formatError' + +interface DetectionReviewPanelProps { + projectId: string + qualityCheckId: string + onOpenEvidenceMap?: (qualityCheckId: string) => void +} + +const ROLE_LABELS: Record = { + false_positive: 'Onterecht gevonden', + false_negative: 'Gemist gebouw', +} + +const DECISION_LABELS: Record = { + confirmed_model_false_positive: 'Bevestigde foutdetectie', + confirmed_model_false_negative: 'Bevestigd gemist gebouw', + reference_gap_or_change: 'Referentie ontbreekt of is verouderd', + qa_alignment_mismatch: 'Vormvergelijking is te streng', + imagery_obscured_or_uncertain: 'Luchtbeeld is onduidelijk', + uncertain: 'Verder onderzoek nodig', + unreviewed: 'Nog niet beoordeeld', +} + +const ROLE_DECISIONS: Record = { + false_positive: [ + 'unreviewed', + 'confirmed_model_false_positive', + 'reference_gap_or_change', + 'qa_alignment_mismatch', + 'uncertain', + ], + false_negative: [ + 'unreviewed', + 'confirmed_model_false_negative', + 'reference_gap_or_change', + 'qa_alignment_mismatch', + 'imagery_obscured_or_uncertain', + 'uncertain', + ], +} + +function shortId(value: string): string { + return value.length > 18 ? `${value.slice(0, 8)}...${value.slice(-6)}` : value +} + +export function DetectionReviewPanel({ + projectId, + qualityCheckId, + onOpenEvidenceMap, +}: DetectionReviewPanelProps): JSX.Element { + const [queue, setQueue] = useState(null) + const [loading, setLoading] = useState(false) + const [savingKey, setSavingKey] = useState(null) + const [error, setError] = useState(null) + const [roleFilter, setRoleFilter] = useState<'all' | DetectionEvidenceRole>('all') + const [statusFilter, setStatusFilter] = useState<'all' | 'reviewed' | 'unreviewed'>('unreviewed') + const [offset, setOffset] = useState(0) + const [draftDecisions, setDraftDecisions] = useState>({}) + const [draftNotes, setDraftNotes] = useState>({}) + + const load = async () => { + setLoading(true) + setError(null) + try { + const result = await qaApi.listDetectionReviews(projectId, qualityCheckId, { + evidenceRole: roleFilter === 'all' ? undefined : roleFilter, + reviewed: statusFilter === 'all' ? undefined : statusFilter === 'reviewed', + limit: 50, + offset, + }) + setQueue(result) + setDraftDecisions(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.decision]))) + setDraftNotes(Object.fromEntries(result.items.map((item) => [reviewKey(item), item.notes ?? '']))) + } catch (caught) { + setError(formatError(caught, 'De controlelijst kon niet worden geladen')) + } finally { + setLoading(false) + } + } + + useEffect(() => { + void load() + }, [projectId, qualityCheckId, roleFilter, statusFilter, offset]) + + const save = async (item: DetectionReviewRead) => { + const key = reviewKey(item) + setSavingKey(key) + setError(null) + try { + await qaApi.upsertDetectionReview(projectId, qualityCheckId, { + evidence_role: item.evidence_role, + evidence_feature_id: item.evidence_feature_id, + decision: draftDecisions[key] ?? item.decision, + notes: draftNotes[key]?.trim() || null, + reviewed_by: 'operator', + }) + await load() + } catch (caught) { + setError(formatError(caught, 'De beoordeling kon niet worden bewaard')) + } finally { + setSavingKey(null) + } + } + + return ( +
+
+
+

Fouten controleren

+

Beoordeel alleen twijfelgevallen. Bevestigde fouten kunnen later veilig als trainingsfeedback worden gebruikt.

+
+ +
+ + {queue ? ( +
+
Te beoordelen{queue.summary.total}
+
Afgerond{queue.summary.reviewed}
+
Resterend{queue.summary.remaining}
+
Fout gevonden{queue.summary.false_positive_total}
+
Gemist{queue.summary.false_negative_total}
+
+ ) : null} + +
+ + + +
+ + {error ?

{error}

: null} + {!loading && queue && queue.items.length === 0 ? ( +
+ Geen objecten in deze selectie +

Pas de filters aan of open de bewijslaag op de kaart.

+
+ ) : null} + +
    + {queue?.items.map((item) => { + const key = reviewKey(item) + const decision = draftDecisions[key] ?? item.decision + return ( +
  1. +
    +
    + {ROLE_LABELS[item.evidence_role]} + {item.class_name ?? 'gebouw'} / {shortId(item.evidence_feature_id)} +
    + {typeof item.confidence === 'number' ? {Math.round(item.confidence * 100)}% vertrouwen : null} +
    +
    + + + +
    +
    2. + ) + })} +
+ {queue && queue.total > queue.limit ? ( +
+ + {offset + 1}-{Math.min(offset + queue.limit, queue.total)} van {queue.total} + +
+ ) : null} +
+ ) +} + +function reviewKey(item: Pick): string { + return `${item.evidence_role}:${item.evidence_feature_id}` +} diff --git a/frontend/src/components/quality/QualityResultsPanel.tsx b/frontend/src/components/quality/QualityResultsPanel.tsx index bef43fd3..236a5710 100644 --- a/frontend/src/components/quality/QualityResultsPanel.tsx +++ b/frontend/src/components/quality/QualityResultsPanel.tsx @@ -1,5 +1,6 @@ import { useMemo, useState } from 'react' import type { DatasetCreateResponse, MetricRead, QualityCheckRead } from '../../types' +import { DetectionReviewPanel } from './DetectionReviewPanel' const CORE_METRIC_ORDER = [ 'precision', @@ -304,6 +305,13 @@ export function QualityResultsPanel({ + {selectedQualityCheck.check_type === 'detections_vs_reference' && selectedProjectId ? ( + + ) : null}
Overeenkomende object-ID's diff --git a/frontend/src/hooks/useDetectionWorkflow.ts b/frontend/src/hooks/useDetectionWorkflow.ts index 1fb49af2..cae25236 100644 --- a/frontend/src/hooks/useDetectionWorkflow.ts +++ b/frontend/src/hooks/useDetectionWorkflow.ts @@ -405,6 +405,7 @@ export function useDetectionWorkflow({ analysisRunId: string, referenceDatasetId: string, useCurrentFilters = true, + iouThresholdOverride?: number, ): Promise => { if (!analysisRunId) { setDetectionQaError('Select a detection run') @@ -422,7 +423,7 @@ export function useDetectionWorkflow({ try { const result = await detectionApi.compareWithReference(analysisRunId, { reference_dataset_id: referenceDatasetId, - iou_threshold: qaIouThreshold, + iou_threshold: iouThresholdOverride ?? qaIouThreshold, class_name: useCurrentFilters ? detectionClassFilter || null : null, min_confidence: useCurrentFilters && detectionMinConfidenceFilter > 0 ? detectionMinConfidenceFilter : null, }) diff --git a/frontend/src/hooks/useMapOrthophotoAnalysis.ts b/frontend/src/hooks/useMapOrthophotoAnalysis.ts index 2c02f660..ea742b66 100644 --- a/frontend/src/hooks/useMapOrthophotoAnalysis.ts +++ b/frontend/src/hooks/useMapOrthophotoAnalysis.ts @@ -17,6 +17,8 @@ export type MapOrthophotoAnalysisStage = | 'complete' | 'failed' +export const MAP_BUILDING_QA_IOU_THRESHOLD = 0.25 + interface MapOrthophotoAnalysisOptions { selectedProjectId: string | null selectedAreaId: string @@ -27,6 +29,7 @@ interface MapOrthophotoAnalysisOptions { compareDetectionRunWithReference: ( analysisRunId: string, referenceDatasetId: string, + iouThreshold: number, ) => Promise onAnalysisReady: () => void } @@ -61,6 +64,12 @@ function formatOrthophotoError(caught: unknown): string { return formatError(caught, 'De kaartgestuurde beeldanalyse is mislukt') } +function formatQualityPercentage(value: number | null | undefined): string { + return typeof value === 'number' && Number.isFinite(value) + ? `${(value * 100).toLocaleString('nl-BE', { maximumFractionDigits: 1 })}%` + : 'n.v.t.' +} + export function useMapOrthophotoAnalysis({ selectedProjectId, selectedAreaId, @@ -75,12 +84,18 @@ export function useMapOrthophotoAnalysis({ const [status, setStatus] = useState('') const [error, setError] = useState(null) const [lastResult, setLastResult] = useState(null) + const [lastQuality, setLastQuality] = useState(null) + const [lastDetectionCount, setLastDetectionCount] = useState(null) + const [lastAnalysisRunId, setLastAnalysisRunId] = useState(null) useEffect(() => { setStage('idle') setStatus('') setError(null) setLastResult(null) + setLastQuality(null) + setLastDetectionCount(null) + setLastAnalysisRunId(null) }, [selectionBbox?.min_x, selectionBbox?.min_y, selectionBbox?.max_x, selectionBbox?.max_y]) const run = async (bbox: VectorSelectionBBox): Promise => { @@ -91,6 +106,9 @@ export function useMapOrthophotoAnalysis({ } setError(null) setLastResult(null) + setLastQuality(null) + setLastDetectionCount(null) + setLastAnalysisRunId(null) setStage('acquiring') setStatus('1/3 Officieel luchtbeeld voor de rechthoek ophalen...') try { @@ -112,20 +130,27 @@ export function useMapOrthophotoAnalysis({ if (!detection) { throw new Error('De beeldanalyse stopte. Open Beeldanalyse voor de technische oorzaak.') } + setLastDetectionCount(detection.detection_count) + setLastAnalysisRunId(detection.analysis_run_id) const reference = findBuildingReference(datasets) if (reference) { setStage('validating') setStatus('3/3 Resultaat vergelijken met officiële GRB-gebouwen...') - const quality = await compareDetectionRunWithReference(detection.analysis_run_id, reference.id) + const quality = await compareDetectionRunWithReference( + detection.analysis_run_id, + reference.id, + MAP_BUILDING_QA_IOU_THRESHOLD, + ) + setLastQuality(quality) setStatus( quality - ? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend en gecontroleerd.` - : `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend; kwaliteitscontrole kon niet afronden.`, + ? `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten, ${quality.matches.toLocaleString('nl-BE')} gekoppeld aan GRB. Precision ${formatQualityPercentage(quality.precision)}, recall ${formatQualityPercentage(quality.recall)}.` + : `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaten; kwaliteitscontrole kon niet afronden.`, ) } else { setStatus( - `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} gebouwen herkend. De GRB-referentielaag ontbreekt voor automatische controle.`, + `Analyse klaar: ${detection.detection_count.toLocaleString('nl-BE')} kandidaatvormen. De GRB-referentielaag ontbreekt voor automatische controle.`, ) } setStage('complete') @@ -144,6 +169,9 @@ export function useMapOrthophotoAnalysis({ status, error, lastResult, + lastQuality, + lastDetectionCount, + lastAnalysisRunId, running: stage === 'acquiring' || stage === 'detecting' || stage === 'validating', run, } diff --git a/frontend/src/services/api/qa.ts b/frontend/src/services/api/qa.ts index f53d9528..71127716 100644 --- a/frontend/src/services/api/qa.ts +++ b/frontend/src/services/api/qa.ts @@ -1,5 +1,13 @@ import { apiGet, apiPost } from './client' -import type { QaComparisonRequest, JobRead, QualityCheckListResponse, QualityEvidenceGeoJsonResponse } from '../../types' +import type { + DetectionReviewList, + DetectionReviewRead, + DetectionReviewUpsert, + JobRead, + QaComparisonRequest, + QualityCheckListResponse, + QualityEvidenceGeoJsonResponse, +} from '../../types' export const qaApi = { runQa: (payload: QaComparisonRequest): Promise => @@ -8,4 +16,23 @@ export const qaApi = { apiGet(`/api/v1/projects/${projectId}/quality-checks`), getQualityEvidenceGeoJson: (projectId: string, qualityCheckId: string): Promise => apiGet(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/evidence/geojson`), + listDetectionReviews: ( + projectId: string, + qualityCheckId: string, + options: { evidenceRole?: string; reviewed?: boolean; limit?: number; offset?: number } = {}, + ): Promise => { + const query = new URLSearchParams({ + limit: String(options.limit ?? 50), + offset: String(options.offset ?? 0), + }) + if (options.evidenceRole) query.set('evidence_role', options.evidenceRole) + if (typeof options.reviewed === 'boolean') query.set('reviewed', String(options.reviewed)) + return apiGet(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews?${query}`) + }, + upsertDetectionReview: ( + projectId: string, + qualityCheckId: string, + payload: DetectionReviewUpsert, + ): Promise => + apiPost(`/api/v1/projects/${projectId}/quality-checks/${qualityCheckId}/reviews`, payload), } diff --git a/frontend/src/styles/premium.css b/frontend/src/styles/premium.css index 52fd5bbc..9372dff2 100644 --- a/frontend/src/styles/premium.css +++ b/frontend/src/styles/premium.css @@ -1862,3 +1862,139 @@ details.ai-lab-model-surface > summary strong { background: var(--accent-soft); } } + +.geo-image-quality-metrics, +.detection-review-summary { + display: grid; + grid-template-columns: repeat(6, minmax(0, 1fr)); + gap: 0.5rem; +} + +.geo-image-quality-metrics { + grid-column: 1 / -1; + margin-top: 0.35rem; + border-top: 1px solid var(--line); + padding-top: 0.65rem; +} + +.geo-image-quality-metrics > div, +.detection-review-summary > div { + display: grid; + min-width: 0; + gap: 0.15rem; + border: 1px solid var(--line); + border-radius: 6px; + padding: 0.55rem; + background: var(--panel-soft); +} + +.geo-image-quality-metrics span, +.detection-review-summary span, +.detection-review-item-heading span { + color: var(--muted); + font-size: 0.7rem; +} + +.geo-image-quality-metrics strong, +.detection-review-summary strong { + color: var(--text); + font-size: 0.9rem; +} + +.detection-review-panel { + display: grid; + gap: 0.75rem; + margin-top: 0.85rem; + border-top: 1px solid var(--line); + padding-top: 0.85rem; +} + +.detection-review-summary { + grid-template-columns: repeat(5, minmax(0, 1fr)); +} + +.detection-review-filters, +.detection-review-editor, +.detection-review-pagination { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)) auto; + gap: 0.65rem; + align-items: end; +} + +.detection-review-filters label, +.detection-review-editor label { + display: grid; + min-width: 0; + gap: 0.3rem; + color: var(--muted); + font-size: 0.72rem; +} + +.detection-review-list { + display: grid; + max-height: 34rem; + gap: 0.5rem; + margin: 0; + overflow: auto; + padding: 0; + list-style: none; +} + +.detection-review-item { + display: grid; + gap: 0.6rem; + border: 1px solid var(--line); + border-radius: 6px; + padding: 0.7rem; + background: #ffffff; +} + +.detection-review-item-heading { + display: flex; + min-width: 0; + gap: 0.75rem; + align-items: center; + justify-content: space-between; +} + +.detection-review-item-heading > div { + display: grid; + min-width: 0; + gap: 0.15rem; +} + +.detection-review-item-heading strong { + overflow-wrap: anywhere; + font-size: 0.82rem; +} + +.detection-review-pagination { + display: flex; + align-items: center; + justify-content: flex-end; +} + +.detection-review-pagination span { + color: var(--muted); + font-size: 0.76rem; +} + +@media (max-width: 900px) { + .geo-image-quality-metrics, + .detection-review-summary { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } + + .detection-review-editor, + .detection-review-filters { + grid-template-columns: 1fr; + } +} + +@media (max-width: 560px) { + .geo-image-quality-metrics, + .detection-review-summary { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 1e7cfbd5..ae759a91 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -894,6 +894,61 @@ export interface QualityEvidenceGeoJsonResponse { geojson: GeoJSON.FeatureCollection } +export type DetectionEvidenceRole = 'false_positive' | 'false_negative' + +export type DetectionReviewDecision = + | 'confirmed_model_false_positive' + | 'confirmed_model_false_negative' + | 'reference_gap_or_change' + | 'qa_alignment_mismatch' + | 'imagery_obscured_or_uncertain' + | 'uncertain' + | 'unreviewed' + +export interface DetectionReviewUpsert { + evidence_role: DetectionEvidenceRole + evidence_feature_id: string + decision: DetectionReviewDecision + notes?: string | null + reviewed_by?: string +} + +export interface DetectionReviewRead { + id?: string | null + project_id: string + quality_check_id: string + analysis_run_id?: string | null + evidence_role: DetectionEvidenceRole + evidence_feature_id: string + detection_id?: string | null + reference_feature_id?: string | null + decision: DetectionReviewDecision + notes?: string | null + reviewed_by?: string | null + confidence?: number | null + class_name?: string | null + source_tile_path?: string | null + created_at?: string | null + updated_at?: string | null +} + +export interface DetectionReviewSummary { + total: number + reviewed: number + remaining: number + false_positive_total: number + false_negative_total: number + decision_counts: Record +} + +export interface DetectionReviewList { + items: DetectionReviewRead[] + total: number + limit: number + offset: number + summary: DetectionReviewSummary +} + export type ExportKind = 'dataset' | 'detection_run' | 'segmentation_run' | 'vector_selection' export interface ExportRead {