from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any from uuid import UUID from geoalchemy2.shape import to_shape from shapely.geometry import GeometryCollection from shapely.geometry.base import BaseGeometry from shapely.strtree import STRtree from shapely.ops import unary_union from shapely.validation import make_valid from app.core.errors import AppError from app.models import Area, Dataset from app.schemas.qa import QaProviderComparisonResult from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.vector_operations_service import VectorOperationsService @dataclass class QaMatchEvidence: matches: int = 0 false_positives: int = 0 false_negatives: int = 0 match_iou_values: list[float] = field(default_factory=list) warnings: list[str] = field(default_factory=list) unsupported: bool = False match_evidence: list[dict[str, Any]] = field(default_factory=list) false_positive_evidence: list[dict[str, Any]] = field(default_factory=list) false_negative_evidence: list[dict[str, Any]] = field(default_factory=list) def _extract_crs_warnings(source_dataset: Dataset, reference_dataset: Dataset) -> list[str]: warnings: list[str] = [] for dataset, label in ((source_dataset, "candidate"), (reference_dataset, "reference")): metadata = dataset.metadata_json crs_assumed = None if isinstance(metadata, dict): crs_assumed = metadata.get("crs_assumed") if crs_assumed: warnings.append(f"CRS assumption is weak for {label} dataset ({dataset.id}); geometry metrics are approximate") if dataset.crs is None: warnings.append(f"Missing CRS on {label} dataset ({dataset.id})") return warnings class QaService: SUPPORTED_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"} @staticmethod def _load_dataset_payload(db, dataset_id: UUID, *, expected_project_id: UUID | None = None) -> tuple[Dataset, dict[str, Any], list[tuple[dict[str, Any], BaseGeometry]]]: dataset = db.get(Dataset, dataset_id) if not dataset: raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) if expected_project_id is not None and dataset.project_id != expected_project_id: raise AppError(code="INVALID_DATASET_SCOPE", message="Dataset does not belong to this project", status_code=400) if dataset.dataset_type not in {"vector", "geojson"}: raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) payload, raw_features = VectorOperationsService._load_dataset_payload(dataset) geometries = VectorOperationsService._extract_geometries(raw_features) return dataset, payload, geometries @staticmethod def _apply_area_filter( geometries: list[tuple[dict[str, Any], BaseGeometry]], area_geometry: BaseGeometry, *, dataset_id: UUID, ) -> list[tuple[dict[str, Any], BaseGeometry]]: area_geom = area_geometry if isinstance(area_geom, GeometryCollection): area_geom = unary_union(area_geom.geoms) filtered: list[tuple[dict[str, Any], BaseGeometry]] = [] for feature, feature_geometry in geometries: clipped = feature_geometry.intersection(area_geom) if clipped.is_empty: continue if not clipped.is_valid: clipped = make_valid(clipped) if not clipped.is_valid: raise AppError( code="INVALID_GEOMETRY", message=f"Area filtering produced invalid geometry for feature in dataset {dataset_id}", status_code=400, ) filtered.append((feature, clipped)) return filtered @staticmethod def _validate_area(db, area_id: UUID | None, project_id: UUID, *, dataset_ids: tuple[UUID, UUID]) -> BaseGeometry | None: if not area_id: return None area = db.get(Area, area_id) if not area: raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) if area.project_id != project_id: raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) if area.id in dataset_ids: raise AppError(code="INVALID_PARAMETERS", message="area_id must reference an area, not a dataset", status_code=400) area_geometry = to_shape(area.geometry) if area_geometry.is_empty: raise AppError(code="INVALID_GEOMETRY", message="Area geometry is empty", status_code=400) return area_geometry @staticmethod def _feature_identifier(feature: dict[str, Any], fallback_prefix: str, index: int) -> str: feature_id = feature.get("id") if feature_id is not None: return str(feature_id) properties = feature.get("properties") if isinstance(properties, dict): for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "id", "name"): value = properties.get(key) if value is not None: return str(value) for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "class_name", "feature_class"): value = feature.get(key) if value is not None: return str(value) return f"{fallback_prefix}-{index + 1}" @staticmethod def _match_io_u_evidence( source_geometries: list[tuple[dict[str, Any], BaseGeometry]], reference_geometries: list[tuple[dict[str, Any], BaseGeometry]], iou_threshold: float, ) -> QaMatchEvidence: source_supported = [ (index, feature, geom) for index, (feature, geom) in enumerate(source_geometries) if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES ] reference_supported = [ (index, feature, geom) for index, (feature, geom) in enumerate(reference_geometries) if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES ] unsupported = sorted( { geom.geom_type for _, geom in source_geometries + reference_geometries if geom.geom_type not in QaService.SUPPORTED_GEOMETRY_TYPES } ) if not source_supported or not reference_supported: return QaMatchEvidence( false_positives=len(source_supported), false_negatives=len(reference_supported), warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], unsupported=True, false_positive_evidence=[ {"candidate_feature_id": QaService._feature_identifier(feature, "candidate", source_index)} for source_index, feature, _ in source_supported ], false_negative_evidence=[ {"reference_feature_id": QaService._feature_identifier(feature, "reference", reference_index)} for reference_index, feature, _ in reference_supported ], ) reference_tree = STRtree([geometry for _, _, geometry in reference_supported]) unmatched_reference_indices = { index for index, (_, _, geometry) in enumerate(reference_supported) if geometry.area > 0 } evidence = QaMatchEvidence(warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], unsupported=bool(unsupported)) for source_index, source_feature, source_geom in source_supported: source_feature_id = QaService._feature_identifier(source_feature, "candidate", source_index) if source_geom.area <= 0: evidence.false_positives += 1 evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id}) continue best_iou = 0.0 best_index = None candidate_reference_indices = sorted(int(index) for index in reference_tree.query(source_geom)) for reference_index in candidate_reference_indices: if reference_index not in unmatched_reference_indices: continue _, _, reference_geom = reference_supported[reference_index] try: intersection = source_geom.intersection(reference_geom) except Exception as exc: # pragma: no cover - robustness path raise AppError(code="GEOMETRY_OPERATION_UNSUPPORTED", message="Geometry operations failed", details={"reason": str(exc)}, status_code=422) if intersection.is_empty: continue intersection_area = intersection.area if intersection_area < 0: intersection_area = 0.0 union_area = source_geom.area + reference_geom.area - intersection_area if union_area <= 0: continue candidate_iou = intersection_area / union_area if candidate_iou > best_iou: best_iou = candidate_iou best_index = reference_index if best_index is not None and best_iou >= iou_threshold: reference_original_index, reference_feature, _ = reference_supported[best_index] evidence.matches += 1 evidence.match_iou_values.append(best_iou) evidence.match_evidence.append( { "candidate_feature_id": source_feature_id, "reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index), "iou": best_iou, } ) unmatched_reference_indices.discard(best_index) else: evidence.false_positives += 1 evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id}) evidence.false_negatives = len(unmatched_reference_indices) for reference_index in sorted(unmatched_reference_indices): reference_original_index, reference_feature, _ = reference_supported[reference_index] evidence.false_negative_evidence.append( {"reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index)} ) return evidence @staticmethod def _match_io_u_metrics( source_geometries: list[tuple[dict[str, Any], BaseGeometry]], reference_geometries: list[tuple[dict[str, Any], BaseGeometry]], iou_threshold: float, ) -> tuple[int, int, int, list[float], list[str], bool]: evidence = QaService._match_io_u_evidence(source_geometries, reference_geometries, iou_threshold) return ( evidence.matches, evidence.false_positives, evidence.false_negatives, evidence.match_iou_values, evidence.warnings, evidence.unsupported, ) @staticmethod def compare_candidate_with_reference( db, project_id: UUID, candidate_dataset_id: UUID, reference_dataset_id: UUID, iou_threshold: float = 0.5, area_id: UUID | None = None, ) -> QaProviderComparisonResult: if candidate_dataset_id == reference_dataset_id: raise AppError(code="INVALID_PARAMETERS", message="Candidate and reference dataset must differ", status_code=400) candidate_dataset, candidate_payload, candidate_geometries = QaService._load_dataset_payload( db, candidate_dataset_id, expected_project_id=project_id, ) reference_dataset, reference_payload, reference_geometries = QaService._load_dataset_payload( db, reference_dataset_id, expected_project_id=project_id, ) DatasetConsumptionGate.assert_eligible(candidate_dataset, purpose="quality_assessment") DatasetConsumptionGate.assert_eligible( reference_dataset, purpose="reference_validation", reference_task="building_validation", ) area_geometry = QaService._validate_area( db, area_id=area_id, project_id=project_id, dataset_ids=(candidate_dataset_id, reference_dataset_id), ) if area_geometry is not None: candidate_geometries = QaService._apply_area_filter(candidate_geometries, area_geometry, dataset_id=candidate_dataset.id) reference_geometries = QaService._apply_area_filter(reference_geometries, area_geometry, dataset_id=reference_dataset.id) evidence = QaService._match_io_u_evidence( candidate_geometries, reference_geometries, iou_threshold, ) candidate_feature_count = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0 reference_feature_count = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0 mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) precision = None if evidence.matches + evidence.false_positives > 0: precision = evidence.matches / (evidence.matches + evidence.false_positives) recall = None if evidence.matches + evidence.false_negatives > 0: recall = evidence.matches / (evidence.matches + evidence.false_negatives) f1_score = None if precision is not None and recall is not None and precision + recall > 0: f1_score = (2 * precision * recall) / (precision + recall) status = "unsupported" if evidence.unsupported else "ok" return QaProviderComparisonResult( status=status, warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + evidence.warnings, candidate_feature_count=candidate_feature_count, reference_feature_count=reference_feature_count, matches=evidence.matches, false_positives=evidence.false_positives, false_negatives=evidence.false_negatives, precision=precision, recall=recall, f1_score=f1_score, mean_iou=mean_iou, iou_threshold=iou_threshold, unsupported_geometry=evidence.unsupported, unsupported_geometries=evidence.warnings, match_evidence=evidence.match_evidence, false_positive_evidence=evidence.false_positive_evidence, false_negative_evidence=evidence.false_negative_evidence, generated_at=datetime.now(timezone.utc), )