from __future__ import annotations from datetime import datetime, timezone from typing import Any from uuid import UUID from geoalchemy2.shape import from_shape, to_shape from shapely.geometry import mapping from shapely.geometry.base import BaseGeometry from shapely.strtree import STRtree from shapely.validation import make_valid from sqlalchemy import func from sqlalchemy.orm import Session from app.core.errors import AppError from app.models import Dataset, VectorFeature from app.schemas.analysis import ChangeDetectionSummary from app.services.vector_operations_service import VectorOperationsService class ChangeDetectionService: SUPPORTED_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"} @staticmethod def compare_vector_datasets( db: Session, *, project_id: UUID, source_dataset_id: UUID, target_dataset_id: UUID, iou_threshold: float = 0.8, include_unchanged: bool = True, modified_threshold: float = 0.3, bbox: dict[str, Any] | None = None, area_id: UUID | None = None, preview_limit: int = 2_000, ) -> ChangeDetectionSummary: if source_dataset_id == target_dataset_id: raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400) if iou_threshold < 0 or iou_threshold > 1: raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400) if modified_threshold < 0 or modified_threshold > iou_threshold: raise AppError( code="INVALID_PARAMETERS", message="modified_threshold must be between 0 and iou_threshold", status_code=400, ) source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source") target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target") selection_geometry = ChangeDetectionService._selection_geometry(db, project_id, bbox=bbox, area_id=area_id) source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset, selection_geometry) target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset, selection_geometry) if not source_features: raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422) if not target_features: raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422) source_features = ChangeDetectionService.restrict_to_selection(source_features, selection_geometry, label="Source") target_features = ChangeDetectionService.restrict_to_selection(target_features, selection_geometry, label="Target") classified = ChangeDetectionService._classify_features( source_features, target_features, iou_threshold=iou_threshold, modified_threshold=modified_threshold, ) buckets: dict[str, list[dict[str, Any]]] = {"added": [], "removed": [], "modified": [], "unchanged": []} for item in classified: buckets[item["change_type"]].append( ChangeDetectionService._feature( geometry=item["geometry"], change_type=item["change_type"], source_dataset_id=source_dataset_id, target_dataset_id=target_dataset_id, source_feature_id=item["source_feature_id"], target_feature_id=item["target_feature_id"], iou=item["iou"], properties=item["properties"], ) ) unchanged_count = len(buckets["unchanged"]) if not include_unchanged: buckets["unchanged"] = [] geojson_features, preview_truncated = ChangeDetectionService.limit_preview( buckets["added"] + buckets["removed"] + buckets["modified"] + buckets["unchanged"], limit=preview_limit, ) warnings = source_warnings + target_warnings edge_count = sum( 1 for feature in source_features + target_features if feature.get("partially_covered") ) if edge_count: warnings.append( f"{edge_count} objecten liggen deels buiten de selectie. Ze zijn volledig vergeleken, zodat de " "selectierand zelf geen wijziging veroorzaakt." ) if preview_truncated: warnings.append( f"De tellingen gelden voor de volledige selectie; de kaart toont maximaal {preview_limit} objecten, " "wijzigingen eerst." ) return ChangeDetectionSummary( source_dataset_id=source_dataset_id, target_dataset_id=target_dataset_id, source_feature_count=len(source_features), target_feature_count=len(target_features), added_count=len(buckets["added"]), removed_count=len(buckets["removed"]), modified_count=len(buckets["modified"]), unchanged_count=unchanged_count, iou_threshold=iou_threshold, modified_iou_threshold=modified_threshold, selection_area_id=area_id, preview_limit=preview_limit, preview_truncated=preview_truncated, warnings=warnings, generated_at=datetime.now(timezone.utc), geojson={"type": "FeatureCollection", "features": geojson_features}, ) @staticmethod def _selection_geometry( db: Session, project_id: UUID, *, bbox: dict[str, Any] | None, area_id: UUID | None, ) -> BaseGeometry | None: """Resolve the drawn rectangle against the named work area, if any.""" from app.models import Area from shapely.geometry import box as shapely_box selection = None if bbox: selection = shapely_box( float(bbox["min_x"]), float(bbox["min_y"]), float(bbox["max_x"]), float(bbox["max_y"]) ) if area_id is None: return selection area = db.get(Area, area_id) if area is None or area.project_id != project_id: raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) area_geometry = to_shape(area.geometry) if selection is None: return area_geometry intersection = selection.intersection(area_geometry) if intersection.is_empty or intersection.area <= 0: raise AppError( code="CHANGE_DETECTION_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422, ) return intersection # Order the preview spends its budget in. An operator asking what changed # is not helped by a cap filled with unchanged footprints. PREVIEW_PRIORITY = {"modified": 0, "added": 1, "removed": 2, "unchanged": 3} @staticmethod def restrict_to_selection( features: list[dict[str, Any]], selection_geometry: BaseGeometry | None, *, label: str = "Dataset", ) -> list[dict[str, Any]]: """Keep the features a drawn selection reaches, and say which it cuts. Geometry is deliberately *not* clipped. A change class describes a whole object: comparing a clipped 2020 footprint against an unclipped 2024 one would manufacture "modified" along the selection edge. Clipping is right for an area metric and wrong for an identity comparison. """ if selection_geometry is None: return features kept: list[dict[str, Any]] = [] for feature in features: geometry = feature["geometry"] if not geometry.intersects(selection_geometry): continue kept.append({**feature, "partially_covered": not selection_geometry.covers(geometry)}) if not kept: raise AppError( code="CHANGE_DETECTION_SELECTION_EMPTY", message=f"{label} dataset has no features inside this selection", status_code=422, ) return kept @staticmethod def limit_preview( features: list[dict[str, Any]], *, limit: int, ) -> tuple[list[dict[str, Any]], bool]: """Cap the returned geometry without capping the counts. ``include_unchanged`` defaulted to true and nothing bounded the result, so a regional comparison returned a FeatureCollection holding both datasets in full. The counts describe the whole selection; the preview describes what a map can usefully draw. """ if limit <= 0 or len(features) <= limit: return features, False ordered = sorted( features, key=lambda item: ChangeDetectionService.PREVIEW_PRIORITY.get(item["change_type"], 9), ) return ordered[:limit], True @staticmethod def _classify_features( source_features: list[dict[str, Any]], target_features: list[dict[str, Any]], *, iou_threshold: float, modified_threshold: float, ) -> list[dict[str, Any]]: """Pair source with target footprints and label how each one changed. Matching is indexed rather than a full cross product: comparing two municipal building layers is otherwise hundreds of millions of geometry intersections. Sources are considered largest first so a big footprint is not left over after a small neighbour claimed its counterpart. """ target_geometries = [feature["geometry"] for feature in target_features] tree = STRtree(target_geometries) if target_geometries else None claimed: set[int] = set() classified: list[dict[str, Any]] = [] order = sorted( range(len(source_features)), key=lambda index: (-source_features[index]["geometry"].area, str(source_features[index]["feature_id"])), ) for source_index in order: source_feature = source_features[source_index] geometry = source_feature["geometry"] best_iou = 0.0 best_index: int | None = None candidates = [] if tree is None else sorted(int(value) for value in tree.query(geometry)) for target_index in candidates: if target_index in claimed: continue candidate_iou = ChangeDetectionService._iou(geometry, target_geometries[target_index]) if candidate_iou > best_iou: best_iou = candidate_iou best_index = target_index if best_index is not None and best_iou >= iou_threshold: claimed.add(best_index) change_type = "unchanged" elif best_index is not None and best_iou >= modified_threshold: # The same object, redrawn: an annexe, a demolition of one wing, # or a resurvey. Reporting it as removed + added would hide it. claimed.add(best_index) change_type = "modified" else: change_type = "removed" classified.append( { "change_type": change_type, "geometry": geometry if change_type != "modified" else target_geometries[best_index], "source_feature_id": source_feature["feature_id"], "target_feature_id": target_features[best_index]["feature_id"] if change_type != "removed" else None, "iou": best_iou if best_iou > 0 else None, "properties": source_feature["properties"], } ) classified.extend( { "change_type": "added", "geometry": target_feature["geometry"], "source_feature_id": None, "target_feature_id": target_feature["feature_id"], "iou": None, "properties": target_feature["properties"], } for target_index, target_feature in enumerate(target_features) if target_index not in claimed ) return classified @staticmethod def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset: dataset = db.get(Dataset, dataset_id) if not dataset: raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404) if dataset.project_id != project_id: raise AppError(code="INVALID_DATASET_SCOPE", message=f"{label} dataset does not belong to this project", status_code=400) VectorOperationsService._require_vector_dataset(dataset) return dataset @staticmethod def _load_features( db: Session, dataset: Dataset, selection_geometry: BaseGeometry | None = None, ) -> tuple[list[dict[str, Any]], list[str]]: query = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id) if selection_geometry is not None and hasattr(query, "filter"): # Bound the load in the database. Pulling a regional building layer # into Python to then discard most of it costs memory and time for # nothing, and the fallback below has no such option. try: query = query.filter( func.ST_Intersects(VectorFeature.geometry, from_shape(selection_geometry, srid=4326)) ) except Exception: # Lightweight unit-test sessions do not implement every spatial # predicate; restrict_to_selection still bounds the population. pass rows = query.all() warnings: list[str] = [] if rows: return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings warnings.append(f"Dataset {dataset.id} has no persisted vector_features; falling back to stored GeoJSON artifact") _payload, raw_features = VectorOperationsService._load_dataset_payload(dataset) extracted = VectorOperationsService._extract_geometries(raw_features) return [ ChangeDetectionService._raw_feature_to_feature(index, raw_feature, geometry) for index, (raw_feature, geometry) in enumerate(extracted) ], warnings @staticmethod def _row_to_feature(row: VectorFeature) -> dict[str, Any]: geometry = ChangeDetectionService._valid_comparable_geometry(to_shape(row.geometry)) return { "feature_id": str(row.source_feature_id or row.id), "properties": dict(row.properties_json or {}), "geometry": geometry, } @staticmethod def _raw_feature_to_feature(index: int, raw_feature: dict[str, Any], geometry: BaseGeometry) -> dict[str, Any]: properties = raw_feature.get("properties") if isinstance(raw_feature.get("properties"), dict) else {} source_id = raw_feature.get("id") or properties.get("id") or properties.get("source_feature_id") or str(index) return { "feature_id": str(source_id), "properties": dict(properties), "geometry": ChangeDetectionService._valid_comparable_geometry(geometry), } @staticmethod def _valid_comparable_geometry(geometry: BaseGeometry) -> BaseGeometry: if geometry.is_empty: raise AppError(code="INVALID_GEOMETRY", message="Empty geometry cannot be compared", status_code=400) if not geometry.is_valid: geometry = make_valid(geometry) if geometry.is_empty or not geometry.is_valid: raise AppError(code="INVALID_GEOMETRY", message="Geometry cannot be repaired for comparison", status_code=400) if geometry.geom_type not in ChangeDetectionService.SUPPORTED_GEOMETRY_TYPES: raise AppError( code="UNSUPPORTED_GEOMETRY", message="Change detection supports Polygon and MultiPolygon geometries only", details={"geometry_type": geometry.geom_type}, status_code=422, ) return geometry @staticmethod def _iou(left: BaseGeometry, right: BaseGeometry) -> float: if left.area <= 0 or right.area <= 0: return 0.0 intersection = left.intersection(right) if intersection.is_empty: return 0.0 union_area = left.area + right.area - intersection.area if union_area <= 0: return 0.0 return float(intersection.area / union_area) @staticmethod def _feature( *, geometry: BaseGeometry, change_type: str, source_dataset_id: UUID, target_dataset_id: UUID, source_feature_id: str | None, target_feature_id: str | None, iou: float | None, properties: dict[str, Any], ) -> dict[str, Any]: return { "type": "Feature", "geometry": mapping(geometry), "properties": { **properties, "change_type": change_type, "source_dataset_id": str(source_dataset_id), "target_dataset_id": str(target_dataset_id), "source_feature_id": source_feature_id, "target_feature_id": target_feature_id, "iou": iou, }, }