Add vector change detection foundation

backend/app/api/routes/__init__.py

@@ -1 +1 @@
-__all__ = ["areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa"]
+__all__ = ["analysis", "areas", "datasets", "health", "projects", "exports", "jobs", "external", "qa"]

backend/app/api/routes/analysis.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends
+from sqlalchemy.orm import Session
+
+from app.core.errors import AppError
+from app.db.session import get_db
+from app.models import Dataset
+from app.schemas.analysis import ChangeDetectionRequest
+from app.services.change_detection_service import ChangeDetectionService
+from app.services.job_service import JobService
+from app.utils.response import envelope
+
+router = APIRouter(prefix="/analysis", tags=["analysis"])
+
+
+@router.post("/change-detection", response_model=dict)
+def run_change_detection(
+    payload: ChangeDetectionRequest,
+    db: Session = Depends(get_db),
+) -> dict:
+    source_dataset = db.get(Dataset, payload.source_dataset_id)
+    if not source_dataset:
+        raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
+    ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
+    job = JobService.run_sync_job(
+        db=db,
+        project_id=source_dataset.project_id,
+        job_type="analysis.change-detection",
+        parameters=payload.model_dump(mode="json"),
+        input_dataset_id=payload.source_dataset_id,
+        operation=lambda: ChangeDetectionService.compare_vector_datasets(
+            db=db,
+            project_id=source_dataset.project_id,
+            source_dataset_id=payload.source_dataset_id,
+            target_dataset_id=payload.target_dataset_id,
+            iou_threshold=payload.iou_threshold,
+            include_unchanged=payload.include_unchanged,
+        ).model_dump(mode="json"),
+    )
+    return envelope(job)

backend/app/main.py

@@ -5,7 +5,7 @@ from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 
-from app.api.routes import areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation
+from app.api.routes import analysis, areas, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation
 from app.core.config import get_settings
 from app.core.errors import AppError
 from app.core.logging import configure_logging
@@ -41,6 +41,7 @@ def create_app() -> FastAPI:
     )
 
     app.include_router(health.router)
+    app.include_router(analysis.router, prefix=settings.api_prefix)
     app.include_router(projects.router, prefix=settings.api_prefix)
     app.include_router(areas.router, prefix=settings.api_prefix)
     app.include_router(datasets.router, prefix=settings.api_prefix)

backend/app/schemas/__init__.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from .common import ApiErrorEnvelope, ApiErrorItem, Envelope, PaginationEnvelope
 from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate
 from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
+from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
 from .dataset import DatasetCreateResponse, DatasetList
 from .detection import (
     DetectionListResponse,
@@ -87,6 +88,8 @@ __all__ = [
     "AreaRead",
     "AreaUpdate",
     "AreaList",
+    "ChangeDetectionRequest",
+    "ChangeDetectionSummary",
     "DatasetCreateResponse",
     "DatasetList",
     "DetectionListResponse",

backend/app/schemas/analysis.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+
+class ChangeDetectionRequest(BaseModel):
+    source_dataset_id: UUID
+    target_dataset_id: UUID
+    iou_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
+    include_unchanged: bool = True
+
+
+class ChangeDetectionSummary(BaseModel):
+    source_dataset_id: UUID
+    target_dataset_id: UUID
+    source_feature_count: int
+    target_feature_count: int
+    added_count: int
+    removed_count: int
+    unchanged_count: int
+    iou_threshold: float
+    warnings: list[str] = Field(default_factory=list)
+    generated_at: datetime
+    geojson: dict

backend/app/services/change_detection_service.py

@@ -0,0 +1,219 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+from uuid import UUID
+
+from geoalchemy2.shape import to_shape
+from shapely.geometry import mapping
+from shapely.geometry.base import BaseGeometry
+from shapely.validation import make_valid
+from sqlalchemy.orm import Session
+
+from app.core.errors import AppError
+from app.models import Dataset, VectorFeature
+from app.schemas.analysis import ChangeDetectionSummary
+from app.services.vector_operations_service import VectorOperationsService
+
+
+class ChangeDetectionService:
+    SUPPORTED_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"}
+
+    @staticmethod
+    def compare_vector_datasets(
+        db: Session,
+        *,
+        project_id: UUID,
+        source_dataset_id: UUID,
+        target_dataset_id: UUID,
+        iou_threshold: float = 0.8,
+        include_unchanged: bool = True,
+    ) -> ChangeDetectionSummary:
+        if source_dataset_id == target_dataset_id:
+            raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400)
+        if iou_threshold < 0 or iou_threshold > 1:
+            raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400)
+
+        source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source")
+        target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target")
+
+        source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset)
+        target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset)
+
+        if not source_features:
+            raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422)
+        if not target_features:
+            raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422)
+
+        matched_target_indices: set[int] = set()
+        unchanged: list[dict[str, Any]] = []
+        removed: list[dict[str, Any]] = []
+
+        for source_feature in source_features:
+            best_iou = 0.0
+            best_index: int | None = None
+            for target_index, target_feature in enumerate(target_features):
+                if target_index in matched_target_indices:
+                    continue
+                candidate_iou = ChangeDetectionService._iou(source_feature["geometry"], target_feature["geometry"])
+                if candidate_iou > best_iou:
+                    best_iou = candidate_iou
+                    best_index = target_index
+
+            if best_index is not None and best_iou >= iou_threshold:
+                matched_target_indices.add(best_index)
+                if include_unchanged:
+                    unchanged.append(
+                        ChangeDetectionService._feature(
+                            geometry=source_feature["geometry"],
+                            change_type="unchanged",
+                            source_dataset_id=source_dataset_id,
+                            target_dataset_id=target_dataset_id,
+                            source_feature_id=source_feature["feature_id"],
+                            target_feature_id=target_features[best_index]["feature_id"],
+                            iou=best_iou,
+                            properties=source_feature["properties"],
+                        )
+                    )
+            else:
+                removed.append(
+                    ChangeDetectionService._feature(
+                        geometry=source_feature["geometry"],
+                        change_type="removed",
+                        source_dataset_id=source_dataset_id,
+                        target_dataset_id=target_dataset_id,
+                        source_feature_id=source_feature["feature_id"],
+                        target_feature_id=None,
+                        iou=best_iou if best_iou > 0 else None,
+                        properties=source_feature["properties"],
+                    )
+                )
+
+        added = [
+            ChangeDetectionService._feature(
+                geometry=target_feature["geometry"],
+                change_type="added",
+                source_dataset_id=source_dataset_id,
+                target_dataset_id=target_dataset_id,
+                source_feature_id=None,
+                target_feature_id=target_feature["feature_id"],
+                iou=None,
+                properties=target_feature["properties"],
+            )
+            for target_index, target_feature in enumerate(target_features)
+            if target_index not in matched_target_indices
+        ]
+
+        geojson_features = added + removed + unchanged
+        return ChangeDetectionSummary(
+            source_dataset_id=source_dataset_id,
+            target_dataset_id=target_dataset_id,
+            source_feature_count=len(source_features),
+            target_feature_count=len(target_features),
+            added_count=len(added),
+            removed_count=len(removed),
+            unchanged_count=len(unchanged) if include_unchanged else len(matched_target_indices),
+            iou_threshold=iou_threshold,
+            warnings=source_warnings + target_warnings,
+            generated_at=datetime.now(timezone.utc),
+            geojson={"type": "FeatureCollection", "features": geojson_features},
+        )
+
+    @staticmethod
+    def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset:
+        dataset = db.get(Dataset, dataset_id)
+        if not dataset:
+            raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
+        if dataset.project_id != project_id:
+            raise AppError(code="INVALID_DATASET_SCOPE", message=f"{label} dataset does not belong to this project", status_code=400)
+        VectorOperationsService._require_vector_dataset(dataset)
+        return dataset
+
+    @staticmethod
+    def _load_features(db: Session, dataset: Dataset) -> tuple[list[dict[str, Any]], list[str]]:
+        rows = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id).all()
+        warnings: list[str] = []
+        if rows:
+            return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings
+
+        warnings.append(f"Dataset {dataset.id} has no persisted vector_features; falling back to stored GeoJSON artifact")
+        _payload, raw_features = VectorOperationsService._load_dataset_payload(dataset)
+        extracted = VectorOperationsService._extract_geometries(raw_features)
+        return [
+            ChangeDetectionService._raw_feature_to_feature(index, raw_feature, geometry)
+            for index, (raw_feature, geometry) in enumerate(extracted)
+        ], warnings
+
+    @staticmethod
+    def _row_to_feature(row: VectorFeature) -> dict[str, Any]:
+        geometry = ChangeDetectionService._valid_comparable_geometry(to_shape(row.geometry))
+        return {
+            "feature_id": str(row.source_feature_id or row.id),
+            "properties": dict(row.properties_json or {}),
+            "geometry": geometry,
+        }
+
+    @staticmethod
+    def _raw_feature_to_feature(index: int, raw_feature: dict[str, Any], geometry: BaseGeometry) -> dict[str, Any]:
+        properties = raw_feature.get("properties") if isinstance(raw_feature.get("properties"), dict) else {}
+        source_id = raw_feature.get("id") or properties.get("id") or properties.get("source_feature_id") or str(index)
+        return {
+            "feature_id": str(source_id),
+            "properties": dict(properties),
+            "geometry": ChangeDetectionService._valid_comparable_geometry(geometry),
+        }
+
+    @staticmethod
+    def _valid_comparable_geometry(geometry: BaseGeometry) -> BaseGeometry:
+        if geometry.is_empty:
+            raise AppError(code="INVALID_GEOMETRY", message="Empty geometry cannot be compared", status_code=400)
+        if not geometry.is_valid:
+            geometry = make_valid(geometry)
+        if geometry.is_empty or not geometry.is_valid:
+            raise AppError(code="INVALID_GEOMETRY", message="Geometry cannot be repaired for comparison", status_code=400)
+        if geometry.geom_type not in ChangeDetectionService.SUPPORTED_GEOMETRY_TYPES:
+            raise AppError(
+                code="UNSUPPORTED_GEOMETRY",
+                message="Change detection supports Polygon and MultiPolygon geometries only",
+                details={"geometry_type": geometry.geom_type},
+                status_code=422,
+            )
+        return geometry
+
+    @staticmethod
+    def _iou(left: BaseGeometry, right: BaseGeometry) -> float:
+        if left.area <= 0 or right.area <= 0:
+            return 0.0
+        intersection = left.intersection(right)
+        if intersection.is_empty:
+            return 0.0
+        union_area = left.area + right.area - intersection.area
+        if union_area <= 0:
+            return 0.0
+        return float(intersection.area / union_area)
+
+    @staticmethod
+    def _feature(
+        *,
+        geometry: BaseGeometry,
+        change_type: str,
+        source_dataset_id: UUID,
+        target_dataset_id: UUID,
+        source_feature_id: str | None,
+        target_feature_id: str | None,
+        iou: float | None,
+        properties: dict[str, Any],
+    ) -> dict[str, Any]:
+        return {
+            "type": "Feature",
+            "geometry": mapping(geometry),
+            "properties": {
+                **properties,
+                "change_type": change_type,
+                "source_dataset_id": str(source_dataset_id),
+                "target_dataset_id": str(target_dataset_id),
+                "source_feature_id": source_feature_id,
+                "target_feature_id": target_feature_id,
+                "iou": iou,
+            },
+        }

backend/tests/test_sprint18_change_detection.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from uuid import uuid4
+
+from fastapi.testclient import TestClient
+from geoalchemy2.shape import from_shape
+from shapely.geometry import box
+
+from app.db.session import get_db
+from app.main import app
+from app.models import Dataset, Job, VectorFeature
+from app.schemas.analysis import ChangeDetectionSummary
+from app.services.change_detection_service import ChangeDetectionService
+
+
+class FakeQuery:
+    def __init__(self, rows):
+        self.rows = list(rows)
+
+    def filter(self, *criteria):
+        for criterion in criteria:
+            left = getattr(criterion, "left", None)
+            right = getattr(criterion, "right", None)
+            operator = getattr(criterion, "operator", None)
+            name = getattr(left, "name", None)
+            value = getattr(right, "value", right)
+            if name and operator and operator.__name__ == "eq":
+                self.rows = [row for row in self.rows if getattr(row, name) == value]
+        return self
+
+    def all(self):
+        return list(self.rows)
+
+
+class FakeSession:
+    def __init__(self, objects=None, query_rows=None) -> None:
+        self.objects = objects or {}
+        self.query_rows = query_rows or {}
+        self.added = []
+        self.commits = 0
+        self.refreshes = []
+
+    def get(self, model, item_id):
+        return self.objects.get((model, item_id))
+
+    def query(self, model):
+        return FakeQuery(self.query_rows.get(model, []))
+
+    def add(self, item) -> None:
+        self.added.append(item)
+        if getattr(item, "id", None) is not None:
+            self.objects[(item.__class__, item.id)] = item
+
+    def commit(self) -> None:
+        self.commits += 1
+
+    def refresh(self, item) -> None:
+        self.refreshes.append(item)
+
+
+def _dataset(dataset_id, project_id, name):
+    return Dataset(
+        id=dataset_id,
+        project_id=project_id,
+        name=name,
+        dataset_type="vector",
+        source="manual",
+        dataset_role="source",
+    )
+
+
+def _feature(dataset_id, source_feature_id, geometry):
+    return VectorFeature(
+        id=uuid4(),
+        dataset_id=dataset_id,
+        source_feature_id=source_feature_id,
+        geometry=from_shape(geometry, srid=4326),
+        properties_json={"source_feature_id": source_feature_id},
+    )
+
+
+def test_change_detection_compares_persisted_vector_features() -> None:
+    project_id = uuid4()
+    source_dataset_id = uuid4()
+    target_dataset_id = uuid4()
+    source_dataset = _dataset(source_dataset_id, project_id, "before.geojson")
+    target_dataset = _dataset(target_dataset_id, project_id, "after.geojson")
+    rows = [
+        _feature(source_dataset_id, "source-unchanged", box(0, 0, 1, 1)),
+        _feature(source_dataset_id, "source-removed", box(10, 10, 11, 11)),
+        _feature(target_dataset_id, "target-unchanged", box(0, 0, 1, 1)),
+        _feature(target_dataset_id, "target-added", box(20, 20, 21, 21)),
+    ]
+    db = FakeSession(
+        objects={(Dataset, source_dataset_id): source_dataset, (Dataset, target_dataset_id): target_dataset},
+        query_rows={VectorFeature: rows},
+    )
+
+    result = ChangeDetectionService.compare_vector_datasets(
+        db=db,
+        project_id=project_id,
+        source_dataset_id=source_dataset_id,
+        target_dataset_id=target_dataset_id,
+        iou_threshold=0.8,
+    )
+
+    change_types = [feature["properties"]["change_type"] for feature in result.geojson["features"]]
+    assert result.source_feature_count == 2
+    assert result.target_feature_count == 2
+    assert result.added_count == 1
+    assert result.removed_count == 1
+    assert result.unchanged_count == 1
+    assert sorted(change_types) == ["added", "removed", "unchanged"]
+    assert result.warnings == []
+
+
+def test_change_detection_endpoint_returns_canonical_envelope(monkeypatch) -> None:
+    project_id = uuid4()
+    source_dataset_id = uuid4()
+    target_dataset_id = uuid4()
+    source_dataset = _dataset(source_dataset_id, project_id, "before.geojson")
+    target_dataset = _dataset(target_dataset_id, project_id, "after.geojson")
+    db = FakeSession(objects={(Dataset, source_dataset_id): source_dataset, (Dataset, target_dataset_id): target_dataset})
+    summary = ChangeDetectionSummary(
+        source_dataset_id=source_dataset_id,
+        target_dataset_id=target_dataset_id,
+        source_feature_count=1,
+        target_feature_count=1,
+        added_count=0,
+        removed_count=0,
+        unchanged_count=1,
+        iou_threshold=0.8,
+        warnings=[],
+        generated_at=datetime.now(timezone.utc),
+        geojson={"type": "FeatureCollection", "features": []},
+    )
+
+    monkeypatch.setattr(
+        "app.api.routes.analysis.ChangeDetectionService.compare_vector_datasets",
+        lambda **_kwargs: summary,
+    )
+    app.dependency_overrides[get_db] = lambda: db
+    try:
+        response = TestClient(app).post(
+            "/api/v1/analysis/change-detection",
+            json={
+                "source_dataset_id": str(source_dataset_id),
+                "target_dataset_id": str(target_dataset_id),
+                "iou_threshold": 0.8,
+                "include_unchanged": True,
+            },
+        )
+    finally:
+        app.dependency_overrides.pop(get_db, None)
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert set(payload) == {"data"}
+    assert payload["data"]["job_type"] == "analysis.change-detection"
+    assert payload["data"]["status"] == "success"
+    assert payload["data"]["result_json"]["unchanged_count"] == 1
+    assert any(isinstance(item, Job) for item in db.added)