Initial public release

backend/app/services/demo_workflow_service.py

@@ -0,0 +1,563 @@
+from __future__ import annotations
+
+import json
+import os
+import importlib
+from datetime import datetime, timezone
+from pathlib import Path
+from uuid import UUID, uuid4
+
+from geoalchemy2.shape import from_shape
+from sqlalchemy.orm import Session
+
+from app.core.errors import AppError
+from app.core.public_demo import (
+    PUBLIC_DEMO_PROJECT_ID,
+    PUBLIC_DEMO_PROJECT_MARKER,
+    PUBLIC_DEMO_PROJECT_NAME,
+)
+from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
+from app.schemas.demo import DemoWorkflowResponse
+from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
+from app.services.geojson_service import parse_geojson_payload
+from app.services.qa_service import QaService
+from app.services.quality_service import QualityService
+from app.services.raster_service import extract_raster_metadata
+from app.services.storage_service import StorageService
+from app.services.vector_feature_service import VectorFeatureService
+from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon
+
+
+class DemoWorkflowService:
+    PROJECT_ID = PUBLIC_DEMO_PROJECT_ID
+    PROJECT_NAME = PUBLIC_DEMO_PROJECT_NAME
+    AREA_NAME = "Demo AOI - Geel buildings"
+    REFERENCE_FILENAME = "demo_reference_buildings.geojson"
+    CANDIDATE_FILENAME = "demo_predicted_buildings.geojson"
+    RASTER_FILENAME = "demo_context_raster.tif"
+    EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
+
+    @staticmethod
+    def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
+        version = DatasetVersion(
+            dataset_id=dataset.id,
+            version=1,
+            storage_path=dataset.storage_path,
+            checksum_sha256=dataset.checksum_sha256,
+            source_metadata=dataset.source_metadata,
+            provenance_metadata=dataset.provenance_metadata,
+        )
+        db.add(version)
+        return version
+
+    @staticmethod
+    def _repo_root() -> Path:
+        return Path(__file__).resolve().parents[3]
+
+    @staticmethod
+    def _fixture_path(filename: str) -> Path:
+        roots: list[Path] = []
+        if os.getenv("GEOINTEL_FIXTURES_ROOT"):
+            roots.append(Path(os.environ["GEOINTEL_FIXTURES_ROOT"]))
+        roots.extend(parent / "fixtures" / "golden" for parent in Path(__file__).resolve().parents)
+        roots.append(Path("/app/fixtures/golden"))
+
+        for root in roots:
+            path = root / filename
+            if path.exists():
+                return path
+        return DemoWorkflowService._repo_root() / "fixtures" / "golden" / filename
+
+    @staticmethod
+    def _load_fixture(filename: str) -> tuple[dict, bytes]:
+        path = DemoWorkflowService._fixture_path(filename)
+        raw = path.read_bytes()
+        return json.loads(raw.decode("utf-8")), raw
+
+    @staticmethod
+    def _load_expected_metrics() -> dict:
+        payload, _raw = DemoWorkflowService._load_fixture(DemoWorkflowService.EXPECTED_METRICS_FILENAME)
+        return payload
+
+    @staticmethod
+    def _demo_area_geometry() -> dict:
+        return {
+            "type": "MultiPolygon",
+            "coordinates": [
+                [
+                    [
+                        [4.9895, 51.1595],
+                        [4.9930, 51.1595],
+                        [4.9930, 51.1615],
+                        [4.9895, 51.1615],
+                        [4.9895, 51.1595],
+                    ]
+                ]
+            ],
+        }
+
+    @staticmethod
+    def _find_existing_project(db: Session) -> Project | None:
+        project = db.get(Project, DemoWorkflowService.PROJECT_ID)
+        if project is None:
+            return None
+        if (
+            project.name != DemoWorkflowService.PROJECT_NAME
+            or project.description != PUBLIC_DEMO_PROJECT_MARKER
+        ):
+            raise AppError(
+                code="PUBLIC_DEMO_IDENTITY_CONFLICT",
+                message="The reserved public-demo project identity is already in use.",
+                status_code=409,
+            )
+        return project
+
+    @staticmethod
+    def _activate_explicit_demo_project(db: Session, project: Project | None) -> Project | None:
+        if project is None or project.status == "active":
+            return project
+        project.status = "active"
+        db.add(project)
+        db.commit()
+        db.refresh(project)
+        return project
+
+    @staticmethod
+    def _has_complete_demo_state(db: Session, project_id: UUID) -> bool:
+        area = db.query(Area).filter(Area.project_id == project_id).first()
+        reference = (
+            db.query(Dataset)
+            .filter(Dataset.project_id == project_id)
+            .filter(Dataset.dataset_role == "reference")
+            .filter(Dataset.source_name == "fixture")
+            .first()
+        )
+        candidate = (
+            db.query(Dataset)
+            .filter(Dataset.project_id == project_id)
+            .filter(Dataset.dataset_role == "source")
+            .filter(Dataset.source_name == "fixture")
+            .filter(Dataset.dataset_type == "vector")
+            .first()
+        )
+        raster = DemoWorkflowService._find_demo_raster_dataset(db, project_id)
+        quality_check = (
+            db.query(QualityCheck)
+            .filter(QualityCheck.project_id == project_id)
+            .filter(QualityCheck.check_type == "demo_candidate_vs_reference")
+            .first()
+        )
+        return bool(area and reference and candidate and raster and quality_check)
+
+    @staticmethod
+    def _find_demo_raster_dataset(db: Session, project_id: UUID) -> Dataset | None:
+        return (
+            db.query(Dataset)
+            .filter(Dataset.project_id == project_id)
+            .filter(Dataset.dataset_type == "raster")
+            .filter(Dataset.source_name == "fixture")
+            .filter(Dataset.name == DemoWorkflowService.RASTER_FILENAME)
+            .first()
+        )
+
+    @staticmethod
+    def _create_area(db: Session, project_id: UUID) -> Area:
+        geometry = DemoWorkflowService._demo_area_geometry()
+        multipolygon = normalize_to_multipolygon(geometry)
+        area = Area(
+            id=uuid4(),
+            project_id=project_id,
+            name=DemoWorkflowService.AREA_NAME,
+            geometry=from_shape(multipolygon, srid=4326),
+            original_crs="EPSG:4326",
+            area_m2=area_m2(multipolygon),
+            bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
+        )
+        db.add(area)
+        db.commit()
+        db.refresh(area)
+        return area
+
+    @staticmethod
+    def _sync_demo_area(db: Session, area: Area) -> Area:
+        multipolygon = normalize_to_multipolygon(DemoWorkflowService._demo_area_geometry())
+        area.name = DemoWorkflowService.AREA_NAME
+        area.geometry = from_shape(multipolygon, srid=4326)
+        area.original_crs = "EPSG:4326"
+        area.area_m2 = area_m2(multipolygon)
+        area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
+        db.add(area)
+        db.commit()
+        db.refresh(area)
+        return area
+
+    @staticmethod
+    def _create_dataset(
+        db: Session,
+        *,
+        project_id: UUID,
+        area_id: UUID,
+        filename: str,
+        payload: dict,
+        raw: bytes,
+        role: str,
+        source_name: str,
+        reference_layer_name: str | None,
+    ) -> Dataset:
+        dataset_id = uuid4()
+        storage_info = StorageService.persist_dataset_file(
+            project_id=str(project_id),
+            dataset_id=str(dataset_id),
+            dataset_type="vector",
+            original_filename=filename,
+            content=raw,
+            content_type="application/geo+json",
+        )
+        metadata = parse_geojson_payload(payload)
+        dataset = Dataset(
+            id=dataset_id,
+            project_id=project_id,
+            area_id=area_id,
+            name=filename,
+            dataset_type="vector",
+            source="fixture",
+            dataset_role=role,
+            source_name=source_name,
+            reference_layer_name=reference_layer_name,
+            source_metadata={
+                "fixture": True,
+                "fixture_name": filename,
+                "usage": "offline demo workflow only",
+            },
+            provenance_metadata={
+                "created_by": "demo_workflow",
+                "source_path": str(DemoWorkflowService._fixture_path(filename)),
+            },
+            imported_at=datetime.now(timezone.utc),
+            storage_path=storage_info["storage_path"],
+            original_filename=storage_info["original_filename"],
+            stored_filename=storage_info["stored_filename"],
+            content_type=storage_info["content_type"],
+            size_bytes=storage_info["size_bytes"],
+            checksum_sha256=storage_info["checksum_sha256"],
+            crs=metadata.get("crs"),
+            bounds_json=metadata.get("bounds_json"),
+            metadata_json=metadata,
+            status="validating",
+        )
+        db.add(dataset)
+        version = DemoWorkflowService._add_initial_version(db, dataset)
+        is_ready = DerivedDatasetGovernanceService.govern_vector(
+            db,
+            dataset=dataset,
+            dataset_version=version,
+            feature_collection=payload,
+            source_key="fixture",
+            operation="demo.fixture_vector",
+            operation_parameters={"fixture_name": filename, "role": role},
+        )
+        if is_ready:
+            VectorFeatureService.persist_geojson_features(
+                db=db,
+                dataset_id=dataset.id,
+                payload=payload,
+                feature_class=reference_layer_name or "building",
+                commit=False,
+            )
+        db.commit()
+        db.refresh(dataset)
+        return dataset
+
+    @staticmethod
+    def _create_demo_raster_bytes() -> bytes:
+        numpy = importlib.import_module("numpy")
+        rasterio_io = importlib.import_module("rasterio.io")
+        rasterio_transform = importlib.import_module("rasterio.transform")
+
+        width = 64
+        height = 48
+        data = numpy.linspace(20, 220, num=width * height, dtype=numpy.uint8).reshape((height, width))
+        transform = rasterio_transform.from_bounds(4.9895, 51.1595, 4.9930, 51.1615, width, height)
+        with rasterio_io.MemoryFile() as memfile:
+            with memfile.open(
+                driver="GTiff",
+                width=width,
+                height=height,
+                count=1,
+                dtype="uint8",
+                crs="EPSG:4326",
+                transform=transform,
+                nodata=0,
+            ) as dataset:
+                dataset.write(data, 1)
+            return memfile.read()
+
+    @staticmethod
+    def _create_raster_dataset(db: Session, *, project_id: UUID, area_id: UUID) -> Dataset:
+        dataset_id = uuid4()
+        raw = DemoWorkflowService._create_demo_raster_bytes()
+        storage_info = StorageService.persist_dataset_file(
+            project_id=str(project_id),
+            dataset_id=str(dataset_id),
+            dataset_type="raster",
+            original_filename=DemoWorkflowService.RASTER_FILENAME,
+            content=raw,
+            content_type="image/tiff",
+        )
+        metadata = extract_raster_metadata(storage_info["storage_path"])
+        bounds = metadata.get("bounds")
+        bounds_json = None
+        if isinstance(bounds, list) and len(bounds) == 4:
+            bounds_json = {"minx": bounds[0], "miny": bounds[1], "maxx": bounds[2], "maxy": bounds[3]}
+        dataset = Dataset(
+            id=dataset_id,
+            project_id=project_id,
+            area_id=area_id,
+            name=DemoWorkflowService.RASTER_FILENAME,
+            dataset_type="raster",
+            source="fixture",
+            dataset_role="source",
+            source_name="fixture",
+            reference_layer_name=None,
+            source_metadata={
+                "fixture": True,
+                "fixture_name": DemoWorkflowService.RASTER_FILENAME,
+                "usage": "offline demo raster workflow only",
+            },
+            provenance_metadata={
+                "created_by": "demo_workflow",
+                "source_path": "generated:demo_context_raster",
+            },
+            imported_at=datetime.now(timezone.utc),
+            storage_path=storage_info["storage_path"],
+            original_filename=storage_info["original_filename"],
+            stored_filename=storage_info["stored_filename"],
+            content_type=storage_info["content_type"],
+            size_bytes=storage_info["size_bytes"],
+            checksum_sha256=storage_info["checksum_sha256"],
+            crs=metadata.get("crs"),
+            bounds_json=bounds_json,
+            metadata_json=metadata,
+            status="validating",
+        )
+        db.add(dataset)
+        version = DemoWorkflowService._add_initial_version(db, dataset)
+        DerivedDatasetGovernanceService.govern_raster(
+            db,
+            dataset=dataset,
+            dataset_version=version,
+            raster_metadata=metadata,
+            source_key="fixture",
+            operation="demo.fixture_raster",
+            operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
+        )
+        db.commit()
+        db.refresh(dataset)
+        return dataset
+
+    @staticmethod
+    def _persist_qa(
+        db: Session,
+        *,
+        project_id: UUID,
+        candidate_dataset_id: UUID,
+        reference_dataset_id: UUID,
+        area_id: UUID,
+    ) -> QualityCheck:
+        result = QaService.compare_candidate_with_reference(
+            db=db,
+            project_id=project_id,
+            candidate_dataset_id=candidate_dataset_id,
+            reference_dataset_id=reference_dataset_id,
+            iou_threshold=0.5,
+            area_id=area_id,
+        )
+        return QualityService.persist_quality_check(
+            db=db,
+            project_id=project_id,
+            candidate_dataset_id=candidate_dataset_id,
+            reference_dataset_id=reference_dataset_id,
+            check_type="demo_candidate_vs_reference",
+            status=result.status,
+            score=result.f1_score,
+            parameters={
+                "iou_threshold": result.iou_threshold,
+                "area_id": str(area_id),
+                "fixture_workflow": True,
+            },
+            findings={
+                "matches": result.matches,
+                "false_positives": result.false_positives,
+                "false_negatives": result.false_negatives,
+                "warnings": result.warnings,
+                "unsupported_geometry": result.unsupported_geometry,
+                "unsupported_geometries": result.unsupported_geometries,
+            },
+            metrics={
+                "precision": result.precision,
+                "recall": result.recall,
+                "f1": result.f1_score,
+                "mean_iou": result.mean_iou,
+                "false_positive_count": result.false_positives,
+                "false_negative_count": result.false_negatives,
+            },
+        )
+
+    @staticmethod
+    def _quality_check_matches_expected(db: Session, quality_check: QualityCheck | None) -> bool:
+        if not quality_check or quality_check.status != "ok":
+            return False
+        expected = DemoWorkflowService._load_expected_metrics()
+        tolerance = float(expected.get("tolerance", 1e-9))
+        if quality_check.score is None or abs(float(quality_check.score) - float(expected["f1"])) > tolerance:
+            return False
+        findings = quality_check.findings_json or {}
+        if int(findings.get("matches", -1)) != int(expected["matches"]):
+            return False
+        if int(findings.get("false_positives", -1)) != int(expected["false_positive_count"]):
+            return False
+        if int(findings.get("false_negatives", -1)) != int(expected["false_negative_count"]):
+            return False
+
+        metrics = db.query(Metric).filter(Metric.quality_check_id == quality_check.id).all()
+        metric_values = {metric.metric_key: metric.metric_value for metric in metrics}
+        required = {
+            "precision": expected["precision"],
+            "recall": expected["recall"],
+            "f1": expected["f1"],
+            "mean_iou": expected["mean_iou"],
+            "false_positive_count": expected["false_positive_count"],
+            "false_negative_count": expected["false_negative_count"],
+        }
+        for key, expected_value in required.items():
+            actual = metric_values.get(key)
+            if actual is None or abs(float(actual) - float(expected_value)) > tolerance:
+                return False
+        return True
+
+    @staticmethod
+    def seed(db: Session) -> DemoWorkflowResponse:
+        existing = DemoWorkflowService._activate_explicit_demo_project(
+            db,
+            DemoWorkflowService._find_existing_project(db),
+        )
+        reference_payload, reference_raw = DemoWorkflowService._load_fixture("reference_buildings.geojson")
+        candidate_payload, candidate_raw = DemoWorkflowService._load_fixture("predicted_buildings.geojson")
+        if existing:
+            area = db.query(Area).filter(Area.project_id == existing.id).order_by(Area.created_at.asc()).first()
+            reference = (
+                db.query(Dataset)
+                .filter(Dataset.project_id == existing.id)
+                .filter(Dataset.dataset_role == "reference")
+                .filter(Dataset.source_name == "fixture")
+                .first()
+            )
+            candidate = (
+                db.query(Dataset)
+                .filter(Dataset.project_id == existing.id)
+                .filter(Dataset.dataset_role == "source")
+                .filter(Dataset.source_name == "fixture")
+                .filter(Dataset.dataset_type == "vector")
+                .first()
+            )
+            raster = DemoWorkflowService._find_demo_raster_dataset(db, existing.id)
+            quality_check = (
+                db.query(QualityCheck)
+                .filter(QualityCheck.project_id == existing.id)
+                .filter(QualityCheck.check_type == "demo_candidate_vs_reference")
+                .order_by(QualityCheck.created_at.desc())
+                .first()
+            )
+            if area and reference and candidate and quality_check:
+                if not raster:
+                    raster = DemoWorkflowService._create_raster_dataset(db=db, project_id=existing.id, area_id=area.id)
+                if not DemoWorkflowService._quality_check_matches_expected(db, quality_check):
+                    area = DemoWorkflowService._sync_demo_area(db, area)
+                    quality_check = DemoWorkflowService._persist_qa(
+                        db=db,
+                        project_id=existing.id,
+                        candidate_dataset_id=candidate.id,
+                        reference_dataset_id=reference.id,
+                        area_id=area.id,
+                    )
+                return DemoWorkflowResponse(
+                    project_id=existing.id,
+                    area_id=area.id,
+                    reference_dataset_id=reference.id,
+                    candidate_dataset_id=candidate.id,
+                    raster_dataset_id=raster.id,
+                    quality_check_id=quality_check.id,
+                    metric_count=db.query(Metric).filter(Metric.quality_check_id == quality_check.id).count(),
+                    status="ready",
+                    message="Demo workflow already exists.",
+                    created=False,
+                )
+            project = existing
+            created = True
+        else:
+            project = Project(
+                id=DemoWorkflowService.PROJECT_ID,
+                name=DemoWorkflowService.PROJECT_NAME,
+                description=PUBLIC_DEMO_PROJECT_MARKER,
+                region="Kempen",
+                status="active",
+            )
+            db.add(project)
+            db.commit()
+            db.refresh(project)
+            area = None
+            reference = None
+            candidate = None
+            raster = None
+            quality_check = None
+            created = True
+
+        if not area:
+            area = DemoWorkflowService._create_area(db, project.id)
+        if not reference:
+            reference = DemoWorkflowService._create_dataset(
+                db=db,
+                project_id=project.id,
+                area_id=area.id,
+                filename=DemoWorkflowService.REFERENCE_FILENAME,
+                payload=reference_payload,
+                raw=reference_raw,
+                role="reference",
+                source_name="fixture",
+                reference_layer_name="buildings",
+            )
+        if not candidate:
+            candidate = DemoWorkflowService._create_dataset(
+                db=db,
+                project_id=project.id,
+                area_id=area.id,
+                filename=DemoWorkflowService.CANDIDATE_FILENAME,
+                payload=candidate_payload,
+                raw=candidate_raw,
+                role="source",
+                source_name="fixture",
+                reference_layer_name=None,
+            )
+        if not raster:
+            raster = DemoWorkflowService._create_raster_dataset(db=db, project_id=project.id, area_id=area.id)
+        if not quality_check:
+            quality_check = DemoWorkflowService._persist_qa(
+                db=db,
+                project_id=project.id,
+                candidate_dataset_id=candidate.id,
+                reference_dataset_id=reference.id,
+                area_id=area.id,
+            )
+
+        return DemoWorkflowResponse(
+            project_id=project.id,
+            area_id=area.id,
+            reference_dataset_id=reference.id,
+            candidate_dataset_id=candidate.id,
+            raster_dataset_id=raster.id,
+            quality_check_id=quality_check.id,
+            metric_count=6,
+            status="ready",
+            message="Demo workflow seeded from explicit local fixtures.",
+            created=created,
+        )