from __future__ import annotations import json import pathlib from datetime import datetime, timezone from pathlib import Path from typing import Any from uuid import UUID import uuid from fastapi import UploadFile from sqlalchemy.orm import Session from app.core.errors import AppError from app.models import Dataset, Project from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary from app.services.geojson_service import parse_geojson_payload, load_dataset_text from app.services.raster_service import extract_raster_metadata from app.services.storage_service import StorageService from app.services.vector_feature_service import VectorFeatureService class DatasetService: VECTOR_EXTENSIONS = {".geojson", ".json"} RASTER_EXTENSIONS = {".tif", ".tiff", ".geotiff"} VECTOR_TYPES = {"vector", "geojson"} RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"} VALID_DATASET_ROLES = {"source", "derived", "reference"} @staticmethod def _canonical_dataset_type(dataset_type: str) -> str: normalized = (dataset_type or "").strip().lower() if normalized in DatasetService.VECTOR_TYPES: return "vector" if normalized in DatasetService.RASTER_TYPES: return "raster" raise AppError( code="INVALID_DATASET_TYPE", message="dataset_type must be 'vector' or 'raster' (or legacy 'geojson')", status_code=400, ) @staticmethod def _normalize_stored_dataset_type(dataset_type: str) -> str: normalized = (dataset_type or "").strip().lower() if normalized in DatasetService.VECTOR_TYPES: return "vector" if normalized in DatasetService.RASTER_TYPES: return "raster" return normalized @staticmethod def _is_vector_type(dataset_type: str) -> bool: return DatasetService._normalize_stored_dataset_type(dataset_type) == "vector" @staticmethod def _is_raster_type(dataset_type: str) -> bool: return DatasetService._normalize_stored_dataset_type(dataset_type) == "raster" @staticmethod def _normalize_dataset_role(dataset_role: str | None) -> str: normalized = (dataset_role or "").strip().lower() or "source" if normalized not in DatasetService.VALID_DATASET_ROLES: raise AppError( code="INVALID_DATASET_ROLE", message="dataset_role must be one of: source, derived, reference", status_code=400, ) return normalized @staticmethod def _extension_for_path(filename: str) -> str: return Path(filename).suffix.lower() @staticmethod def _validate_upload_filename(filename: str | None) -> str: if not filename: raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400) return filename @staticmethod def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]: total = db.query(Dataset).filter(Dataset.project_id == project_id).count() rows = ( db.query(Dataset) .filter(Dataset.project_id == project_id) .order_by(Dataset.created_at.desc()) .offset(offset) .limit(limit) .all() ) response_items = [] for row in rows: feature_count = None metadata_json = row.metadata_json or {} vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json) if isinstance(metadata_json, dict): feature_count = metadata_json.get("feature_count") response_items.append( DatasetCreateResponse( id=row.id, name=row.name, dataset_type=row.dataset_type, source=row.source, dataset_role=row.dataset_role, source_name=row.source_name, reference_layer_name=row.reference_layer_name, source_metadata=row.source_metadata, provenance_metadata=row.provenance_metadata, imported_at=row.imported_at, project_id=row.project_id, area_id=row.area_id, storage_path=row.storage_path, original_filename=row.original_filename, stored_filename=row.stored_filename, content_type=row.content_type, size_bytes=row.size_bytes, checksum_sha256=row.checksum_sha256, crs=row.crs, bounds_json=row.bounds_json, metadata_json=row.metadata_json, vector_summary=vector_summary, status=row.status, derived_from_dataset_id=row.derived_from_dataset_id, created_at=row.created_at, feature_count=feature_count, ) ) return response_items, total @staticmethod def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None: if not DatasetService._is_vector_type(dataset_type): return None if not isinstance(metadata_json, dict): return None return DatasetVectorSummary( feature_count=metadata_json.get("feature_count"), geometry_types=metadata_json.get("geometry_types"), bounds_json=metadata_json.get("bounds_json"), approximate_area_m2=metadata_json.get("approximate_area_m2"), crs=metadata_json.get("crs"), feature_geometry_count=metadata_json.get("feature_geometry_count"), invalid_features=metadata_json.get("invalid_features"), crs_assumed=metadata_json.get("crs_assumed"), ) @staticmethod def _extract_raster_bounds_json(metadata_json: dict[str, Any]) -> dict[str, float] | None: existing = metadata_json.get("bounds_json") if isinstance(existing, dict): return existing bounds = metadata_json.get("bounds") if isinstance(bounds, (list, tuple)) and len(bounds) == 4: return { "minx": float(bounds[0]), "miny": float(bounds[1]), "maxx": float(bounds[2]), "maxy": float(bounds[3]), } return None @staticmethod def _extract_raster_resolution_json(metadata_json: dict[str, Any]) -> dict[str, float] | None: existing = metadata_json.get("resolution_json") if isinstance(existing, dict): return existing resolution = metadata_json.get("resolution") if isinstance(resolution, (list, tuple)) and len(resolution) >= 2: return {"x": float(resolution[0]), "y": float(resolution[1])} return None @staticmethod def _extract_raster_bands_json(metadata_json: dict[str, Any]) -> dict[str, Any] | None: existing = metadata_json.get("bands_json") if isinstance(existing, dict): return existing bands_json: dict[str, Any] = {} if metadata_json.get("band_count") is not None: bands_json["band_count"] = int(metadata_json["band_count"]) if metadata_json.get("dtype") is not None: bands_json["dtype"] = metadata_json["dtype"] return bands_json or None @staticmethod async def upload_dataset( db: Session, project_id: UUID, file: UploadFile, dataset_type: str, source: str, dataset_role: str = "source", source_name: str | None = None, reference_layer_name: str | None = None, source_metadata: dict | None = None, provenance_metadata: dict | None = None, area_id: UUID | None = None, ) -> DatasetCreateResponse: if not db.get(Project, project_id): raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) filename = DatasetService._validate_upload_filename(file.filename) canonical_type = DatasetService._canonical_dataset_type(dataset_type) normalized_role = DatasetService._normalize_dataset_role(dataset_role) normalized_source_name = source_name if normalized_role == "reference" and not normalized_source_name: normalized_source_name = "manual" if normalized_role == "reference" and canonical_type == "raster": raise AppError(code="INVALID_DATASET_ROLE", message="reference role requires vector dataset type", status_code=400) extension = DatasetService._extension_for_path(filename) if canonical_type == "vector" and extension not in DatasetService.VECTOR_EXTENSIONS: raise AppError(code="INVALID_UPLOAD", message="Vector uploads require .geojson or .json files", status_code=415) if canonical_type == "raster" and extension not in DatasetService.RASTER_EXTENSIONS: raise AppError( code="INVALID_UPLOAD", message="Raster uploads require .tif, .tiff or .geotiff files", status_code=415, ) raw = await file.read() storage_info = StorageService.persist_dataset_file( project_id=str(project_id), dataset_id=str(dataset_id := uuid.uuid4()), dataset_type=canonical_type, original_filename=filename, content=raw, content_type=file.content_type, ) metadata: dict[str, Any] = {} vector_payload: dict[str, Any] | None = None status = "uploaded" try: status = "validating" if canonical_type == "vector": try: text = raw.decode("utf-8") except UnicodeDecodeError as exc: raise AppError(code="INVALID_UPLOAD", message="Upload must be UTF-8 encoded", status_code=400) from exc metadata = parse_geojson_payload(text) vector_payload = json.loads(text) status = "ready" else: metadata = extract_raster_metadata(storage_info["storage_path"]) status = "ready" except ValueError as exc: status = "failed" StorageService.remove_dataset_file(storage_info["storage_path"]) raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc except AppError as exc: if canonical_type == "raster" and exc.code == "RASTER_PROCESSING_UNAVAILABLE": status = "failed" metadata = { "processing_error": exc.message, "processing_code": exc.code, } else: StorageService.remove_dataset_file(storage_info["storage_path"]) raise bounds_json = metadata.get("bounds_json") if isinstance(metadata, dict) else None resolution_json = metadata.get("resolution_json") if isinstance(metadata, dict) else None bands_json = metadata.get("bands_json") if isinstance(metadata, dict) else None if canonical_type == "raster" and isinstance(metadata, dict): bounds_json = DatasetService._extract_raster_bounds_json(metadata) resolution_json = DatasetService._extract_raster_resolution_json(metadata) bands_json = DatasetService._extract_raster_bands_json(metadata) dataset = Dataset( id=dataset_id, project_id=project_id, area_id=area_id, name=filename, dataset_type=canonical_type, source=source, dataset_role=normalized_role, source_name=normalized_source_name, reference_layer_name=reference_layer_name if normalized_role == "reference" else None, source_metadata=source_metadata, provenance_metadata=provenance_metadata, imported_at=datetime.now(timezone.utc), storage_path=storage_info["storage_path"], original_filename=storage_info["original_filename"], stored_filename=storage_info["stored_filename"], content_type=storage_info["content_type"], size_bytes=storage_info["size_bytes"], checksum_sha256=storage_info["checksum_sha256"], crs=metadata.get("crs") if isinstance(metadata, dict) else None, bounds_json=bounds_json, resolution_json=resolution_json, bands_json=bands_json, metadata_json=metadata, status=status, ) db.add(dataset) db.commit() db.refresh(dataset) if canonical_type == "vector" and vector_payload is not None and status == "ready": feature_class = reference_layer_name if normalized_role == "reference" else None VectorFeatureService.persist_geojson_features( db=db, dataset_id=dataset.id, payload=vector_payload, feature_class=feature_class, ) return DatasetCreateResponse( id=dataset.id, name=dataset.name, dataset_type=dataset.dataset_type, source=dataset.source, dataset_role=dataset.dataset_role, source_name=dataset.source_name, reference_layer_name=dataset.reference_layer_name, source_metadata=dataset.source_metadata, provenance_metadata=dataset.provenance_metadata, imported_at=dataset.imported_at, project_id=dataset.project_id, area_id=dataset.area_id, storage_path=dataset.storage_path, original_filename=dataset.original_filename, stored_filename=dataset.stored_filename, content_type=dataset.content_type, size_bytes=dataset.size_bytes, checksum_sha256=dataset.checksum_sha256, crs=dataset.crs, derived_from_dataset_id=dataset.derived_from_dataset_id, bounds_json=dataset.bounds_json, metadata_json=dataset.metadata_json, vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}), status=dataset.status, created_at=dataset.created_at, feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None, ) @staticmethod def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse: dataset = DatasetService._get_dataset(db, dataset_id) if not dataset.storage_path: raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) if not Path(dataset.storage_path).exists(): raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) try: if DatasetService._is_vector_type(dataset.dataset_type): metadata = parse_geojson_payload(load_dataset_text(dataset.storage_path)) elif DatasetService._is_raster_type(dataset.dataset_type): metadata = extract_raster_metadata(dataset.storage_path) else: raise AppError(code="INVALID_DATASET_TYPE", message="Cannot refresh metadata for this dataset type", status_code=400) dataset.status = "ready" except ValueError as exc: dataset.status = "failed" raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc except AppError as exc: if DatasetService._is_raster_type(dataset.dataset_type) and exc.code == "RASTER_PROCESSING_UNAVAILABLE": dataset.status = "failed" metadata = {"processing_error": exc.message, "processing_code": exc.code} else: dataset.status = "failed" raise bounds_json = metadata.get("bounds_json") if isinstance(metadata, dict) else dataset.bounds_json resolution_json = metadata.get("resolution_json") if isinstance(metadata, dict) else dataset.resolution_json bands_json = metadata.get("bands_json") if isinstance(metadata, dict) else dataset.bands_json if DatasetService._is_raster_type(dataset.dataset_type) and isinstance(metadata, dict): bounds_json = DatasetService._extract_raster_bounds_json(metadata) resolution_json = DatasetService._extract_raster_resolution_json(metadata) bands_json = DatasetService._extract_raster_bands_json(metadata) dataset.crs = metadata.get("crs") if isinstance(metadata, dict) else dataset.crs dataset.bounds_json = bounds_json dataset.metadata_json = metadata dataset.resolution_json = resolution_json dataset.bands_json = bands_json db.add(dataset) db.commit() db.refresh(dataset) return DatasetCreateResponse( id=dataset.id, name=dataset.name, dataset_type=dataset.dataset_type, source=dataset.source, dataset_role=dataset.dataset_role, source_name=dataset.source_name, reference_layer_name=dataset.reference_layer_name, source_metadata=dataset.source_metadata, provenance_metadata=dataset.provenance_metadata, imported_at=dataset.imported_at, project_id=dataset.project_id, area_id=dataset.area_id, storage_path=dataset.storage_path, original_filename=dataset.original_filename, stored_filename=dataset.stored_filename, content_type=dataset.content_type, size_bytes=dataset.size_bytes, checksum_sha256=dataset.checksum_sha256, crs=dataset.crs, derived_from_dataset_id=dataset.derived_from_dataset_id, bounds_json=dataset.bounds_json, metadata_json=dataset.metadata_json, vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}), status=dataset.status, created_at=dataset.created_at, feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None, ) @staticmethod def get_dataset(db: Session, dataset_id: UUID) -> Dataset: dataset = db.get(Dataset, dataset_id) if not dataset: raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) return dataset @staticmethod def _get_dataset(db: Session, dataset_id: UUID) -> Dataset: return DatasetService.get_dataset(db, dataset_id) @staticmethod def get_dataset_geojson(db: Session, dataset_id: UUID) -> dict: dataset = DatasetService._get_dataset(db, dataset_id) if not DatasetService._is_vector_type(dataset.dataset_type): raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) if not dataset.storage_path: raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) if not pathlib.Path(dataset.storage_path).exists(): raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) raw = load_dataset_text(dataset.storage_path) try: return json.loads(raw) except Exception as exc: raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=500) from exc @staticmethod def inspect_vector_dataset(db: Session, dataset_id: UUID) -> dict[str, Any]: dataset = DatasetService._get_dataset(db, dataset_id) if not DatasetService._is_vector_type(dataset.dataset_type): raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) if not dataset.storage_path or not Path(dataset.storage_path).exists(): raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) metadata = dataset.metadata_json or {} if not isinstance(metadata, dict): metadata = {} summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata) return { "dataset": { "id": str(dataset.id), "name": dataset.name, "dataset_type": dataset.dataset_type, "status": dataset.status, "source": dataset.source, "storage": DatasetStorageResponse( original_filename=dataset.original_filename, stored_filename=dataset.stored_filename, content_type=dataset.content_type, size_bytes=dataset.size_bytes, checksum_sha256=dataset.checksum_sha256, ).model_dump(), "feature_count": metadata.get("feature_count"), "crs": metadata.get("crs"), }, "summary": summary.model_dump() if summary else None, "metadata": metadata, } @staticmethod def vector_summary(db: Session, dataset_id: UUID) -> dict[str, Any]: dataset = DatasetService._get_dataset(db, dataset_id) if not DatasetService._is_vector_type(dataset.dataset_type): raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400) metadata = dataset.metadata_json or {} if not isinstance(metadata, dict): metadata = {} summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata) if not summary: raise AppError(code="INVALID_GEOJSON", message="Vector summary unavailable", status_code=422) return summary.model_dump() @staticmethod def raster_metadata(db: Session, dataset_id: UUID) -> dict[str, Any]: dataset = DatasetService._get_dataset(db, dataset_id) if not DatasetService._is_raster_type(dataset.dataset_type): raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a raster dataset", status_code=400) if not dataset.storage_path: raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) if not Path(dataset.storage_path).exists(): raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) if isinstance(dataset.metadata_json, dict) and dataset.metadata_json.get("driver"): return dataset.metadata_json metadata = extract_raster_metadata(dataset.storage_path) dataset.metadata_json = dict(dataset.metadata_json or {}) dataset.metadata_json.update(metadata) dataset.status = "ready" db.add(dataset) db.commit() db.refresh(dataset) return metadata