453 lines
20 KiB
Python
453 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import pathlib
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import UUID
|
|
import uuid
|
|
|
|
from fastapi import UploadFile
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.errors import AppError
|
|
from app.models import Dataset, Project
|
|
from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary
|
|
from app.services.geojson_service import parse_geojson_payload, load_dataset_text
|
|
from app.services.raster_service import extract_raster_metadata
|
|
from app.services.storage_service import StorageService
|
|
from app.services.vector_feature_service import VectorFeatureService
|
|
|
|
|
|
class DatasetService:
|
|
VECTOR_EXTENSIONS = {".geojson", ".json"}
|
|
RASTER_EXTENSIONS = {".tif", ".tiff", ".geotiff"}
|
|
VECTOR_TYPES = {"vector", "geojson"}
|
|
RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"}
|
|
VALID_DATASET_ROLES = {"source", "derived", "reference"}
|
|
|
|
@staticmethod
|
|
def _canonical_dataset_type(dataset_type: str) -> str:
|
|
normalized = (dataset_type or "").strip().lower()
|
|
if normalized in DatasetService.VECTOR_TYPES:
|
|
return "vector"
|
|
if normalized in DatasetService.RASTER_TYPES:
|
|
return "raster"
|
|
raise AppError(
|
|
code="INVALID_DATASET_TYPE",
|
|
message="dataset_type must be 'vector' or 'raster' (or legacy 'geojson')",
|
|
status_code=400,
|
|
)
|
|
|
|
@staticmethod
|
|
def _normalize_stored_dataset_type(dataset_type: str) -> str:
|
|
normalized = (dataset_type or "").strip().lower()
|
|
if normalized in DatasetService.VECTOR_TYPES:
|
|
return "vector"
|
|
if normalized in DatasetService.RASTER_TYPES:
|
|
return "raster"
|
|
return normalized
|
|
|
|
@staticmethod
|
|
def _is_vector_type(dataset_type: str) -> bool:
|
|
return DatasetService._normalize_stored_dataset_type(dataset_type) == "vector"
|
|
|
|
@staticmethod
|
|
def _is_raster_type(dataset_type: str) -> bool:
|
|
return DatasetService._normalize_stored_dataset_type(dataset_type) == "raster"
|
|
|
|
@staticmethod
|
|
def _normalize_dataset_role(dataset_role: str | None) -> str:
|
|
normalized = (dataset_role or "").strip().lower() or "source"
|
|
if normalized not in DatasetService.VALID_DATASET_ROLES:
|
|
raise AppError(
|
|
code="INVALID_DATASET_ROLE",
|
|
message="dataset_role must be one of: source, derived, reference",
|
|
status_code=400,
|
|
)
|
|
return normalized
|
|
|
|
@staticmethod
|
|
def _extension_for_path(filename: str) -> str:
|
|
return Path(filename).suffix.lower()
|
|
|
|
@staticmethod
|
|
def _validate_upload_filename(filename: str | None) -> str:
|
|
if not filename:
|
|
raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
|
|
return filename
|
|
|
|
@staticmethod
|
|
def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
|
|
total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
|
|
rows = (
|
|
db.query(Dataset)
|
|
.filter(Dataset.project_id == project_id)
|
|
.order_by(Dataset.created_at.desc())
|
|
.offset(offset)
|
|
.limit(limit)
|
|
.all()
|
|
)
|
|
response_items = []
|
|
for row in rows:
|
|
feature_count = None
|
|
metadata_json = row.metadata_json or {}
|
|
vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json)
|
|
if isinstance(metadata_json, dict):
|
|
feature_count = metadata_json.get("feature_count")
|
|
response_items.append(
|
|
DatasetCreateResponse(
|
|
id=row.id,
|
|
name=row.name,
|
|
dataset_type=row.dataset_type,
|
|
source=row.source,
|
|
dataset_role=row.dataset_role,
|
|
source_name=row.source_name,
|
|
reference_layer_name=row.reference_layer_name,
|
|
source_metadata=row.source_metadata,
|
|
provenance_metadata=row.provenance_metadata,
|
|
imported_at=row.imported_at,
|
|
project_id=row.project_id,
|
|
area_id=row.area_id,
|
|
storage_path=row.storage_path,
|
|
original_filename=row.original_filename,
|
|
stored_filename=row.stored_filename,
|
|
content_type=row.content_type,
|
|
size_bytes=row.size_bytes,
|
|
checksum_sha256=row.checksum_sha256,
|
|
crs=row.crs,
|
|
bounds_json=row.bounds_json,
|
|
metadata_json=row.metadata_json,
|
|
vector_summary=vector_summary,
|
|
status=row.status,
|
|
derived_from_dataset_id=row.derived_from_dataset_id,
|
|
created_at=row.created_at,
|
|
feature_count=feature_count,
|
|
)
|
|
)
|
|
return response_items, total
|
|
|
|
@staticmethod
|
|
def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None:
|
|
if not DatasetService._is_vector_type(dataset_type):
|
|
return None
|
|
if not isinstance(metadata_json, dict):
|
|
return None
|
|
return DatasetVectorSummary(
|
|
feature_count=metadata_json.get("feature_count"),
|
|
geometry_types=metadata_json.get("geometry_types"),
|
|
bounds_json=metadata_json.get("bounds_json"),
|
|
approximate_area_m2=metadata_json.get("approximate_area_m2"),
|
|
crs=metadata_json.get("crs"),
|
|
feature_geometry_count=metadata_json.get("feature_geometry_count"),
|
|
invalid_features=metadata_json.get("invalid_features"),
|
|
crs_assumed=metadata_json.get("crs_assumed"),
|
|
)
|
|
|
|
@staticmethod
|
|
async def upload_dataset(
|
|
db: Session,
|
|
project_id: UUID,
|
|
file: UploadFile,
|
|
dataset_type: str,
|
|
source: str,
|
|
dataset_role: str = "source",
|
|
source_name: str | None = None,
|
|
reference_layer_name: str | None = None,
|
|
source_metadata: dict | None = None,
|
|
provenance_metadata: dict | None = None,
|
|
area_id: UUID | None = None,
|
|
) -> DatasetCreateResponse:
|
|
if not db.get(Project, project_id):
|
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
|
|
|
filename = DatasetService._validate_upload_filename(file.filename)
|
|
canonical_type = DatasetService._canonical_dataset_type(dataset_type)
|
|
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
|
|
normalized_source_name = source_name
|
|
if normalized_role == "reference" and not normalized_source_name:
|
|
normalized_source_name = "manual"
|
|
if normalized_role == "reference" and canonical_type == "raster":
|
|
raise AppError(code="INVALID_DATASET_ROLE", message="reference role requires vector dataset type", status_code=400)
|
|
extension = DatasetService._extension_for_path(filename)
|
|
|
|
if canonical_type == "vector" and extension not in DatasetService.VECTOR_EXTENSIONS:
|
|
raise AppError(code="INVALID_UPLOAD", message="Vector uploads require .geojson or .json files", status_code=415)
|
|
if canonical_type == "raster" and extension not in DatasetService.RASTER_EXTENSIONS:
|
|
raise AppError(
|
|
code="INVALID_UPLOAD",
|
|
message="Raster uploads require .tif, .tiff or .geotiff files",
|
|
status_code=415,
|
|
)
|
|
|
|
raw = await file.read()
|
|
storage_info = StorageService.persist_dataset_file(
|
|
project_id=str(project_id),
|
|
dataset_id=str(dataset_id := uuid.uuid4()),
|
|
dataset_type=canonical_type,
|
|
original_filename=filename,
|
|
content=raw,
|
|
content_type=file.content_type,
|
|
)
|
|
|
|
metadata: dict[str, Any] = {}
|
|
vector_payload: dict[str, Any] | None = None
|
|
status = "uploaded"
|
|
try:
|
|
status = "validating"
|
|
if canonical_type == "vector":
|
|
try:
|
|
text = raw.decode("utf-8")
|
|
except UnicodeDecodeError as exc:
|
|
raise AppError(code="INVALID_UPLOAD", message="Upload must be UTF-8 encoded", status_code=400) from exc
|
|
metadata = parse_geojson_payload(text)
|
|
vector_payload = json.loads(text)
|
|
status = "ready"
|
|
else:
|
|
metadata = extract_raster_metadata(storage_info["storage_path"])
|
|
status = "ready"
|
|
except ValueError as exc:
|
|
status = "failed"
|
|
StorageService.remove_dataset_file(storage_info["storage_path"])
|
|
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
|
|
except AppError as exc:
|
|
if canonical_type == "raster" and exc.code == "RASTER_PROCESSING_UNAVAILABLE":
|
|
status = "failed"
|
|
metadata = {
|
|
"processing_error": exc.message,
|
|
"processing_code": exc.code,
|
|
}
|
|
else:
|
|
StorageService.remove_dataset_file(storage_info["storage_path"])
|
|
raise
|
|
|
|
dataset = Dataset(
|
|
id=dataset_id,
|
|
project_id=project_id,
|
|
area_id=area_id,
|
|
name=filename,
|
|
dataset_type=canonical_type,
|
|
source=source,
|
|
dataset_role=normalized_role,
|
|
source_name=normalized_source_name,
|
|
reference_layer_name=reference_layer_name if normalized_role == "reference" else None,
|
|
source_metadata=source_metadata,
|
|
provenance_metadata=provenance_metadata,
|
|
imported_at=datetime.now(timezone.utc),
|
|
storage_path=storage_info["storage_path"],
|
|
original_filename=storage_info["original_filename"],
|
|
stored_filename=storage_info["stored_filename"],
|
|
content_type=storage_info["content_type"],
|
|
size_bytes=storage_info["size_bytes"],
|
|
checksum_sha256=storage_info["checksum_sha256"],
|
|
crs=metadata.get("crs") if isinstance(metadata, dict) else None,
|
|
bounds_json=metadata.get("bounds_json") if isinstance(metadata, dict) else None,
|
|
resolution_json=metadata.get("resolution_json") if isinstance(metadata, dict) else None,
|
|
bands_json=metadata.get("bands_json") if isinstance(metadata, dict) else None,
|
|
metadata_json=metadata,
|
|
status=status,
|
|
)
|
|
db.add(dataset)
|
|
db.commit()
|
|
db.refresh(dataset)
|
|
|
|
if canonical_type == "vector" and vector_payload is not None and status == "ready":
|
|
feature_class = reference_layer_name if normalized_role == "reference" else None
|
|
VectorFeatureService.persist_geojson_features(
|
|
db=db,
|
|
dataset_id=dataset.id,
|
|
payload=vector_payload,
|
|
feature_class=feature_class,
|
|
)
|
|
|
|
return DatasetCreateResponse(
|
|
id=dataset.id,
|
|
name=dataset.name,
|
|
dataset_type=dataset.dataset_type,
|
|
source=dataset.source,
|
|
dataset_role=dataset.dataset_role,
|
|
source_name=dataset.source_name,
|
|
reference_layer_name=dataset.reference_layer_name,
|
|
source_metadata=dataset.source_metadata,
|
|
provenance_metadata=dataset.provenance_metadata,
|
|
imported_at=dataset.imported_at,
|
|
project_id=dataset.project_id,
|
|
area_id=dataset.area_id,
|
|
storage_path=dataset.storage_path,
|
|
original_filename=dataset.original_filename,
|
|
stored_filename=dataset.stored_filename,
|
|
content_type=dataset.content_type,
|
|
size_bytes=dataset.size_bytes,
|
|
checksum_sha256=dataset.checksum_sha256,
|
|
crs=dataset.crs,
|
|
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
|
bounds_json=dataset.bounds_json,
|
|
metadata_json=dataset.metadata_json,
|
|
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
|
status=dataset.status,
|
|
created_at=dataset.created_at,
|
|
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
|
)
|
|
|
|
@staticmethod
|
|
def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse:
|
|
dataset = DatasetService._get_dataset(db, dataset_id)
|
|
if not dataset.storage_path:
|
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
|
if not Path(dataset.storage_path).exists():
|
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
|
|
|
try:
|
|
if DatasetService._is_vector_type(dataset.dataset_type):
|
|
metadata = parse_geojson_payload(load_dataset_text(dataset.storage_path))
|
|
elif DatasetService._is_raster_type(dataset.dataset_type):
|
|
metadata = extract_raster_metadata(dataset.storage_path)
|
|
else:
|
|
raise AppError(code="INVALID_DATASET_TYPE", message="Cannot refresh metadata for this dataset type", status_code=400)
|
|
dataset.status = "ready"
|
|
except ValueError as exc:
|
|
dataset.status = "failed"
|
|
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
|
|
except AppError as exc:
|
|
if DatasetService._is_raster_type(dataset.dataset_type) and exc.code == "RASTER_PROCESSING_UNAVAILABLE":
|
|
dataset.status = "failed"
|
|
metadata = {"processing_error": exc.message, "processing_code": exc.code}
|
|
else:
|
|
dataset.status = "failed"
|
|
raise
|
|
|
|
dataset.crs = metadata.get("crs") if isinstance(metadata, dict) else dataset.crs
|
|
dataset.bounds_json = metadata.get("bounds_json") if isinstance(metadata, dict) else dataset.bounds_json
|
|
dataset.metadata_json = metadata
|
|
dataset.resolution_json = metadata.get("resolution_json") if isinstance(metadata, dict) else dataset.resolution_json
|
|
dataset.bands_json = metadata.get("bands_json") if isinstance(metadata, dict) else dataset.bands_json
|
|
|
|
db.add(dataset)
|
|
db.commit()
|
|
db.refresh(dataset)
|
|
|
|
return DatasetCreateResponse(
|
|
id=dataset.id,
|
|
name=dataset.name,
|
|
dataset_type=dataset.dataset_type,
|
|
source=dataset.source,
|
|
dataset_role=dataset.dataset_role,
|
|
source_name=dataset.source_name,
|
|
reference_layer_name=dataset.reference_layer_name,
|
|
source_metadata=dataset.source_metadata,
|
|
provenance_metadata=dataset.provenance_metadata,
|
|
imported_at=dataset.imported_at,
|
|
project_id=dataset.project_id,
|
|
area_id=dataset.area_id,
|
|
storage_path=dataset.storage_path,
|
|
original_filename=dataset.original_filename,
|
|
stored_filename=dataset.stored_filename,
|
|
content_type=dataset.content_type,
|
|
size_bytes=dataset.size_bytes,
|
|
checksum_sha256=dataset.checksum_sha256,
|
|
crs=dataset.crs,
|
|
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
|
bounds_json=dataset.bounds_json,
|
|
metadata_json=dataset.metadata_json,
|
|
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
|
status=dataset.status,
|
|
created_at=dataset.created_at,
|
|
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
|
)
|
|
|
|
@staticmethod
|
|
def get_dataset(db: Session, dataset_id: UUID) -> Dataset:
|
|
dataset = db.get(Dataset, dataset_id)
|
|
if not dataset:
|
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
|
return dataset
|
|
|
|
@staticmethod
|
|
def _get_dataset(db: Session, dataset_id: UUID) -> Dataset:
|
|
return DatasetService.get_dataset(db, dataset_id)
|
|
|
|
@staticmethod
|
|
def get_dataset_geojson(db: Session, dataset_id: UUID) -> dict:
|
|
dataset = DatasetService._get_dataset(db, dataset_id)
|
|
if not DatasetService._is_vector_type(dataset.dataset_type):
|
|
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
|
|
if not dataset.storage_path:
|
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
|
if not pathlib.Path(dataset.storage_path).exists():
|
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
|
|
|
raw = load_dataset_text(dataset.storage_path)
|
|
try:
|
|
return json.loads(raw)
|
|
except Exception as exc:
|
|
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=500) from exc
|
|
|
|
@staticmethod
|
|
def inspect_vector_dataset(db: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
dataset = DatasetService._get_dataset(db, dataset_id)
|
|
if not DatasetService._is_vector_type(dataset.dataset_type):
|
|
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
|
|
if not dataset.storage_path or not Path(dataset.storage_path).exists():
|
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
|
metadata = dataset.metadata_json or {}
|
|
if not isinstance(metadata, dict):
|
|
metadata = {}
|
|
summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata)
|
|
return {
|
|
"dataset": {
|
|
"id": str(dataset.id),
|
|
"name": dataset.name,
|
|
"dataset_type": dataset.dataset_type,
|
|
"status": dataset.status,
|
|
"source": dataset.source,
|
|
"storage": DatasetStorageResponse(
|
|
original_filename=dataset.original_filename,
|
|
stored_filename=dataset.stored_filename,
|
|
content_type=dataset.content_type,
|
|
size_bytes=dataset.size_bytes,
|
|
checksum_sha256=dataset.checksum_sha256,
|
|
).model_dump(),
|
|
"feature_count": metadata.get("feature_count"),
|
|
"crs": metadata.get("crs"),
|
|
},
|
|
"summary": summary.model_dump() if summary else None,
|
|
"metadata": metadata,
|
|
}
|
|
|
|
@staticmethod
|
|
def vector_summary(db: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
dataset = DatasetService._get_dataset(db, dataset_id)
|
|
if not DatasetService._is_vector_type(dataset.dataset_type):
|
|
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
|
|
|
|
metadata = dataset.metadata_json or {}
|
|
if not isinstance(metadata, dict):
|
|
metadata = {}
|
|
summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata)
|
|
if not summary:
|
|
raise AppError(code="INVALID_GEOJSON", message="Vector summary unavailable", status_code=422)
|
|
return summary.model_dump()
|
|
|
|
@staticmethod
|
|
def raster_metadata(db: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
dataset = DatasetService._get_dataset(db, dataset_id)
|
|
if not DatasetService._is_raster_type(dataset.dataset_type):
|
|
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a raster dataset", status_code=400)
|
|
if not dataset.storage_path:
|
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
|
if not Path(dataset.storage_path).exists():
|
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
|
|
|
if isinstance(dataset.metadata_json, dict) and dataset.metadata_json.get("driver"):
|
|
return dataset.metadata_json
|
|
|
|
metadata = extract_raster_metadata(dataset.storage_path)
|
|
dataset.metadata_json = dict(dataset.metadata_json or {})
|
|
dataset.metadata_json.update(metadata)
|
|
dataset.status = "ready"
|
|
db.add(dataset)
|
|
db.commit()
|
|
db.refresh(dataset)
|
|
return metadata
|