Initial GeoIntel V1 foundation
This commit is contained in:
Codex
@@ -0,0 +1,452 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
import uuid
|
||||
|
||||
from fastapi import UploadFile
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, Project
|
||||
from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary
|
||||
from app.services.geojson_service import parse_geojson_payload, load_dataset_text
|
||||
from app.services.raster_service import extract_raster_metadata
|
||||
from app.services.storage_service import StorageService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class DatasetService:
|
||||
VECTOR_EXTENSIONS = {".geojson", ".json"}
|
||||
RASTER_EXTENSIONS = {".tif", ".tiff", ".geotiff"}
|
||||
VECTOR_TYPES = {"vector", "geojson"}
|
||||
RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"}
|
||||
VALID_DATASET_ROLES = {"source", "derived", "reference"}
|
||||
|
||||
@staticmethod
|
||||
def _canonical_dataset_type(dataset_type: str) -> str:
|
||||
normalized = (dataset_type or "").strip().lower()
|
||||
if normalized in DatasetService.VECTOR_TYPES:
|
||||
return "vector"
|
||||
if normalized in DatasetService.RASTER_TYPES:
|
||||
return "raster"
|
||||
raise AppError(
|
||||
code="INVALID_DATASET_TYPE",
|
||||
message="dataset_type must be 'vector' or 'raster' (or legacy 'geojson')",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_stored_dataset_type(dataset_type: str) -> str:
|
||||
normalized = (dataset_type or "").strip().lower()
|
||||
if normalized in DatasetService.VECTOR_TYPES:
|
||||
return "vector"
|
||||
if normalized in DatasetService.RASTER_TYPES:
|
||||
return "raster"
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _is_vector_type(dataset_type: str) -> bool:
|
||||
return DatasetService._normalize_stored_dataset_type(dataset_type) == "vector"
|
||||
|
||||
@staticmethod
|
||||
def _is_raster_type(dataset_type: str) -> bool:
|
||||
return DatasetService._normalize_stored_dataset_type(dataset_type) == "raster"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_dataset_role(dataset_role: str | None) -> str:
|
||||
normalized = (dataset_role or "").strip().lower() or "source"
|
||||
if normalized not in DatasetService.VALID_DATASET_ROLES:
|
||||
raise AppError(
|
||||
code="INVALID_DATASET_ROLE",
|
||||
message="dataset_role must be one of: source, derived, reference",
|
||||
status_code=400,
|
||||
)
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _extension_for_path(filename: str) -> str:
|
||||
return Path(filename).suffix.lower()
|
||||
|
||||
@staticmethod
|
||||
def _validate_upload_filename(filename: str | None) -> str:
|
||||
if not filename:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
|
||||
return filename
|
||||
|
||||
@staticmethod
|
||||
def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
|
||||
total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
|
||||
rows = (
|
||||
db.query(Dataset)
|
||||
.filter(Dataset.project_id == project_id)
|
||||
.order_by(Dataset.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
response_items = []
|
||||
for row in rows:
|
||||
feature_count = None
|
||||
metadata_json = row.metadata_json or {}
|
||||
vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json)
|
||||
if isinstance(metadata_json, dict):
|
||||
feature_count = metadata_json.get("feature_count")
|
||||
response_items.append(
|
||||
DatasetCreateResponse(
|
||||
id=row.id,
|
||||
name=row.name,
|
||||
dataset_type=row.dataset_type,
|
||||
source=row.source,
|
||||
dataset_role=row.dataset_role,
|
||||
source_name=row.source_name,
|
||||
reference_layer_name=row.reference_layer_name,
|
||||
source_metadata=row.source_metadata,
|
||||
provenance_metadata=row.provenance_metadata,
|
||||
imported_at=row.imported_at,
|
||||
project_id=row.project_id,
|
||||
area_id=row.area_id,
|
||||
storage_path=row.storage_path,
|
||||
original_filename=row.original_filename,
|
||||
stored_filename=row.stored_filename,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
checksum_sha256=row.checksum_sha256,
|
||||
crs=row.crs,
|
||||
bounds_json=row.bounds_json,
|
||||
metadata_json=row.metadata_json,
|
||||
vector_summary=vector_summary,
|
||||
status=row.status,
|
||||
derived_from_dataset_id=row.derived_from_dataset_id,
|
||||
created_at=row.created_at,
|
||||
feature_count=feature_count,
|
||||
)
|
||||
)
|
||||
return response_items, total
|
||||
|
||||
@staticmethod
|
||||
def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None:
|
||||
if not DatasetService._is_vector_type(dataset_type):
|
||||
return None
|
||||
if not isinstance(metadata_json, dict):
|
||||
return None
|
||||
return DatasetVectorSummary(
|
||||
feature_count=metadata_json.get("feature_count"),
|
||||
geometry_types=metadata_json.get("geometry_types"),
|
||||
bounds_json=metadata_json.get("bounds_json"),
|
||||
approximate_area_m2=metadata_json.get("approximate_area_m2"),
|
||||
crs=metadata_json.get("crs"),
|
||||
feature_geometry_count=metadata_json.get("feature_geometry_count"),
|
||||
invalid_features=metadata_json.get("invalid_features"),
|
||||
crs_assumed=metadata_json.get("crs_assumed"),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def upload_dataset(
|
||||
db: Session,
|
||||
project_id: UUID,
|
||||
file: UploadFile,
|
||||
dataset_type: str,
|
||||
source: str,
|
||||
dataset_role: str = "source",
|
||||
source_name: str | None = None,
|
||||
reference_layer_name: str | None = None,
|
||||
source_metadata: dict | None = None,
|
||||
provenance_metadata: dict | None = None,
|
||||
area_id: UUID | None = None,
|
||||
) -> DatasetCreateResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
filename = DatasetService._validate_upload_filename(file.filename)
|
||||
canonical_type = DatasetService._canonical_dataset_type(dataset_type)
|
||||
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
|
||||
normalized_source_name = source_name
|
||||
if normalized_role == "reference" and not normalized_source_name:
|
||||
normalized_source_name = "manual"
|
||||
if normalized_role == "reference" and canonical_type == "raster":
|
||||
raise AppError(code="INVALID_DATASET_ROLE", message="reference role requires vector dataset type", status_code=400)
|
||||
extension = DatasetService._extension_for_path(filename)
|
||||
|
||||
if canonical_type == "vector" and extension not in DatasetService.VECTOR_EXTENSIONS:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Vector uploads require .geojson or .json files", status_code=415)
|
||||
if canonical_type == "raster" and extension not in DatasetService.RASTER_EXTENSIONS:
|
||||
raise AppError(
|
||||
code="INVALID_UPLOAD",
|
||||
message="Raster uploads require .tif, .tiff or .geotiff files",
|
||||
status_code=415,
|
||||
)
|
||||
|
||||
raw = await file.read()
|
||||
storage_info = StorageService.persist_dataset_file(
|
||||
project_id=str(project_id),
|
||||
dataset_id=str(dataset_id := uuid.uuid4()),
|
||||
dataset_type=canonical_type,
|
||||
original_filename=filename,
|
||||
content=raw,
|
||||
content_type=file.content_type,
|
||||
)
|
||||
|
||||
metadata: dict[str, Any] = {}
|
||||
vector_payload: dict[str, Any] | None = None
|
||||
status = "uploaded"
|
||||
try:
|
||||
status = "validating"
|
||||
if canonical_type == "vector":
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Upload must be UTF-8 encoded", status_code=400) from exc
|
||||
metadata = parse_geojson_payload(text)
|
||||
vector_payload = json.loads(text)
|
||||
status = "ready"
|
||||
else:
|
||||
metadata = extract_raster_metadata(storage_info["storage_path"])
|
||||
status = "ready"
|
||||
except ValueError as exc:
|
||||
status = "failed"
|
||||
StorageService.remove_dataset_file(storage_info["storage_path"])
|
||||
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
|
||||
except AppError as exc:
|
||||
if canonical_type == "raster" and exc.code == "RASTER_PROCESSING_UNAVAILABLE":
|
||||
status = "failed"
|
||||
metadata = {
|
||||
"processing_error": exc.message,
|
||||
"processing_code": exc.code,
|
||||
}
|
||||
else:
|
||||
StorageService.remove_dataset_file(storage_info["storage_path"])
|
||||
raise
|
||||
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
name=filename,
|
||||
dataset_type=canonical_type,
|
||||
source=source,
|
||||
dataset_role=normalized_role,
|
||||
source_name=normalized_source_name,
|
||||
reference_layer_name=reference_layer_name if normalized_role == "reference" else None,
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
storage_path=storage_info["storage_path"],
|
||||
original_filename=storage_info["original_filename"],
|
||||
stored_filename=storage_info["stored_filename"],
|
||||
content_type=storage_info["content_type"],
|
||||
size_bytes=storage_info["size_bytes"],
|
||||
checksum_sha256=storage_info["checksum_sha256"],
|
||||
crs=metadata.get("crs") if isinstance(metadata, dict) else None,
|
||||
bounds_json=metadata.get("bounds_json") if isinstance(metadata, dict) else None,
|
||||
resolution_json=metadata.get("resolution_json") if isinstance(metadata, dict) else None,
|
||||
bands_json=metadata.get("bands_json") if isinstance(metadata, dict) else None,
|
||||
metadata_json=metadata,
|
||||
status=status,
|
||||
)
|
||||
db.add(dataset)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
|
||||
if canonical_type == "vector" and vector_payload is not None and status == "ready":
|
||||
feature_class = reference_layer_name if normalized_role == "reference" else None
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=vector_payload,
|
||||
feature_class=feature_class,
|
||||
)
|
||||
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
||||
status=dataset.status,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse:
|
||||
dataset = DatasetService._get_dataset(db, dataset_id)
|
||||
if not dataset.storage_path:
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
if not Path(dataset.storage_path).exists():
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
|
||||
try:
|
||||
if DatasetService._is_vector_type(dataset.dataset_type):
|
||||
metadata = parse_geojson_payload(load_dataset_text(dataset.storage_path))
|
||||
elif DatasetService._is_raster_type(dataset.dataset_type):
|
||||
metadata = extract_raster_metadata(dataset.storage_path)
|
||||
else:
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Cannot refresh metadata for this dataset type", status_code=400)
|
||||
dataset.status = "ready"
|
||||
except ValueError as exc:
|
||||
dataset.status = "failed"
|
||||
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
|
||||
except AppError as exc:
|
||||
if DatasetService._is_raster_type(dataset.dataset_type) and exc.code == "RASTER_PROCESSING_UNAVAILABLE":
|
||||
dataset.status = "failed"
|
||||
metadata = {"processing_error": exc.message, "processing_code": exc.code}
|
||||
else:
|
||||
dataset.status = "failed"
|
||||
raise
|
||||
|
||||
dataset.crs = metadata.get("crs") if isinstance(metadata, dict) else dataset.crs
|
||||
dataset.bounds_json = metadata.get("bounds_json") if isinstance(metadata, dict) else dataset.bounds_json
|
||||
dataset.metadata_json = metadata
|
||||
dataset.resolution_json = metadata.get("resolution_json") if isinstance(metadata, dict) else dataset.resolution_json
|
||||
dataset.bands_json = metadata.get("bands_json") if isinstance(metadata, dict) else dataset.bands_json
|
||||
|
||||
db.add(dataset)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
|
||||
return DatasetCreateResponse(
|
||||
id=dataset.id,
|
||||
name=dataset.name,
|
||||
dataset_type=dataset.dataset_type,
|
||||
source=dataset.source,
|
||||
dataset_role=dataset.dataset_role,
|
||||
source_name=dataset.source_name,
|
||||
reference_layer_name=dataset.reference_layer_name,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
imported_at=dataset.imported_at,
|
||||
project_id=dataset.project_id,
|
||||
area_id=dataset.area_id,
|
||||
storage_path=dataset.storage_path,
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
crs=dataset.crs,
|
||||
derived_from_dataset_id=dataset.derived_from_dataset_id,
|
||||
bounds_json=dataset.bounds_json,
|
||||
metadata_json=dataset.metadata_json,
|
||||
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
|
||||
status=dataset.status,
|
||||
created_at=dataset.created_at,
|
||||
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_dataset(db: Session, dataset_id: UUID) -> Dataset:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _get_dataset(db: Session, dataset_id: UUID) -> Dataset:
|
||||
return DatasetService.get_dataset(db, dataset_id)
|
||||
|
||||
@staticmethod
|
||||
def get_dataset_geojson(db: Session, dataset_id: UUID) -> dict:
|
||||
dataset = DatasetService._get_dataset(db, dataset_id)
|
||||
if not DatasetService._is_vector_type(dataset.dataset_type):
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
|
||||
if not dataset.storage_path:
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
if not pathlib.Path(dataset.storage_path).exists():
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
|
||||
raw = load_dataset_text(dataset.storage_path)
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except Exception as exc:
|
||||
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=500) from exc
|
||||
|
||||
@staticmethod
|
||||
def inspect_vector_dataset(db: Session, dataset_id: UUID) -> dict[str, Any]:
|
||||
dataset = DatasetService._get_dataset(db, dataset_id)
|
||||
if not DatasetService._is_vector_type(dataset.dataset_type):
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
|
||||
if not dataset.storage_path or not Path(dataset.storage_path).exists():
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
metadata = dataset.metadata_json or {}
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata)
|
||||
return {
|
||||
"dataset": {
|
||||
"id": str(dataset.id),
|
||||
"name": dataset.name,
|
||||
"dataset_type": dataset.dataset_type,
|
||||
"status": dataset.status,
|
||||
"source": dataset.source,
|
||||
"storage": DatasetStorageResponse(
|
||||
original_filename=dataset.original_filename,
|
||||
stored_filename=dataset.stored_filename,
|
||||
content_type=dataset.content_type,
|
||||
size_bytes=dataset.size_bytes,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
).model_dump(),
|
||||
"feature_count": metadata.get("feature_count"),
|
||||
"crs": metadata.get("crs"),
|
||||
},
|
||||
"summary": summary.model_dump() if summary else None,
|
||||
"metadata": metadata,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def vector_summary(db: Session, dataset_id: UUID) -> dict[str, Any]:
|
||||
dataset = DatasetService._get_dataset(db, dataset_id)
|
||||
if not DatasetService._is_vector_type(dataset.dataset_type):
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
|
||||
|
||||
metadata = dataset.metadata_json or {}
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata)
|
||||
if not summary:
|
||||
raise AppError(code="INVALID_GEOJSON", message="Vector summary unavailable", status_code=422)
|
||||
return summary.model_dump()
|
||||
|
||||
@staticmethod
|
||||
def raster_metadata(db: Session, dataset_id: UUID) -> dict[str, Any]:
|
||||
dataset = DatasetService._get_dataset(db, dataset_id)
|
||||
if not DatasetService._is_raster_type(dataset.dataset_type):
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a raster dataset", status_code=400)
|
||||
if not dataset.storage_path:
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
if not Path(dataset.storage_path).exists():
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
|
||||
if isinstance(dataset.metadata_json, dict) and dataset.metadata_json.get("driver"):
|
||||
return dataset.metadata_json
|
||||
|
||||
metadata = extract_raster_metadata(dataset.storage_path)
|
||||
dataset.metadata_json = dict(dataset.metadata_json or {})
|
||||
dataset.metadata_json.update(metadata)
|
||||
dataset.status = "ready"
|
||||
db.add(dataset)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return metadata
|
||||
Reference in New Issue
Block a user