Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial GeoIntel V1 foundation
GeoIntel CI / docs-smoke (push) Has been cancelled
Details
GeoIntel CI / contract-smoke (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Codex
2026-06-16 23:36:32 +02:00
commit 6ea3586a3e
605 changed files with 45284 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/.gitkeep
View File
backend/app/services/area_service.py
+77
View File
@@ -0,0 +1,77 @@
from __future__ import annotations
import uuid
from sqlalchemy.orm import Session
from geoalchemy2.shape import from_shape
from app.core.errors import AppError
from app.models import Area, Project
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon
class AreaService:
@staticmethod
def list_areas(db: Session, project_id: uuid.UUID, limit: int = 50, offset: int = 0) -> tuple[list[AreaRead], int]:
total = db.query(Area).filter(Area.project_id == project_id).count()
areas = (
db.query(Area)
.filter(Area.project_id == project_id)
.order_by(Area.created_at.desc())
.offset(offset)
.limit(limit)
.all()
)
return [AreaRead.model_validate(area) for area in areas], total
@staticmethod
def create_area(db: Session, project_id: uuid.UUID, payload: AreaCreate) -> AreaRead:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
try:
multipolygon = normalize_to_multipolygon(payload.geometry)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
area = Area(
project_id=project_id,
name=payload.name.strip() or "Unnamed area",
geometry=from_shape(multipolygon, srid=4326),
original_crs=payload.crs or "EPSG:4326",
area_m2=area_m2(multipolygon),
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
)
db.add(area)
db.commit()
db.refresh(area)
return AreaRead.model_validate(area)
@staticmethod
def get_area(db: Session, area_id: uuid.UUID) -> AreaRead:
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
return AreaRead.model_validate(area)
@staticmethod
def update_area(db: Session, area_id: uuid.UUID, payload: AreaUpdate) -> AreaRead:
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
changed = False
if payload.name:
area.name = payload.name.strip() or area.name
changed = True
if payload.crs:
area.original_crs = payload.crs
changed = True
if not changed:
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
db.add(area)
db.commit()
db.refresh(area)
return AreaRead.model_validate(area)
backend/app/services/dataset_service.py
+452
View File
@@ -0,0 +1,452 @@
from __future__ import annotations
import json
import pathlib
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from uuid import UUID
import uuid
from fastapi import UploadFile
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Dataset, Project
from app.schemas.dataset import DatasetCreateResponse, DatasetStorageResponse, DatasetVectorSummary
from app.services.geojson_service import parse_geojson_payload, load_dataset_text
from app.services.raster_service import extract_raster_metadata
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
class DatasetService:
VECTOR_EXTENSIONS = {".geojson", ".json"}
RASTER_EXTENSIONS = {".tif", ".tiff", ".geotiff"}
VECTOR_TYPES = {"vector", "geojson"}
RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"}
VALID_DATASET_ROLES = {"source", "derived", "reference"}
@staticmethod
def _canonical_dataset_type(dataset_type: str) -> str:
normalized = (dataset_type or "").strip().lower()
if normalized in DatasetService.VECTOR_TYPES:
return "vector"
if normalized in DatasetService.RASTER_TYPES:
return "raster"
raise AppError(
code="INVALID_DATASET_TYPE",
message="dataset_type must be 'vector' or 'raster' (or legacy 'geojson')",
status_code=400,
)
@staticmethod
def _normalize_stored_dataset_type(dataset_type: str) -> str:
normalized = (dataset_type or "").strip().lower()
if normalized in DatasetService.VECTOR_TYPES:
return "vector"
if normalized in DatasetService.RASTER_TYPES:
return "raster"
return normalized
@staticmethod
def _is_vector_type(dataset_type: str) -> bool:
return DatasetService._normalize_stored_dataset_type(dataset_type) == "vector"
@staticmethod
def _is_raster_type(dataset_type: str) -> bool:
return DatasetService._normalize_stored_dataset_type(dataset_type) == "raster"
@staticmethod
def _normalize_dataset_role(dataset_role: str | None) -> str:
normalized = (dataset_role or "").strip().lower() or "source"
if normalized not in DatasetService.VALID_DATASET_ROLES:
raise AppError(
code="INVALID_DATASET_ROLE",
message="dataset_role must be one of: source, derived, reference",
status_code=400,
)
return normalized
@staticmethod
def _extension_for_path(filename: str) -> str:
return Path(filename).suffix.lower()
@staticmethod
def _validate_upload_filename(filename: str | None) -> str:
if not filename:
raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
return filename
@staticmethod
def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
rows = (
db.query(Dataset)
.filter(Dataset.project_id == project_id)
.order_by(Dataset.created_at.desc())
.offset(offset)
.limit(limit)
.all()
)
response_items = []
for row in rows:
feature_count = None
metadata_json = row.metadata_json or {}
vector_summary = DatasetService._extract_vector_summary(row.dataset_type, metadata_json)
if isinstance(metadata_json, dict):
feature_count = metadata_json.get("feature_count")
response_items.append(
DatasetCreateResponse(
id=row.id,
name=row.name,
dataset_type=row.dataset_type,
source=row.source,
dataset_role=row.dataset_role,
source_name=row.source_name,
reference_layer_name=row.reference_layer_name,
source_metadata=row.source_metadata,
provenance_metadata=row.provenance_metadata,
imported_at=row.imported_at,
project_id=row.project_id,
area_id=row.area_id,
storage_path=row.storage_path,
original_filename=row.original_filename,
stored_filename=row.stored_filename,
content_type=row.content_type,
size_bytes=row.size_bytes,
checksum_sha256=row.checksum_sha256,
crs=row.crs,
bounds_json=row.bounds_json,
metadata_json=row.metadata_json,
vector_summary=vector_summary,
status=row.status,
derived_from_dataset_id=row.derived_from_dataset_id,
created_at=row.created_at,
feature_count=feature_count,
)
)
return response_items, total
@staticmethod
def _extract_vector_summary(dataset_type: str, metadata_json: dict) -> DatasetVectorSummary | None:
if not DatasetService._is_vector_type(dataset_type):
return None
if not isinstance(metadata_json, dict):
return None
return DatasetVectorSummary(
feature_count=metadata_json.get("feature_count"),
geometry_types=metadata_json.get("geometry_types"),
bounds_json=metadata_json.get("bounds_json"),
approximate_area_m2=metadata_json.get("approximate_area_m2"),
crs=metadata_json.get("crs"),
feature_geometry_count=metadata_json.get("feature_geometry_count"),
invalid_features=metadata_json.get("invalid_features"),
crs_assumed=metadata_json.get("crs_assumed"),
)
@staticmethod
async def upload_dataset(
db: Session,
project_id: UUID,
file: UploadFile,
dataset_type: str,
source: str,
dataset_role: str = "source",
source_name: str | None = None,
reference_layer_name: str | None = None,
source_metadata: dict | None = None,
provenance_metadata: dict | None = None,
area_id: UUID | None = None,
) -> DatasetCreateResponse:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
filename = DatasetService._validate_upload_filename(file.filename)
canonical_type = DatasetService._canonical_dataset_type(dataset_type)
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
normalized_source_name = source_name
if normalized_role == "reference" and not normalized_source_name:
normalized_source_name = "manual"
if normalized_role == "reference" and canonical_type == "raster":
raise AppError(code="INVALID_DATASET_ROLE", message="reference role requires vector dataset type", status_code=400)
extension = DatasetService._extension_for_path(filename)
if canonical_type == "vector" and extension not in DatasetService.VECTOR_EXTENSIONS:
raise AppError(code="INVALID_UPLOAD", message="Vector uploads require .geojson or .json files", status_code=415)
if canonical_type == "raster" and extension not in DatasetService.RASTER_EXTENSIONS:
raise AppError(
code="INVALID_UPLOAD",
message="Raster uploads require .tif, .tiff or .geotiff files",
status_code=415,
)
raw = await file.read()
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id := uuid.uuid4()),
dataset_type=canonical_type,
original_filename=filename,
content=raw,
content_type=file.content_type,
)
metadata: dict[str, Any] = {}
vector_payload: dict[str, Any] | None = None
status = "uploaded"
try:
status = "validating"
if canonical_type == "vector":
try:
text = raw.decode("utf-8")
except UnicodeDecodeError as exc:
raise AppError(code="INVALID_UPLOAD", message="Upload must be UTF-8 encoded", status_code=400) from exc
metadata = parse_geojson_payload(text)
vector_payload = json.loads(text)
status = "ready"
else:
metadata = extract_raster_metadata(storage_info["storage_path"])
status = "ready"
except ValueError as exc:
status = "failed"
StorageService.remove_dataset_file(storage_info["storage_path"])
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
except AppError as exc:
if canonical_type == "raster" and exc.code == "RASTER_PROCESSING_UNAVAILABLE":
status = "failed"
metadata = {
"processing_error": exc.message,
"processing_code": exc.code,
}
else:
StorageService.remove_dataset_file(storage_info["storage_path"])
raise
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
name=filename,
dataset_type=canonical_type,
source=source,
dataset_role=normalized_role,
source_name=normalized_source_name,
reference_layer_name=reference_layer_name if normalized_role == "reference" else None,
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
storage_path=storage_info["storage_path"],
original_filename=storage_info["original_filename"],
stored_filename=storage_info["stored_filename"],
content_type=storage_info["content_type"],
size_bytes=storage_info["size_bytes"],
checksum_sha256=storage_info["checksum_sha256"],
crs=metadata.get("crs") if isinstance(metadata, dict) else None,
bounds_json=metadata.get("bounds_json") if isinstance(metadata, dict) else None,
resolution_json=metadata.get("resolution_json") if isinstance(metadata, dict) else None,
bands_json=metadata.get("bands_json") if isinstance(metadata, dict) else None,
metadata_json=metadata,
status=status,
)
db.add(dataset)
db.commit()
db.refresh(dataset)
if canonical_type == "vector" and vector_payload is not None and status == "ready":
feature_class = reference_layer_name if normalized_role == "reference" else None
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=vector_payload,
feature_class=feature_class,
)
return DatasetCreateResponse(
id=dataset.id,
name=dataset.name,
dataset_type=dataset.dataset_type,
source=dataset.source,
dataset_role=dataset.dataset_role,
source_name=dataset.source_name,
reference_layer_name=dataset.reference_layer_name,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
imported_at=dataset.imported_at,
project_id=dataset.project_id,
area_id=dataset.area_id,
storage_path=dataset.storage_path,
original_filename=dataset.original_filename,
stored_filename=dataset.stored_filename,
content_type=dataset.content_type,
size_bytes=dataset.size_bytes,
checksum_sha256=dataset.checksum_sha256,
crs=dataset.crs,
derived_from_dataset_id=dataset.derived_from_dataset_id,
bounds_json=dataset.bounds_json,
metadata_json=dataset.metadata_json,
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
status=dataset.status,
created_at=dataset.created_at,
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
)
@staticmethod
def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse:
dataset = DatasetService._get_dataset(db, dataset_id)
if not dataset.storage_path:
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
if not Path(dataset.storage_path).exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
try:
if DatasetService._is_vector_type(dataset.dataset_type):
metadata = parse_geojson_payload(load_dataset_text(dataset.storage_path))
elif DatasetService._is_raster_type(dataset.dataset_type):
metadata = extract_raster_metadata(dataset.storage_path)
else:
raise AppError(code="INVALID_DATASET_TYPE", message="Cannot refresh metadata for this dataset type", status_code=400)
dataset.status = "ready"
except ValueError as exc:
dataset.status = "failed"
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
except AppError as exc:
if DatasetService._is_raster_type(dataset.dataset_type) and exc.code == "RASTER_PROCESSING_UNAVAILABLE":
dataset.status = "failed"
metadata = {"processing_error": exc.message, "processing_code": exc.code}
else:
dataset.status = "failed"
raise
dataset.crs = metadata.get("crs") if isinstance(metadata, dict) else dataset.crs
dataset.bounds_json = metadata.get("bounds_json") if isinstance(metadata, dict) else dataset.bounds_json
dataset.metadata_json = metadata
dataset.resolution_json = metadata.get("resolution_json") if isinstance(metadata, dict) else dataset.resolution_json
dataset.bands_json = metadata.get("bands_json") if isinstance(metadata, dict) else dataset.bands_json
db.add(dataset)
db.commit()
db.refresh(dataset)
return DatasetCreateResponse(
id=dataset.id,
name=dataset.name,
dataset_type=dataset.dataset_type,
source=dataset.source,
dataset_role=dataset.dataset_role,
source_name=dataset.source_name,
reference_layer_name=dataset.reference_layer_name,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
imported_at=dataset.imported_at,
project_id=dataset.project_id,
area_id=dataset.area_id,
storage_path=dataset.storage_path,
original_filename=dataset.original_filename,
stored_filename=dataset.stored_filename,
content_type=dataset.content_type,
size_bytes=dataset.size_bytes,
checksum_sha256=dataset.checksum_sha256,
crs=dataset.crs,
derived_from_dataset_id=dataset.derived_from_dataset_id,
bounds_json=dataset.bounds_json,
metadata_json=dataset.metadata_json,
vector_summary=DatasetService._extract_vector_summary(dataset.dataset_type, dataset.metadata_json or {}),
status=dataset.status,
created_at=dataset.created_at,
feature_count=metadata.get("feature_count") if isinstance(metadata, dict) else None,
)
@staticmethod
def get_dataset(db: Session, dataset_id: UUID) -> Dataset:
dataset = db.get(Dataset, dataset_id)
if not dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
return dataset
@staticmethod
def _get_dataset(db: Session, dataset_id: UUID) -> Dataset:
return DatasetService.get_dataset(db, dataset_id)
@staticmethod
def get_dataset_geojson(db: Session, dataset_id: UUID) -> dict:
dataset = DatasetService._get_dataset(db, dataset_id)
if not DatasetService._is_vector_type(dataset.dataset_type):
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
if not dataset.storage_path:
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
if not pathlib.Path(dataset.storage_path).exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
raw = load_dataset_text(dataset.storage_path)
try:
return json.loads(raw)
except Exception as exc:
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=500) from exc
@staticmethod
def inspect_vector_dataset(db: Session, dataset_id: UUID) -> dict[str, Any]:
dataset = DatasetService._get_dataset(db, dataset_id)
if not DatasetService._is_vector_type(dataset.dataset_type):
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
if not dataset.storage_path or not Path(dataset.storage_path).exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
metadata = dataset.metadata_json or {}
if not isinstance(metadata, dict):
metadata = {}
summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata)
return {
"dataset": {
"id": str(dataset.id),
"name": dataset.name,
"dataset_type": dataset.dataset_type,
"status": dataset.status,
"source": dataset.source,
"storage": DatasetStorageResponse(
original_filename=dataset.original_filename,
stored_filename=dataset.stored_filename,
content_type=dataset.content_type,
size_bytes=dataset.size_bytes,
checksum_sha256=dataset.checksum_sha256,
).model_dump(),
"feature_count": metadata.get("feature_count"),
"crs": metadata.get("crs"),
},
"summary": summary.model_dump() if summary else None,
"metadata": metadata,
}
@staticmethod
def vector_summary(db: Session, dataset_id: UUID) -> dict[str, Any]:
dataset = DatasetService._get_dataset(db, dataset_id)
if not DatasetService._is_vector_type(dataset.dataset_type):
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
metadata = dataset.metadata_json or {}
if not isinstance(metadata, dict):
metadata = {}
summary = DatasetService._extract_vector_summary(dataset.dataset_type, metadata)
if not summary:
raise AppError(code="INVALID_GEOJSON", message="Vector summary unavailable", status_code=422)
return summary.model_dump()
@staticmethod
def raster_metadata(db: Session, dataset_id: UUID) -> dict[str, Any]:
dataset = DatasetService._get_dataset(db, dataset_id)
if not DatasetService._is_raster_type(dataset.dataset_type):
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a raster dataset", status_code=400)
if not dataset.storage_path:
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
if not Path(dataset.storage_path).exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
if isinstance(dataset.metadata_json, dict) and dataset.metadata_json.get("driver"):
return dataset.metadata_json
metadata = extract_raster_metadata(dataset.storage_path)
dataset.metadata_json = dict(dataset.metadata_json or {})
dataset.metadata_json.update(metadata)
dataset.status = "ready"
db.add(dataset)
db.commit()
db.refresh(dataset)
return metadata
backend/app/services/demo_workflow_service.py
+288
View File
@@ -0,0 +1,288 @@
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from uuid import UUID, uuid4
from geoalchemy2.shape import from_shape
from sqlalchemy.orm import Session
from app.models import Area, Dataset, Metric, Project, QualityCheck
from app.schemas.demo import DemoWorkflowResponse
from app.services.geojson_service import parse_geojson_payload
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon
class DemoWorkflowService:
PROJECT_NAME = "GeoIntel Demo - Building QA"
AREA_NAME = "Demo AOI - Geel buildings"
REFERENCE_FILENAME = "demo_reference_buildings.geojson"
CANDIDATE_FILENAME = "demo_predicted_buildings.geojson"
@staticmethod
def _repo_root() -> Path:
return Path(__file__).resolve().parents[3]
@staticmethod
def _fixture_path(filename: str) -> Path:
return DemoWorkflowService._repo_root() / "fixtures" / "golden" / filename
@staticmethod
def _load_fixture(filename: str) -> tuple[dict, bytes]:
path = DemoWorkflowService._fixture_path(filename)
raw = path.read_bytes()
return json.loads(raw.decode("utf-8")), raw
@staticmethod
def _find_existing_project(db: Session) -> Project | None:
return (
db.query(Project)
.filter(Project.name == DemoWorkflowService.PROJECT_NAME)
.filter(Project.status != "deleted")
.first()
)
@staticmethod
def _create_area(db: Session, project_id: UUID) -> Area:
geometry = {
"type": "MultiPolygon",
"coordinates": [
[
[
[4.30, 51.18],
[4.45, 51.18],
[4.45, 51.33],
[4.30, 51.33],
[4.30, 51.18],
]
]
],
}
multipolygon = normalize_to_multipolygon(geometry)
area = Area(
id=uuid4(),
project_id=project_id,
name=DemoWorkflowService.AREA_NAME,
geometry=from_shape(multipolygon, srid=4326),
original_crs="EPSG:4326",
area_m2=area_m2(multipolygon),
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
)
db.add(area)
db.commit()
db.refresh(area)
return area
@staticmethod
def _create_dataset(
db: Session,
*,
project_id: UUID,
area_id: UUID,
filename: str,
payload: dict,
raw: bytes,
role: str,
source_name: str,
reference_layer_name: str | None,
) -> Dataset:
dataset_id = uuid4()
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type="vector",
original_filename=filename,
content=raw,
content_type="application/geo+json",
)
metadata = parse_geojson_payload(payload)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
name=filename,
dataset_type="vector",
source="fixture",
dataset_role=role,
source_name=source_name,
reference_layer_name=reference_layer_name,
source_metadata={
"fixture": True,
"fixture_name": filename,
"usage": "offline demo workflow only",
},
provenance_metadata={
"created_by": "demo_workflow",
"source_path": str(DemoWorkflowService._fixture_path(filename)),
},
imported_at=datetime.now(timezone.utc),
storage_path=storage_info["storage_path"],
original_filename=storage_info["original_filename"],
stored_filename=storage_info["stored_filename"],
content_type=storage_info["content_type"],
size_bytes=storage_info["size_bytes"],
checksum_sha256=storage_info["checksum_sha256"],
crs=metadata.get("crs"),
bounds_json=metadata.get("bounds_json"),
metadata_json=metadata,
status="ready",
)
db.add(dataset)
db.commit()
db.refresh(dataset)
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
)
return dataset
@staticmethod
def _persist_qa(
db: Session,
*,
project_id: UUID,
candidate_dataset_id: UUID,
reference_dataset_id: UUID,
area_id: UUID,
) -> QualityCheck:
result = QaService.compare_candidate_with_reference(
db=db,
project_id=project_id,
candidate_dataset_id=candidate_dataset_id,
reference_dataset_id=reference_dataset_id,
iou_threshold=0.5,
area_id=area_id,
)
return QualityService.persist_quality_check(
db=db,
project_id=project_id,
candidate_dataset_id=candidate_dataset_id,
reference_dataset_id=reference_dataset_id,
check_type="demo_candidate_vs_reference",
status=result.status,
score=result.f1_score,
parameters={
"iou_threshold": result.iou_threshold,
"area_id": str(area_id),
"fixture_workflow": True,
},
findings={
"matches": result.matches,
"false_positives": result.false_positives,
"false_negatives": result.false_negatives,
"warnings": result.warnings,
"unsupported_geometry": result.unsupported_geometry,
"unsupported_geometries": result.unsupported_geometries,
},
metrics={
"precision": result.precision,
"recall": result.recall,
"f1": result.f1_score,
"mean_iou": result.mean_iou,
"false_positive_count": result.false_positives,
"false_negative_count": result.false_negatives,
},
)
@staticmethod
def seed(db: Session) -> DemoWorkflowResponse:
existing = DemoWorkflowService._find_existing_project(db)
if existing:
area = db.query(Area).filter(Area.project_id == existing.id).order_by(Area.created_at.asc()).first()
reference = (
db.query(Dataset)
.filter(Dataset.project_id == existing.id)
.filter(Dataset.dataset_role == "reference")
.filter(Dataset.source_name == "fixture")
.first()
)
candidate = (
db.query(Dataset)
.filter(Dataset.project_id == existing.id)
.filter(Dataset.dataset_role == "source")
.filter(Dataset.source_name == "fixture")
.first()
)
quality_check = (
db.query(QualityCheck)
.filter(QualityCheck.project_id == existing.id)
.filter(QualityCheck.check_type == "demo_candidate_vs_reference")
.order_by(QualityCheck.created_at.desc())
.first()
)
if area and reference and candidate and quality_check:
return DemoWorkflowResponse(
project_id=existing.id,
area_id=area.id,
reference_dataset_id=reference.id,
candidate_dataset_id=candidate.id,
quality_check_id=quality_check.id,
metric_count=db.query(Metric).filter(Metric.quality_check_id == quality_check.id).count(),
status="ready",
message="Demo workflow already exists.",
created=False,
)
project = Project(
id=uuid4(),
name=DemoWorkflowService.PROJECT_NAME,
description="Offline fixture workflow: reference buildings, predicted buildings and persisted QA metrics.",
region="Kempen",
status="active",
)
db.add(project)
db.commit()
db.refresh(project)
area = DemoWorkflowService._create_area(db, project.id)
reference_payload, reference_raw = DemoWorkflowService._load_fixture("reference_buildings.geojson")
candidate_payload, candidate_raw = DemoWorkflowService._load_fixture("predicted_buildings.geojson")
reference = DemoWorkflowService._create_dataset(
db=db,
project_id=project.id,
area_id=area.id,
filename=DemoWorkflowService.REFERENCE_FILENAME,
payload=reference_payload,
raw=reference_raw,
role="reference",
source_name="fixture",
reference_layer_name="buildings",
)
candidate = DemoWorkflowService._create_dataset(
db=db,
project_id=project.id,
area_id=area.id,
filename=DemoWorkflowService.CANDIDATE_FILENAME,
payload=candidate_payload,
raw=candidate_raw,
role="source",
source_name="fixture",
reference_layer_name=None,
)
quality_check = DemoWorkflowService._persist_qa(
db=db,
project_id=project.id,
candidate_dataset_id=candidate.id,
reference_dataset_id=reference.id,
area_id=area.id,
)
return DemoWorkflowResponse(
project_id=project.id,
area_id=area.id,
reference_dataset_id=reference.id,
candidate_dataset_id=candidate.id,
quality_check_id=quality_check.id,
metric_count=6,
status="ready",
message="Demo workflow seeded from explicit local fixtures.",
created=True,
)
backend/app/services/detection_georeferencing.py
+73
View File
@@ -0,0 +1,73 @@
from __future__ import annotations
from typing import Any
from pyproj import Transformer
from shapely.geometry import Polygon
from app.core.errors import AppError
def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs: str | None = None) -> Polygon:
if len(bbox) != 4:
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must contain four pixel coordinates", status_code=422)
x_min, y_min, x_max, y_max = [float(value) for value in bbox]
if x_max <= x_min or y_max <= y_min:
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must have positive width and height", status_code=422)
transform = tile.get("transform")
if isinstance(transform, list) and len(transform) >= 6:
corners = [
_apply_gdal_transform(transform, x_min, y_min),
_apply_gdal_transform(transform, x_max, y_min),
_apply_gdal_transform(transform, x_max, y_max),
_apply_gdal_transform(transform, x_min, y_max),
_apply_gdal_transform(transform, x_min, y_min),
]
else:
corners = _corners_from_bounds(bbox=[x_min, y_min, x_max, y_max], tile=tile)
source_crs = crs or tile.get("crs") or tile.get("source_crs") or "EPSG:4326"
if str(source_crs).upper() not in {"EPSG:4326", "4326"}:
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
corners = [transformer.transform(x, y) for x, y in corners]
polygon = Polygon(corners)
if polygon.is_empty or not polygon.is_valid:
raise AppError(code="DETECTION_INVALID_GEOMETRY", message="Georeferenced detection geometry is invalid", status_code=422)
return polygon
def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]:
c, a, b, f, d, e = [float(value) for value in transform[:6]]
return (a * x + b * y + c, d * x + e * y + f)
def _corners_from_bounds(bbox: list[float], tile: dict[str, Any]) -> list[tuple[float, float]]:
bounds = tile.get("bounds")
pixel_window = tile.get("pixel_window")
if not (isinstance(bounds, list) and len(bounds) == 4 and isinstance(pixel_window, list) and len(pixel_window) == 4):
raise AppError(
code="DETECTION_TILE_MANIFEST_INVALID",
message="Tile manifest entries require transform or bounds plus pixel_window for georeferencing",
status_code=422,
)
x_min, y_min, x_max, y_max = bbox
left, bottom, right, top = [float(value) for value in bounds]
_, _, width, height = [float(value) for value in pixel_window]
if width <= 0 or height <= 0:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile pixel_window must have positive size", status_code=422)
def project(px: float, py: float) -> tuple[float, float]:
x = left + (px / width) * (right - left)
y = top - (py / height) * (top - bottom)
return (x, y)
return [
project(x_min, y_min),
project(x_max, y_min),
project(x_max, y_max),
project(x_min, y_max),
project(x_min, y_min),
]
backend/app/services/detection_service.py
+618
View File
@@ -0,0 +1,618 @@
from __future__ import annotations
import uuid
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from typing import Type
from geoalchemy2.shape import from_shape, to_shape
from shapely.geometry import mapping, shape
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import AnalysisRun, Dataset, Detection, Job, Project, VectorFeature
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.yolo_adapter import YoloDetectionAdapter
class DetectionService:
@staticmethod
def _now() -> datetime:
return datetime.now(UTC)
@staticmethod
def run_detection(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
model_id: str,
confidence_threshold: float,
class_filter: list[str] | None = None,
tile_manifest_path: str | None = None,
parameters_json: dict[str, Any] | None = None,
settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
) -> DetectionRunResponse:
parameters = dict(parameters_json or {})
resolved_settings = settings or get_settings()
project = db.get(Project, project_id)
if not project:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster":
raise AppError(
code="INVALID_DATASET_TYPE",
message="Detection requires a raster dataset",
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
model = ModelRegistryService.get_model_capability(
model_id,
settings=resolved_settings,
yolo_adapter_class=yolo_adapter_class,
)
if model is None:
raise AppError(code="DETECTION_MODEL_NOT_FOUND", message="Detection model not found", status_code=404)
if model.model_id == "manual-fixture-detector" and parameters.get("fixture_mode") is not True:
raise AppError(
code="FIXTURE_MODE_REQUIRED",
message="Fixture detector requires explicit fixture_mode=true",
status_code=400,
)
if model.model_id == resolved_settings.yolo_model_id and not tile_manifest_path:
raise AppError(
code="DETECTION_TILE_MANIFEST_REQUIRED",
message="Configured YOLO inference requires an existing raster tile manifest path",
status_code=400,
)
run_parameters = {
"model_id": model.model_id,
"confidence_threshold": confidence_threshold,
"class_filter": class_filter or [],
"tile_manifest_path": tile_manifest_path,
"parameters_json": parameters,
}
job = DetectionService._create_job(db, project_id, dataset_id, run_parameters)
analysis_run = DetectionService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters)
if not model.configured:
message = model.limitation_message
code = "DETECTION_DEPENDENCY_UNAVAILABLE" if model.status == "dependency_unavailable" else "DETECTION_MODEL_UNAVAILABLE"
DetectionService._mark_failed(db, analysis_run, job, code=code, message=message)
return DetectionRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="failed",
detection_count=0,
error_code=code,
message=message,
)
if model.model_id == "manual-fixture-detector":
detections = DetectionService._persist_fixture_detections(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
raw_detections=parameters.get("fixture_detections"),
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
)
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
return DetectionRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="success",
detection_count=len(detections),
message="Fixture detections persisted.",
)
if model.model_id == resolved_settings.yolo_model_id:
try:
detections = DetectionService._run_configured_yolo(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
tile_manifest_path=tile_manifest_path,
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
settings=resolved_settings,
yolo_adapter_class=yolo_adapter_class,
)
except AppError as exc:
DetectionService._mark_failed(db, analysis_run, job, code=exc.code, message=exc.message)
return DetectionRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="failed",
detection_count=0,
error_code=exc.code,
message=exc.message,
)
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
return DetectionRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="success",
detection_count=len(detections),
message="YOLO detections persisted.",
)
raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503)
@staticmethod
def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection":
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
return DetectionRunRead.model_validate(run)
@staticmethod
def list_runs(
db,
*,
project_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None,
) -> DetectionRunListResponse:
query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "detection")
if project_id is not None:
query = query.filter(AnalysisRun.project_id == project_id)
if dataset_id is not None:
query = query.filter(AnalysisRun.dataset_id == dataset_id)
rows = query.order_by(AnalysisRun.created_at.desc()).all()
return DetectionRunListResponse(items=[DetectionRunRead.model_validate(row) for row in rows], total=len(rows))
@staticmethod
def list_detections(
db,
analysis_run_id: uuid.UUID | None = None,
*,
dataset_id: uuid.UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
) -> DetectionListResponse:
if analysis_run_id is not None:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection":
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
rows = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
items = [DetectionRead.model_validate(row) for row in rows]
return DetectionListResponse(items=items, total=len(items))
@staticmethod
def get_detection(db, detection_id: uuid.UUID) -> DetectionRead:
detection = db.get(Detection, detection_id)
if not detection:
raise AppError(code="DETECTION_NOT_FOUND", message="Detection not found", status_code=404)
return DetectionRead.model_validate(detection)
@staticmethod
def detections_to_geojson(
db,
*,
analysis_run_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
) -> dict[str, Any]:
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
return {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": str(detection.id),
"properties": DetectionService._detection_properties(detection),
"geometry": mapping(to_shape(detection.geometry)),
}
for detection in detections
],
}
@staticmethod
def compare_detections_with_reference(
db,
analysis_run_id: uuid.UUID,
reference_dataset_id: uuid.UUID,
iou_threshold: float = 0.5,
class_name: str | None = None,
min_confidence: float | None = None,
) -> dict[str, Any]:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection":
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
reference_dataset = db.get(Dataset, reference_dataset_id)
if not reference_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Reference dataset not found", status_code=404)
if reference_dataset.project_id != run.project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Reference dataset does not belong to detection project", status_code=400)
if reference_dataset.dataset_type not in {"vector", "geojson"}:
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
references = db.query(VectorFeature).filter(VectorFeature.dataset_id == reference_dataset_id).all()
if not references:
raise AppError(
code="REFERENCE_FEATURES_NOT_FOUND",
message="Reference dataset has no persisted vector features for QA",
status_code=422,
)
candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references]
matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
candidate_geometries,
reference_geometries,
iou_threshold,
)
mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
precision = matches / (matches + false_positives) if matches + false_positives > 0 else None
recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None
f1_score = None
if precision is not None and recall is not None:
f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0
status = "unsupported" if unsupported else "ok"
quality_check = QualityService.persist_quality_check(
db=db,
project_id=run.project_id,
analysis_run_id=analysis_run_id,
candidate_dataset_id=run.dataset_id,
reference_dataset_id=reference_dataset_id,
check_type="detections_vs_reference",
status=status,
score=f1_score,
parameters={
"analysis_run_id": str(analysis_run_id),
"reference_dataset_id": str(reference_dataset_id),
"iou_threshold": iou_threshold,
"class_name": class_name,
"min_confidence": min_confidence,
},
findings={
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"warnings": warnings,
"unsupported_geometry": unsupported,
},
metrics={
"precision": precision,
"recall": recall,
"f1": f1_score,
"mean_iou": mean_iou,
"false_positive_count": false_positives,
"false_negative_count": false_negatives,
},
)
return {
"status": status,
"quality_check_id": str(quality_check.id),
"analysis_run_id": str(analysis_run_id),
"reference_dataset_id": str(reference_dataset_id),
"candidate_feature_count": len(candidate_geometries),
"reference_feature_count": len(reference_geometries),
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"precision": precision,
"recall": recall,
"f1_score": f1_score,
"mean_iou": mean_iou,
"iou_threshold": iou_threshold,
"warnings": warnings,
}
@staticmethod
def _create_job(db, project_id: uuid.UUID, dataset_id: uuid.UUID, parameters: dict[str, Any]) -> Job:
job = Job(
id=uuid.uuid4(),
job_type="detection.run",
status="running",
project_id=project_id,
dataset_id=dataset_id,
input_dataset_id=dataset_id,
parameters_json=parameters,
started_at=DetectionService._now(),
)
db.add(job)
db.commit()
db.refresh(job)
return job
@staticmethod
def _query_detection_rows(
db,
*,
analysis_run_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
) -> list[Detection]:
query = db.query(Detection)
if analysis_run_id is not None:
query = query.filter(Detection.analysis_run_id == analysis_run_id)
if dataset_id is not None:
query = query.filter(Detection.dataset_id == dataset_id)
if class_name:
query = query.filter(Detection.class_name == class_name)
if min_confidence is not None:
query = query.filter(Detection.confidence >= min_confidence)
return query.order_by(Detection.created_at.desc()).all()
@staticmethod
def _detection_properties(detection: Detection) -> dict[str, Any]:
return {
"detection_id": str(detection.id),
"class_name": detection.class_name,
"confidence": detection.confidence,
"model_name": detection.model_name,
"model_version": detection.model_version,
"analysis_run_id": str(detection.analysis_run_id) if detection.analysis_run_id else None,
"dataset_id": str(detection.dataset_id) if detection.dataset_id else None,
"job_id": str(detection.job_id) if detection.job_id else None,
"source_tile_path": detection.source_tile_path,
"bbox_json": detection.bbox_json,
}
@staticmethod
def _create_analysis_run(db, project_id, dataset_id, job_id, model, parameters: dict[str, Any]) -> AnalysisRun:
analysis_run = AnalysisRun(
id=uuid.uuid4(),
project_id=project_id,
dataset_id=dataset_id,
job_id=job_id,
analysis_type="detection",
status="running",
model_name=model.model_id,
model_version=model.version,
parameters_json=parameters,
started_at=DetectionService._now(),
)
db.add(analysis_run)
db.commit()
db.refresh(analysis_run)
return analysis_run
@staticmethod
def _mark_failed(db, analysis_run: AnalysisRun, job: Job, code: str, message: str) -> None:
result = {"error_code": code, "message": message, "detection_count": 0}
analysis_run.status = "failed"
analysis_run.finished_at = DetectionService._now()
analysis_run.error_message = message
analysis_run.result_json = result
job.status = "failed"
job.finished_at = analysis_run.finished_at
job.error_message = message
job.result_json = result
db.add(analysis_run)
db.add(job)
db.commit()
db.refresh(analysis_run)
db.refresh(job)
@staticmethod
def _mark_success(db, analysis_run: AnalysisRun, job: Job, detection_count: int) -> None:
result = {"detection_count": detection_count}
analysis_run.status = "success"
analysis_run.finished_at = DetectionService._now()
analysis_run.result_json = result
job.status = "success"
job.finished_at = analysis_run.finished_at
job.result_json = result
db.add(analysis_run)
db.add(job)
db.commit()
db.refresh(analysis_run)
db.refresh(job)
@staticmethod
def _persist_fixture_detections(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
analysis_run: AnalysisRun,
job: Job,
model_name: str,
model_version: str | None,
raw_detections: Any,
confidence_threshold: float,
class_filter: list[str],
) -> list[Detection]:
if not isinstance(raw_detections, list):
raise AppError(code="INVALID_FIXTURE_DETECTIONS", message="fixture_detections must be a list", status_code=400)
persisted: list[Detection] = []
allowed_classes = set(class_filter)
for raw in raw_detections:
if not isinstance(raw, dict):
raise AppError(code="INVALID_FIXTURE_DETECTION", message="Each fixture detection must be an object", status_code=400)
class_name = str(raw.get("class_name") or "")
confidence = float(raw.get("confidence", 0.0))
if allowed_classes and class_name not in allowed_classes:
continue
if confidence < confidence_threshold:
continue
geometry_payload = raw.get("geometry")
if not isinstance(geometry_payload, dict):
raise AppError(code="INVALID_FIXTURE_DETECTION", message="Fixture detection geometry is required", status_code=400)
geometry = shape(geometry_payload)
if geometry.is_empty or not geometry.is_valid:
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture detection geometry must be valid", status_code=400)
detection = Detection(
id=uuid.uuid4(),
project_id=project_id,
dataset_id=dataset_id,
analysis_run_id=analysis_run.id,
job_id=job.id,
model_name=model_name,
model_version=model_version,
class_name=class_name,
confidence=confidence,
geometry=from_shape(geometry, srid=4326),
bbox_json=raw.get("bbox_json"),
source_tile_path=raw.get("source_tile_path"),
properties_json=raw.get("properties_json"),
)
db.add(detection)
persisted.append(detection)
db.commit()
for detection in persisted:
db.refresh(detection)
return persisted
@staticmethod
def _run_configured_yolo(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
analysis_run: AnalysisRun,
job: Job,
model_name: str,
model_version: str | None,
tile_manifest_path: str | None,
confidence_threshold: float,
class_filter: list[str],
settings: Settings,
yolo_adapter_class: Type[YoloDetectionAdapter],
) -> list[Detection]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
model_path = Path(settings.yolo_model_path or "").expanduser()
adapter = yolo_adapter_class(settings)
model = adapter.load_model(model_path)
allowed_classes = set(class_filter)
persisted: list[Detection] = []
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326"
for tile in manifest["tiles"]:
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser())
for raw in adapter.predict_tile(model, tile_path, confidence_threshold):
class_name = str(raw.get("class_name") or "")
confidence = float(raw.get("confidence", 0.0))
if allowed_classes and class_name not in allowed_classes:
continue
if confidence < confidence_threshold:
continue
bbox = raw.get("bbox")
if not isinstance(bbox, list):
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO adapter returned a detection without bbox", status_code=422)
geometry = pixel_bbox_to_epsg4326_polygon(bbox=bbox, tile=tile, crs=tile.get("crs") or manifest_crs)
detection = Detection(
id=uuid.uuid4(),
project_id=project_id,
dataset_id=dataset_id,
analysis_run_id=analysis_run.id,
job_id=job.id,
model_name=model_name,
model_version=model_version,
class_name=class_name,
confidence=confidence,
geometry=from_shape(geometry, srid=4326),
bbox_json={
"x_min": float(bbox[0]),
"y_min": float(bbox[1]),
"x_max": float(bbox[2]),
"y_max": float(bbox[3]),
},
source_tile_path=str(tile_path),
properties_json={**dict(raw.get("properties") or {}), "tile_index": tile.get("index")},
)
db.add(detection)
persisted.append(detection)
db.commit()
for detection in persisted:
db.refresh(detection)
return persisted
@staticmethod
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]:
if not tile_manifest_path:
raise AppError(
code="DETECTION_TILE_MANIFEST_REQUIRED",
message="Configured YOLO inference requires an existing raster tile manifest path",
status_code=400,
)
manifest_path = Path(tile_manifest_path).expanduser()
if not manifest_path.exists() or not manifest_path.is_file():
raise AppError(
code="DETECTION_TILE_MANIFEST_NOT_FOUND",
message="Raster tile manifest path does not exist",
details={"tile_manifest_path": str(manifest_path)},
status_code=422,
)
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Raster tile manifest must be valid JSON", status_code=422) from exc
tiles = manifest.get("tiles")
if not isinstance(tiles, list) or not tiles:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Raster tile manifest must contain tiles", status_code=422)
if len(tiles) > max_tiles:
raise AppError(
code="DETECTION_TILE_LIMIT_EXCEEDED",
message="Raster tile manifest exceeds configured YOLO tile limit",
details={"tile_count": len(tiles), "max_tiles": max_tiles},
status_code=422,
)
return manifest
@staticmethod
def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path) -> Path:
raw_path = tile.get("path")
if not isinstance(raw_path, str) or not raw_path:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile manifest entries require a path", status_code=422)
tile_path = Path(raw_path).expanduser()
if not tile_path.is_absolute():
tile_path = manifest_path.parent / tile_path
if not tile_path.exists() or not tile_path.is_file():
raise AppError(
code="DETECTION_TILE_NOT_FOUND",
message="Tile referenced by manifest does not exist",
details={"tile_path": str(tile_path)},
status_code=422,
)
return tile_path
backend/app/services/export_service.py
+431
View File
@@ -0,0 +1,431 @@
from __future__ import annotations
import json
import re
import uuid
from html import escape
from pathlib import Path
from typing import Any
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import AnalysisRun, Dataset, Export, Project, QualityCheck
from app.schemas.export import ExportContentResponse, ExportCreateResponse, ExportListResponse, ExportRead
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService
from app.services.storage_service import StorageService
class ExportService:
@staticmethod
def export_dataset_geojson(db: Session, dataset_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse:
dataset = db.get(Dataset, dataset_id)
if not dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type not in DatasetService.VECTOR_TYPES:
raise AppError(
code="INVALID_DATASET_TYPE",
message="GeoJSON dataset export requires a vector dataset",
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
metadata = {
"source": "dataset",
"dataset_id": str(dataset.id),
"project_id": str(dataset.project_id),
"dataset_type": dataset.dataset_type,
"feature_count": len(feature_collection.get("features", [])),
}
export = ExportService._write_json_export(
db,
project_id=dataset.project_id,
analysis_run_id=None,
export_type="dataset_geojson",
storage_path=export_path,
content=feature_collection,
metadata=metadata,
)
return ExportService._create_response(export)
@staticmethod
def export_detection_run_geojson(db: Session, analysis_run_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection":
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
metadata = {
"source": "detection_run",
"analysis_run_id": str(run.id),
"project_id": str(run.project_id),
"dataset_id": str(run.dataset_id) if run.dataset_id else None,
"feature_count": len(feature_collection.get("features", [])),
}
export = ExportService._write_json_export(
db,
project_id=run.project_id,
analysis_run_id=run.id,
export_type="detection_geojson",
storage_path=export_path,
content=feature_collection,
metadata=metadata,
)
return ExportService._create_response(export)
@staticmethod
def export_segmentation_run_geojson(db: Session, analysis_run_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "segmentation":
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
metadata = {
"source": "segmentation_run",
"analysis_run_id": str(run.id),
"project_id": str(run.project_id),
"dataset_id": str(run.dataset_id) if run.dataset_id else None,
"feature_count": len(feature_collection.get("features", [])),
}
export = ExportService._write_json_export(
db,
project_id=run.project_id,
analysis_run_id=run.id,
export_type="segmentation_geojson",
storage_path=export_path,
content=feature_collection,
metadata=metadata,
)
return ExportService._create_response(export)
@staticmethod
def export_project_metadata(db: Session, project_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse:
project = db.get(Project, project_id)
if not project:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
content = ExportService._project_summary(db, project)
filename = ExportService._filename(name, f"{project.id}-metadata.json", ".json")
export_path = StorageService.dataset_export_path(str(project.id), "project", filename)
metadata = {
"source": "project_metadata",
"project_id": str(project.id),
"dataset_count": len(content["datasets"]),
"quality_check_count": len(content["quality_checks"]),
"export_count": len(content["exports"]),
}
export = ExportService._write_json_export(
db,
project_id=project.id,
analysis_run_id=None,
export_type="project_metadata_json",
storage_path=export_path,
content=content,
metadata=metadata,
)
return ExportService._create_response(export)
@staticmethod
def export_project_report(db: Session, project_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse:
project = db.get(Project, project_id)
if not project:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
summary = ExportService._project_summary(db, project)
html = ExportService._render_project_report_html(summary)
filename = ExportService._filename(name, f"{project.id}-report.html", ".html")
export_path = StorageService.dataset_export_path(str(project.id), "project", filename)
metadata = {
"source": "project_report",
"project_id": str(project.id),
"dataset_count": len(summary["datasets"]),
"quality_check_count": len(summary["quality_checks"]),
"export_count": len(summary["exports"]),
"format": "html",
}
export = ExportService._write_text_export(
db,
project_id=project.id,
analysis_run_id=None,
export_type="project_report_html",
storage_path=export_path,
content=html,
metadata=metadata,
)
return ExportService._create_response(export)
@staticmethod
def list_project_exports(db: Session, project_id: uuid.UUID, limit: int = 50, offset: int = 0) -> ExportListResponse:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
query = db.query(Export).filter(Export.project_id == project_id).order_by(Export.created_at.desc())
rows = query.offset(offset).limit(limit).all()
total = query.count()
return ExportListResponse(
items=[ExportRead.model_validate(row) for row in rows],
total=total,
limit=limit,
offset=offset,
)
@staticmethod
def get_export(db: Session, export_id: uuid.UUID) -> ExportRead:
export = db.get(Export, export_id)
if not export:
raise AppError(code="EXPORT_NOT_FOUND", message="Export not found", status_code=404)
return ExportRead.model_validate(export)
@staticmethod
def get_export_content(db: Session, export_id: uuid.UUID) -> ExportContentResponse:
export = db.get(Export, export_id)
if not export:
raise AppError(code="EXPORT_NOT_FOUND", message="Export not found", status_code=404)
path = ExportService.get_export_download_path(db, export_id)
try:
content = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise AppError(code="EXPORT_CONTENT_INVALID", message="Export artifact is not valid JSON", status_code=422) from exc
return ExportContentResponse(export_id=export.id, export_type=export.export_type, content=content)
@staticmethod
def get_export_download_path(db: Session, export_id: uuid.UUID) -> Path:
export = db.get(Export, export_id)
if not export:
raise AppError(code="EXPORT_NOT_FOUND", message="Export not found", status_code=404)
path = Path(export.storage_path)
if not path.exists() or not path.is_file():
raise AppError(
code="EXPORT_CONTENT_NOT_FOUND",
message="Export artifact is missing from storage",
details={"storage_path": export.storage_path},
status_code=404,
)
return path
@staticmethod
def _write_json_export(
db: Session,
*,
project_id: uuid.UUID,
analysis_run_id: uuid.UUID | None,
export_type: str,
storage_path: str,
content: dict[str, Any],
metadata: dict[str, Any],
) -> Export:
path = Path(storage_path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(content, ensure_ascii=False, indent=2), encoding="utf-8")
return ExportService._persist_export(
db,
project_id=project_id,
analysis_run_id=analysis_run_id,
export_type=export_type,
storage_path=str(path),
metadata=metadata,
)
@staticmethod
def _write_text_export(
db: Session,
*,
project_id: uuid.UUID,
analysis_run_id: uuid.UUID | None,
export_type: str,
storage_path: str,
content: str,
metadata: dict[str, Any],
) -> Export:
path = Path(storage_path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return ExportService._persist_export(
db,
project_id=project_id,
analysis_run_id=analysis_run_id,
export_type=export_type,
storage_path=str(path),
metadata=metadata,
)
@staticmethod
def _persist_export(
db: Session,
*,
project_id: uuid.UUID,
analysis_run_id: uuid.UUID | None,
export_type: str,
storage_path: str,
metadata: dict[str, Any],
) -> Export:
export = Export(
id=uuid.uuid4(),
project_id=project_id,
analysis_run_id=analysis_run_id,
export_type=export_type,
storage_path=storage_path,
metadata_json=metadata,
)
db.add(export)
db.commit()
db.refresh(export)
return export
@staticmethod
def _project_summary(db: Session, project: Project) -> dict[str, Any]:
datasets = db.query(Dataset).filter(Dataset.project_id == project.id).order_by(Dataset.created_at.desc()).all()
quality_checks = (
db.query(QualityCheck)
.filter(QualityCheck.project_id == project.id)
.order_by(QualityCheck.created_at.desc())
.all()
)
exports = db.query(Export).filter(Export.project_id == project.id).order_by(Export.created_at.desc()).all()
return {
"project": {
"id": str(project.id),
"name": project.name,
"description": project.description,
"region": project.region,
"status": project.status,
},
"datasets": [
{
"id": str(dataset.id),
"name": dataset.name,
"dataset_type": dataset.dataset_type,
"dataset_role": dataset.dataset_role,
"source_name": dataset.source_name,
"reference_layer_name": dataset.reference_layer_name,
"status": dataset.status,
"crs": dataset.crs,
"bounds_json": dataset.bounds_json,
"feature_count": (dataset.metadata_json or {}).get("feature_count"),
}
for dataset in datasets
],
"quality_checks": [
{
"id": str(check.id),
"analysis_run_id": str(check.analysis_run_id) if check.analysis_run_id else None,
"candidate_dataset_id": str(check.candidate_dataset_id) if check.candidate_dataset_id else None,
"reference_dataset_id": str(check.reference_dataset_id),
"check_type": check.check_type,
"status": check.status,
"score": check.score,
}
for check in quality_checks
],
"exports": [
{
"id": str(export.id),
"analysis_run_id": str(export.analysis_run_id) if export.analysis_run_id else None,
"export_type": export.export_type,
"storage_path": export.storage_path,
"metadata_json": export.metadata_json,
"created_at": export.created_at.isoformat() if export.created_at else None,
}
for export in exports
],
}
@staticmethod
def _render_project_report_html(summary: dict[str, Any]) -> str:
project = summary["project"]
datasets = summary["datasets"]
quality_checks = summary["quality_checks"]
exports = summary["exports"]
dataset_rows = "\n".join(
"<tr>"
f"<td>{escape(str(item['name']))}</td>"
f"<td>{escape(str(item['dataset_type']))}</td>"
f"<td>{escape(str(item['dataset_role']))}</td>"
f"<td>{escape(str(item['status']))}</td>"
f"<td>{escape(str(item['feature_count'] if item['feature_count'] is not None else 'n/a'))}</td>"
"</tr>"
for item in datasets
)
quality_rows = "\n".join(
"<tr>"
f"<td>{escape(str(item['check_type']))}</td>"
f"<td>{escape(str(item['status']))}</td>"
f"<td>{escape(str(item['score'] if item['score'] is not None else 'n/a'))}</td>"
f"<td>{escape(str(item['reference_dataset_id']))}</td>"
"</tr>"
for item in quality_checks
)
export_rows = "\n".join(
"<tr>"
f"<td>{escape(str(item['export_type']))}</td>"
f"<td>{escape(str(item['storage_path']))}</td>"
f"<td>{escape(str(item['created_at'] or 'n/a'))}</td>"
"</tr>"
for item in exports
)
return f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>GeoIntel Project Report - {escape(str(project["name"]))}</title>
<style>
body {{ font-family: Arial, sans-serif; color: #0f172a; margin: 2rem; }}
h1, h2 {{ margin-bottom: 0.4rem; }}
table {{ width: 100%; border-collapse: collapse; margin: 1rem 0 2rem; }}
th, td {{ border: 1px solid #cbd5e1; padding: 0.5rem; text-align: left; }}
th {{ background: #e2e8f0; }}
.muted {{ color: #475569; }}
</style>
</head>
<body>
<h1>{escape(str(project["name"]))}</h1>
<p class="muted">GeoIntel project report artifact</p>
<p>Region: {escape(str(project["region"]))}</p>
<p>Status: {escape(str(project["status"]))}</p>
<p>Description: {escape(str(project["description"] or "n/a"))}</p>
<h2>Datasets ({len(datasets)})</h2>
<table>
<thead><tr><th>Name</th><th>Type</th><th>Role</th><th>Status</th><th>Features</th></tr></thead>
<tbody>{dataset_rows or '<tr><td colspan="5">No datasets</td></tr>'}</tbody>
</table>
<h2>QA/QC Results ({len(quality_checks)})</h2>
<table>
<thead><tr><th>Check</th><th>Status</th><th>Score</th><th>Reference dataset</th></tr></thead>
<tbody>{quality_rows or '<tr><td colspan="4">No QA/QC results</td></tr>'}</tbody>
</table>
<h2>Export History ({len(exports)})</h2>
<table>
<thead><tr><th>Type</th><th>Storage path</th><th>Created</th></tr></thead>
<tbody>{export_rows or '<tr><td colspan="3">No exports</td></tr>'}</tbody>
</table>
</body>
</html>
"""
@staticmethod
def _create_response(export: Export) -> ExportCreateResponse:
return ExportCreateResponse(
export_id=export.id,
path=export.storage_path,
status="ready",
export_type=export.export_type,
metadata_json=export.metadata_json,
)
@staticmethod
def _filename(name: str | None, fallback: str, suffix: str) -> str:
raw_name = name or fallback
cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "_", raw_name).strip("._")
if not cleaned:
cleaned = fallback
if not cleaned.lower().endswith(suffix):
cleaned = f"{cleaned}{suffix}"
return cleaned
backend/app/services/geojson_service.py
+134
View File
@@ -0,0 +1,134 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import json
from pyproj import Transformer, CRS
from shapely.geometry import shape
from shapely.geometry.base import BaseGeometry
from shapely.ops import unary_union
from shapely.ops import transform as _transform_geometry
from shapely.validation import make_valid
def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]:
if isinstance(raw_text, dict):
payload = raw_text
else:
try:
payload = json.loads(raw_text)
except Exception as exc:
raise ValueError("Uploaded dataset is not valid JSON") from exc
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection":
raise ValueError("Upload must be a GeoJSON FeatureCollection")
features = payload.get("features") or []
if not isinstance(features, list):
raise ValueError("FeatureCollection features is invalid")
geometry_types: set[str] = set()
geometries = []
invalid_features = 0
polygon_area_m2: float | None = None
crs_assumed = None
for feature in features:
if not isinstance(feature, dict):
continue
geometry = feature.get("geometry")
if not geometry:
continue
try:
geom = shape(geometry)
except Exception as exc:
raise ValueError("Invalid feature geometry") from exc
if not geom.is_valid:
geom = make_valid(geom)
if not geom.is_valid:
invalid_features += 1
raise ValueError("Invalid geometry remains after repair")
geometry_types.add(str(geom.geom_type))
geometries.append(geom)
if geometries:
unioned = unary_union(geometries)
bounds = unioned.bounds
bounds_json = {
"min_x": float(bounds[0]),
"min_y": float(bounds[1]),
"max_x": float(bounds[2]),
"max_y": float(bounds[3]),
}
else:
bounds_json = None
crs = None
crs_assumed = False
raw_crs = payload.get("crs")
if isinstance(raw_crs, dict):
raw_name = raw_crs.get("properties", {}).get("name")
if isinstance(raw_name, str):
crs = raw_name
elif isinstance(raw_crs, str):
crs = raw_crs
if not crs:
crs = "EPSG:4326"
crs_assumed = True
polygon_area_m2 = _approximate_polygon_area_m2(geometries, crs)
return {
"feature_count": len(features),
"geometry_types": sorted(geometry_types),
"bounds_json": bounds_json,
"approximate_area_m2": polygon_area_m2,
"invalid_features": invalid_features,
"crs": crs,
"crs_assumed": crs_assumed,
"extracted_at": datetime.now(timezone.utc).isoformat(),
"feature_geometry_count": len(geometries),
}
def load_dataset_text(file_path: str) -> str:
return Path(file_path).read_text(encoding="utf-8")
def _approximate_polygon_area_m2(geometries: list[BaseGeometry], crs: str | None) -> float | None:
if not geometries:
return 0.0
try:
polygons = [geometry for geometry in geometries if geometry.geom_type.lower() in {"polygon", "multipolygon"}]
if not polygons:
return None
target_crs = CRS.from_epsg(31370)
source_crs = _crs_to_epsg(crs)
transformer = Transformer.from_crs(source_crs, target_crs, always_xy=True)
projected = [_transform_polygon_for_area(geometry, transformer) for geometry in polygons]
area = sum(item.area for item in projected)
if area < 0:
area = 0.0
return float(area)
except Exception:
return None
def _crs_to_epsg(value: str | None) -> str:
if not value:
return "EPSG:4326"
normalized = value.upper().strip().replace(" ", "")
if normalized.startswith("EPSG:"):
return normalized
if normalized.replace("-", "").isdigit():
return f"EPSG:{normalized}"
return "EPSG:4326"
def _transform_polygon_for_area(geometry: BaseGeometry, transformer: Transformer):
if geometry.is_empty:
return geometry
if geometry.geom_type.lower() in {"polygon", "multipolygon"}:
return _transform_geometry(transformer.transform, geometry)
return geometry
backend/app/services/job_service.py
+190
View File
@@ -0,0 +1,190 @@
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from collections.abc import Callable
from typing import Any
from app.core.errors import AppError
from app.models import Job
from app.schemas.job import JobCreate, JobRead
class JobService:
VALID_STATUSES = {"queued", "running", "success", "failed"}
@staticmethod
def run_sync_job(
db,
project_id: uuid.UUID,
job_type: str,
parameters: dict[str, Any] | None,
operation: Callable[[], Any],
input_dataset_id: uuid.UUID | None = None,
) -> dict[str, Any]:
created = JobService.create_job(
db,
JobCreate(
job_type=job_type,
project_id=project_id,
input_dataset_id=input_dataset_id,
parameters_json=JobService._coerce_payload(parameters),
),
)
try:
JobService.mark_running(db, created.id)
result = operation()
output_dataset_id = None
if isinstance(result, uuid.UUID):
output_dataset_id = result
result = {"output_dataset_id": str(result)}
if isinstance(result, dict):
candidate_output_dataset_id = result.get("output_dataset_id")
if isinstance(candidate_output_dataset_id, str):
try:
output_dataset_id = uuid.UUID(candidate_output_dataset_id)
except ValueError:
output_dataset_id = None
elif isinstance(candidate_output_dataset_id, uuid.UUID):
output_dataset_id = candidate_output_dataset_id
if isinstance(result, dict):
job = JobService.mark_success(db, created.id, result=result, output_dataset_id=output_dataset_id)
else:
job = JobService.mark_success(db, created.id, result={"result": result}, output_dataset_id=output_dataset_id)
job_payload = job.model_dump()
if isinstance(job_payload.get("output_dataset_id"), uuid.UUID):
job_payload["output_dataset_id"] = str(job_payload["output_dataset_id"])
result_json = job_payload.get("result_json")
if isinstance(result_json, dict):
if isinstance(result_json.get("output_dataset_id"), uuid.UUID):
result_json["output_dataset_id"] = str(result_json["output_dataset_id"])
job_payload["result_json"] = result_json
return job_payload
except AppError as exc:
failed = JobService.mark_failed(
db,
created.id,
error_message=exc.message,
details={"code": exc.code, "details": exc.details},
)
payload = failed.model_dump()
if isinstance(payload.get("output_dataset_id"), uuid.UUID):
payload["output_dataset_id"] = str(payload["output_dataset_id"])
result_json = payload.get("result_json")
if isinstance(result_json, dict):
if isinstance(result_json.get("output_dataset_id"), uuid.UUID):
result_json["output_dataset_id"] = str(result_json["output_dataset_id"])
payload["result_json"] = result_json
raise
@staticmethod
def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]:
return dict(payload or {})
@staticmethod
def create_job(db, payload: JobCreate) -> JobRead:
job = Job(
id=uuid.uuid4(),
job_type=payload.job_type,
status="queued",
project_id=payload.project_id,
dataset_id=payload.dataset_id,
input_dataset_id=payload.input_dataset_id,
output_dataset_id=payload.output_dataset_id,
parameters_json=JobService._coerce_payload(payload.parameters_json),
result_json=None,
error_message=None,
)
db.add(job)
db.commit()
db.refresh(job)
return JobRead.model_validate(job)
@staticmethod
def mark_running(db, job_id: uuid.UUID) -> JobRead:
job = JobService._get_job(db, job_id)
job.status = "running"
job.started_at = datetime.now(timezone.utc)
job.error_message = None
db.add(job)
db.commit()
db.refresh(job)
return JobRead.model_validate(job)
@staticmethod
def mark_success(
db,
job_id: uuid.UUID,
result: dict[str, Any] | None = None,
output_dataset_id: uuid.UUID | None = None,
) -> JobRead:
job = JobService._get_job(db, job_id)
job.status = "success"
job.finished_at = datetime.now(timezone.utc)
if output_dataset_id is not None:
job.output_dataset_id = output_dataset_id
job.result_json = result
job.error_message = None
db.add(job)
db.commit()
db.refresh(job)
return JobRead.model_validate(job)
@staticmethod
def mark_failed(db, job_id: uuid.UUID, error_message: str, details: dict[str, Any] | None = None) -> JobRead:
job = JobService._get_job(db, job_id)
job.status = "failed"
job.finished_at = datetime.now(timezone.utc)
if details:
job.result_json = details
job.error_message = error_message
db.add(job)
db.commit()
db.refresh(job)
return JobRead.model_validate(job)
@staticmethod
def get_job(db, job_id: uuid.UUID) -> JobRead:
return JobRead.model_validate(JobService._get_job(db, job_id))
@staticmethod
def get_job_status(db, job_id: uuid.UUID) -> dict:
job = JobService._get_job(db, job_id)
return {
"id": job.id,
"project_id": str(job.project_id),
"status": job.status,
"error_message": job.error_message,
"started_at": job.started_at,
"finished_at": job.finished_at,
"result_json": job.result_json,
}
@staticmethod
def list_jobs(
db,
project_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None,
limit: int = 50,
offset: int = 0,
) -> tuple[list[JobRead], int]:
query = db.query(Job)
if project_id is not None:
query = query.filter(Job.project_id == project_id)
if dataset_id is not None:
query = query.filter((Job.dataset_id == dataset_id) | (Job.input_dataset_id == dataset_id) | (Job.output_dataset_id == dataset_id))
total = query.count()
rows = query.order_by(Job.created_at.desc()).offset(offset).limit(limit).all()
return [JobRead.model_validate(row) for row in rows], total
@staticmethod
def _get_job(db, job_id: uuid.UUID) -> Job:
job = db.get(Job, job_id)
if not job:
raise AppError(code="JOB_NOT_FOUND", message="Job not found", status_code=404)
return job
@staticmethod
def validate_status(status: str) -> None:
if status not in JobService.VALID_STATUSES:
raise AppError(code="INVALID_JOB_STATUS", message="Invalid job status", status_code=400)
backend/app/services/model_registry_service.py
+144
View File
@@ -0,0 +1,144 @@
from __future__ import annotations
from pathlib import Path
from typing import Type
from app.core.config import Settings, get_settings
from app.schemas.detection import DetectionModelCapability
from app.services.yolo_adapter import YoloDetectionAdapter
class ModelRegistryService:
@staticmethod
def list_model_capabilities(
settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
task_type: str = "object_detection",
) -> list[DetectionModelCapability]:
resolved_settings = settings or get_settings()
if task_type == "segmentation":
return ModelRegistryService.list_segmentation_model_capabilities()
if task_type != "object_detection":
return []
return [
DetectionModelCapability(
model_id="yolo-placeholder",
display_name="YOLO detector placeholder",
framework="ultralytics/pytorch",
task_type="object_detection",
supported_classes=["building", "road", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed.",
version=None,
),
ModelRegistryService._configured_yolo_capability(resolved_settings, yolo_adapter_class),
DetectionModelCapability(
model_id="manual-fixture-detector",
display_name="Manual fixture detector",
framework="fixture",
task_type="object_detection",
supported_classes=["building"],
configured=True,
status="configured",
limitation_message="Fixture detector is for explicit tests/demo fixtures only and is not production inference.",
version="fixture-v1",
),
]
@staticmethod
def get_model_capability(
model_id: str,
settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
task_type: str = "object_detection",
) -> DetectionModelCapability | None:
normalized = model_id.strip()
for model in ModelRegistryService.list_model_capabilities(settings=settings, yolo_adapter_class=yolo_adapter_class, task_type=task_type):
if model.model_id == normalized:
return model
return None
@staticmethod
def list_segmentation_model_capabilities() -> list[DetectionModelCapability]:
return [
DetectionModelCapability(
model_id="segmentation-placeholder",
display_name="Segmentation placeholder",
framework="placeholder",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="Segmentation inference is not configured in Sprint 9; no SAM/YOLO-seg model is downloaded or executed.",
version=None,
),
DetectionModelCapability(
model_id="fixture-segmenter",
display_name="Fixture segmenter",
framework="fixture",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=True,
status="configured",
limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.",
version="fixture-v1",
),
DetectionModelCapability(
model_id="yolo-seg-configured",
display_name="Configured YOLO segmentation",
framework="ultralytics/pytorch",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="YOLO-seg is not configured in Sprint 9. GeoIntel will not download segmentation model weights automatically.",
version=None,
),
DetectionModelCapability(
model_id="sam-configured",
display_name="Configured SAM segmentation",
framework="sam",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="SAM is not configured in Sprint 9 and is not installed as a backend dependency.",
version=None,
),
]
@staticmethod
def _configured_yolo_capability(
settings: Settings,
yolo_adapter_class: Type[YoloDetectionAdapter],
) -> DetectionModelCapability:
configured = False
status = "not_configured"
limitation = "YOLO is disabled. Set YOLO_ENABLED=true and YOLO_MODEL_PATH to a local model file to enable inference."
model_path = Path(settings.yolo_model_path).expanduser() if settings.yolo_model_path else None
if settings.yolo_enabled:
if not yolo_adapter_class.dependencies_available():
status = "dependency_unavailable"
limitation = "YOLO dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
elif model_path is None:
limitation = "YOLO_MODEL_PATH is not set. GeoIntel will not download model weights automatically."
elif not model_path.exists() or not model_path.is_file():
limitation = "YOLO_MODEL_PATH does not point to an existing local model file. GeoIntel will not download model weights automatically."
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO inference over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.yolo_model_id,
display_name=settings.yolo_model_display_name,
framework="ultralytics/pytorch",
task_type="object_detection",
supported_classes=["building", "road", "water", "landuse"],
configured=configured,
status=status,
limitation_message=limitation,
version=settings.yolo_model_version,
)
backend/app/services/project_service.py
+64
View File
@@ -0,0 +1,64 @@
from __future__ import annotations
import uuid
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Project
from app.schemas.project import ProjectCreate, ProjectRead, ProjectUpdate
class ProjectService:
@staticmethod
def list_projects(db: Session, limit: int = 50, offset: int = 0) -> tuple[list[ProjectRead], int]:
query = db.query(Project).filter(Project.status != "deleted").order_by(Project.created_at.desc())
total = query.count()
items = query.offset(offset).limit(limit).all()
return [ProjectRead.model_validate(item) for item in items], total
@staticmethod
def create_project(db: Session, payload: ProjectCreate) -> ProjectRead:
project = Project(name=payload.name.strip(), description=(payload.description or "").strip() or None, region=payload.region or "Kempen")
db.add(project)
db.commit()
db.refresh(project)
return ProjectRead.model_validate(project)
@staticmethod
def get_project(db: Session, project_id: uuid.UUID) -> ProjectRead | None:
project = db.get(Project, project_id)
if not project or project.status == "deleted":
return None
return ProjectRead.model_validate(project)
@staticmethod
def update_project(db: Session, project_id: uuid.UUID, payload: ProjectUpdate) -> ProjectRead | None:
project = db.get(Project, project_id)
if not project or project.status == "deleted":
return None
payload_data = payload.model_dump(exclude_unset=True)
changed = False
for key, value in payload_data.items():
if value is None:
continue
setattr(project, key, value)
changed = True
if not changed:
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
db.add(project)
db.commit()
db.refresh(project)
return ProjectRead.model_validate(project)
@staticmethod
def delete_project(db: Session, project_id: uuid.UUID) -> bool:
project = db.get(Project, project_id)
if not project or project.status == "deleted":
return False
project.status = "deleted"
db.add(project)
db.commit()
return True
backend/app/services/qa_service.py
+248
View File
@@ -0,0 +1,248 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from uuid import UUID
from geoalchemy2.shape import to_shape
from shapely.geometry import GeometryCollection
from shapely.geometry.base import BaseGeometry
from shapely.ops import unary_union
from shapely.validation import make_valid
from shapely.geometry import shape
from app.core.errors import AppError
from app.models import Area, Dataset
from app.schemas.qa import QaProviderComparisonResult
from app.services.vector_operations_service import VectorOperationsService
def _extract_crs_warnings(source_dataset: Dataset, reference_dataset: Dataset) -> list[str]:
warnings: list[str] = []
for dataset, label in ((source_dataset, "candidate"), (reference_dataset, "reference")):
metadata = dataset.metadata_json
crs_assumed = None
if isinstance(metadata, dict):
crs_assumed = metadata.get("crs_assumed")
if crs_assumed:
warnings.append(f"CRS assumption is weak for {label} dataset ({dataset.id}); geometry metrics are approximate")
if dataset.crs is None:
warnings.append(f"Missing CRS on {label} dataset ({dataset.id})")
return warnings
class QaService:
SUPPORTED_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"}
@staticmethod
def _load_dataset_payload(db, dataset_id: UUID, *, expected_project_id: UUID | None = None) -> tuple[Dataset, dict[str, Any], list[tuple[dict[str, Any], BaseGeometry]]]:
dataset = db.get(Dataset, dataset_id)
if not dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if expected_project_id is not None and dataset.project_id != expected_project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Dataset does not belong to this project", status_code=400)
if dataset.dataset_type not in {"vector", "geojson"}:
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
payload, raw_features = VectorOperationsService._load_dataset_payload(dataset)
geometries = VectorOperationsService._extract_geometries(raw_features)
return dataset, payload, geometries
@staticmethod
def _apply_area_filter(
geometries: list[tuple[dict[str, Any], BaseGeometry]],
area_geometry: BaseGeometry,
*,
dataset_id: UUID,
) -> list[tuple[dict[str, Any], BaseGeometry]]:
area_geom = area_geometry
if isinstance(area_geom, GeometryCollection):
area_geom = unary_union(area_geom.geoms)
filtered: list[tuple[dict[str, Any], BaseGeometry]] = []
for feature, feature_geometry in geometries:
clipped = feature_geometry.intersection(area_geom)
if clipped.is_empty:
continue
if not clipped.is_valid:
clipped = make_valid(clipped)
if not clipped.is_valid:
raise AppError(
code="INVALID_GEOMETRY",
message=f"Area filtering produced invalid geometry for feature in dataset {dataset_id}",
status_code=400,
)
filtered.append((feature, clipped))
return filtered
@staticmethod
def _validate_area(db, area_id: UUID | None, project_id: UUID, *, dataset_ids: tuple[UUID, UUID]) -> BaseGeometry | None:
if not area_id:
return None
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
if area.id in dataset_ids:
raise AppError(code="INVALID_PARAMETERS", message="area_id must reference an area, not a dataset", status_code=400)
area_geometry = to_shape(area.geometry)
if area_geometry.is_empty:
raise AppError(code="INVALID_GEOMETRY", message="Area geometry is empty", status_code=400)
return area_geometry
@staticmethod
def _match_io_u_metrics(
source_geometries: list[tuple[dict[str, Any], BaseGeometry]],
reference_geometries: list[tuple[dict[str, Any], BaseGeometry]],
iou_threshold: float,
) -> tuple[int, int, int, list[float], list[str], bool]:
source_supported = [
(feature, geom) for feature, geom in source_geometries if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
]
reference_supported = [
(feature, geom)
for feature, geom in reference_geometries
if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
]
unsupported = sorted(
{
geom.geom_type
for _, geom in source_geometries + reference_geometries
if geom.geom_type not in QaService.SUPPORTED_GEOMETRY_TYPES
}
)
if not source_supported or not reference_supported:
return (
0,
len(source_supported),
len(reference_supported),
[],
[f"Unsupported geometry types: {unsupported}"] if unsupported else [],
True,
)
unmatched_reference_indices = set(range(len(reference_supported)))
matches = 0
match_iou_values: list[float] = []
false_positives = 0
for _, source_geom in source_supported:
if source_geom.area <= 0:
false_positives += 1
continue
best_iou = 0.0
best_index = None
for reference_index in list(unmatched_reference_indices):
_, reference_geom = reference_supported[reference_index]
if reference_geom.area <= 0:
unmatched_reference_indices.discard(reference_index)
continue
try:
intersection = source_geom.intersection(reference_geom)
except Exception as exc: # pragma: no cover - robustness path
raise AppError(code="GEOMETRY_OPERATION_UNSUPPORTED", message="Geometry operations failed", details={"reason": str(exc)}, status_code=422)
if intersection.is_empty:
continue
intersection_area = intersection.area
if intersection_area < 0:
intersection_area = 0.0
union_area = source_geom.area + reference_geom.area - intersection_area
if union_area <= 0:
continue
candidate_iou = intersection_area / union_area
if candidate_iou > best_iou:
best_iou = candidate_iou
best_index = reference_index
if best_index is not None and best_iou >= iou_threshold:
matches += 1
match_iou_values.append(best_iou)
unmatched_reference_indices.discard(best_index)
else:
false_positives += 1
false_negatives = len(unmatched_reference_indices)
warnings: list[str] = [f"Unsupported geometry types: {unsupported}"] if unsupported else []
return matches, false_positives, false_negatives, match_iou_values, warnings, bool(unsupported)
@staticmethod
def compare_candidate_with_reference(
db,
project_id: UUID,
candidate_dataset_id: UUID,
reference_dataset_id: UUID,
iou_threshold: float = 0.5,
area_id: UUID | None = None,
) -> QaProviderComparisonResult:
if candidate_dataset_id == reference_dataset_id:
raise AppError(code="INVALID_PARAMETERS", message="Candidate and reference dataset must differ", status_code=400)
candidate_dataset, candidate_payload, candidate_geometries = QaService._load_dataset_payload(
db,
candidate_dataset_id,
expected_project_id=project_id,
)
reference_dataset, reference_payload, reference_geometries = QaService._load_dataset_payload(
db,
reference_dataset_id,
expected_project_id=project_id,
)
area_geometry = QaService._validate_area(
db,
area_id=area_id,
project_id=project_id,
dataset_ids=(candidate_dataset_id, reference_dataset_id),
)
if area_geometry is not None:
candidate_geometries = QaService._apply_area_filter(candidate_geometries, area_geometry, dataset_id=candidate_dataset.id)
reference_geometries = QaService._apply_area_filter(reference_geometries, area_geometry, dataset_id=reference_dataset.id)
matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
candidate_geometries,
reference_geometries,
iou_threshold,
)
candidate_feature_count = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0
reference_feature_count = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0
mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
precision = None
if matches + false_positives > 0:
precision = matches / (matches + false_positives)
recall = None
if matches + false_negatives > 0:
recall = matches / (matches + false_negatives)
f1_score = None
if precision is not None and recall is not None and precision + recall > 0:
f1_score = (2 * precision * recall) / (precision + recall)
status = "unsupported" if unsupported else "ok"
return QaProviderComparisonResult(
status=status,
warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + warnings,
candidate_feature_count=candidate_feature_count,
reference_feature_count=reference_feature_count,
matches=matches,
false_positives=false_positives,
false_negatives=false_negatives,
precision=precision,
recall=recall,
f1_score=f1_score,
mean_iou=mean_iou,
iou_threshold=iou_threshold,
unsupported_geometry=unsupported,
unsupported_geometries=warnings,
generated_at=datetime.now(timezone.utc),
)
backend/app/services/quality_check_service.py
+47
View File
@@ -0,0 +1,47 @@
from __future__ import annotations
from uuid import UUID
from sqlalchemy.orm import Session
from app.models import Metric, QualityCheck
from app.schemas.qa import MetricRead, QualityCheckRead
class QualityCheckService:
@staticmethod
def list_quality_checks(
db: Session,
*,
project_id: UUID,
limit: int = 50,
offset: int = 0,
) -> tuple[list[QualityCheckRead], int]:
query = (
db.query(QualityCheck)
.filter(QualityCheck.project_id == project_id)
.order_by(QualityCheck.created_at.desc())
)
total = query.count()
rows = query.offset(offset).limit(limit).all()
if not rows:
return [], total
quality_check_ids = [row.id for row in rows]
metrics_by_quality_check: dict[UUID, list[MetricRead]] = {row.id: [] for row in rows}
metrics = (
db.query(Metric)
.filter(Metric.quality_check_id.in_(quality_check_ids))
.order_by(Metric.created_at.asc())
.all()
)
for metric in metrics:
if metric.quality_check_id in metrics_by_quality_check:
metrics_by_quality_check[metric.quality_check_id].append(MetricRead.model_validate(metric))
return [
QualityCheckRead.model_validate(row).model_copy(
update={"metrics": metrics_by_quality_check.get(row.id, [])}
)
for row in rows
], total
backend/app/services/quality_service.py
+58
View File
@@ -0,0 +1,58 @@
from __future__ import annotations
from datetime import datetime, timezone
from uuid import UUID, uuid4
from app.models import Metric, QualityCheck
class QualityService:
@staticmethod
def persist_quality_check(
db,
project_id: UUID,
reference_dataset_id: UUID,
check_type: str,
status: str,
score: float | None,
parameters: dict | None,
findings: dict | None,
*,
job_id: UUID | None = None,
analysis_run_id: UUID | None = None,
candidate_dataset_id: UUID | None = None,
metrics: dict[str, float | int | None] | None = None,
commit: bool = True,
) -> QualityCheck:
quality_check = QualityCheck(
id=uuid4(),
project_id=project_id,
job_id=job_id,
analysis_run_id=analysis_run_id,
candidate_dataset_id=candidate_dataset_id,
reference_dataset_id=reference_dataset_id,
check_type=check_type,
status=status,
score=score,
parameters_json=parameters or {},
findings_json=findings or {},
completed_at=datetime.now(timezone.utc),
)
db.add(quality_check)
for key, value in (metrics or {}).items():
db.add(
Metric(
id=uuid4(),
quality_check_id=quality_check.id,
analysis_run_id=analysis_run_id,
metric_key=key,
metric_value=float(value) if value is not None else None,
metadata_json={},
)
)
if commit:
db.commit()
db.refresh(quality_check)
return quality_check
backend/app/services/raster_operations_service.py
+1022
View File
File diff suppressed because it is too large Load Diff
backend/app/services/raster_service.py
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from pathlib import Path
from app.core.errors import AppError
def _import_rasterio():
import importlib
rasterio = importlib.import_module("rasterio")
errors = importlib.import_module("rasterio.errors")
return rasterio, errors
def extract_raster_metadata(path: str) -> dict:
try:
rasterio, errors = _import_rasterio()
except Exception as exc: # pragma: no cover - exercised via API-level fallback tests
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Raster processing unavailable. Install rasterio and GDAL-compatible drivers to enable raster metadata extraction.",
status_code=503,
) from exc
dataset_path = Path(path)
try:
with rasterio.open(dataset_path) as dataset:
nodata = dataset.nodata
if isinstance(nodata, (list, tuple)):
nodata_value = [None if value is None else float(value) for value in nodata]
else:
nodata_value = None if nodata is None else float(nodata)
transform = dataset.transform.to_gdal() if hasattr(dataset, "transform") else None
return {
"driver": dataset.driver,
"width": int(dataset.width),
"height": int(dataset.height),
"band_count": int(dataset.count),
"crs": str(dataset.crs) if dataset.crs else None,
"bounds": list(dataset.bounds),
"resolution": list(dataset.res),
"dtype": list(dataset.dtypes),
"nodata": nodata_value,
"transform": list(transform) if transform is not None else None,
}
except Exception as exc:
if isinstance(exc, errors.RasterioIOError):
raise AppError(code="INVALID_RASTER", message="Uploaded raster file is invalid", status_code=400) from exc
raise AppError(code="RASTER_METADATA_ERROR", message="Unable to read raster metadata", status_code=400) from exc
backend/app/services/segmentation_adapter.py
+48
View File
@@ -0,0 +1,48 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Protocol
@dataclass(frozen=True)
class SegmentationAdapterResult:
class_name: str
confidence: float | None
geometry: dict[str, Any]
bbox_json: dict[str, Any] | None = None
mask_path: str | None = None
source_tile_path: str | None = None
tile_index: int | None = None
properties_json: dict[str, Any] | None = None
provenance_json: dict[str, Any] | None = None
area_m2: float | None = None
class SegmentationAdapter(Protocol):
def segment(self, *args: Any, **kwargs: Any) -> list[SegmentationAdapterResult]:
"""Future segmentation adapters must local-import model dependencies inside execution paths."""
class FixtureSegmentationAdapter:
def segment(self, raw_segmentations: Any) -> list[SegmentationAdapterResult]:
if not isinstance(raw_segmentations, list):
return []
results: list[SegmentationAdapterResult] = []
for raw in raw_segmentations:
if not isinstance(raw, dict):
continue
results.append(
SegmentationAdapterResult(
class_name=str(raw.get("class_name") or ""),
confidence=float(raw["confidence"]) if raw.get("confidence") is not None else None,
geometry=raw.get("geometry"),
bbox_json=raw.get("bbox_json"),
mask_path=raw.get("mask_path"),
source_tile_path=raw.get("source_tile_path"),
tile_index=raw.get("tile_index"),
properties_json=raw.get("properties_json"),
provenance_json=raw.get("provenance_json"),
area_m2=raw.get("area_m2"),
)
)
return results
backend/app/services/segmentation_service.py
+512
View File
@@ -0,0 +1,512 @@
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from geoalchemy2.shape import from_shape, to_shape
from shapely.geometry import MultiPolygon, Polygon, mapping, shape
from shapely.validation import make_valid
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import AnalysisRun, Dataset, Job, Project, Segmentation, VectorFeature
from app.schemas.segmentation import (
SegmentationListResponse,
SegmentationRead,
SegmentationRunListResponse,
SegmentationRunRead,
SegmentationRunResponse,
)
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.segmentation_adapter import FixtureSegmentationAdapter
class SegmentationService:
@staticmethod
def _now() -> datetime:
return datetime.now(UTC)
@staticmethod
def run_segmentation(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
model_id: str,
confidence_threshold: float,
class_filter: list[str] | None = None,
tile_manifest_path: str | None = None,
parameters_json: dict[str, Any] | None = None,
settings: Settings | None = None,
) -> SegmentationRunResponse:
parameters = dict(parameters_json or {})
resolved_settings = settings or get_settings()
project = db.get(Project, project_id)
if not project:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster":
raise AppError(
code="INVALID_DATASET_TYPE",
message="Segmentation requires a raster dataset",
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
model = ModelRegistryService.get_model_capability(model_id, task_type="segmentation")
if model is None:
raise AppError(code="SEGMENTATION_MODEL_NOT_FOUND", message="Segmentation model not found", status_code=404)
if model.model_id == "fixture-segmenter" and parameters.get("fixture_mode") is not True:
raise AppError(
code="FIXTURE_MODE_REQUIRED",
message="Fixture segmenter requires explicit fixture_mode=true",
status_code=400,
)
run_parameters = {
"model_id": model.model_id,
"confidence_threshold": confidence_threshold,
"class_filter": class_filter or [],
"tile_manifest_path": tile_manifest_path,
"parameters_json": parameters,
}
job = SegmentationService._create_job(db, project_id, dataset_id, run_parameters)
analysis_run = SegmentationService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters)
if not model.configured:
message = model.limitation_message
SegmentationService._mark_failed(
db,
analysis_run,
job,
code="SEGMENTATION_MODEL_UNAVAILABLE",
message=message,
)
return SegmentationRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="failed",
segmentation_count=0,
error_code="SEGMENTATION_MODEL_UNAVAILABLE",
message=message,
)
if model.model_id == "fixture-segmenter":
segmentations = SegmentationService._persist_fixture_segmentations(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
raw_segmentations=parameters.get("fixture_segmentations"),
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
settings=resolved_settings,
)
SegmentationService._mark_success(db, analysis_run, job, segmentation_count=len(segmentations))
return SegmentationRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="success",
segmentation_count=len(segmentations),
message="Fixture segmentations persisted.",
)
raise AppError(code="SEGMENTATION_MODEL_UNAVAILABLE", message="Segmentation model is unavailable", status_code=503)
@staticmethod
def get_run(db, analysis_run_id: uuid.UUID) -> SegmentationRunRead:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "segmentation":
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
return SegmentationRunRead.model_validate(run)
@staticmethod
def list_runs(
db,
*,
project_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None,
) -> SegmentationRunListResponse:
query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "segmentation")
if project_id is not None:
query = query.filter(AnalysisRun.project_id == project_id)
if dataset_id is not None:
query = query.filter(AnalysisRun.dataset_id == dataset_id)
rows = query.order_by(AnalysisRun.created_at.desc()).all()
return SegmentationRunListResponse(items=[SegmentationRunRead.model_validate(row) for row in rows], total=len(rows))
@staticmethod
def list_segmentations(
db,
analysis_run_id: uuid.UUID | None = None,
*,
dataset_id: uuid.UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
) -> SegmentationListResponse:
if analysis_run_id is not None:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "segmentation":
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
rows = SegmentationService._query_segmentation_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
items = [SegmentationRead.model_validate(row) for row in rows]
return SegmentationListResponse(items=items, total=len(items))
@staticmethod
def get_segmentation(db, segmentation_id: uuid.UUID) -> SegmentationRead:
segmentation = db.get(Segmentation, segmentation_id)
if not segmentation:
raise AppError(code="SEGMENTATION_NOT_FOUND", message="Segmentation not found", status_code=404)
return SegmentationRead.model_validate(segmentation)
@staticmethod
def segmentations_to_geojson(
db,
*,
analysis_run_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
) -> dict[str, Any]:
segmentations = SegmentationService._query_segmentation_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
return {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": str(segmentation.id),
"properties": SegmentationService._segmentation_properties(segmentation),
"geometry": mapping(to_shape(segmentation.geometry)),
}
for segmentation in segmentations
],
}
@staticmethod
def compare_segmentations_with_reference(
db,
analysis_run_id: uuid.UUID,
reference_dataset_id: uuid.UUID,
iou_threshold: float = 0.5,
class_name: str | None = None,
min_confidence: float | None = None,
) -> dict[str, Any]:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "segmentation":
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
reference_dataset = db.get(Dataset, reference_dataset_id)
if not reference_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Reference dataset not found", status_code=404)
if reference_dataset.project_id != run.project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Reference dataset does not belong to segmentation project", status_code=400)
if reference_dataset.dataset_type not in {"vector", "geojson"}:
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
segmentations = SegmentationService._query_segmentation_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
if not segmentations:
raise AppError(
code="SEGMENTATIONS_NOT_FOUND",
message="Segmentation run has no persisted geometries for QA",
status_code=422,
)
references = db.query(VectorFeature).filter(VectorFeature.dataset_id == reference_dataset_id).all()
if not references:
raise AppError(
code="REFERENCE_FEATURES_NOT_FOUND",
message="Reference dataset has no persisted vector features for QA",
status_code=422,
)
candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in segmentations]
reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references]
matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
candidate_geometries,
reference_geometries,
iou_threshold,
)
mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
precision = matches / (matches + false_positives) if matches + false_positives > 0 else None
recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None
f1_score = None
if precision is not None and recall is not None:
f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0
status = "unsupported" if unsupported else "ok"
quality_check = QualityService.persist_quality_check(
db=db,
project_id=run.project_id,
analysis_run_id=analysis_run_id,
candidate_dataset_id=run.dataset_id,
reference_dataset_id=reference_dataset_id,
check_type="segmentations_vs_reference",
status=status,
score=f1_score,
parameters={
"analysis_run_id": str(analysis_run_id),
"reference_dataset_id": str(reference_dataset_id),
"iou_threshold": iou_threshold,
"class_name": class_name,
"min_confidence": min_confidence,
},
findings={
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"warnings": warnings,
"unsupported_geometry": unsupported,
},
metrics={
"precision": precision,
"recall": recall,
"f1": f1_score,
"mean_iou": mean_iou,
"false_positive_count": false_positives,
"false_negative_count": false_negatives,
},
)
return {
"status": status,
"quality_check_id": str(quality_check.id),
"analysis_run_id": str(analysis_run_id),
"reference_dataset_id": str(reference_dataset_id),
"candidate_feature_count": len(candidate_geometries),
"reference_feature_count": len(reference_geometries),
"matches": matches,
"false_positives": false_positives,
"false_negatives": false_negatives,
"precision": precision,
"recall": recall,
"f1_score": f1_score,
"mean_iou": mean_iou,
"iou_threshold": iou_threshold,
"warnings": warnings,
}
@staticmethod
def mask_artifact_path(storage_root: str, project_id: uuid.UUID, analysis_run_id: uuid.UUID, tile_index: int | None, segmentation_id: uuid.UUID) -> str:
tile_folder = f"tile_{tile_index if tile_index is not None else 0}"
return (Path(storage_root) / "masks" / str(project_id) / str(analysis_run_id) / tile_folder / f"mask_{segmentation_id}.png").as_posix()
@staticmethod
def _create_job(db, project_id: uuid.UUID, dataset_id: uuid.UUID, parameters: dict[str, Any]) -> Job:
job = Job(
id=uuid.uuid4(),
job_type="segmentation.run",
status="running",
project_id=project_id,
dataset_id=dataset_id,
input_dataset_id=dataset_id,
parameters_json=parameters,
started_at=SegmentationService._now(),
)
db.add(job)
db.commit()
db.refresh(job)
return job
@staticmethod
def _create_analysis_run(db, project_id, dataset_id, job_id, model, parameters: dict[str, Any]) -> AnalysisRun:
analysis_run = AnalysisRun(
id=uuid.uuid4(),
project_id=project_id,
dataset_id=dataset_id,
job_id=job_id,
analysis_type="segmentation",
status="running",
model_name=model.model_id,
model_version=model.version,
parameters_json=parameters,
started_at=SegmentationService._now(),
)
db.add(analysis_run)
db.commit()
db.refresh(analysis_run)
return analysis_run
@staticmethod
def _mark_failed(db, analysis_run: AnalysisRun, job: Job, code: str, message: str) -> None:
result = {"error_code": code, "message": message, "segmentation_count": 0}
analysis_run.status = "failed"
analysis_run.finished_at = SegmentationService._now()
analysis_run.error_message = message
analysis_run.result_json = result
job.status = "failed"
job.finished_at = analysis_run.finished_at
job.error_message = message
job.result_json = result
db.add(analysis_run)
db.add(job)
db.commit()
db.refresh(analysis_run)
db.refresh(job)
@staticmethod
def _mark_success(db, analysis_run: AnalysisRun, job: Job, segmentation_count: int) -> None:
result = {"segmentation_count": segmentation_count}
analysis_run.status = "success"
analysis_run.finished_at = SegmentationService._now()
analysis_run.result_json = result
job.status = "success"
job.finished_at = analysis_run.finished_at
job.result_json = result
db.add(analysis_run)
db.add(job)
db.commit()
db.refresh(analysis_run)
db.refresh(job)
@staticmethod
def _persist_fixture_segmentations(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
analysis_run: AnalysisRun,
job: Job,
model_name: str,
model_version: str | None,
raw_segmentations: Any,
confidence_threshold: float,
class_filter: list[str],
settings: Settings,
) -> list[Segmentation]:
if not isinstance(raw_segmentations, list):
raise AppError(code="INVALID_FIXTURE_SEGMENTATIONS", message="fixture_segmentations must be a list", status_code=400)
adapter = FixtureSegmentationAdapter()
adapter_results = adapter.segment(raw_segmentations)
if len(adapter_results) != len(raw_segmentations):
raise AppError(code="INVALID_FIXTURE_SEGMENTATION", message="Each fixture segmentation must be an object", status_code=400)
persisted: list[Segmentation] = []
allowed_classes = set(class_filter)
for raw in adapter_results:
class_name = raw.class_name
confidence = raw.confidence
if allowed_classes and class_name not in allowed_classes:
continue
if confidence is not None and confidence < confidence_threshold:
continue
if not isinstance(raw.geometry, dict):
raise AppError(code="INVALID_FIXTURE_SEGMENTATION", message="Fixture segmentation geometry is required", status_code=400)
geometry = SegmentationService._validated_multipolygon(raw.geometry)
segmentation_id = uuid.uuid4()
mask_path = raw.mask_path or SegmentationService.mask_artifact_path(
settings.storage_root,
project_id,
analysis_run.id,
raw.tile_index,
segmentation_id,
)
segmentation = Segmentation(
id=segmentation_id,
project_id=project_id,
dataset_id=dataset_id,
analysis_run_id=analysis_run.id,
job_id=job.id,
model_name=model_name,
model_version=model_version,
class_name=class_name,
confidence=confidence,
geometry=from_shape(geometry, srid=4326),
bbox_json=raw.bbox_json,
area_m2=raw.area_m2,
mask_path=mask_path,
source_tile_path=raw.source_tile_path,
tile_index=raw.tile_index,
properties_json=raw.properties_json,
provenance_json={**dict(raw.provenance_json or {}), "fixture_mode": True},
)
db.add(segmentation)
persisted.append(segmentation)
db.commit()
for segmentation in persisted:
db.refresh(segmentation)
return persisted
@staticmethod
def _validated_multipolygon(geometry_payload: dict[str, Any]) -> MultiPolygon:
try:
geometry = shape(geometry_payload)
except Exception as exc:
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be valid GeoJSON", status_code=400) from exc
if geometry.is_empty:
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must not be empty", status_code=400)
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be valid", status_code=400)
if isinstance(geometry, Polygon):
geometry = MultiPolygon([geometry])
if not isinstance(geometry, MultiPolygon):
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be Polygon or MultiPolygon", status_code=400)
if geometry.area <= 0:
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must have positive area", status_code=400)
return geometry
@staticmethod
def _query_segmentation_rows(
db,
*,
analysis_run_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
) -> list[Segmentation]:
query = db.query(Segmentation)
if analysis_run_id is not None:
query = query.filter(Segmentation.analysis_run_id == analysis_run_id)
if dataset_id is not None:
query = query.filter(Segmentation.dataset_id == dataset_id)
if class_name:
query = query.filter(Segmentation.class_name == class_name)
if min_confidence is not None:
query = query.filter(Segmentation.confidence >= min_confidence)
return query.order_by(Segmentation.created_at.desc()).all()
@staticmethod
def _segmentation_properties(segmentation: Segmentation) -> dict[str, Any]:
return {
"segmentation_id": str(segmentation.id),
"class_name": segmentation.class_name,
"confidence": segmentation.confidence,
"area_m2": segmentation.area_m2,
"model_name": segmentation.model_name,
"model_version": segmentation.model_version,
"analysis_run_id": str(segmentation.analysis_run_id) if segmentation.analysis_run_id else None,
"dataset_id": str(segmentation.dataset_id) if segmentation.dataset_id else None,
"job_id": str(segmentation.job_id) if segmentation.job_id else None,
"source_tile_path": segmentation.source_tile_path,
"tile_index": segmentation.tile_index,
"mask_path": segmentation.mask_path,
"bbox_json": segmentation.bbox_json,
"provenance_json": segmentation.provenance_json,
}
backend/app/services/storage_service.py
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
import hashlib
import shutil
from pathlib import Path
from typing import Any
from app.core.config import get_settings
class StorageService:
@staticmethod
def _base_dir() -> Path:
return Path(get_settings().storage_root).resolve()
@staticmethod
def normalize_dataset_type(dataset_type: str) -> str:
normalized = dataset_type.strip().lower()
if normalized == "geojson":
return "vector"
return normalized
@staticmethod
def _safe_filename(value: str) -> str:
value = value.strip().replace("\\", "/").split("/")[-1]
fallback = "upload"
if not value:
return fallback
allowed = []
for char in value:
if char.isalnum() or char in "-_ .":
allowed.append(char)
else:
allowed.append("_")
cleaned = "".join(allowed)
cleaned = cleaned.strip(" .")
return cleaned or fallback
@staticmethod
def dataset_root(project_id: str, dataset_id: str, dataset_type: str) -> Path:
return StorageService._base_dir() / "uploads" / project_id / dataset_type / dataset_id
@staticmethod
def derived_raster_root(project_id: str, dataset_id: str) -> Path:
return StorageService._base_dir() / "rasters" / "derived" / project_id / dataset_id
@staticmethod
def preview_root(project_id: str, dataset_id: str) -> Path:
return StorageService._base_dir() / "previews" / project_id / dataset_id
@staticmethod
def raster_tiles_root(project_id: str, source_dataset_id: str, tile_set_id: str) -> Path:
return StorageService._base_dir() / "tiles" / project_id / source_dataset_id / tile_set_id
@staticmethod
def dataset_file_path(
project_id: str,
dataset_id: str,
dataset_type: str,
original_filename: str,
) -> str:
safe_original = StorageService._safe_filename(original_filename)
stored_filename = f"{dataset_id}_{safe_original}"
return str(StorageService.dataset_root(project_id, dataset_id, dataset_type) / stored_filename)
@staticmethod
def calculate_checksum_sha256(content: bytes) -> str:
digest = hashlib.sha256()
digest.update(content)
return digest.hexdigest()
@staticmethod
def persist_dataset_file(
project_id: str,
dataset_id: str,
dataset_type: str,
original_filename: str,
content: bytes,
content_type: str | None,
) -> dict[str, Any]:
normalized_type = StorageService.normalize_dataset_type(dataset_type)
file_path = Path(StorageService.dataset_file_path(project_id, dataset_id, normalized_type, original_filename))
file_path.parent.mkdir(parents=True, exist_ok=True)
with file_path.open("wb") as stream:
stream.write(content)
metadata: dict[str, Any] = {
"original_filename": StorageService._safe_filename(original_filename),
"stored_filename": file_path.name,
"content_type": content_type or "application/octet-stream",
"size_bytes": len(content),
"checksum_sha256": StorageService.calculate_checksum_sha256(content),
"storage_path": str(file_path),
}
return metadata
@staticmethod
def persist_file(
storage_path: str,
content: bytes,
original_filename: str,
content_type: str | None,
) -> dict[str, Any]:
target = Path(storage_path)
target.parent.mkdir(parents=True, exist_ok=True)
with target.open("wb") as stream:
stream.write(content)
metadata: dict[str, Any] = {
"original_filename": StorageService._safe_filename(original_filename),
"stored_filename": target.name,
"content_type": content_type or "application/octet-stream",
"size_bytes": len(content),
"checksum_sha256": StorageService.calculate_checksum_sha256(content),
"storage_path": str(target),
}
return metadata
@staticmethod
def remove_dataset_file(path: str) -> None:
target = Path(path)
if target.exists():
target.unlink(missing_ok=True)
dataset_parent = target.parent
if dataset_parent.exists() and dataset_parent.is_dir():
has_files = any(dataset_parent.iterdir())
if not has_files:
shutil.rmtree(dataset_parent, ignore_errors=True)
@staticmethod
def dataset_export_path(project_id: str, dataset_id: str, filename: str) -> str:
output_dir = StorageService._base_dir() / "exports" / project_id / "datasets"
output_dir.mkdir(parents=True, exist_ok=True)
return str(output_dir / f"{dataset_id}_{StorageService._safe_filename(filename)}")
backend/app/services/vector_feature_service.py
+65
View File
@@ -0,0 +1,65 @@
from __future__ import annotations
from typing import Any
from uuid import UUID
from geoalchemy2.shape import from_shape
from shapely.geometry import shape
from shapely.validation import make_valid
from app.core.errors import AppError
from app.models import VectorFeature
class VectorFeatureService:
@staticmethod
def persist_geojson_features(
db,
dataset_id: UUID,
payload: dict[str, Any],
feature_class: str | None = None,
*,
commit: bool = True,
) -> list[VectorFeature]:
features = payload.get("features")
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400)
persisted: list[VectorFeature] = []
for index, feature in enumerate(features):
if not isinstance(feature, dict):
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
geometry_payload = feature.get("geometry")
if geometry_payload is None:
continue
try:
geometry = shape(geometry_payload)
except Exception as exc:
raise AppError(code="INVALID_GEOJSON", message=f"Invalid feature geometry at index {index}", status_code=400) from exc
if geometry.is_empty:
continue
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
source_feature_id = feature.get("id")
if source_feature_id is None:
source_feature_id = properties.get("id") or properties.get("source_feature_id")
row = VectorFeature(
dataset_id=dataset_id,
feature_class=feature_class,
source_feature_id=str(source_feature_id) if source_feature_id is not None else None,
properties_json=properties,
geometry=from_shape(geometry, srid=4326),
)
db.add(row)
persisted.append(row)
if commit:
db.commit()
for row in persisted:
db.refresh(row)
return persisted
backend/app/services/vector_operations_service.py
+329
View File
@@ -0,0 +1,329 @@
from __future__ import annotations
import json
import uuid
from pathlib import Path
from typing import Any
from geoalchemy2.shape import to_shape
from shapely.geometry import GeometryCollection, MultiPolygon, shape
from shapely.geometry.base import BaseGeometry
from shapely.geometry import mapping
from shapely.ops import unary_union
from shapely.validation import make_valid
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Area, Dataset
from app.schemas.operations import VectorOperationResult
from app.services.geojson_service import parse_geojson_payload
from app.services.storage_service import StorageService
class VectorOperationsService:
@staticmethod
def _require_vector_dataset(dataset: Dataset) -> None:
if dataset.dataset_type not in {"vector", "geojson"}:
raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a vector dataset", status_code=400)
@staticmethod
def _load_dataset_payload(dataset: Dataset) -> tuple[dict[str, Any], list[dict[str, Any]]]:
if not dataset.storage_path:
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
path = Path(dataset.storage_path)
if not path.exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception as exc:
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection":
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is not a FeatureCollection", status_code=400)
features = payload.get("features")
if not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400)
return payload, [feature for feature in features if isinstance(feature, dict)]
@staticmethod
def _extract_geometries(features: list[dict[str, Any]]) -> list[tuple[dict[str, Any], BaseGeometry]]:
geometries: list[tuple[dict[str, Any], BaseGeometry]] = []
for feature in features:
if not isinstance(feature, dict):
continue
geometry = feature.get("geometry")
if not geometry:
continue
try:
shapely_geom = shape(geometry)
except Exception as exc:
raise AppError(code="INVALID_GEOMETRY", message="Feature geometry invalid", status_code=400) from exc
if not shapely_geom.is_valid:
shapely_geom = make_valid(shapely_geom)
if not shapely_geom.is_valid:
raise AppError(code="INVALID_GEOMETRY", message="Feature geometry cannot be repaired", status_code=400)
geometries.append((feature, shapely_geom))
if not geometries:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422)
return geometries
@staticmethod
def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult:
dataset = db.get(Dataset, dataset_id)
if not dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
VectorOperationsService._require_vector_dataset(dataset)
payload, features = VectorOperationsService._load_dataset_payload(dataset)
geometries = VectorOperationsService._extract_geometries(features)
geometry_type_summary: dict[str, int] = {}
for _, geometry in geometries:
geometry_type_summary[geometry.geom_type] = geometry_type_summary.get(geometry.geom_type, 0) + 1
unioned = unary_union([geometry for _, geometry in geometries])
bounds = unioned.bounds
return VectorOperationResult(
source_dataset_id=str(dataset_id),
feature_count=len(geometries),
geometry_type_summary=geometry_type_summary,
bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])},
crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs,
)
@staticmethod
def bbox(db: Session, dataset_id: uuid.UUID) -> dict[str, Any]:
summary = VectorOperationsService.inspect(db, dataset_id)
return {
"dataset_id": str(dataset_id),
"bounds_json": summary.bounds_json,
"feature_count": summary.feature_count,
"crs": summary.crs,
}
@staticmethod
def stats(db: Session, dataset_id: uuid.UUID) -> dict[str, Any]:
summary = VectorOperationsService.inspect(db, dataset_id)
return {
"dataset_id": str(dataset_id),
"feature_count": summary.feature_count,
"geometry_type_summary": summary.geometry_type_summary,
"bounds_json": summary.bounds_json,
"crs": summary.crs,
}
@staticmethod
def clip_by_area(db: Session, dataset_id: uuid.UUID, area_id: uuid.UUID, output_name: str | None) -> uuid.UUID:
source_dataset = db.get(Dataset, dataset_id)
if not source_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
VectorOperationsService._require_vector_dataset(source_dataset)
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != source_dataset.project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to dataset project", status_code=400)
payload, features = VectorOperationsService._load_dataset_payload(source_dataset)
geometries = VectorOperationsService._extract_geometries(features)
area_geom = to_shape(area.geometry)
if area_geom.is_empty:
raise AppError(code="INVALID_AREA_GEOMETRY", message="Area geometry is empty", status_code=400)
if isinstance(area_geom, GeometryCollection):
area_geom = unary_union(area_geom.geoms)
if area_geom.geom_type == "MultiPolygon":
area_geom = MultiPolygon(area_geom.geoms)
if not area_geom.is_valid:
area_geom = make_valid(area_geom)
if not area_geom.is_valid:
raise AppError(code="INVALID_AREA_GEOMETRY", message="Area geometry cannot be repaired", status_code=400)
output_features: list[dict[str, Any]] = []
for feature, source_geom in geometries:
clipped = source_geom.intersection(area_geom)
if clipped.is_empty:
continue
if not clipped.is_valid:
clipped = make_valid(clipped)
if not clipped.is_valid:
raise AppError(code="INVALID_GEOMETRY", message="Clipped geometry became invalid", status_code=400)
output_features.append({
"type": "Feature",
"geometry": mapping(clipped),
"properties": feature.get("properties", {}) or {},
})
if not output_features:
raise AppError(code="VECTOR_OPERATION_EMPTY_RESULT", message="Clip operation produced no output features", status_code=422)
return VectorOperationsService._persist_derived_dataset(
db=db,
source_dataset=source_dataset,
source_id=dataset_id,
operation="clip",
feature_collection={"type": "FeatureCollection", "features": output_features},
output_name=output_name,
default_name="vector_clipped",
)
@staticmethod
def buffer(db: Session, dataset_id: uuid.UUID, distance_m: float, dissolve: bool, output_name: str | None) -> uuid.UUID:
source_dataset = db.get(Dataset, dataset_id)
if not source_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
VectorOperationsService._require_vector_dataset(source_dataset)
if distance_m <= 0:
raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400)
_, features = VectorOperationsService._load_dataset_payload(source_dataset)
geometries = VectorOperationsService._extract_geometries(features)
buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries]
output_features: list[dict[str, Any]] = []
for feature, geometry in buffered_features:
if geometry.is_empty:
continue
if not geometry.is_valid:
geometry = make_valid(geometry)
if not geometry.is_valid:
raise AppError(code="INVALID_GEOMETRY", message="Buffer geometry became invalid", status_code=400)
output_features.append({
"type": "Feature",
"geometry": mapping(geometry),
"properties": feature.get("properties", {}) or {},
})
if dissolve:
dissolved = unary_union([shape(feature["geometry"]) for feature in output_features])
output_features = [{
"type": "Feature",
"geometry": mapping(dissolved),
"properties": {"operation": "vector_buffer", "distance_m": distance_m, "dissolve": True},
}]
if not output_features:
raise AppError(code="VECTOR_OPERATION_EMPTY_RESULT", message="Buffer operation produced no output features", status_code=422)
return VectorOperationsService._persist_derived_dataset(
db=db,
source_dataset=source_dataset,
source_id=dataset_id,
operation="buffer",
feature_collection={"type": "FeatureCollection", "features": output_features},
output_name=output_name,
default_name="vector_buffered",
)
@staticmethod
def intersect(
db: Session,
source_dataset_id: uuid.UUID,
target_dataset_id: uuid.UUID,
output_name: str | None,
) -> uuid.UUID:
if source_dataset_id == target_dataset_id:
raise AppError(code="INVALID_PARAMETERS", message="other_dataset_id must be different from source dataset", status_code=400)
source_dataset = db.get(Dataset, source_dataset_id)
if not source_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
VectorOperationsService._require_vector_dataset(source_dataset)
target_dataset = db.get(Dataset, target_dataset_id)
if not target_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Target dataset not found", status_code=404)
VectorOperationsService._require_vector_dataset(target_dataset)
if target_dataset.project_id != source_dataset.project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Datasets must belong to same project", status_code=400)
source_payload, source_features = VectorOperationsService._load_dataset_payload(source_dataset)
target_payload, _ = VectorOperationsService._load_dataset_payload(target_dataset)
source_geometries = VectorOperationsService._extract_geometries(source_features)
target_geometries = VectorOperationsService._extract_geometries(target_payload.get("features", []))
target_union = unary_union([geometry for _, geometry in target_geometries])
output_features: list[dict[str, Any]] = []
for source_feature, source_geometry in source_geometries:
intersection = source_geometry.intersection(target_union)
if intersection.is_empty:
continue
if not intersection.is_valid:
intersection = make_valid(intersection)
if not intersection.is_valid:
raise AppError(code="INVALID_GEOMETRY", message="Intersection geometry became invalid", status_code=400)
output_features.append({
"type": "Feature",
"geometry": mapping(intersection),
"properties": source_feature.get("properties", {}) or {},
})
if not output_features:
raise AppError(code="VECTOR_OPERATION_EMPTY_RESULT", message="Intersection operation produced no output features", status_code=422)
return VectorOperationsService._persist_derived_dataset(
db=db,
source_dataset=source_dataset,
source_id=source_dataset_id,
operation="intersect",
feature_collection={"type": "FeatureCollection", "features": output_features},
output_name=output_name,
default_name="vector_intersect",
)
@staticmethod
def _persist_derived_dataset(
db: Session,
source_dataset: Dataset,
source_id: uuid.UUID,
operation: str,
feature_collection: dict[str, Any],
output_name: str | None,
default_name: str,
) -> uuid.UUID:
derived_id = uuid.uuid4()
output_name_value = f"{(output_name or default_name)}.geojson"
if not output_name_value.strip():
output_name_value = f"{default_name}.geojson"
stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
storage_info = StorageService.persist_dataset_file(
project_id=str(source_dataset.project_id),
dataset_id=str(derived_id),
dataset_type="vector",
original_filename=output_name_value,
content=stored,
content_type="application/geo+json",
)
metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")))
derived_dataset = Dataset(
id=derived_id,
project_id=source_dataset.project_id,
area_id=source_dataset.area_id,
name=output_name_value,
dataset_type="vector",
source=f"operation:{operation}",
storage_path=storage_info["storage_path"],
original_filename=storage_info["original_filename"],
stored_filename=storage_info["stored_filename"],
content_type=storage_info["content_type"],
size_bytes=storage_info["size_bytes"],
checksum_sha256=storage_info["checksum_sha256"],
derived_from_dataset_id=source_id,
crs=metadata.get("crs"),
bounds_json=metadata.get("bounds_json"),
resolution_json=metadata.get("resolution_json"),
bands_json=metadata.get("bands_json"),
metadata_json=metadata,
status="ready",
)
db.add(derived_dataset)
db.commit()
db.refresh(derived_dataset)
return derived_id
backend/app/services/yolo_adapter.py
+100
View File
@@ -0,0 +1,100 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
from typing import Any
from app.core.config import Settings
from app.core.errors import AppError
class YoloDetectionAdapter:
def __init__(self, settings: Settings) -> None:
self.settings = settings
@staticmethod
def dependencies_available() -> bool:
return importlib.util.find_spec("ultralytics") is not None and importlib.util.find_spec("torch") is not None
def load_model(self, model_path: Path):
if not model_path.exists() or not model_path.is_file():
raise AppError(
code="DETECTION_MODEL_UNAVAILABLE",
message="Configured YOLO model file does not exist",
details={"model_path": str(model_path)},
status_code=503,
)
if not self.dependencies_available():
raise AppError(
code="DETECTION_DEPENDENCY_UNAVAILABLE",
message="YOLO dependencies are not installed. Install backend optional extras with geointel-backend[ai].",
status_code=503,
)
try:
from ultralytics import YOLO
except ImportError as exc:
raise AppError(
code="DETECTION_DEPENDENCY_UNAVAILABLE",
message="YOLO dependencies are not importable. Install backend optional extras with geointel-backend[ai].",
status_code=503,
) from exc
try:
return YOLO(str(model_path))
except Exception as exc:
raise AppError(
code="DETECTION_MODEL_LOAD_FAILED",
message="Configured YOLO model could not be loaded",
details={"model_path": str(model_path)},
status_code=503,
) from exc
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict[str, Any]]:
if not tile_path.exists() or not tile_path.is_file():
raise AppError(
code="DETECTION_TILE_NOT_FOUND",
message="Tile referenced by manifest does not exist",
details={"tile_path": str(tile_path)},
status_code=422,
)
results = model.predict(
source=str(tile_path),
conf=float(confidence_threshold),
imgsz=int(self.settings.yolo_image_size),
device=self.settings.yolo_device,
verbose=False,
)
detections: list[dict[str, Any]] = []
for result in results:
names = getattr(result, "names", {}) or {}
boxes = getattr(result, "boxes", None)
if boxes is None:
continue
xyxy_values = _to_list(getattr(boxes, "xyxy", []))
confidence_values = _to_list(getattr(boxes, "conf", []))
class_values = _to_list(getattr(boxes, "cls", []))
for index, bbox in enumerate(xyxy_values):
class_id = int(class_values[index]) if index < len(class_values) else -1
detections.append(
{
"class_name": str(names.get(class_id, class_id)),
"confidence": float(confidence_values[index]) if index < len(confidence_values) else 0.0,
"bbox": [float(value) for value in bbox],
"properties": {"class_id": class_id},
}
)
return detections
def _to_list(value: Any) -> list[Any]:
if hasattr(value, "detach"):
value = value.detach()
if hasattr(value, "cpu"):
value = value.cpu()
if hasattr(value, "numpy"):
value = value.numpy()
if hasattr(value, "tolist"):
return value.tolist()
return list(value)
backend/app/services/yolo_preflight_service.py
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Type
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.detection_service import DetectionService
from app.services.yolo_adapter import YoloDetectionAdapter
class YoloPreflightService:
@staticmethod
def run(
*,
settings: Settings | None = None,
tile_manifest_path: str | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
assume_dependencies: bool = False,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
result: dict[str, Any] = {
"model_id": resolved_settings.yolo_model_id,
"model_path": resolved_settings.yolo_model_path,
"tile_manifest_path": tile_manifest_path,
"status": "not_configured",
"message": "",
"checks": {
"enabled": resolved_settings.yolo_enabled,
"dependencies_available": None,
"model_path_set": None,
"model_file_exists": None,
"manifest_path_set": None,
"manifest_valid": None,
"tile_paths_exist": None,
"tile_limit_ok": None,
},
"tile_count": 0,
"max_tiles": resolved_settings.yolo_max_tiles,
"will_download_models": False,
"will_run_inference": False,
}
if not resolved_settings.yolo_enabled:
result["message"] = "YOLO is disabled. Set YOLO_ENABLED=true for configured local inference."
return result
dependencies_available = True if assume_dependencies else yolo_adapter_class.dependencies_available()
result["checks"]["dependencies_available"] = dependencies_available
if not dependencies_available:
result["status"] = "dependency_unavailable"
result["message"] = "YOLO dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
return result
result["checks"]["model_path_set"] = bool(resolved_settings.yolo_model_path)
if not resolved_settings.yolo_model_path:
result["message"] = "YOLO_MODEL_PATH is not set. GeoIntel will not download model weights automatically."
return result
model_path = Path(resolved_settings.yolo_model_path).expanduser()
model_exists = model_path.exists() and model_path.is_file()
result["checks"]["model_file_exists"] = model_exists
if not model_exists:
result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file."
return result
result["checks"]["manifest_path_set"] = bool(tile_manifest_path)
if not tile_manifest_path:
result["status"] = "manifest_unavailable"
result["message"] = "Configured YOLO inference requires an existing raster tile manifest path."
return result
try:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, resolved_settings.yolo_max_tiles)
tile_paths = [DetectionService._resolve_tile_path(tile, Path(tile_manifest_path).expanduser()) for tile in manifest["tiles"]]
except AppError as exc:
result["status"] = "manifest_invalid"
result["message"] = exc.message
result["error_code"] = exc.code
result["checks"]["manifest_valid"] = False
if exc.code != "DETECTION_TILE_LIMIT_EXCEEDED":
result["checks"]["tile_limit_ok"] = None
else:
result["checks"]["tile_limit_ok"] = False
return result
result["checks"]["manifest_valid"] = True
result["checks"]["tile_paths_exist"] = all(path.exists() and path.is_file() for path in tile_paths)
result["checks"]["tile_limit_ok"] = len(tile_paths) <= resolved_settings.yolo_max_tiles
result["tile_count"] = len(tile_paths)
result["status"] = "ready"
result["message"] = "Configured YOLO preflight passed. No model was loaded and no inference was run."
return result