Initial GeoIntel V1 foundation
This commit is contained in:
Codex
@@ -0,0 +1,512 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from geoalchemy2.shape import from_shape, to_shape
|
||||
from shapely.geometry import MultiPolygon, Polygon, mapping, shape
|
||||
from shapely.validation import make_valid
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Dataset, Job, Project, Segmentation, VectorFeature
|
||||
from app.schemas.segmentation import (
|
||||
SegmentationListResponse,
|
||||
SegmentationRead,
|
||||
SegmentationRunListResponse,
|
||||
SegmentationRunRead,
|
||||
SegmentationRunResponse,
|
||||
)
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.quality_service import QualityService
|
||||
from app.services.segmentation_adapter import FixtureSegmentationAdapter
|
||||
|
||||
|
||||
class SegmentationService:
|
||||
@staticmethod
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
@staticmethod
|
||||
def run_segmentation(
|
||||
db,
|
||||
project_id: uuid.UUID,
|
||||
dataset_id: uuid.UUID,
|
||||
model_id: str,
|
||||
confidence_threshold: float,
|
||||
class_filter: list[str] | None = None,
|
||||
tile_manifest_path: str | None = None,
|
||||
parameters_json: dict[str, Any] | None = None,
|
||||
settings: Settings | None = None,
|
||||
) -> SegmentationRunResponse:
|
||||
parameters = dict(parameters_json or {})
|
||||
resolved_settings = settings or get_settings()
|
||||
project = db.get(Project, project_id)
|
||||
if not project:
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
if dataset.dataset_type != "raster":
|
||||
raise AppError(
|
||||
code="INVALID_DATASET_TYPE",
|
||||
message="Segmentation requires a raster dataset",
|
||||
details={"dataset_type": dataset.dataset_type},
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
model = ModelRegistryService.get_model_capability(model_id, task_type="segmentation")
|
||||
if model is None:
|
||||
raise AppError(code="SEGMENTATION_MODEL_NOT_FOUND", message="Segmentation model not found", status_code=404)
|
||||
if model.model_id == "fixture-segmenter" and parameters.get("fixture_mode") is not True:
|
||||
raise AppError(
|
||||
code="FIXTURE_MODE_REQUIRED",
|
||||
message="Fixture segmenter requires explicit fixture_mode=true",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
run_parameters = {
|
||||
"model_id": model.model_id,
|
||||
"confidence_threshold": confidence_threshold,
|
||||
"class_filter": class_filter or [],
|
||||
"tile_manifest_path": tile_manifest_path,
|
||||
"parameters_json": parameters,
|
||||
}
|
||||
job = SegmentationService._create_job(db, project_id, dataset_id, run_parameters)
|
||||
analysis_run = SegmentationService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters)
|
||||
|
||||
if not model.configured:
|
||||
message = model.limitation_message
|
||||
SegmentationService._mark_failed(
|
||||
db,
|
||||
analysis_run,
|
||||
job,
|
||||
code="SEGMENTATION_MODEL_UNAVAILABLE",
|
||||
message=message,
|
||||
)
|
||||
return SegmentationRunResponse(
|
||||
analysis_run_id=analysis_run.id,
|
||||
job_id=job.id,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id=model.model_id,
|
||||
status="failed",
|
||||
segmentation_count=0,
|
||||
error_code="SEGMENTATION_MODEL_UNAVAILABLE",
|
||||
message=message,
|
||||
)
|
||||
|
||||
if model.model_id == "fixture-segmenter":
|
||||
segmentations = SegmentationService._persist_fixture_segmentations(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
analysis_run=analysis_run,
|
||||
job=job,
|
||||
model_name=model.model_id,
|
||||
model_version=model.version,
|
||||
raw_segmentations=parameters.get("fixture_segmentations"),
|
||||
confidence_threshold=confidence_threshold,
|
||||
class_filter=class_filter or [],
|
||||
settings=resolved_settings,
|
||||
)
|
||||
SegmentationService._mark_success(db, analysis_run, job, segmentation_count=len(segmentations))
|
||||
return SegmentationRunResponse(
|
||||
analysis_run_id=analysis_run.id,
|
||||
job_id=job.id,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id=model.model_id,
|
||||
status="success",
|
||||
segmentation_count=len(segmentations),
|
||||
message="Fixture segmentations persisted.",
|
||||
)
|
||||
|
||||
raise AppError(code="SEGMENTATION_MODEL_UNAVAILABLE", message="Segmentation model is unavailable", status_code=503)
|
||||
|
||||
@staticmethod
|
||||
def get_run(db, analysis_run_id: uuid.UUID) -> SegmentationRunRead:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "segmentation":
|
||||
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
||||
return SegmentationRunRead.model_validate(run)
|
||||
|
||||
@staticmethod
|
||||
def list_runs(
|
||||
db,
|
||||
*,
|
||||
project_id: uuid.UUID | None = None,
|
||||
dataset_id: uuid.UUID | None = None,
|
||||
) -> SegmentationRunListResponse:
|
||||
query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "segmentation")
|
||||
if project_id is not None:
|
||||
query = query.filter(AnalysisRun.project_id == project_id)
|
||||
if dataset_id is not None:
|
||||
query = query.filter(AnalysisRun.dataset_id == dataset_id)
|
||||
rows = query.order_by(AnalysisRun.created_at.desc()).all()
|
||||
return SegmentationRunListResponse(items=[SegmentationRunRead.model_validate(row) for row in rows], total=len(rows))
|
||||
|
||||
@staticmethod
|
||||
def list_segmentations(
|
||||
db,
|
||||
analysis_run_id: uuid.UUID | None = None,
|
||||
*,
|
||||
dataset_id: uuid.UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
) -> SegmentationListResponse:
|
||||
if analysis_run_id is not None:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "segmentation":
|
||||
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
||||
rows = SegmentationService._query_segmentation_rows(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
items = [SegmentationRead.model_validate(row) for row in rows]
|
||||
return SegmentationListResponse(items=items, total=len(items))
|
||||
|
||||
@staticmethod
|
||||
def get_segmentation(db, segmentation_id: uuid.UUID) -> SegmentationRead:
|
||||
segmentation = db.get(Segmentation, segmentation_id)
|
||||
if not segmentation:
|
||||
raise AppError(code="SEGMENTATION_NOT_FOUND", message="Segmentation not found", status_code=404)
|
||||
return SegmentationRead.model_validate(segmentation)
|
||||
|
||||
@staticmethod
|
||||
def segmentations_to_geojson(
|
||||
db,
|
||||
*,
|
||||
analysis_run_id: uuid.UUID | None = None,
|
||||
dataset_id: uuid.UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
segmentations = SegmentationService._query_segmentation_rows(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": str(segmentation.id),
|
||||
"properties": SegmentationService._segmentation_properties(segmentation),
|
||||
"geometry": mapping(to_shape(segmentation.geometry)),
|
||||
}
|
||||
for segmentation in segmentations
|
||||
],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def compare_segmentations_with_reference(
|
||||
db,
|
||||
analysis_run_id: uuid.UUID,
|
||||
reference_dataset_id: uuid.UUID,
|
||||
iou_threshold: float = 0.5,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "segmentation":
|
||||
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
||||
reference_dataset = db.get(Dataset, reference_dataset_id)
|
||||
if not reference_dataset:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Reference dataset not found", status_code=404)
|
||||
if reference_dataset.project_id != run.project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Reference dataset does not belong to segmentation project", status_code=400)
|
||||
if reference_dataset.dataset_type not in {"vector", "geojson"}:
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
|
||||
|
||||
segmentations = SegmentationService._query_segmentation_rows(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=run.dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
if not segmentations:
|
||||
raise AppError(
|
||||
code="SEGMENTATIONS_NOT_FOUND",
|
||||
message="Segmentation run has no persisted geometries for QA",
|
||||
status_code=422,
|
||||
)
|
||||
references = db.query(VectorFeature).filter(VectorFeature.dataset_id == reference_dataset_id).all()
|
||||
if not references:
|
||||
raise AppError(
|
||||
code="REFERENCE_FEATURES_NOT_FOUND",
|
||||
message="Reference dataset has no persisted vector features for QA",
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in segmentations]
|
||||
reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references]
|
||||
matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
|
||||
candidate_geometries,
|
||||
reference_geometries,
|
||||
iou_threshold,
|
||||
)
|
||||
mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
|
||||
precision = matches / (matches + false_positives) if matches + false_positives > 0 else None
|
||||
recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None
|
||||
f1_score = None
|
||||
if precision is not None and recall is not None:
|
||||
f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0
|
||||
status = "unsupported" if unsupported else "ok"
|
||||
quality_check = QualityService.persist_quality_check(
|
||||
db=db,
|
||||
project_id=run.project_id,
|
||||
analysis_run_id=analysis_run_id,
|
||||
candidate_dataset_id=run.dataset_id,
|
||||
reference_dataset_id=reference_dataset_id,
|
||||
check_type="segmentations_vs_reference",
|
||||
status=status,
|
||||
score=f1_score,
|
||||
parameters={
|
||||
"analysis_run_id": str(analysis_run_id),
|
||||
"reference_dataset_id": str(reference_dataset_id),
|
||||
"iou_threshold": iou_threshold,
|
||||
"class_name": class_name,
|
||||
"min_confidence": min_confidence,
|
||||
},
|
||||
findings={
|
||||
"matches": matches,
|
||||
"false_positives": false_positives,
|
||||
"false_negatives": false_negatives,
|
||||
"warnings": warnings,
|
||||
"unsupported_geometry": unsupported,
|
||||
},
|
||||
metrics={
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"f1": f1_score,
|
||||
"mean_iou": mean_iou,
|
||||
"false_positive_count": false_positives,
|
||||
"false_negative_count": false_negatives,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"status": status,
|
||||
"quality_check_id": str(quality_check.id),
|
||||
"analysis_run_id": str(analysis_run_id),
|
||||
"reference_dataset_id": str(reference_dataset_id),
|
||||
"candidate_feature_count": len(candidate_geometries),
|
||||
"reference_feature_count": len(reference_geometries),
|
||||
"matches": matches,
|
||||
"false_positives": false_positives,
|
||||
"false_negatives": false_negatives,
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"f1_score": f1_score,
|
||||
"mean_iou": mean_iou,
|
||||
"iou_threshold": iou_threshold,
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def mask_artifact_path(storage_root: str, project_id: uuid.UUID, analysis_run_id: uuid.UUID, tile_index: int | None, segmentation_id: uuid.UUID) -> str:
|
||||
tile_folder = f"tile_{tile_index if tile_index is not None else 0}"
|
||||
return (Path(storage_root) / "masks" / str(project_id) / str(analysis_run_id) / tile_folder / f"mask_{segmentation_id}.png").as_posix()
|
||||
|
||||
@staticmethod
|
||||
def _create_job(db, project_id: uuid.UUID, dataset_id: uuid.UUID, parameters: dict[str, Any]) -> Job:
|
||||
job = Job(
|
||||
id=uuid.uuid4(),
|
||||
job_type="segmentation.run",
|
||||
status="running",
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
input_dataset_id=dataset_id,
|
||||
parameters_json=parameters,
|
||||
started_at=SegmentationService._now(),
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
@staticmethod
|
||||
def _create_analysis_run(db, project_id, dataset_id, job_id, model, parameters: dict[str, Any]) -> AnalysisRun:
|
||||
analysis_run = AnalysisRun(
|
||||
id=uuid.uuid4(),
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
job_id=job_id,
|
||||
analysis_type="segmentation",
|
||||
status="running",
|
||||
model_name=model.model_id,
|
||||
model_version=model.version,
|
||||
parameters_json=parameters,
|
||||
started_at=SegmentationService._now(),
|
||||
)
|
||||
db.add(analysis_run)
|
||||
db.commit()
|
||||
db.refresh(analysis_run)
|
||||
return analysis_run
|
||||
|
||||
@staticmethod
|
||||
def _mark_failed(db, analysis_run: AnalysisRun, job: Job, code: str, message: str) -> None:
|
||||
result = {"error_code": code, "message": message, "segmentation_count": 0}
|
||||
analysis_run.status = "failed"
|
||||
analysis_run.finished_at = SegmentationService._now()
|
||||
analysis_run.error_message = message
|
||||
analysis_run.result_json = result
|
||||
job.status = "failed"
|
||||
job.finished_at = analysis_run.finished_at
|
||||
job.error_message = message
|
||||
job.result_json = result
|
||||
db.add(analysis_run)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(analysis_run)
|
||||
db.refresh(job)
|
||||
|
||||
@staticmethod
|
||||
def _mark_success(db, analysis_run: AnalysisRun, job: Job, segmentation_count: int) -> None:
|
||||
result = {"segmentation_count": segmentation_count}
|
||||
analysis_run.status = "success"
|
||||
analysis_run.finished_at = SegmentationService._now()
|
||||
analysis_run.result_json = result
|
||||
job.status = "success"
|
||||
job.finished_at = analysis_run.finished_at
|
||||
job.result_json = result
|
||||
db.add(analysis_run)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(analysis_run)
|
||||
db.refresh(job)
|
||||
|
||||
@staticmethod
|
||||
def _persist_fixture_segmentations(
|
||||
db,
|
||||
project_id: uuid.UUID,
|
||||
dataset_id: uuid.UUID,
|
||||
analysis_run: AnalysisRun,
|
||||
job: Job,
|
||||
model_name: str,
|
||||
model_version: str | None,
|
||||
raw_segmentations: Any,
|
||||
confidence_threshold: float,
|
||||
class_filter: list[str],
|
||||
settings: Settings,
|
||||
) -> list[Segmentation]:
|
||||
if not isinstance(raw_segmentations, list):
|
||||
raise AppError(code="INVALID_FIXTURE_SEGMENTATIONS", message="fixture_segmentations must be a list", status_code=400)
|
||||
adapter = FixtureSegmentationAdapter()
|
||||
adapter_results = adapter.segment(raw_segmentations)
|
||||
if len(adapter_results) != len(raw_segmentations):
|
||||
raise AppError(code="INVALID_FIXTURE_SEGMENTATION", message="Each fixture segmentation must be an object", status_code=400)
|
||||
persisted: list[Segmentation] = []
|
||||
allowed_classes = set(class_filter)
|
||||
for raw in adapter_results:
|
||||
class_name = raw.class_name
|
||||
confidence = raw.confidence
|
||||
if allowed_classes and class_name not in allowed_classes:
|
||||
continue
|
||||
if confidence is not None and confidence < confidence_threshold:
|
||||
continue
|
||||
if not isinstance(raw.geometry, dict):
|
||||
raise AppError(code="INVALID_FIXTURE_SEGMENTATION", message="Fixture segmentation geometry is required", status_code=400)
|
||||
geometry = SegmentationService._validated_multipolygon(raw.geometry)
|
||||
segmentation_id = uuid.uuid4()
|
||||
mask_path = raw.mask_path or SegmentationService.mask_artifact_path(
|
||||
settings.storage_root,
|
||||
project_id,
|
||||
analysis_run.id,
|
||||
raw.tile_index,
|
||||
segmentation_id,
|
||||
)
|
||||
segmentation = Segmentation(
|
||||
id=segmentation_id,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
analysis_run_id=analysis_run.id,
|
||||
job_id=job.id,
|
||||
model_name=model_name,
|
||||
model_version=model_version,
|
||||
class_name=class_name,
|
||||
confidence=confidence,
|
||||
geometry=from_shape(geometry, srid=4326),
|
||||
bbox_json=raw.bbox_json,
|
||||
area_m2=raw.area_m2,
|
||||
mask_path=mask_path,
|
||||
source_tile_path=raw.source_tile_path,
|
||||
tile_index=raw.tile_index,
|
||||
properties_json=raw.properties_json,
|
||||
provenance_json={**dict(raw.provenance_json or {}), "fixture_mode": True},
|
||||
)
|
||||
db.add(segmentation)
|
||||
persisted.append(segmentation)
|
||||
db.commit()
|
||||
for segmentation in persisted:
|
||||
db.refresh(segmentation)
|
||||
return persisted
|
||||
|
||||
@staticmethod
|
||||
def _validated_multipolygon(geometry_payload: dict[str, Any]) -> MultiPolygon:
|
||||
try:
|
||||
geometry = shape(geometry_payload)
|
||||
except Exception as exc:
|
||||
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be valid GeoJSON", status_code=400) from exc
|
||||
if geometry.is_empty:
|
||||
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must not be empty", status_code=400)
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be valid", status_code=400)
|
||||
if isinstance(geometry, Polygon):
|
||||
geometry = MultiPolygon([geometry])
|
||||
if not isinstance(geometry, MultiPolygon):
|
||||
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must be Polygon or MultiPolygon", status_code=400)
|
||||
if geometry.area <= 0:
|
||||
raise AppError(code="INVALID_FIXTURE_GEOMETRY", message="Fixture segmentation geometry must have positive area", status_code=400)
|
||||
return geometry
|
||||
|
||||
@staticmethod
|
||||
def _query_segmentation_rows(
|
||||
db,
|
||||
*,
|
||||
analysis_run_id: uuid.UUID | None = None,
|
||||
dataset_id: uuid.UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
) -> list[Segmentation]:
|
||||
query = db.query(Segmentation)
|
||||
if analysis_run_id is not None:
|
||||
query = query.filter(Segmentation.analysis_run_id == analysis_run_id)
|
||||
if dataset_id is not None:
|
||||
query = query.filter(Segmentation.dataset_id == dataset_id)
|
||||
if class_name:
|
||||
query = query.filter(Segmentation.class_name == class_name)
|
||||
if min_confidence is not None:
|
||||
query = query.filter(Segmentation.confidence >= min_confidence)
|
||||
return query.order_by(Segmentation.created_at.desc()).all()
|
||||
|
||||
@staticmethod
|
||||
def _segmentation_properties(segmentation: Segmentation) -> dict[str, Any]:
|
||||
return {
|
||||
"segmentation_id": str(segmentation.id),
|
||||
"class_name": segmentation.class_name,
|
||||
"confidence": segmentation.confidence,
|
||||
"area_m2": segmentation.area_m2,
|
||||
"model_name": segmentation.model_name,
|
||||
"model_version": segmentation.model_version,
|
||||
"analysis_run_id": str(segmentation.analysis_run_id) if segmentation.analysis_run_id else None,
|
||||
"dataset_id": str(segmentation.dataset_id) if segmentation.dataset_id else None,
|
||||
"job_id": str(segmentation.job_id) if segmentation.job_id else None,
|
||||
"source_tile_path": segmentation.source_tile_path,
|
||||
"tile_index": segmentation.tile_index,
|
||||
"mask_path": segmentation.mask_path,
|
||||
"bbox_json": segmentation.bbox_json,
|
||||
"provenance_json": segmentation.provenance_json,
|
||||
}
|
||||
Reference in New Issue
Block a user