Add local model asset catalog
This commit is contained in:
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Session
|
||||
from app.db.session import get_db
|
||||
from app.schemas import DetectionQaRequest, DetectionRunRequest
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.yolo_preflight_service import YoloPreflightService
|
||||
from app.utils.response import envelope
|
||||
@@ -20,12 +21,22 @@ def list_detection_models() -> dict:
|
||||
return envelope({"models": [model.model_dump() for model in ModelRegistryService.list_model_capabilities()]})
|
||||
|
||||
|
||||
@router.get("/model-assets", response_model=dict)
|
||||
def list_detection_model_assets() -> dict:
|
||||
return envelope(ModelAssetCatalogService.list_assets().model_dump())
|
||||
|
||||
|
||||
@router.get("/yolo/preflight", response_model=dict)
|
||||
def get_yolo_preflight(tile_manifest_path: str | None = None, check_model_load: bool = False) -> dict:
|
||||
def get_yolo_preflight(
|
||||
tile_manifest_path: str | None = None,
|
||||
check_model_load: bool = False,
|
||||
model_asset_id: str | None = None,
|
||||
) -> dict:
|
||||
return envelope(
|
||||
YoloPreflightService.run(
|
||||
tile_manifest_path=tile_manifest_path,
|
||||
check_model_load=check_model_load,
|
||||
model_asset_id=model_asset_id,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -37,6 +48,7 @@ def run_detection(payload: DetectionRunRequest, db: Session = Depends(get_db)) -
|
||||
project_id=payload.project_id,
|
||||
dataset_id=payload.dataset_id,
|
||||
model_id=payload.model_id,
|
||||
model_asset_id=payload.model_asset_id,
|
||||
confidence_threshold=payload.confidence_threshold,
|
||||
class_filter=payload.class_filter,
|
||||
tile_manifest_path=payload.tile_manifest_path,
|
||||
|
||||
@@ -23,6 +23,7 @@ class Settings(BaseSettings):
|
||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
||||
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
|
||||
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
|
||||
yolo_model_path: str | None = Field(default=None, validation_alias="YOLO_MODEL_PATH")
|
||||
yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID")
|
||||
yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME")
|
||||
|
||||
@@ -15,6 +15,8 @@ from .detection import (
|
||||
DetectionRunRead,
|
||||
DetectionRunRequest,
|
||||
DetectionRunResponse,
|
||||
ModelAssetListResponse,
|
||||
ModelAssetRead,
|
||||
)
|
||||
from .segmentation import (
|
||||
SegmentationListResponse,
|
||||
@@ -105,6 +107,8 @@ __all__ = [
|
||||
"DetectionRunRead",
|
||||
"DetectionRunRequest",
|
||||
"DetectionRunResponse",
|
||||
"ModelAssetListResponse",
|
||||
"ModelAssetRead",
|
||||
"SegmentationListResponse",
|
||||
"SegmentationModelCapability",
|
||||
"SegmentationModelsResponse",
|
||||
|
||||
@@ -22,10 +22,33 @@ class DetectionModelsResponse(BaseModel):
|
||||
models: list[DetectionModelCapability]
|
||||
|
||||
|
||||
class ModelAssetRead(BaseModel):
|
||||
model_asset_id: str
|
||||
filename: str
|
||||
display_name: str
|
||||
model_path: str
|
||||
suffix: str
|
||||
framework: str
|
||||
task_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
active: bool
|
||||
status: str
|
||||
limitation_message: str
|
||||
will_download_models: bool = False
|
||||
|
||||
|
||||
class ModelAssetListResponse(BaseModel):
|
||||
items: list[ModelAssetRead]
|
||||
total: int
|
||||
model_directory: str
|
||||
|
||||
|
||||
class DetectionRunRequest(BaseModel):
|
||||
project_id: UUID
|
||||
dataset_id: UUID
|
||||
model_id: str
|
||||
model_asset_id: str | None = None
|
||||
confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
class_filter: list[str] | None = None
|
||||
tile_manifest_path: str | None = None
|
||||
|
||||
@@ -15,6 +15,7 @@ from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project, VectorFeature
|
||||
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
|
||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.quality_service import QualityService
|
||||
@@ -33,6 +34,7 @@ class DetectionService:
|
||||
dataset_id: uuid.UUID,
|
||||
model_id: str,
|
||||
confidence_threshold: float,
|
||||
model_asset_id: str | None = None,
|
||||
class_filter: list[str] | None = None,
|
||||
tile_manifest_path: str | None = None,
|
||||
parameters_json: dict[str, Any] | None = None,
|
||||
@@ -55,6 +57,11 @@ class DetectionService:
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
selected_model_asset = None
|
||||
if model_id == resolved_settings.yolo_model_id and model_asset_id:
|
||||
selected_model_asset = ModelAssetCatalogService.resolve_asset(model_asset_id, settings=resolved_settings)
|
||||
resolved_settings = ModelAssetCatalogService.settings_for_asset(resolved_settings, selected_model_asset)
|
||||
|
||||
model = ModelRegistryService.get_model_capability(
|
||||
model_id,
|
||||
settings=resolved_settings,
|
||||
@@ -77,6 +84,9 @@ class DetectionService:
|
||||
|
||||
run_parameters = {
|
||||
"model_id": model.model_id,
|
||||
"model_asset_id": selected_model_asset.model_asset_id if selected_model_asset else None,
|
||||
"model_asset_path": selected_model_asset.model_path if selected_model_asset else None,
|
||||
"model_asset_sha256": selected_model_asset.sha256 if selected_model_asset else None,
|
||||
"confidence_threshold": confidence_threshold,
|
||||
"class_filter": class_filter or [],
|
||||
"tile_manifest_path": tile_manifest_path,
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.schemas.detection import ModelAssetListResponse, ModelAssetRead
|
||||
|
||||
|
||||
class ModelAssetCatalogService:
|
||||
SUPPORTED_SUFFIXES = {
|
||||
".pt": "ultralytics/pytorch",
|
||||
".onnx": "onnx",
|
||||
".engine": "tensorrt",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def list_assets(settings: Settings | None = None) -> ModelAssetListResponse:
|
||||
resolved_settings = settings or get_settings()
|
||||
model_directory = ModelAssetCatalogService._model_directory(resolved_settings)
|
||||
active_model_path = ModelAssetCatalogService._resolved_file_path(resolved_settings.yolo_model_path)
|
||||
if not model_directory.exists() or not model_directory.is_dir():
|
||||
return ModelAssetListResponse(items=[], total=0, model_directory=str(model_directory))
|
||||
|
||||
items = [
|
||||
ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path)
|
||||
for path in sorted(model_directory.iterdir(), key=lambda item: item.name.lower())
|
||||
if path.is_file() and path.suffix.lower() in ModelAssetCatalogService.SUPPORTED_SUFFIXES
|
||||
]
|
||||
return ModelAssetListResponse(items=items, total=len(items), model_directory=str(model_directory))
|
||||
|
||||
@staticmethod
|
||||
def resolve_asset(model_asset_id: str, settings: Settings | None = None) -> ModelAssetRead:
|
||||
normalized = model_asset_id.strip()
|
||||
for asset in ModelAssetCatalogService.list_assets(settings=settings).items:
|
||||
if asset.model_asset_id == normalized:
|
||||
return asset
|
||||
raise AppError(
|
||||
code="DETECTION_MODEL_ASSET_NOT_FOUND",
|
||||
message="Selected local model asset was not found in the configured model directory",
|
||||
details={"model_asset_id": normalized},
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def settings_for_asset(settings: Settings, asset: ModelAssetRead) -> Settings:
|
||||
return settings.model_copy(update={"yolo_model_path": asset.model_path})
|
||||
|
||||
@staticmethod
|
||||
def _model_directory(settings: Settings) -> Path:
|
||||
configured_directory = Path(settings.yolo_models_dir).expanduser()
|
||||
if configured_directory.exists() and configured_directory.is_dir():
|
||||
return configured_directory.resolve()
|
||||
active_model_path = ModelAssetCatalogService._resolved_file_path(settings.yolo_model_path)
|
||||
if active_model_path and active_model_path.parent.exists() and active_model_path.parent.is_dir():
|
||||
return active_model_path.parent.resolve()
|
||||
return configured_directory.resolve()
|
||||
|
||||
@staticmethod
|
||||
def _asset_from_file(path: Path, *, active_model_path: Path | None) -> ModelAssetRead:
|
||||
resolved_path = path.resolve()
|
||||
return ModelAssetRead(
|
||||
model_asset_id=ModelAssetCatalogService._asset_id(path),
|
||||
filename=path.name,
|
||||
display_name=path.stem,
|
||||
model_path=str(resolved_path),
|
||||
suffix=path.suffix.lower(),
|
||||
framework=ModelAssetCatalogService.SUPPORTED_SUFFIXES[path.suffix.lower()],
|
||||
task_type="object_detection",
|
||||
size_bytes=path.stat().st_size,
|
||||
sha256=ModelAssetCatalogService._sha256(path),
|
||||
active=active_model_path == resolved_path,
|
||||
status="available",
|
||||
limitation_message="Local runtime model asset. GeoIntel will not download or mutate model weights.",
|
||||
will_download_models=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _asset_id(path: Path) -> str:
|
||||
raw = f"{path.stem}-{path.suffix.lower().lstrip('.')}"
|
||||
normalized = re.sub(r"[^a-z0-9]+", "-", raw.lower()).strip("-")
|
||||
return normalized or "model-asset"
|
||||
|
||||
@staticmethod
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _resolved_file_path(raw_path: str | None) -> Path | None:
|
||||
if not raw_path:
|
||||
return None
|
||||
path = Path(raw_path).expanduser()
|
||||
if not path.exists() or not path.is_file():
|
||||
return None
|
||||
return path.resolve()
|
||||
@@ -8,6 +8,7 @@ from typing import Any, Type
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||
|
||||
|
||||
@@ -20,10 +21,16 @@ class YoloPreflightService:
|
||||
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
|
||||
assume_dependencies: bool = False,
|
||||
check_model_load: bool = False,
|
||||
model_asset_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
resolved_settings = settings or get_settings()
|
||||
selected_asset = None
|
||||
if model_asset_id:
|
||||
selected_asset = ModelAssetCatalogService.resolve_asset(model_asset_id, settings=resolved_settings)
|
||||
resolved_settings = ModelAssetCatalogService.settings_for_asset(resolved_settings, selected_asset)
|
||||
result: dict[str, Any] = {
|
||||
"model_id": resolved_settings.yolo_model_id,
|
||||
"model_asset_id": selected_asset.model_asset_id if selected_asset else None,
|
||||
"model_path": resolved_settings.yolo_model_path,
|
||||
"tile_manifest_path": tile_manifest_path,
|
||||
"status": "not_configured",
|
||||
|
||||
Reference in New Issue
Block a user