Initial public release
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
from app.models.entities import AnalysisRun, Area, Dataset, Export, Project
|
||||
|
||||
__all__ = ["AnalysisRun", "Area", "Dataset", "Export", "Project"]
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from app.core.errors import AppError
|
||||
|
||||
|
||||
def guest_project_scope(request: Request) -> UUID | None:
|
||||
principal = getattr(request.state, "auth_principal", None)
|
||||
if getattr(principal, "role", None) != "guest":
|
||||
return None
|
||||
project_id = getattr(principal, "project_id", None)
|
||||
if isinstance(project_id, UUID):
|
||||
return project_id
|
||||
raise AppError(
|
||||
code="GUEST_PROJECT_SCOPE_REQUIRED",
|
||||
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||
status_code=403,
|
||||
)
|
||||
|
||||
|
||||
def assert_guest_project_scope(request: Request, project_id: UUID) -> None:
|
||||
guest_project_id = guest_project_scope(request)
|
||||
if guest_project_id is not None and project_id != guest_project_id:
|
||||
raise AppError(
|
||||
code="GUEST_PROJECT_SCOPE_REQUIRED",
|
||||
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||
status_code=403,
|
||||
)
|
||||
|
||||
|
||||
def guest_scoped_project_filter(
|
||||
request: Request,
|
||||
requested_project_id: UUID | None,
|
||||
) -> UUID | None:
|
||||
guest_project_id = guest_project_scope(request)
|
||||
if guest_project_id is None:
|
||||
return requested_project_id
|
||||
if requested_project_id is not None:
|
||||
assert_guest_project_scope(request, requested_project_id)
|
||||
return guest_project_id
|
||||
@@ -0,0 +1,15 @@
|
||||
__all__ = [
|
||||
"analysis",
|
||||
"areas",
|
||||
"assistant",
|
||||
"auth",
|
||||
"datasets",
|
||||
"exports",
|
||||
"external",
|
||||
"health",
|
||||
"jobs",
|
||||
"projects",
|
||||
"qa",
|
||||
"source_registry",
|
||||
"temporal",
|
||||
]
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.guest_scope import assert_guest_project_scope
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.models import Dataset
|
||||
from app.schemas import Envelope, JobRead
|
||||
from app.schemas.analysis import ChangeDetectionRequest
|
||||
from app.services.change_detection_service import ChangeDetectionService
|
||||
from app.services.job_service import JobService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/analysis", tags=["analysis"])
|
||||
|
||||
|
||||
@router.post("/change-detection", response_model=Envelope[JobRead])
|
||||
def run_change_detection(
|
||||
payload: ChangeDetectionRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
source_dataset = db.get(Dataset, payload.source_dataset_id)
|
||||
if not source_dataset:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
|
||||
assert_guest_project_scope(request, source_dataset.project_id)
|
||||
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
|
||||
ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target")
|
||||
job = JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=source_dataset.project_id,
|
||||
job_type="analysis.change-detection",
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
input_dataset_id=payload.source_dataset_id,
|
||||
operation=lambda: ChangeDetectionService.compare_vector_datasets(
|
||||
db=db,
|
||||
project_id=source_dataset.project_id,
|
||||
source_dataset_id=payload.source_dataset_id,
|
||||
target_dataset_id=payload.target_dataset_id,
|
||||
iou_threshold=payload.iou_threshold,
|
||||
modified_threshold=payload.modified_threshold,
|
||||
include_unchanged=payload.include_unchanged,
|
||||
bbox=payload.bbox.model_dump() if payload.bbox is not None else None,
|
||||
area_id=payload.area_id,
|
||||
preview_limit=payload.preview_limit,
|
||||
).model_dump(mode="json"),
|
||||
)
|
||||
return envelope(job)
|
||||
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas.aoi_operation import AoiOperationCreate, AoiOperationList, AoiOperationRead, AoiPartitionCheckpoint, AoiPartitionComplete, AoiPartitionFail, AoiPartitionRead
|
||||
from app.schemas.common import Envelope
|
||||
from app.services.aoi_operation_service import AoiOperationService
|
||||
from app.services.aoi_operation_executor import AoiOperationExecutor
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/aoi-operations", tags=["aoi-operations"])
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=Envelope[AoiOperationRead])
|
||||
def create_operation(project_id: UUID, payload: AoiOperationCreate, db: Session = Depends(get_db)):
|
||||
return envelope(AoiOperationService.create(db, project_id, payload))
|
||||
|
||||
|
||||
@router.get("", response_model=Envelope[AoiOperationList])
|
||||
def list_operations(project_id: UUID, limit: int = Query(default=50, ge=1, le=200), db: Session = Depends(get_db)):
|
||||
return envelope(AoiOperationService.list(db, project_id, limit))
|
||||
|
||||
|
||||
@router.get("/{operation_id}", response_model=Envelope[AoiOperationRead])
|
||||
def read_operation(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
|
||||
return envelope(AoiOperationService.read(db, project_id, operation_id))
|
||||
|
||||
|
||||
@router.post("/{operation_id}/partitions/claim", response_model=Envelope[AoiPartitionRead | None])
|
||||
def claim_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
|
||||
partition = AoiOperationService.claim_next(db, project_id, operation_id)
|
||||
return envelope(AoiPartitionRead.model_validate(partition).model_dump() if partition else None)
|
||||
|
||||
|
||||
@router.post("/{operation_id}/execute-next", response_model=Envelope[AoiOperationRead])
|
||||
def execute_next_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
|
||||
return envelope(AoiOperationExecutor.execute_next(db, project_id, operation_id))
|
||||
|
||||
|
||||
@router.put("/{operation_id}/partitions/{partition_id}/checkpoint", response_model=Envelope[AoiPartitionRead])
|
||||
def checkpoint_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionCheckpoint, db: Session = Depends(get_db)):
|
||||
partition = AoiOperationService.checkpoint(db, project_id, operation_id, partition_id, payload.checkpoint_json)
|
||||
return envelope(AoiPartitionRead.model_validate(partition).model_dump())
|
||||
|
||||
|
||||
@router.post("/{operation_id}/partitions/{partition_id}/complete", response_model=Envelope[AoiOperationRead])
|
||||
def complete_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionComplete, db: Session = Depends(get_db)):
|
||||
return envelope(AoiOperationService.complete(db, project_id, operation_id, partition_id, payload.result_json, payload.skipped))
|
||||
|
||||
|
||||
@router.post("/{operation_id}/partitions/{partition_id}/fail", response_model=Envelope[AoiOperationRead])
|
||||
def fail_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionFail, db: Session = Depends(get_db)):
|
||||
return envelope(AoiOperationService.fail(db, project_id, operation_id, partition_id, payload.error_message, payload.retryable, payload.details))
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.models import Area
|
||||
from app.schemas import Envelope
|
||||
from app.schemas.area import AreaCreate, AreaList, AreaRead, AreaUpdate, MunicipalitySearchList
|
||||
from app.services.area_service import AreaService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/areas", tags=["areas"])
|
||||
|
||||
|
||||
@router.get("", response_model=Envelope[AreaList])
|
||||
def list_areas(
|
||||
project_id: UUID,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
areas, total = AreaService.list_areas(db, project_id=project_id, limit=limit, offset=offset)
|
||||
return envelope({"items": [AreaService.serialize_area(area) for area in areas], "total": total, "limit": limit, "offset": offset})
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=Envelope[AreaRead])
|
||||
def create_area(project_id: UUID, payload: AreaCreate, db: Session = Depends(get_db)):
|
||||
area = AreaService.create_area(db, project_id, payload)
|
||||
return envelope(AreaService.serialize_area(area))
|
||||
|
||||
|
||||
@router.get("/municipalities", response_model=Envelope[MunicipalitySearchList])
|
||||
def search_municipalities(
|
||||
project_id: UUID,
|
||||
query: str = Query(default="", max_length=120),
|
||||
limit: int = Query(default=20, ge=1, le=50),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
items, total = AreaService.search_municipalities(db, project_id, query, limit)
|
||||
return envelope({"items": items, "total": total})
|
||||
|
||||
|
||||
@router.post("/municipalities/{niscode}/activate", response_model=Envelope[AreaRead])
|
||||
def activate_municipality(project_id: UUID, niscode: str, db: Session = Depends(get_db)):
|
||||
area = AreaService.activate_municipality(db, project_id, niscode)
|
||||
return envelope(AreaService.serialize_area(area))
|
||||
|
||||
|
||||
@router.get("/{area_id}", response_model=Envelope[AreaRead])
|
||||
def get_area(
|
||||
project_id: UUID,
|
||||
area_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
area = AreaService.get_area(db, area_id)
|
||||
if area.project_id != project_id:
|
||||
raise HTTPException(status_code=404, detail="Area not found")
|
||||
return envelope(AreaService.serialize_area(area))
|
||||
|
||||
|
||||
@router.patch("/{area_id}", response_model=Envelope[AreaRead])
|
||||
def update_area(
|
||||
project_id: UUID,
|
||||
area_id: UUID,
|
||||
payload: AreaUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
existing = db.get(Area, area_id)
|
||||
if not existing or existing.project_id != project_id:
|
||||
raise HTTPException(status_code=404, detail="Area not found")
|
||||
area = AreaService.update_area(db, area_id, payload)
|
||||
return envelope(AreaService.serialize_area(area))
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas import Envelope
|
||||
from app.schemas.assistant import (
|
||||
AssistantModelList,
|
||||
AssistantQueryRequest,
|
||||
AssistantQueryResponse,
|
||||
AssistantStatus,
|
||||
)
|
||||
from app.services.geo_assistant_service import GeoAssistantService
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(tags=["assistant"])
|
||||
|
||||
|
||||
@router.get("/assistant/status", response_model=Envelope[AssistantStatus])
|
||||
def assistant_status() -> dict:
|
||||
return envelope(GeoAssistantService().status().model_dump())
|
||||
|
||||
|
||||
@router.get("/assistant/models", response_model=Envelope[AssistantModelList])
|
||||
def assistant_models() -> dict:
|
||||
service = GeoAssistantService()
|
||||
models = service.list_models()
|
||||
return envelope(
|
||||
{
|
||||
"items": [model.model_dump() for model in models],
|
||||
"total": len(models),
|
||||
"default_model": service.settings.ollama_default_model,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/projects/{project_id}/assistant/query",
|
||||
response_model=Envelope[AssistantQueryResponse],
|
||||
)
|
||||
def assistant_query(
|
||||
project_id: UUID,
|
||||
payload: AssistantQueryRequest,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(GeoAssistantService().query(db, project_id=project_id, payload=payload).model_dump())
|
||||
@@ -0,0 +1,326 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from ipaddress import ip_address, ip_network
|
||||
|
||||
from fastapi import APIRouter, Depends, Request, Response, status
|
||||
from fastapi.responses import RedirectResponse
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.schemas.auth import AuthLoginRequest, AuthSession, AuthSessionEnvelope
|
||||
from app.services.auth_service import AuthPrincipal, AuthService
|
||||
from app.services.authentik_oidc_service import AuthentikOidcService
|
||||
from app.services.demo_workflow_service import DemoWorkflowService
|
||||
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||
COOKIE_NAME = "geointel_session"
|
||||
OIDC_FLOW_COOKIE_NAME = "geointel_oidc_flow"
|
||||
logger = logging.getLogger("geointel.auth")
|
||||
_TRUSTED_PROXY_NETWORKS = (
|
||||
ip_network("127.0.0.0/8"),
|
||||
ip_network("::1/128"),
|
||||
ip_network("172.16.0.0/12"),
|
||||
)
|
||||
|
||||
|
||||
def _peer_is_trusted_proxy(request: Request) -> bool:
|
||||
if request.client is None:
|
||||
return False
|
||||
try:
|
||||
peer_address = ip_address(request.client.host)
|
||||
except ValueError:
|
||||
return False
|
||||
return any(peer_address in network for network in _TRUSTED_PROXY_NETWORKS)
|
||||
|
||||
|
||||
def _request_is_https(request: Request) -> bool:
|
||||
if request.url.scheme == "https":
|
||||
return True
|
||||
if not _peer_is_trusted_proxy(request):
|
||||
return False
|
||||
forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",", 1)[0].strip().lower()
|
||||
return forwarded_proto == "https"
|
||||
|
||||
|
||||
def _client_host(request: Request) -> str:
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
if not _peer_is_trusted_proxy(request):
|
||||
return peer
|
||||
forwarded = request.headers.get("x-real-ip", "").strip()
|
||||
if not forwarded:
|
||||
return peer
|
||||
try:
|
||||
return str(ip_address(forwarded))
|
||||
except ValueError:
|
||||
return peer
|
||||
|
||||
|
||||
def _session_from_principal(
|
||||
principal: AuthPrincipal,
|
||||
*,
|
||||
guest_access_enabled: bool,
|
||||
authentik_enabled: bool,
|
||||
) -> AuthSession:
|
||||
return AuthSession(
|
||||
authentication_required=True,
|
||||
authenticated=True,
|
||||
username=principal.username,
|
||||
expires_at=datetime.fromtimestamp(principal.expires_at, tz=UTC),
|
||||
role=principal.role,
|
||||
guest_access_enabled=guest_access_enabled,
|
||||
authentik_enabled=authentik_enabled,
|
||||
guest_project_id=principal.project_id,
|
||||
)
|
||||
|
||||
|
||||
def _session_payload(request: Request) -> AuthSession:
|
||||
settings = get_settings()
|
||||
guest_access_enabled = settings.auth_enabled and settings.guest_access_enabled
|
||||
authentik_enabled = AuthentikOidcService(settings).enabled
|
||||
if not settings.auth_enabled:
|
||||
return AuthSession(
|
||||
authentication_required=False,
|
||||
authenticated=True,
|
||||
guest_access_enabled=False,
|
||||
authentik_enabled=False,
|
||||
)
|
||||
principal = AuthService.verify_session_token(request.cookies.get(COOKIE_NAME), settings)
|
||||
if principal is None:
|
||||
return AuthSession(
|
||||
authentication_required=True,
|
||||
authenticated=False,
|
||||
guest_access_enabled=guest_access_enabled,
|
||||
authentik_enabled=authentik_enabled,
|
||||
)
|
||||
return _session_from_principal(
|
||||
principal,
|
||||
guest_access_enabled=guest_access_enabled,
|
||||
authentik_enabled=authentik_enabled,
|
||||
)
|
||||
|
||||
|
||||
def _set_session_cookie(
|
||||
*,
|
||||
request: Request,
|
||||
response: Response,
|
||||
token: str,
|
||||
max_age: int,
|
||||
) -> None:
|
||||
response.set_cookie(
|
||||
key=COOKIE_NAME,
|
||||
value=token,
|
||||
max_age=max_age,
|
||||
httponly=True,
|
||||
secure=_request_is_https(request),
|
||||
samesite="strict",
|
||||
path="/",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/session", response_model=AuthSessionEnvelope)
|
||||
def session(request: Request) -> AuthSessionEnvelope:
|
||||
return AuthSessionEnvelope(data=_session_payload(request))
|
||||
|
||||
|
||||
@router.post("/login", response_model=AuthSessionEnvelope)
|
||||
def login(payload: AuthLoginRequest, request: Request, response: Response) -> AuthSessionEnvelope:
|
||||
settings = get_settings()
|
||||
if not settings.auth_enabled:
|
||||
raise AppError(
|
||||
code="AUTHENTICATION_DISABLED",
|
||||
message="Operator authentication is not enabled on this runtime",
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
)
|
||||
if settings.auth_require_https and not _request_is_https(request):
|
||||
raise AppError(
|
||||
code="AUTH_HTTPS_REQUIRED",
|
||||
message="Operator authentication requires HTTPS on this runtime",
|
||||
status_code=status.HTTP_426_UPGRADE_REQUIRED,
|
||||
)
|
||||
client_host = _client_host(request)
|
||||
throttle_key = f"{client_host}:{payload.username.casefold()}"
|
||||
retry_after = AuthService.retry_after_seconds(throttle_key)
|
||||
if retry_after:
|
||||
raise AppError(
|
||||
code="LOGIN_RATE_LIMITED",
|
||||
message="Te veel mislukte aanmeldpogingen. Probeer later opnieuw.",
|
||||
details={"retry_after_seconds": retry_after},
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
)
|
||||
if not AuthService.credentials_match(payload.username, payload.password, settings):
|
||||
AuthService.record_failure(throttle_key)
|
||||
raise AppError(
|
||||
code="INVALID_CREDENTIALS",
|
||||
message="Gebruikersnaam of wachtwoord is onjuist.",
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
AuthService.clear_failures(throttle_key)
|
||||
token = AuthService.create_session_token(payload.username, settings)
|
||||
principal = AuthService.verify_session_token(token, settings)
|
||||
if principal is None: # pragma: no cover - defensive invariant
|
||||
raise AppError(
|
||||
code="SESSION_CREATION_FAILED",
|
||||
message="De beveiligde sessie kon niet worden aangemaakt.",
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
_set_session_cookie(
|
||||
request=request,
|
||||
response=response,
|
||||
token=token,
|
||||
max_age=settings.auth_session_ttl_seconds,
|
||||
)
|
||||
return AuthSessionEnvelope(
|
||||
data=_session_from_principal(
|
||||
principal,
|
||||
guest_access_enabled=settings.guest_access_enabled,
|
||||
authentik_enabled=AuthentikOidcService(settings).enabled,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/authentik/start")
|
||||
def authentik_start(request: Request) -> RedirectResponse:
|
||||
settings = get_settings()
|
||||
service = AuthentikOidcService(settings)
|
||||
try:
|
||||
location, flow = service.start()
|
||||
except Exception as exc:
|
||||
logger.warning("Authentik authorization start failed: %s", type(exc).__name__)
|
||||
raise AppError(
|
||||
code="AUTHENTIK_UNAVAILABLE",
|
||||
message="Authentik is momenteel niet beschikbaar.",
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
) from exc
|
||||
response = RedirectResponse(location, status_code=status.HTTP_302_FOUND)
|
||||
response.set_cookie(
|
||||
OIDC_FLOW_COOKIE_NAME,
|
||||
flow,
|
||||
max_age=600,
|
||||
httponly=True,
|
||||
secure=True,
|
||||
samesite="lax",
|
||||
path=f"{settings.api_prefix}/auth/authentik",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/authentik/callback")
|
||||
def authentik_callback(
|
||||
request: Request,
|
||||
code: str = "",
|
||||
state: str = "",
|
||||
) -> RedirectResponse:
|
||||
settings = get_settings()
|
||||
service = AuthentikOidcService(settings)
|
||||
base_url = settings.public_base_url.rstrip("/")
|
||||
try:
|
||||
service.finish(
|
||||
code=code,
|
||||
state=state,
|
||||
flow_cookie=request.cookies.get(OIDC_FLOW_COOKIE_NAME, ""),
|
||||
)
|
||||
token = AuthService.create_session_token(
|
||||
settings.auth_username or "operator",
|
||||
settings,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Authentik callback rejected: %s", type(exc).__name__)
|
||||
response = RedirectResponse(
|
||||
f"{base_url}/?authentik=error",
|
||||
status_code=status.HTTP_302_FOUND,
|
||||
)
|
||||
else:
|
||||
response = RedirectResponse(
|
||||
f"{base_url}/",
|
||||
status_code=status.HTTP_302_FOUND,
|
||||
)
|
||||
_set_session_cookie(
|
||||
request=request,
|
||||
response=response,
|
||||
token=token,
|
||||
max_age=settings.auth_session_ttl_seconds,
|
||||
)
|
||||
response.delete_cookie(
|
||||
OIDC_FLOW_COOKIE_NAME,
|
||||
path=f"{settings.api_prefix}/auth/authentik",
|
||||
secure=True,
|
||||
httponly=True,
|
||||
samesite="lax",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/guest", response_model=AuthSessionEnvelope)
|
||||
def guest_login(
|
||||
request: Request,
|
||||
response: Response,
|
||||
db: Session = Depends(get_db),
|
||||
) -> AuthSessionEnvelope:
|
||||
settings = get_settings()
|
||||
if not settings.auth_enabled or not settings.guest_access_enabled:
|
||||
raise AppError(
|
||||
code="GUEST_ACCESS_DISABLED",
|
||||
message="Gasttoegang is niet ingeschakeld op deze GeoIntel-installatie.",
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
|
||||
client_host = _client_host(request)
|
||||
retry_after = AuthService.consume_guest_request(
|
||||
f"guest-login:{client_host}",
|
||||
max_requests=settings.guest_login_requests_per_minute,
|
||||
)
|
||||
if retry_after:
|
||||
raise AppError(
|
||||
code="GUEST_LOGIN_RATE_LIMITED",
|
||||
message="Too many guest sessions were requested. Try again later.",
|
||||
details={"retry_after_seconds": retry_after},
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
)
|
||||
|
||||
demo = DemoWorkflowService.seed(db)
|
||||
token = AuthService.create_session_token(
|
||||
settings.guest_display_name,
|
||||
settings,
|
||||
role="guest",
|
||||
project_id=demo.project_id,
|
||||
ttl_seconds=settings.guest_session_ttl_seconds,
|
||||
)
|
||||
principal = AuthService.verify_session_token(token, settings)
|
||||
if principal is None: # pragma: no cover - defensive invariant
|
||||
raise AppError(
|
||||
code="SESSION_CREATION_FAILED",
|
||||
message="De tijdelijke gastensessie kon niet worden aangemaakt.",
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
_set_session_cookie(
|
||||
request=request,
|
||||
response=response,
|
||||
token=token,
|
||||
max_age=settings.guest_session_ttl_seconds,
|
||||
)
|
||||
return AuthSessionEnvelope(
|
||||
data=_session_from_principal(
|
||||
principal,
|
||||
guest_access_enabled=True,
|
||||
authentik_enabled=AuthentikOidcService(settings).enabled,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/logout", response_model=AuthSessionEnvelope)
|
||||
def logout(response: Response) -> AuthSessionEnvelope:
|
||||
settings = get_settings()
|
||||
response.delete_cookie(key=COOKIE_NAME, path="/", httponly=True, samesite="strict")
|
||||
return AuthSessionEnvelope(
|
||||
data=AuthSession(
|
||||
authentication_required=settings.auth_enabled,
|
||||
authenticated=not settings.auth_enabled,
|
||||
guest_access_enabled=settings.auth_enabled and settings.guest_access_enabled,
|
||||
authentik_enabled=AuthentikOidcService(settings).enabled,
|
||||
)
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas import Envelope
|
||||
from app.schemas.demo import DemoWorkflowResponse
|
||||
from app.services.demo_workflow_service import DemoWorkflowService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/demo", tags=["demo"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/workflow",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
response_model=Envelope[DemoWorkflowResponse],
|
||||
)
|
||||
def seed_demo_workflow(db: Session = Depends(get_db)) -> dict:
|
||||
result: DemoWorkflowResponse = DemoWorkflowService.seed(db)
|
||||
return envelope(result.model_dump())
|
||||
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.guest_scope import (
|
||||
assert_guest_project_scope,
|
||||
guest_project_scope,
|
||||
guest_scoped_project_filter,
|
||||
)
|
||||
from app.db.session import get_db
|
||||
from app.schemas import (
|
||||
AnalysisQaResponse,
|
||||
DetectionListResponse,
|
||||
DetectionModelsResponse,
|
||||
DetectionComparisonRequest,
|
||||
DetectionComparisonResponse,
|
||||
DetectionQaRequest,
|
||||
DetectionRead,
|
||||
DetectionRunListResponse,
|
||||
DetectionRunRead,
|
||||
DetectionRunRequest,
|
||||
DetectionRunResponse,
|
||||
Envelope,
|
||||
GeoJsonFeatureCollection,
|
||||
JobRead,
|
||||
ModelAssetListResponse,
|
||||
YoloPreflightResponse,
|
||||
)
|
||||
from app.services.detection_comparison_service import DetectionComparisonService
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.yolo_preflight_service import YoloPreflightService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/detection", tags=["detection"])
|
||||
|
||||
|
||||
@router.get("/models", response_model=Envelope[DetectionModelsResponse])
|
||||
def list_detection_models() -> dict:
|
||||
return envelope({"models": [model.model_dump() for model in ModelRegistryService.list_model_capabilities()]})
|
||||
|
||||
|
||||
@router.get("/model-assets", response_model=Envelope[ModelAssetListResponse])
|
||||
def list_detection_model_assets() -> dict:
|
||||
return envelope(ModelAssetCatalogService.list_assets().model_dump())
|
||||
|
||||
|
||||
@router.get("/yolo/preflight", response_model=Envelope[YoloPreflightResponse])
|
||||
def get_yolo_preflight(
|
||||
tile_manifest_path: str | None = None,
|
||||
check_model_load: bool = False,
|
||||
model_asset_id: str | None = None,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(
|
||||
YoloPreflightService.run(
|
||||
tile_manifest_path=tile_manifest_path,
|
||||
check_model_load=check_model_load,
|
||||
model_asset_id=model_asset_id,
|
||||
db=db,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/run", response_model=Envelope[DetectionRunResponse])
|
||||
def run_detection(
|
||||
payload: DetectionRunRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
assert_guest_project_scope(request, payload.project_id)
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
project_id=payload.project_id,
|
||||
dataset_id=payload.dataset_id,
|
||||
model_id=payload.model_id,
|
||||
model_asset_id=payload.model_asset_id,
|
||||
confidence_threshold=payload.confidence_threshold,
|
||||
class_filter=payload.class_filter,
|
||||
tile_manifest_path=payload.tile_manifest_path,
|
||||
parameters_json=payload.parameters_json,
|
||||
)
|
||||
return envelope(result.model_dump())
|
||||
|
||||
|
||||
@router.post("/run-async", response_model=Envelope[JobRead])
|
||||
def queue_detection(
|
||||
payload: DetectionRunRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Queue a detection run for the background worker.
|
||||
|
||||
Tiled GPU inference takes minutes; ``POST /detection/run`` performs it
|
||||
inside the request and is only appropriate for a handful of tiles. Poll
|
||||
``GET /jobs/{id}`` for the queued run instead.
|
||||
"""
|
||||
|
||||
assert_guest_project_scope(request, payload.project_id)
|
||||
job = DetectionService.enqueue_detection(
|
||||
db=db,
|
||||
project_id=payload.project_id,
|
||||
dataset_id=payload.dataset_id,
|
||||
model_id=payload.model_id,
|
||||
model_asset_id=payload.model_asset_id,
|
||||
confidence_threshold=payload.confidence_threshold,
|
||||
class_filter=payload.class_filter,
|
||||
tile_manifest_path=payload.tile_manifest_path,
|
||||
parameters_json=payload.parameters_json,
|
||||
)
|
||||
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.get("/runs", response_model=Envelope[DetectionRunListResponse])
|
||||
def list_detection_runs(
|
||||
request: Request,
|
||||
project_id: UUID | None = None,
|
||||
dataset_id: UUID | None = None,
|
||||
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
project_id = guest_scoped_project_filter(request, project_id)
|
||||
return envelope(
|
||||
DetectionService.list_runs(
|
||||
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/runs/{analysis_run_id}", response_model=Envelope[DetectionRunRead])
|
||||
def get_detection_run(
|
||||
analysis_run_id: UUID,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
run = DetectionService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(run.model_dump())
|
||||
|
||||
|
||||
@router.get(
|
||||
"/runs/{analysis_run_id}/detections",
|
||||
response_model=Envelope[DetectionListResponse],
|
||||
)
|
||||
def list_detection_run_detections(
|
||||
analysis_run_id: UUID,
|
||||
request: Request,
|
||||
dataset_id: UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
run = DetectionService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(
|
||||
DetectionService.list_detections(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/datasets/{dataset_id}/detections",
|
||||
response_model=Envelope[DetectionListResponse],
|
||||
)
|
||||
def list_dataset_detections(
|
||||
dataset_id: UUID,
|
||||
request: Request,
|
||||
analysis_run_id: UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
assert_guest_project_scope(request, dataset.project_id)
|
||||
return envelope(
|
||||
DetectionService.list_detections(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/detections/{detection_id}", response_model=Envelope[DetectionRead])
|
||||
def get_detection(
|
||||
detection_id: UUID,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
detection = DetectionService.get_detection(db, detection_id)
|
||||
assert_guest_project_scope(request, detection.project_id)
|
||||
return envelope(detection.model_dump())
|
||||
|
||||
|
||||
@router.get(
|
||||
"/runs/{analysis_run_id}/geojson",
|
||||
response_model=Envelope[GeoJsonFeatureCollection],
|
||||
)
|
||||
def get_detection_run_geojson(
|
||||
analysis_run_id: UUID,
|
||||
request: Request,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
run = DetectionService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(
|
||||
DetectionService.detections_to_geojson(
|
||||
db,
|
||||
limit=limit,
|
||||
analysis_run_id=analysis_run_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/datasets/{dataset_id}/geojson",
|
||||
response_model=Envelope[GeoJsonFeatureCollection],
|
||||
)
|
||||
def get_dataset_detection_geojson(
|
||||
dataset_id: UUID,
|
||||
request: Request,
|
||||
analysis_run_id: UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
assert_guest_project_scope(request, dataset.project_id)
|
||||
return envelope(
|
||||
DetectionService.detections_to_geojson(
|
||||
db,
|
||||
limit=limit,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/runs/compare", response_model=Envelope[DetectionComparisonResponse])
|
||||
def compare_detection_runs(payload: DetectionComparisonRequest, db: Session = Depends(get_db)) -> dict:
|
||||
"""Rank several runs against one reference on average precision.
|
||||
|
||||
The workbench ranks model variants by a stored F1 measured at each
|
||||
variant's own confidence threshold, which orders the thresholds as much as
|
||||
the models. Average precision describes the whole ranking a model produced.
|
||||
Comparability is reported first: runs over different rasters, different
|
||||
references or different inference coverage are not alternatives.
|
||||
"""
|
||||
|
||||
return envelope(
|
||||
DetectionComparisonService.compare_runs(
|
||||
db,
|
||||
analysis_run_ids=payload.analysis_run_ids,
|
||||
reference_dataset_id=payload.reference_dataset_id,
|
||||
iou_threshold=payload.iou_threshold,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/runs/{analysis_run_id}/qa/reference",
|
||||
response_model=Envelope[AnalysisQaResponse],
|
||||
)
|
||||
def compare_detection_run_with_reference(
|
||||
analysis_run_id: UUID,
|
||||
payload: DetectionQaRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
run = DetectionService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(
|
||||
DetectionService.compare_detections_with_reference(
|
||||
db=db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
reference_dataset_id=payload.reference_dataset_id,
|
||||
iou_threshold=payload.iou_threshold,
|
||||
class_name=payload.class_name,
|
||||
min_confidence=payload.min_confidence,
|
||||
calibration_thresholds=payload.calibration_thresholds,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.guest_scope import assert_guest_project_scope, guest_project_scope
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.schemas import Envelope
|
||||
from app.schemas.export import (
|
||||
ExportContentResponse,
|
||||
ExportCreateResponse,
|
||||
ExportListResponse,
|
||||
ExportRead,
|
||||
GeoJsonExportRequest,
|
||||
MapResultExportRequest,
|
||||
MetadataExportRequest,
|
||||
ReportExportRequest,
|
||||
)
|
||||
from app.services.export_service import ExportService
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/exports", tags=["exports"])
|
||||
|
||||
|
||||
@router.post("/geojson", response_model=Envelope[ExportCreateResponse])
|
||||
def export_geojson(
|
||||
payload: GeoJsonExportRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
if guest_project_scope(request) is not None:
|
||||
if payload.export_kind in {"dataset", "vector_selection"} and payload.dataset_id is not None:
|
||||
dataset = DatasetService.get_dataset(db, payload.dataset_id)
|
||||
assert_guest_project_scope(request, dataset.project_id)
|
||||
elif payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
|
||||
run = DetectionService.get_run(db, payload.analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
elif payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
|
||||
run = SegmentationService.get_run(db, payload.analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
if payload.export_kind == "vector_selection" and payload.dataset_id is not None and payload.bbox is not None:
|
||||
return envelope(
|
||||
ExportService.export_vector_selection_geojson(
|
||||
db,
|
||||
payload.dataset_id,
|
||||
payload.bbox.model_dump(),
|
||||
area_id=payload.area_id,
|
||||
limit=payload.limit,
|
||||
name=payload.name,
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
if payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
|
||||
return envelope(
|
||||
ExportService.export_detection_run_geojson(
|
||||
db,
|
||||
payload.analysis_run_id,
|
||||
payload.name,
|
||||
intended_use=payload.intended_use,
|
||||
).model_dump(mode="json")
|
||||
)
|
||||
if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
|
||||
return envelope(
|
||||
ExportService.export_segmentation_run_geojson(db, payload.analysis_run_id, payload.name).model_dump(mode="json")
|
||||
)
|
||||
if payload.dataset_id is not None:
|
||||
return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json"))
|
||||
raise AppError(
|
||||
code="INVALID_EXPORT_REQUEST",
|
||||
message="GeoJSON export request does not match any supported export target",
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/metadata", response_model=Envelope[ExportCreateResponse])
|
||||
def export_project_metadata(
|
||||
payload: MetadataExportRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
assert_guest_project_scope(request, payload.project_id)
|
||||
return envelope(ExportService.export_project_metadata(db, payload.project_id, payload.name).model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.post("/report", response_model=Envelope[ExportCreateResponse])
|
||||
def export_project_report(
|
||||
payload: ReportExportRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
assert_guest_project_scope(request, payload.project_id)
|
||||
return envelope(ExportService.export_project_report(db, payload.project_id, payload.name).model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.post("/map-result", response_model=Envelope[ExportCreateResponse])
|
||||
def export_map_result(
|
||||
payload: MapResultExportRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
assert_guest_project_scope(request, payload.project_id)
|
||||
return envelope(ExportService.export_map_result(db, payload).model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.get(
|
||||
"/projects/{project_id}/exports",
|
||||
response_model=Envelope[ExportListResponse],
|
||||
)
|
||||
def list_project_exports(
|
||||
project_id: UUID,
|
||||
request: Request,
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
assert_guest_project_scope(request, project_id)
|
||||
return envelope(ExportService.list_project_exports(db, project_id, limit=limit, offset=offset).model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.get("/{export_id}", response_model=Envelope[ExportRead])
|
||||
def get_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
|
||||
export = ExportService.get_export(db, export_id)
|
||||
assert_guest_project_scope(request, export.project_id)
|
||||
return envelope(export.model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.get("/{export_id}/download")
|
||||
def download_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
|
||||
if guest_project_scope(request) is not None:
|
||||
export = ExportService.get_export(db, export_id)
|
||||
assert_guest_project_scope(request, export.project_id)
|
||||
path = ExportService.get_export_download_path(db, export_id)
|
||||
media_type = "text/html" if path.suffix.lower() in {".html", ".htm"} else "application/json"
|
||||
return FileResponse(path, filename=path.name, media_type=media_type)
|
||||
|
||||
|
||||
@router.get("/{export_id}/content", response_model=Envelope[ExportContentResponse])
|
||||
def get_export_content(export_id: UUID, request: Request, db: Session = Depends(get_db)):
|
||||
if guest_project_scope(request) is not None:
|
||||
export = ExportService.get_export(db, export_id)
|
||||
assert_guest_project_scope(request, export.project_id)
|
||||
return envelope(ExportService.get_export_content(db, export_id).model_dump(mode="json"))
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.models import Area, Project
|
||||
from app.providers.registry import fetch_provider_data, get_provider, import_provider_dataset, list_provider_capabilities
|
||||
from app.schemas import (
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolveRequest,
|
||||
CoverageResolveResponse,
|
||||
Envelope,
|
||||
ExternalFetchRequest,
|
||||
ExternalFetchResponse,
|
||||
ProviderCapabilitiesResponse,
|
||||
ProviderCapabilityResponse,
|
||||
ProviderImportRequest,
|
||||
ProviderImportResponse,
|
||||
ProviderLayersResponse,
|
||||
ProviderStatusResponse,
|
||||
)
|
||||
from app.services.coverage_registry_service import CoverageRegistryService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/external", tags=["external"])
|
||||
|
||||
|
||||
def _validate_area_in_project(db: Session, project_id, area_id: str | None) -> None:
|
||||
if area_id is None:
|
||||
return
|
||||
area = db.get(Area, area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
|
||||
|
||||
def _assert_project_exists(db: Session, project_id):
|
||||
project = db.get(Project, project_id)
|
||||
if not project:
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
|
||||
def _assert_guest_project_scope(request: Request, project_id) -> None:
|
||||
principal = getattr(request.state, "auth_principal", None)
|
||||
if (
|
||||
getattr(principal, "role", None) == "guest"
|
||||
and getattr(principal, "project_id", None) != project_id
|
||||
):
|
||||
raise AppError(
|
||||
code="GUEST_PROJECT_SCOPE_REQUIRED",
|
||||
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||
status_code=403,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_layer_input(layers: list[str] | None) -> list[str]:
|
||||
return [layer.strip() for layer in (layers or []) if isinstance(layer, str) and layer.strip()]
|
||||
|
||||
|
||||
|
||||
|
||||
def _provider_payload(provider_name: str) -> dict:
|
||||
return get_provider(provider_name).capability.to_dict()
|
||||
|
||||
|
||||
@router.get("/providers", response_model=Envelope[ProviderCapabilitiesResponse])
|
||||
def list_external_providers() -> dict:
|
||||
return envelope({
|
||||
"providers": [provider.to_dict() for provider in list_provider_capabilities()],
|
||||
})
|
||||
|
||||
|
||||
@router.get("/coverage/catalog", response_model=Envelope[CoverageCatalogResponse])
|
||||
def get_coverage_catalog() -> dict:
|
||||
return envelope(CoverageRegistryService.catalog().model_dump())
|
||||
|
||||
|
||||
@router.post("/coverage/resolve", response_model=Envelope[CoverageResolveResponse])
|
||||
def resolve_project_coverage(
|
||||
payload: CoverageResolveRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
_assert_guest_project_scope(request, payload.project_id)
|
||||
result = CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id=payload.project_id,
|
||||
bbox=payload.bbox,
|
||||
themes=payload.themes,
|
||||
)
|
||||
return envelope(result.model_dump())
|
||||
|
||||
|
||||
@router.get(
|
||||
"/providers/capabilities",
|
||||
response_model=Envelope[ProviderCapabilitiesResponse],
|
||||
)
|
||||
def get_external_provider_capabilities() -> dict:
|
||||
return envelope({
|
||||
"providers": [provider.to_dict() for provider in list_provider_capabilities()],
|
||||
})
|
||||
|
||||
|
||||
@router.get(
|
||||
"/providers/{provider_name}",
|
||||
response_model=Envelope[ProviderCapabilityResponse],
|
||||
)
|
||||
def get_external_provider(provider_name: str) -> dict:
|
||||
return envelope(_provider_payload(provider_name))
|
||||
|
||||
|
||||
@router.get(
|
||||
"/providers/{provider_name}/layers",
|
||||
response_model=Envelope[ProviderLayersResponse],
|
||||
)
|
||||
def get_external_provider_layers(provider_name: str) -> dict:
|
||||
provider = get_provider(provider_name)
|
||||
return envelope({
|
||||
"provider_name": provider.provider_name,
|
||||
"layers": provider.supported_layers,
|
||||
})
|
||||
|
||||
|
||||
@router.get(
|
||||
"/providers/{provider_name}/status",
|
||||
response_model=Envelope[ProviderStatusResponse],
|
||||
)
|
||||
def get_external_provider_status(provider_name: str) -> dict:
|
||||
provider = get_provider(provider_name)
|
||||
return envelope({
|
||||
"provider_name": provider.provider_name,
|
||||
"configured": provider.is_configured,
|
||||
"status": provider.capability.status,
|
||||
"limitation_message": provider.limitation_message,
|
||||
})
|
||||
|
||||
|
||||
@router.post(
|
||||
"/providers/{provider_name}/import",
|
||||
response_model=Envelope[ProviderImportResponse],
|
||||
)
|
||||
def import_external_provider_dataset(provider_name: str, payload: ProviderImportRequest) -> dict:
|
||||
result = import_provider_dataset(
|
||||
provider_name=provider_name,
|
||||
project_id=payload.project_id,
|
||||
area_id=payload.area_id,
|
||||
layers=_normalize_layer_input(payload.layers),
|
||||
requested_dataset_role=payload.dataset_role,
|
||||
)
|
||||
return envelope(result.model_dump())
|
||||
|
||||
|
||||
def _run_fetch(payload: ExternalFetchRequest, provider_name: str) -> ExternalFetchResponse:
|
||||
area_id_str = str(payload.area_id) if payload.area_id else None
|
||||
response = fetch_provider_data(
|
||||
provider_name=provider_name,
|
||||
project_id=str(payload.project_id),
|
||||
area_id=area_id_str,
|
||||
layers=_normalize_layer_input(payload.layers),
|
||||
)
|
||||
return ExternalFetchResponse(
|
||||
provider=provider_name,
|
||||
status=response.get("status", "not_configured"),
|
||||
message=response.get("message", "Provider fetch executed."),
|
||||
requested_layers=_normalize_layer_input(payload.layers),
|
||||
project_id=payload.project_id,
|
||||
area_id=payload.area_id,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/osm/fetch", response_model=Envelope[ExternalFetchResponse])
|
||||
def fetch_osm(payload: ExternalFetchRequest, db: Session = Depends(get_db)) -> dict:
|
||||
_assert_project_exists(db, payload.project_id)
|
||||
_validate_area_in_project(db, payload.project_id, payload.area_id)
|
||||
return envelope(_run_fetch(payload, "osm").model_dump())
|
||||
|
||||
|
||||
@router.post("/grb/fetch", response_model=Envelope[ExternalFetchResponse])
|
||||
def fetch_grb(payload: ExternalFetchRequest, db: Session = Depends(get_db)) -> dict:
|
||||
_assert_project_exists(db, payload.project_id)
|
||||
_validate_area_in_project(db, payload.project_id, payload.area_id)
|
||||
return envelope(_run_fetch(payload, "grb").model_dump())
|
||||
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
from fastapi import APIRouter, Response, status
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.session import get_engine
|
||||
from app.providers.registry import list_provider_capabilities
|
||||
from app.schemas.health import (
|
||||
HealthResponse,
|
||||
SystemCapabilities,
|
||||
SystemCapabilitiesEnvelope,
|
||||
)
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _dependency_enabled(module: str) -> bool:
|
||||
try:
|
||||
import_module(module)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _expected_migration_heads() -> list[str]:
|
||||
backend_root = Path(__file__).resolve().parents[3]
|
||||
config = Config(str(backend_root / "alembic.ini"))
|
||||
config.set_main_option("script_location", str(backend_root / "alembic"))
|
||||
return list(ScriptDirectory.from_config(config).get_heads())
|
||||
|
||||
|
||||
def _database_checks() -> dict[str, str]:
|
||||
checks = {
|
||||
"database": "degraded",
|
||||
"postgis": "degraded",
|
||||
"migration": "degraded",
|
||||
}
|
||||
try:
|
||||
with get_engine().connect() as connection:
|
||||
connection.execute(text("SELECT 1"))
|
||||
checks["database"] = "ok"
|
||||
connection.execute(
|
||||
text("SELECT PostGIS_Version()")
|
||||
).scalar_one()
|
||||
checks["postgis"] = "ok"
|
||||
database_head = connection.execute(
|
||||
text("SELECT version_num FROM alembic_version")
|
||||
).scalar_one()
|
||||
expected_heads = _expected_migration_heads()
|
||||
if len(expected_heads) == 1 and database_head == expected_heads[0]:
|
||||
checks["migration"] = "ok"
|
||||
else:
|
||||
checks["migration"] = "degraded"
|
||||
except Exception:
|
||||
return checks
|
||||
return checks
|
||||
|
||||
|
||||
def _storage_check(storage_root: str) -> str:
|
||||
root = Path(storage_root).expanduser()
|
||||
try:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
with NamedTemporaryFile(
|
||||
prefix=".geointel-readiness-",
|
||||
dir=root,
|
||||
delete=True,
|
||||
) as handle:
|
||||
handle.write(b"ok")
|
||||
handle.flush()
|
||||
return "ok"
|
||||
except OSError:
|
||||
return "degraded"
|
||||
|
||||
|
||||
def _readiness_payload() -> HealthResponse:
|
||||
settings = get_settings()
|
||||
checks = _database_checks()
|
||||
checks["storage"] = _storage_check(settings.storage_root)
|
||||
ready = all(
|
||||
value == "ok" or value.startswith("ok:")
|
||||
for value in checks.values()
|
||||
)
|
||||
return HealthResponse(
|
||||
status="ok" if ready else "degraded",
|
||||
service="geointel-backend",
|
||||
version="public",
|
||||
database=checks["database"],
|
||||
postgis=checks["postgis"],
|
||||
migration=checks["migration"],
|
||||
storage=checks["storage"],
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/health/live", response_model=HealthResponse)
|
||||
def liveness() -> HealthResponse:
|
||||
return HealthResponse(
|
||||
status="ok",
|
||||
service="geointel-backend",
|
||||
version="public",
|
||||
)
|
||||
|
||||
|
||||
def _readiness_response(response: Response) -> HealthResponse:
|
||||
payload = _readiness_payload()
|
||||
if payload.status != "ok":
|
||||
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/health", response_model=HealthResponse)
|
||||
def readiness(response: Response) -> HealthResponse:
|
||||
return _readiness_response(response)
|
||||
|
||||
|
||||
@router.get("/health/ready", response_model=HealthResponse)
|
||||
def readiness_explicit(response: Response) -> HealthResponse:
|
||||
return _readiness_response(response)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/system/capabilities",
|
||||
response_model=SystemCapabilitiesEnvelope,
|
||||
)
|
||||
def capabilities() -> SystemCapabilitiesEnvelope:
|
||||
settings = get_settings()
|
||||
providers = [item.to_dict() for item in list_provider_capabilities()]
|
||||
configured_yolo = ModelRegistryService.get_model_capability(
|
||||
settings.yolo_model_id,
|
||||
settings=settings,
|
||||
)
|
||||
yolo_configured = bool(configured_yolo and configured_yolo.configured)
|
||||
yolo_status = configured_yolo.status if configured_yolo else "not_configured"
|
||||
configured_sam = ModelRegistryService.get_model_capability(
|
||||
settings.sam_model_id,
|
||||
settings=settings,
|
||||
task_type="segmentation",
|
||||
)
|
||||
postgis_ready = _database_checks()["postgis"].startswith("ok:")
|
||||
return SystemCapabilitiesEnvelope(
|
||||
data=SystemCapabilities(
|
||||
postgis=postgis_ready,
|
||||
rasterio=_dependency_enabled("rasterio"),
|
||||
geopandas=_dependency_enabled("geopandas"),
|
||||
yolo=yolo_configured,
|
||||
yolo_status=yolo_status,
|
||||
sam=bool(configured_sam and configured_sam.configured),
|
||||
grb="bounded",
|
||||
sentinel="planned",
|
||||
version=settings.app_version,
|
||||
build_sha=settings.build_sha,
|
||||
providers=providers,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas import Envelope, JobCreate, JobList, JobRead, JobStatus
|
||||
from app.services.job_service import JobService
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}", tags=["jobs"])
|
||||
|
||||
|
||||
@router.post("/jobs", status_code=201, response_model=Envelope[JobRead])
|
||||
def create_job(
|
||||
project_id: UUID,
|
||||
payload: JobCreate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
if payload.project_id != project_id:
|
||||
raise HTTPException(status_code=400, detail="project_id mismatch")
|
||||
return envelope(JobService.create_job(db, payload).model_dump())
|
||||
|
||||
|
||||
@router.get("/jobs", response_model=Envelope[JobList])
|
||||
def list_jobs(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID | None = Query(default=None),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
items, total = JobService.list_jobs(
|
||||
db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return envelope(JobList(items=items, total=total, limit=limit, offset=offset).model_dump())
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}", response_model=Envelope[JobRead])
|
||||
def read_job(
|
||||
project_id: UUID,
|
||||
job_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = JobService.get_job(db, job_id)
|
||||
if job.project_id != project_id:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return envelope(job.model_dump())
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}/status", response_model=Envelope[JobStatus])
|
||||
def read_job_status(
|
||||
project_id: UUID,
|
||||
job_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
status_row = JobService.get_job_status(db, job_id)
|
||||
if status_row["project_id"] != str(project_id):
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return envelope(JobStatus(**status_row).model_dump())
|
||||
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas import Envelope
|
||||
from app.schemas.project import ProjectCreate, ProjectDeleteResult, ProjectList, ProjectRead, ProjectUpdate
|
||||
from app.services.project_service import ProjectService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/projects", tags=["projects"])
|
||||
|
||||
|
||||
@router.get("", response_model=Envelope[ProjectList])
|
||||
def list_projects(
|
||||
request: Request,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
name: str | None = Query(default=None, min_length=1, max_length=255),
|
||||
project_status: Literal["active", "archived", "all"] = Query(default="active", alias="status"),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
principal = getattr(request.state, "auth_principal", None)
|
||||
if principal is not None and principal.role == "guest":
|
||||
project = ProjectService.get_project(db, principal.project_id)
|
||||
status_matches = bool(
|
||||
project is not None
|
||||
and (project_status == "all" or project.status == project_status)
|
||||
)
|
||||
name_matches = bool(
|
||||
project is not None
|
||||
and (name is None or name.casefold() in project.name.casefold())
|
||||
)
|
||||
matches = project is not None and status_matches and name_matches
|
||||
visible = [project] if matches and offset == 0 else []
|
||||
return envelope(
|
||||
{
|
||||
"items": [ProjectRead.model_validate(item).model_dump() for item in visible[:limit]],
|
||||
"total": 1 if matches else 0,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
)
|
||||
projects, total = ProjectService.list_projects(
|
||||
db,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
name=name,
|
||||
project_status=project_status,
|
||||
)
|
||||
return envelope({"items": [ProjectRead.model_validate(item).model_dump() for item in projects], "total": total, "limit": limit, "offset": offset})
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED, response_model=Envelope[ProjectRead])
|
||||
def create_project(payload: ProjectCreate, db: Session = Depends(get_db)):
|
||||
project = ProjectService.create_project(db, payload)
|
||||
return envelope(ProjectRead.model_validate(project).model_dump())
|
||||
|
||||
|
||||
@router.get("/{project_id}", response_model=Envelope[ProjectRead])
|
||||
def get_project(project_id: UUID, db: Session = Depends(get_db)):
|
||||
project = ProjectService.get_project(db, project_id)
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
return envelope(ProjectRead.model_validate(project).model_dump())
|
||||
|
||||
|
||||
@router.patch("/{project_id}", response_model=Envelope[ProjectRead])
|
||||
def update_project(project_id: UUID, payload: ProjectUpdate, db: Session = Depends(get_db)):
|
||||
project = ProjectService.update_project(db, project_id, payload)
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
return envelope(ProjectRead.model_validate(project).model_dump())
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/{project_id}",
|
||||
status_code=status.HTTP_200_OK,
|
||||
response_model=Envelope[ProjectDeleteResult],
|
||||
)
|
||||
def delete_project(project_id: UUID, db: Session = Depends(get_db)):
|
||||
if not ProjectService.delete_project(db, project_id):
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
return envelope({"deleted": True})
|
||||
@@ -0,0 +1,83 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, Job
|
||||
from app.schemas import Envelope, JobRead, QaProviderComparisonRequest
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.job_service import JobService
|
||||
from app.services.quality_service import QualityService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/qa", tags=["qa"])
|
||||
|
||||
|
||||
@router.post("/detections-vs-reference", response_model=Envelope[JobRead])
|
||||
def compare_candidate_with_reference(
|
||||
payload: QaProviderComparisonRequest,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
candidate_dataset = db.get(Dataset, payload.candidate_dataset_id)
|
||||
if not candidate_dataset:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Candidate dataset not found", status_code=404)
|
||||
job = JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=candidate_dataset.project_id,
|
||||
job_type="qa.compare-candidate-with-reference",
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
input_dataset_id=candidate_dataset.id,
|
||||
operation=lambda: QaService.compare_candidate_with_reference(
|
||||
db=db,
|
||||
project_id=candidate_dataset.project_id,
|
||||
candidate_dataset_id=payload.candidate_dataset_id,
|
||||
reference_dataset_id=payload.reference_dataset_id,
|
||||
iou_threshold=payload.iou_threshold,
|
||||
area_id=payload.area_id,
|
||||
).model_dump(mode="json"),
|
||||
)
|
||||
result_json = job.get("result_json") if isinstance(job, dict) else None
|
||||
if isinstance(result_json, dict) and job.get("status") == "success":
|
||||
quality_check = QualityService.persist_quality_check(
|
||||
db=db,
|
||||
project_id=candidate_dataset.project_id,
|
||||
job_id=uuid.UUID(str(job["id"])),
|
||||
candidate_dataset_id=payload.candidate_dataset_id,
|
||||
reference_dataset_id=payload.reference_dataset_id,
|
||||
check_type="candidate_vs_reference",
|
||||
status=str(result_json.get("status", "ok")),
|
||||
score=result_json.get("f1_score"),
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
findings={
|
||||
"matches": result_json.get("matches"),
|
||||
"false_positives": result_json.get("false_positives"),
|
||||
"false_negatives": result_json.get("false_negatives"),
|
||||
"warnings": result_json.get("warnings", []),
|
||||
"unsupported_geometry": result_json.get("unsupported_geometry", False),
|
||||
"unsupported_geometries": result_json.get("unsupported_geometries", []),
|
||||
"match_evidence": result_json.get("match_evidence", []),
|
||||
"false_positive_evidence": result_json.get("false_positive_evidence", []),
|
||||
"false_negative_evidence": result_json.get("false_negative_evidence", []),
|
||||
},
|
||||
metrics={
|
||||
"precision": result_json.get("precision"),
|
||||
"recall": result_json.get("recall"),
|
||||
"f1": result_json.get("f1_score"),
|
||||
"mean_iou": result_json.get("mean_iou"),
|
||||
"false_positive_count": result_json.get("false_positives"),
|
||||
"false_negative_count": result_json.get("false_negatives"),
|
||||
},
|
||||
)
|
||||
result_json["quality_check_id"] = str(quality_check.id)
|
||||
|
||||
job_record = db.get(Job, uuid.UUID(str(job["id"])))
|
||||
if job_record:
|
||||
job_record.result_json = result_json
|
||||
db.add(job_record)
|
||||
db.commit()
|
||||
|
||||
return envelope(job)
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas import Envelope, QualityEvidenceResponse
|
||||
from app.schemas.detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewUpsert
|
||||
from app.schemas.qa import QualityCheckList
|
||||
from app.services.detection_review_service import DetectionReviewService
|
||||
from app.services.quality_evidence_service import QualityEvidenceService
|
||||
from app.services.quality_check_service import QualityCheckService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}", tags=["quality-checks"])
|
||||
|
||||
|
||||
@router.get("/quality-checks", response_model=Envelope[QualityCheckList])
|
||||
def list_quality_checks(
|
||||
project_id: UUID,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
items, total = QualityCheckService.list_quality_checks(
|
||||
db,
|
||||
project_id=project_id,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
return envelope(QualityCheckList(items=items, total=total, limit=limit, offset=offset).model_dump())
|
||||
|
||||
|
||||
@router.get(
|
||||
"/quality-checks/{quality_check_id}/evidence/geojson",
|
||||
response_model=Envelope[QualityEvidenceResponse],
|
||||
)
|
||||
def get_quality_check_evidence_geojson(
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
limit: int = Query(
|
||||
default=QualityEvidenceService.DEFAULT_EVIDENCE_LIMIT,
|
||||
ge=0,
|
||||
le=100_000,
|
||||
description="Maximum evidence features to draw; 0 returns everything. Misses and false positives first.",
|
||||
),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(
|
||||
QualityEvidenceService.evidence_geojson(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/quality-checks/{quality_check_id}/reviews",
|
||||
response_model=Envelope[DetectionReviewList],
|
||||
)
|
||||
def list_detection_reviews(
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
evidence_role: str | None = Query(default=None, pattern="^(false_positive|false_negative)$"),
|
||||
decision: str | None = Query(default=None, max_length=64),
|
||||
reviewed: bool | None = Query(default=None),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(
|
||||
DetectionReviewService.list_reviews(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
evidence_role=evidence_role,
|
||||
decision=decision,
|
||||
reviewed=reviewed,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/quality-checks/{quality_check_id}/reviews",
|
||||
response_model=Envelope[DetectionReviewRead],
|
||||
)
|
||||
def upsert_detection_review(
|
||||
project_id: UUID,
|
||||
quality_check_id: UUID,
|
||||
payload: DetectionReviewUpsert,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
return envelope(
|
||||
DetectionReviewService.upsert_review(
|
||||
db,
|
||||
project_id=project_id,
|
||||
quality_check_id=quality_check_id,
|
||||
payload=payload,
|
||||
).model_dump()
|
||||
)
|
||||
@@ -0,0 +1,284 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.guest_scope import (
|
||||
assert_guest_project_scope,
|
||||
guest_project_scope,
|
||||
guest_scoped_project_filter,
|
||||
)
|
||||
from app.db.session import get_db
|
||||
from app.schemas import (
|
||||
AnalysisQaResponse,
|
||||
Envelope,
|
||||
GeoJsonFeatureCollection,
|
||||
JobRead,
|
||||
SegmentationListResponse,
|
||||
SegmentationModelsResponse,
|
||||
SegmentationQaRequest,
|
||||
SegmentationRead,
|
||||
SegmentationRunListResponse,
|
||||
SegmentationRunRead,
|
||||
SegmentationRunRequest,
|
||||
SegmentationRunResponse,
|
||||
)
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/segmentation", tags=["segmentation"])
|
||||
|
||||
|
||||
@router.get("/models", response_model=Envelope[SegmentationModelsResponse])
|
||||
def list_segmentation_models() -> dict:
|
||||
return envelope({"models": [model.model_dump() for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")]})
|
||||
|
||||
|
||||
@router.post("/run", response_model=Envelope[SegmentationRunResponse])
|
||||
def run_segmentation(
|
||||
payload: SegmentationRunRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
assert_guest_project_scope(request, payload.project_id)
|
||||
result = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=payload.project_id,
|
||||
dataset_id=payload.dataset_id,
|
||||
model_id=payload.model_id,
|
||||
confidence_threshold=payload.confidence_threshold,
|
||||
class_filter=payload.class_filter,
|
||||
tile_manifest_path=payload.tile_manifest_path,
|
||||
parameters_json=payload.parameters_json,
|
||||
)
|
||||
return envelope(result.model_dump())
|
||||
|
||||
|
||||
@router.post("/run-async", response_model=Envelope[JobRead])
|
||||
def queue_segmentation(
|
||||
payload: SegmentationRunRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Queue a segmentation run for the background worker.
|
||||
|
||||
Configured segmentation walks the same tile manifest as detection and is
|
||||
just as unsuited to running inside the request. Poll ``GET /jobs/{id}``.
|
||||
"""
|
||||
|
||||
assert_guest_project_scope(request, payload.project_id)
|
||||
job = SegmentationService.enqueue_segmentation(
|
||||
db=db,
|
||||
project_id=payload.project_id,
|
||||
dataset_id=payload.dataset_id,
|
||||
model_id=payload.model_id,
|
||||
confidence_threshold=payload.confidence_threshold,
|
||||
class_filter=payload.class_filter,
|
||||
tile_manifest_path=payload.tile_manifest_path,
|
||||
parameters_json=payload.parameters_json,
|
||||
)
|
||||
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
|
||||
|
||||
|
||||
@router.get("/runs", response_model=Envelope[SegmentationRunListResponse])
|
||||
def list_segmentation_runs(
|
||||
request: Request,
|
||||
project_id: UUID | None = None,
|
||||
dataset_id: UUID | None = None,
|
||||
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
project_id = guest_scoped_project_filter(request, project_id)
|
||||
return envelope(
|
||||
SegmentationService.list_runs(
|
||||
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/runs/{analysis_run_id}", response_model=Envelope[SegmentationRunRead])
|
||||
def get_segmentation_run(
|
||||
analysis_run_id: UUID,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
run = SegmentationService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(run.model_dump())
|
||||
|
||||
|
||||
@router.get(
|
||||
"/runs/{analysis_run_id}/segmentations",
|
||||
response_model=Envelope[SegmentationListResponse],
|
||||
)
|
||||
def list_segmentation_run_outputs(
|
||||
analysis_run_id: UUID,
|
||||
request: Request,
|
||||
dataset_id: UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
run = SegmentationService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(
|
||||
SegmentationService.list_segmentations(
|
||||
db,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/datasets/{dataset_id}/segmentations",
|
||||
response_model=Envelope[SegmentationListResponse],
|
||||
)
|
||||
def list_dataset_segmentations(
|
||||
dataset_id: UUID,
|
||||
request: Request,
|
||||
analysis_run_id: UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
assert_guest_project_scope(request, dataset.project_id)
|
||||
return envelope(
|
||||
SegmentationService.list_segmentations(
|
||||
db,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/segmentations/{segmentation_id}", response_model=Envelope[SegmentationRead])
|
||||
def get_segmentation(
|
||||
segmentation_id: UUID,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
segmentation = SegmentationService.get_segmentation(db, segmentation_id)
|
||||
assert_guest_project_scope(request, segmentation.project_id)
|
||||
return envelope(segmentation.model_dump())
|
||||
|
||||
|
||||
@router.get(
|
||||
"/runs/{analysis_run_id}/geojson",
|
||||
response_model=Envelope[GeoJsonFeatureCollection],
|
||||
)
|
||||
def get_segmentation_run_geojson(
|
||||
analysis_run_id: UUID,
|
||||
request: Request,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
run = SegmentationService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(
|
||||
SegmentationService.segmentations_to_geojson(
|
||||
db,
|
||||
limit=limit,
|
||||
analysis_run_id=analysis_run_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/datasets/{dataset_id}/geojson",
|
||||
response_model=Envelope[GeoJsonFeatureCollection],
|
||||
)
|
||||
def get_dataset_segmentation_geojson(
|
||||
dataset_id: UUID,
|
||||
request: Request,
|
||||
analysis_run_id: UUID | None = None,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
limit: int = Query(
|
||||
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||
ge=0,
|
||||
le=50_000,
|
||||
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||
),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||
assert_guest_project_scope(request, dataset.project_id)
|
||||
return envelope(
|
||||
SegmentationService.segmentations_to_geojson(
|
||||
db,
|
||||
limit=limit,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/runs/{analysis_run_id}/qa/reference",
|
||||
response_model=Envelope[AnalysisQaResponse],
|
||||
)
|
||||
def compare_segmentation_run_with_reference(
|
||||
analysis_run_id: UUID,
|
||||
payload: SegmentationQaRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if guest_project_scope(request) is not None:
|
||||
run = SegmentationService.get_run(db, analysis_run_id)
|
||||
assert_guest_project_scope(request, run.project_id)
|
||||
return envelope(
|
||||
SegmentationService.compare_segmentations_with_reference(
|
||||
db=db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
reference_dataset_id=payload.reference_dataset_id,
|
||||
iou_threshold=payload.iou_threshold,
|
||||
class_name=payload.class_name,
|
||||
min_confidence=payload.min_confidence,
|
||||
calibration_thresholds=payload.calibration_thresholds,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.models import Area, Dataset
|
||||
from app.schemas.common import Envelope
|
||||
from app.schemas.operations import VectorSelectionResponse
|
||||
from app.schemas.selection_partitions import VectorPartitionSelectionRequest
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}", tags=["selection-partitions"])
|
||||
|
||||
|
||||
def _product_identity(dataset: Dataset) -> str:
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
return str(metadata.get("product_key") or dataset.reference_layer_name or "")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/datasets/vector/partitions/select",
|
||||
response_model=Envelope[VectorSelectionResponse],
|
||||
)
|
||||
def select_vector_partitions(
|
||||
project_id: UUID,
|
||||
payload: VectorPartitionSelectionRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
datasets = db.query(Dataset).filter(Dataset.id.in_(payload.dataset_ids)).all()
|
||||
by_id = {dataset.id: dataset for dataset in datasets}
|
||||
ordered = [by_id.get(dataset_id) for dataset_id in payload.dataset_ids]
|
||||
if any(dataset is None or dataset.project_id != project_id for dataset in ordered):
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="One or more selection partitions were not found", status_code=404)
|
||||
typed_datasets = [dataset for dataset in ordered if dataset is not None]
|
||||
if any(dataset.dataset_type not in {"vector", "geojson"} or dataset.status != "ready" for dataset in typed_datasets):
|
||||
raise AppError(
|
||||
code="INVALID_VECTOR_PARTITIONS",
|
||||
message="Every selection partition must be a ready vector dataset",
|
||||
status_code=409,
|
||||
)
|
||||
source_names = {dataset.source_name for dataset in typed_datasets}
|
||||
product_keys = {_product_identity(dataset) for dataset in typed_datasets}
|
||||
if len(source_names) != 1 or len(product_keys) != 1:
|
||||
raise AppError(
|
||||
code="VECTOR_PARTITION_SOURCE_MISMATCH",
|
||||
message="Selection partitions must belong to one governed source product",
|
||||
details={"source_names": sorted(str(value) for value in source_names), "product_keys": sorted(product_keys)},
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
selection_geometry = None
|
||||
selection_area_id = None
|
||||
if payload.area_id is not None:
|
||||
selection_area = db.get(Area, payload.area_id)
|
||||
if selection_area is None or selection_area.project_id != project_id:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
selection_geometry, _covers_full_area = VectorFeatureService.constrain_bbox_to_area(
|
||||
payload.bbox.model_dump(),
|
||||
selection_area.geometry,
|
||||
)
|
||||
selection_area_id = selection_area.id
|
||||
|
||||
representative = typed_datasets[0]
|
||||
dataset_ids = [dataset.id for dataset in typed_datasets]
|
||||
result = VectorFeatureService.select_features_by_bbox(
|
||||
db,
|
||||
dataset_id=representative.id,
|
||||
dataset_ids=dataset_ids,
|
||||
bbox=payload.bbox.model_dump(),
|
||||
limit=payload.limit,
|
||||
dataset=representative,
|
||||
selection_geometry=selection_geometry,
|
||||
selection_area_id=selection_area_id,
|
||||
deduplicate_source_features=True,
|
||||
)
|
||||
result.update(
|
||||
partition_count=len(dataset_ids),
|
||||
source_name=representative.source_name,
|
||||
dataset_ids=dataset_ids,
|
||||
)
|
||||
return envelope(VectorSelectionResponse(**result).model_dump(exclude_none=True))
|
||||
@@ -0,0 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
|
||||
from app.schemas import (
|
||||
DatasetLineageEdgeRead,
|
||||
DatasetProvenanceRead,
|
||||
DatasetQuarantineRead,
|
||||
Envelope,
|
||||
ItemList,
|
||||
SourceRegistryDetailRead,
|
||||
SourceRegistryRead,
|
||||
SourceSnapshotRead,
|
||||
)
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(tags=["source-registry"])
|
||||
|
||||
|
||||
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
|
||||
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
|
||||
|
||||
|
||||
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
|
||||
def list_source_registry(
|
||||
classification: str | None = None,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query = (
|
||||
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
|
||||
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
|
||||
)
|
||||
if classification:
|
||||
query = query.filter(SourceRegistry.classification == classification.strip().lower())
|
||||
rows = (
|
||||
query.group_by(SourceRegistry.id)
|
||||
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
|
||||
.all()
|
||||
)
|
||||
items = [_source_read(source, snapshot_count=count) for source, count in rows]
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
|
||||
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
|
||||
normalized_key = source_key.strip().lower()
|
||||
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
|
||||
if source is None:
|
||||
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
|
||||
snapshots = (
|
||||
db.query(SourceSnapshot)
|
||||
.filter(SourceSnapshot.source_registry_id == source.id)
|
||||
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
detail = SourceRegistryDetailRead(
|
||||
source=_source_read(source, snapshot_count=len(snapshots)),
|
||||
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
|
||||
)
|
||||
return envelope(detail)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/projects/{project_id}/datasets/{dataset_id}/provenance",
|
||||
response_model=Envelope[DatasetProvenanceRead],
|
||||
)
|
||||
def get_dataset_provenance(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if db.get(Project, project_id) is None:
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if dataset is None or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
|
||||
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
|
||||
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
|
||||
lineage = (
|
||||
db.query(DatasetLineageEdge)
|
||||
.filter(
|
||||
or_(
|
||||
DatasetLineageEdge.parent_dataset_id == dataset.id,
|
||||
DatasetLineageEdge.child_dataset_id == dataset.id,
|
||||
)
|
||||
)
|
||||
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
|
||||
.all()
|
||||
)
|
||||
quarantines = (
|
||||
db.query(DatasetQuarantine)
|
||||
.filter(DatasetQuarantine.dataset_id == dataset.id)
|
||||
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
|
||||
.all()
|
||||
)
|
||||
result = DatasetProvenanceRead(
|
||||
dataset_id=dataset.id,
|
||||
source=_source_read(source) if source else None,
|
||||
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
|
||||
data_contract_key=dataset.data_contract_key,
|
||||
data_contract_version=dataset.data_contract_version,
|
||||
validation_status=dataset.validation_status,
|
||||
validation_report_json=dataset.validation_report_json,
|
||||
provenance_status=dataset.provenance_status,
|
||||
lineage_status=dataset.lineage_status,
|
||||
quarantine_status=dataset.quarantine_status,
|
||||
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
|
||||
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
|
||||
)
|
||||
return envelope(result)
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.schemas import Envelope, ItemList
|
||||
from app.schemas.temporal import (
|
||||
TemporalComparisonRequest,
|
||||
TemporalComparisonResponse,
|
||||
TemporalSeriesRead,
|
||||
)
|
||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}/temporal", tags=["temporal"])
|
||||
|
||||
|
||||
@router.get("/series", response_model=Envelope[ItemList[TemporalSeriesRead]])
|
||||
def list_temporal_series(project_id: UUID, db: Session = Depends(get_db)):
|
||||
series = TemporalAnalysisService.list_series(db, project_id)
|
||||
return envelope({"items": [item.model_dump() for item in series], "total": len(series)})
|
||||
|
||||
|
||||
@router.post("/compare", response_model=Envelope[TemporalComparisonResponse])
|
||||
def compare_temporal_snapshots(
|
||||
project_id: UUID,
|
||||
payload: TemporalComparisonRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return envelope(TemporalAnalysisService.compare(db, project_id=project_id, payload=payload).model_dump())
|
||||
@@ -0,0 +1,584 @@
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import AliasChoices, Field, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
populate_by_name=True,
|
||||
)
|
||||
|
||||
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
|
||||
app_version: str = Field(
|
||||
default="1.0.0",
|
||||
validation_alias="GEOINTEL_APP_VERSION",
|
||||
)
|
||||
build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
|
||||
build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
|
||||
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
|
||||
auth_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_ENABLED")
|
||||
auth_require_https: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_REQUIRE_HTTPS")
|
||||
auth_username: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_USERNAME")
|
||||
auth_password_hash: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_PASSWORD_HASH")
|
||||
auth_session_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_SESSION_SECRET")
|
||||
authentik_issuer: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ISSUER")
|
||||
authentik_client_id: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_ID")
|
||||
authentik_client_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_SECRET")
|
||||
authentik_allowed_email: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ALLOWED_EMAIL")
|
||||
public_base_url: str = Field(
|
||||
default="http://localhost:1202",
|
||||
validation_alias="GEOINTEL_PUBLIC_BASE_URL",
|
||||
)
|
||||
auth_session_ttl_seconds: int = Field(
|
||||
default=43_200,
|
||||
ge=900,
|
||||
le=604_800,
|
||||
validation_alias="GEOINTEL_AUTH_SESSION_TTL_SECONDS",
|
||||
)
|
||||
guest_access_enabled: bool = Field(
|
||||
default=False,
|
||||
validation_alias="GEOINTEL_GUEST_ACCESS_ENABLED",
|
||||
)
|
||||
guest_display_name: str = Field(
|
||||
default="Gast",
|
||||
min_length=1,
|
||||
max_length=64,
|
||||
validation_alias="GEOINTEL_GUEST_DISPLAY_NAME",
|
||||
)
|
||||
guest_session_ttl_seconds: int = Field(
|
||||
default=7_200,
|
||||
ge=900,
|
||||
le=86_400,
|
||||
validation_alias="GEOINTEL_GUEST_SESSION_TTL_SECONDS",
|
||||
)
|
||||
guest_login_requests_per_minute: int = Field(
|
||||
default=3,
|
||||
ge=1,
|
||||
le=60,
|
||||
validation_alias="GEOINTEL_GUEST_LOGIN_REQUESTS_PER_MINUTE",
|
||||
)
|
||||
guest_compute_requests_per_minute: int = Field(
|
||||
default=4,
|
||||
ge=1,
|
||||
le=120,
|
||||
validation_alias="GEOINTEL_GUEST_COMPUTE_REQUESTS_PER_MINUTE",
|
||||
)
|
||||
guest_compute_max_concurrency: int = Field(
|
||||
default=2,
|
||||
ge=1,
|
||||
le=16,
|
||||
validation_alias="GEOINTEL_GUEST_COMPUTE_MAX_CONCURRENCY",
|
||||
)
|
||||
database_url: str = Field(
|
||||
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
|
||||
validation_alias="DATABASE_URL",
|
||||
)
|
||||
storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT")
|
||||
# Analysis consumes only artifacts under storage_root. Provisioning
|
||||
# workflows that stage tiles elsewhere before ingest can opt out.
|
||||
allow_external_artifact_paths: bool = Field(
|
||||
default=False, validation_alias="GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS"
|
||||
)
|
||||
max_upload_mb: int = Field(
|
||||
default=500,
|
||||
ge=1,
|
||||
le=2_048,
|
||||
validation_alias=AliasChoices("GEOINTEL_MAX_UPLOAD_MB", "MAX_UPLOAD_MB"),
|
||||
)
|
||||
max_in_memory_vector_mb: int = Field(
|
||||
default=64,
|
||||
ge=1,
|
||||
le=256,
|
||||
validation_alias="GEOINTEL_MAX_IN_MEMORY_VECTOR_MB",
|
||||
)
|
||||
max_raster_pixels: int = Field(
|
||||
default=40_000_000,
|
||||
ge=1,
|
||||
le=500_000_000,
|
||||
validation_alias="GEOINTEL_MAX_RASTER_PIXELS",
|
||||
)
|
||||
max_raster_bands: int = Field(
|
||||
default=16,
|
||||
ge=1,
|
||||
le=256,
|
||||
validation_alias="GEOINTEL_MAX_RASTER_BANDS",
|
||||
)
|
||||
max_decoded_raster_mb: int = Field(
|
||||
default=1024,
|
||||
ge=16,
|
||||
le=8192,
|
||||
validation_alias="GEOINTEL_MAX_DECODED_RASTER_MB",
|
||||
)
|
||||
orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED")
|
||||
orthophoto_wms_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
||||
validation_alias="ORTHOPHOTO_WMS_URL",
|
||||
)
|
||||
orthophoto_wms_layer: str = Field(default="Ortho", validation_alias="ORTHOPHOTO_WMS_LAYER")
|
||||
spw_orthophoto_wms_url: str = Field(
|
||||
default="https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_LAST/MapServer/WMSServer",
|
||||
validation_alias="SPW_ORTHOPHOTO_WMS_URL",
|
||||
)
|
||||
brussels_orthophoto_wms_url: str = Field(
|
||||
default="https://geoservices-grid.irisnet.be/geoserver/urbisgrid/ows",
|
||||
validation_alias="BRUSSELS_ORTHOPHOTO_WMS_URL",
|
||||
)
|
||||
orthophoto_resolution_m: float = Field(default=1.0, gt=0, validation_alias="ORTHOPHOTO_RESOLUTION_M")
|
||||
orthophoto_min_side_m: float = Field(default=128.0, gt=0, validation_alias="ORTHOPHOTO_MIN_SIDE_M")
|
||||
orthophoto_max_side_m: float = Field(default=1024.0, gt=0, validation_alias="ORTHOPHOTO_MAX_SIDE_M")
|
||||
orthophoto_timeout_seconds: int = Field(default=120, ge=1, validation_alias="ORTHOPHOTO_TIMEOUT_SECONDS")
|
||||
orthophoto_max_response_mb: int = Field(default=32, ge=1, validation_alias="ORTHOPHOTO_MAX_RESPONSE_MB")
|
||||
orthophoto_cache_ttl_hours: int = Field(default=24, ge=0, validation_alias="ORTHOPHOTO_CACHE_TTL_HOURS")
|
||||
source_catalog_probe_enabled: bool = Field(default=True, validation_alias="SOURCE_CATALOG_PROBE_ENABLED")
|
||||
source_catalog_grb_wfs_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/GRB/wfs",
|
||||
validation_alias="SOURCE_CATALOG_GRB_WFS_URL",
|
||||
)
|
||||
source_catalog_alz_release_url: str = Field(
|
||||
default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen",
|
||||
validation_alias="SOURCE_CATALOG_ALZ_RELEASE_URL",
|
||||
)
|
||||
source_catalog_statbel_dcat_url: str = Field(
|
||||
default="https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl",
|
||||
validation_alias="SOURCE_CATALOG_STATBEL_DCAT_URL",
|
||||
)
|
||||
source_catalog_statbel_max_response_mb: int = Field(
|
||||
default=5,
|
||||
ge=1,
|
||||
le=10,
|
||||
validation_alias="SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB",
|
||||
)
|
||||
source_catalog_probe_timeout_seconds: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=60,
|
||||
validation_alias="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS",
|
||||
)
|
||||
source_catalog_probe_max_response_mb: int = Field(
|
||||
default=2,
|
||||
ge=1,
|
||||
le=10,
|
||||
validation_alias="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB",
|
||||
)
|
||||
source_catalog_probe_cache_ttl_seconds: int = Field(
|
||||
default=900,
|
||||
ge=0,
|
||||
le=86_400,
|
||||
validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS",
|
||||
)
|
||||
grb_enabled: bool = Field(default=True, validation_alias="GRB_ENABLED")
|
||||
grb_ogc_api_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/GRB/ogc/features/v1",
|
||||
validation_alias="GRB_OGC_API_URL",
|
||||
)
|
||||
grb_min_side_m: float = Field(default=10.0, gt=0, validation_alias="GRB_MIN_SIDE_M")
|
||||
grb_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="GRB_MAX_SIDE_M")
|
||||
grb_page_size: int = Field(default=1000, ge=1, le=1000, validation_alias="GRB_PAGE_SIZE")
|
||||
grb_max_pages: int = Field(default=200, ge=1, le=1000, validation_alias="GRB_MAX_PAGES")
|
||||
grb_max_features: int = Field(default=150_000, ge=1, validation_alias="GRB_MAX_FEATURES")
|
||||
grb_timeout_seconds: int = Field(default=180, ge=1, le=600, validation_alias="GRB_TIMEOUT_SECONDS")
|
||||
grb_max_response_mb: int = Field(default=20, ge=1, le=100, validation_alias="GRB_MAX_RESPONSE_MB")
|
||||
grb_max_total_response_mb: int = Field(
|
||||
default=256,
|
||||
ge=1,
|
||||
le=2048,
|
||||
validation_alias="GRB_MAX_TOTAL_RESPONSE_MB",
|
||||
)
|
||||
grb_cache_ttl_hours: int = Field(default=24, ge=0, le=8760, validation_alias="GRB_CACHE_TTL_HOURS")
|
||||
official_vector_enabled: bool = Field(default=True, validation_alias="OFFICIAL_VECTOR_ENABLED")
|
||||
bwk_wfs_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/BWK/wfs",
|
||||
validation_alias="BWK_WFS_URL",
|
||||
)
|
||||
dov_soil_wfs_url: str = Field(
|
||||
default="https://www.dov.vlaanderen.be/geoserver/wfs",
|
||||
validation_alias="DOV_SOIL_WFS_URL",
|
||||
)
|
||||
official_vector_min_side_m: float = Field(
|
||||
default=10.0,
|
||||
gt=0,
|
||||
validation_alias="OFFICIAL_VECTOR_MIN_SIDE_M",
|
||||
)
|
||||
official_vector_max_side_m: float = Field(
|
||||
default=20_000.0,
|
||||
gt=0,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_SIDE_M",
|
||||
)
|
||||
official_vector_page_size: int = Field(
|
||||
default=1000,
|
||||
ge=1,
|
||||
le=2000,
|
||||
validation_alias="OFFICIAL_VECTOR_PAGE_SIZE",
|
||||
)
|
||||
official_vector_max_pages: int = Field(
|
||||
default=200,
|
||||
ge=1,
|
||||
le=1000,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_PAGES",
|
||||
)
|
||||
official_vector_max_features: int = Field(
|
||||
default=100_000,
|
||||
ge=1,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_FEATURES",
|
||||
)
|
||||
official_vector_timeout_seconds: int = Field(
|
||||
default=180,
|
||||
ge=1,
|
||||
le=600,
|
||||
validation_alias="OFFICIAL_VECTOR_TIMEOUT_SECONDS",
|
||||
)
|
||||
official_vector_max_response_mb: int = Field(
|
||||
default=20,
|
||||
ge=1,
|
||||
le=100,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_RESPONSE_MB",
|
||||
)
|
||||
official_vector_max_total_response_mb: int = Field(
|
||||
default=256,
|
||||
ge=1,
|
||||
le=2048,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_TOTAL_RESPONSE_MB",
|
||||
)
|
||||
official_vector_cache_ttl_hours: int = Field(
|
||||
default=24,
|
||||
ge=0,
|
||||
le=8760,
|
||||
validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS",
|
||||
)
|
||||
spw_picc_enabled: bool = Field(default=True, validation_alias="SPW_PICC_ENABLED")
|
||||
spw_picc_mapserver_url: str = Field(
|
||||
default=(
|
||||
"https://geoservices.wallonie.be/arcgis/rest/services/"
|
||||
"TOPOGRAPHIE/PICC_VDIFF/MapServer"
|
||||
),
|
||||
validation_alias="SPW_PICC_MAPSERVER_URL",
|
||||
)
|
||||
spw_flood_hazard_enabled: bool = Field(default=True, validation_alias="SPW_FLOOD_HAZARD_ENABLED")
|
||||
spw_flood_hazard_mapserver_url: str = Field(
|
||||
default=(
|
||||
"https://geoservices.wallonie.be/arcgis/rest/services/"
|
||||
"EAU/ALEA_INOND/MapServer"
|
||||
),
|
||||
validation_alias="SPW_FLOOD_HAZARD_MAPSERVER_URL",
|
||||
)
|
||||
urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED")
|
||||
urbis_wfs_url: str = Field(
|
||||
default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows",
|
||||
validation_alias="URBIS_WFS_URL",
|
||||
)
|
||||
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
|
||||
dhmv_wcs_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/DHMV/wcs",
|
||||
validation_alias="DHMV_WCS_URL",
|
||||
)
|
||||
dhmv_resolution_m: float = Field(default=5.0, ge=1.0, le=10.0, validation_alias="DHMV_RESOLUTION_M")
|
||||
dhmv_min_side_m: float = Field(default=10.0, gt=0, validation_alias="DHMV_MIN_SIDE_M")
|
||||
dhmv_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="DHMV_MAX_SIDE_M")
|
||||
dhmv_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="DHMV_MAX_PIXELS")
|
||||
dhmv_timeout_seconds: int = Field(default=300, ge=1, validation_alias="DHMV_TIMEOUT_SECONDS")
|
||||
dhmv_max_response_mb: int = Field(default=160, ge=1, validation_alias="DHMV_MAX_RESPONSE_MB")
|
||||
flood_hazard_enabled: bool = Field(default=True, validation_alias="FLOOD_HAZARD_ENABLED")
|
||||
flood_hazard_wcs_url: str = Field(
|
||||
default="https://geoservice.waterinfo.be/OGRK/wcs",
|
||||
validation_alias="FLOOD_HAZARD_WCS_URL",
|
||||
)
|
||||
flood_hazard_resolution_m: float = Field(default=5.0, ge=2.0, le=20.0, validation_alias="FLOOD_HAZARD_RESOLUTION_M")
|
||||
flood_hazard_min_side_m: float = Field(default=10.0, gt=0, validation_alias="FLOOD_HAZARD_MIN_SIDE_M")
|
||||
flood_hazard_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="FLOOD_HAZARD_MAX_SIDE_M")
|
||||
flood_hazard_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="FLOOD_HAZARD_MAX_PIXELS")
|
||||
flood_hazard_timeout_seconds: int = Field(default=300, ge=1, validation_alias="FLOOD_HAZARD_TIMEOUT_SECONDS")
|
||||
flood_hazard_max_response_mb: int = Field(default=160, ge=1, validation_alias="FLOOD_HAZARD_MAX_RESPONSE_MB")
|
||||
bathymetry_profiles_enabled: bool = Field(default=True, validation_alias="BATHYMETRY_PROFILES_ENABLED")
|
||||
bathymetry_profiles_layer_url: str = Field(
|
||||
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
|
||||
validation_alias="BATHYMETRY_PROFILES_LAYER_URL",
|
||||
)
|
||||
bathymetry_watercourse_layer_url: str = Field(
|
||||
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1",
|
||||
validation_alias="BATHYMETRY_WATERCOURSE_LAYER_URL",
|
||||
)
|
||||
bathymetry_profiles_page_size: int = Field(
|
||||
default=1000,
|
||||
ge=1,
|
||||
le=2000,
|
||||
validation_alias="BATHYMETRY_PROFILES_PAGE_SIZE",
|
||||
)
|
||||
bathymetry_profiles_max_features: int = Field(
|
||||
default=50_000,
|
||||
ge=1,
|
||||
le=250_000,
|
||||
validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES",
|
||||
)
|
||||
bathymetry_profiles_max_pages: int = Field(
|
||||
default=200,
|
||||
ge=1,
|
||||
le=5_000,
|
||||
validation_alias="BATHYMETRY_PROFILES_MAX_PAGES",
|
||||
)
|
||||
bathymetry_profiles_timeout_seconds: int = Field(
|
||||
default=120,
|
||||
ge=1,
|
||||
le=600,
|
||||
validation_alias="BATHYMETRY_PROFILES_TIMEOUT_SECONDS",
|
||||
)
|
||||
bathymetry_profiles_max_response_mb: int = Field(
|
||||
default=32,
|
||||
ge=1,
|
||||
le=256,
|
||||
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
|
||||
)
|
||||
bathymetry_raster_max_pixels: int = Field(
|
||||
default=30_000_000,
|
||||
ge=1,
|
||||
validation_alias="BATHYMETRY_RASTER_MAX_PIXELS",
|
||||
)
|
||||
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
|
||||
mdk_bathymetry_wcs_url: str = Field(
|
||||
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
||||
validation_alias="MDK_BATHYMETRY_WCS_URL",
|
||||
)
|
||||
mdk_bathymetry_probe_timeout_seconds: int = Field(
|
||||
default=20,
|
||||
ge=1,
|
||||
le=120,
|
||||
validation_alias="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS",
|
||||
)
|
||||
mdk_bathymetry_probe_max_response_mb: int = Field(
|
||||
default=4,
|
||||
ge=1,
|
||||
le=16,
|
||||
validation_alias="MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB",
|
||||
)
|
||||
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
|
||||
thematic_raster_wcs_url: str = Field(
|
||||
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
||||
validation_alias="THEMATIC_RASTER_WCS_URL",
|
||||
)
|
||||
mdk_bathymetry_acquisition_enabled: bool = Field(
|
||||
default=False,
|
||||
validation_alias="MDK_BATHYMETRY_ACQUISITION_ENABLED",
|
||||
)
|
||||
mdk_bathymetry_coverage_id: str | None = Field(default=None, validation_alias="MDK_BATHYMETRY_COVERAGE_ID")
|
||||
mdk_bathymetry_request_crs: str = Field(default="EPSG:4326", validation_alias="MDK_BATHYMETRY_REQUEST_CRS")
|
||||
mdk_bathymetry_max_bbox_deg2: float = Field(
|
||||
default=0.25,
|
||||
gt=0,
|
||||
validation_alias="MDK_BATHYMETRY_MAX_BBOX_DEG2",
|
||||
)
|
||||
mdk_bathymetry_acquisition_timeout_seconds: int = Field(
|
||||
default=120,
|
||||
ge=1,
|
||||
validation_alias="MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS",
|
||||
)
|
||||
mdk_bathymetry_acquisition_max_response_mb: int = Field(
|
||||
default=160,
|
||||
ge=1,
|
||||
validation_alias="MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB",
|
||||
)
|
||||
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
|
||||
thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
|
||||
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
|
||||
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
|
||||
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
||||
walous_enabled: bool = Field(default=True, validation_alias="WALOUS_ENABLED")
|
||||
walous_source_dir: str = Field(
|
||||
default="/app/storage/source-cache/walous",
|
||||
validation_alias="WALOUS_SOURCE_DIR",
|
||||
)
|
||||
walous_analysis_resolution_m: float = Field(
|
||||
default=10.0,
|
||||
ge=1.0,
|
||||
le=100.0,
|
||||
validation_alias="WALOUS_ANALYSIS_RESOLUTION_M",
|
||||
)
|
||||
walous_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="WALOUS_MAX_SIDE_M")
|
||||
walous_max_pixels: int = Field(default=36_000_000, ge=1, validation_alias="WALOUS_MAX_PIXELS")
|
||||
spw_terrain_enabled: bool = Field(default=True, validation_alias="SPW_TERRAIN_ENABLED")
|
||||
spw_terrain_source_dir: str = Field(
|
||||
default="/app/storage/source-cache/spw-terrain",
|
||||
validation_alias="SPW_TERRAIN_SOURCE_DIR",
|
||||
)
|
||||
spw_terrain_analysis_resolution_m: float = Field(
|
||||
default=5.0,
|
||||
ge=1.0,
|
||||
le=10.0,
|
||||
validation_alias="SPW_TERRAIN_ANALYSIS_RESOLUTION_M",
|
||||
)
|
||||
spw_terrain_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="SPW_TERRAIN_MAX_SIDE_M")
|
||||
spw_terrain_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="SPW_TERRAIN_MAX_PIXELS")
|
||||
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
|
||||
reconcile_interrupted_runs_on_startup: bool = Field(
|
||||
default=False,
|
||||
validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
|
||||
)
|
||||
aoi_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AOI_WORKER_ENABLED")
|
||||
aoi_worker_poll_seconds: float = Field(default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_AOI_WORKER_POLL_SECONDS")
|
||||
# Executes queued detection.run / segmentation.run jobs so tiled GPU
|
||||
# inference never blocks an HTTP request.
|
||||
analysis_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_ANALYSIS_WORKER_ENABLED")
|
||||
analysis_worker_poll_seconds: float = Field(
|
||||
default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_ANALYSIS_WORKER_POLL_SECONDS"
|
||||
)
|
||||
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
||||
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
|
||||
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
|
||||
yolo_model_path: str | None = Field(default=None, validation_alias="YOLO_MODEL_PATH")
|
||||
yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID")
|
||||
yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME")
|
||||
yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION")
|
||||
yolo_model_classes: str = Field(default="building", validation_alias="YOLO_MODEL_CLASSES")
|
||||
yolo_enforce_validation_scope: bool = Field(default=False, validation_alias="YOLO_ENFORCE_VALIDATION_SCOPE")
|
||||
yolo_validation_scope_manifest_path: str | None = Field(
|
||||
default=None,
|
||||
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_PATH",
|
||||
)
|
||||
yolo_validation_scope_manifest_sha256: str | None = Field(
|
||||
default=None,
|
||||
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_SHA256",
|
||||
)
|
||||
# Deprecated compatibility field. Mutable Area names are never an
|
||||
# inference authorization boundary; deployments must use the immutable
|
||||
# checksum-bound scope manifest above.
|
||||
yolo_validated_area_names: str = Field(default="Mol,Kempen", validation_alias="YOLO_VALIDATED_AREA_NAMES")
|
||||
yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE")
|
||||
yolo_require_cuda: bool = Field(default=False, validation_alias="YOLO_REQUIRE_CUDA")
|
||||
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
|
||||
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
|
||||
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
|
||||
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
|
||||
yolo_suppress_tile_edge_detections: bool = Field(
|
||||
default=True, validation_alias="YOLO_SUPPRESS_TILE_EDGE_DETECTIONS"
|
||||
)
|
||||
# Intersection over the smaller box. The candidate evaluation freezes this
|
||||
# during calibration; serving a promoted model at a different value means
|
||||
# the runtime suppresses detections the gate counted.
|
||||
yolo_containment_nms_threshold: float = Field(
|
||||
default=0.85, ge=0.0, le=1.0, validation_alias="YOLO_CONTAINMENT_NMS_THRESHOLD"
|
||||
)
|
||||
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
|
||||
yolo_seg_enabled: bool = Field(default=False, validation_alias="YOLO_SEG_ENABLED")
|
||||
yolo_seg_model_path: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_PATH")
|
||||
yolo_seg_model_id: str = Field(default="yolo-seg-configured", validation_alias="YOLO_SEG_MODEL_ID")
|
||||
yolo_seg_model_display_name: str = Field(
|
||||
default="Configured YOLO segmentation",
|
||||
validation_alias="YOLO_SEG_MODEL_DISPLAY_NAME",
|
||||
)
|
||||
yolo_seg_model_version: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_VERSION")
|
||||
sam_enabled: bool = Field(default=False, validation_alias="SAM_ENABLED")
|
||||
sam_model_path: str | None = Field(default=None, validation_alias="SAM_MODEL_PATH")
|
||||
sam_model_id: str = Field(default="sam-configured", validation_alias="SAM_MODEL_ID")
|
||||
sam_model_display_name: str = Field(
|
||||
default="Configured SAM segmentation",
|
||||
validation_alias="SAM_MODEL_DISPLAY_NAME",
|
||||
)
|
||||
sam_model_version: str | None = Field(default=None, validation_alias="SAM_MODEL_VERSION")
|
||||
segmentation_max_masks_per_tile: int = Field(default=300, ge=1, validation_alias="SEGMENTATION_MAX_MASKS_PER_TILE")
|
||||
# Masks and boxes overlap differently, so segmentation carries its own
|
||||
# containment value rather than borrowing the detector's.
|
||||
segmentation_containment_nms_threshold: float = Field(
|
||||
default=0.85,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
validation_alias="SEGMENTATION_CONTAINMENT_NMS_THRESHOLD",
|
||||
)
|
||||
segmentation_duplicate_iou_threshold: float = Field(
|
||||
default=0.5,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
validation_alias="SEGMENTATION_DUPLICATE_IOU_THRESHOLD",
|
||||
)
|
||||
ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED")
|
||||
ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL")
|
||||
ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL")
|
||||
ollama_timeout_seconds: int = Field(default=120, ge=5, le=600, validation_alias="OLLAMA_TIMEOUT_SECONDS")
|
||||
ollama_max_output_tokens: int = Field(default=1_200, ge=100, le=4_000, validation_alias="OLLAMA_MAX_OUTPUT_TOKENS")
|
||||
ollama_context_tokens: int = Field(default=16_384, ge=4_096, le=131_072, validation_alias="OLLAMA_CONTEXT_TOKENS")
|
||||
cors_origins: list[str] | str = Field(
|
||||
default=["http://localhost:5173", "http://127.0.0.1:5173"],
|
||||
validation_alias="CORS_ORIGINS",
|
||||
)
|
||||
|
||||
@field_validator("cors_origins", mode="before")
|
||||
@classmethod
|
||||
def parse_cors_origins(cls, value: object) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if value is None:
|
||||
return ["http://localhost:5173", "http://127.0.0.1:5173"]
|
||||
return [str(value)]
|
||||
|
||||
@field_validator("ollama_base_url")
|
||||
@classmethod
|
||||
def validate_ollama_base_url(cls, value: str) -> str:
|
||||
normalized = value.strip().rstrip("/")
|
||||
if not normalized.startswith(("http://", "https://")):
|
||||
raise ValueError("OLLAMA_BASE_URL must use http or https")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_operator_auth(self) -> "Settings":
|
||||
self.guest_display_name = self.guest_display_name.strip()
|
||||
if not self.guest_display_name:
|
||||
raise ValueError("GEOINTEL_GUEST_DISPLAY_NAME must not be blank")
|
||||
for field_name in (
|
||||
"authentik_issuer",
|
||||
"authentik_client_id",
|
||||
"authentik_client_secret",
|
||||
"authentik_allowed_email",
|
||||
):
|
||||
value = getattr(self, field_name)
|
||||
setattr(self, field_name, value.strip() if value else None)
|
||||
self.public_base_url = self.public_base_url.strip().rstrip("/")
|
||||
authentik_values = (
|
||||
self.authentik_issuer,
|
||||
self.authentik_client_id,
|
||||
self.authentik_client_secret,
|
||||
self.authentik_allowed_email,
|
||||
)
|
||||
if any(authentik_values) and not all(authentik_values):
|
||||
raise ValueError("All GEOINTEL_AUTHENTIK_* values must be configured together")
|
||||
if all(authentik_values):
|
||||
if not self.auth_enabled:
|
||||
raise ValueError("GEOINTEL_AUTH_ENABLED must be true when Authentik is configured")
|
||||
for label, value in (
|
||||
("GEOINTEL_AUTHENTIK_ISSUER", self.authentik_issuer),
|
||||
("GEOINTEL_PUBLIC_BASE_URL", self.public_base_url),
|
||||
):
|
||||
parsed = urlsplit(str(value))
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or not parsed.hostname
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise ValueError(f"{label} must be an absolute HTTPS URL without credentials, query or fragment")
|
||||
public_url = urlsplit(self.public_base_url)
|
||||
if public_url.path not in ("", "/"):
|
||||
raise ValueError("GEOINTEL_PUBLIC_BASE_URL must not contain a path")
|
||||
if "@" not in str(self.authentik_allowed_email) or any(
|
||||
character.isspace() for character in str(self.authentik_allowed_email)
|
||||
):
|
||||
raise ValueError("GEOINTEL_AUTHENTIK_ALLOWED_EMAIL must be one valid e-mail address")
|
||||
if not self.auth_enabled:
|
||||
return self
|
||||
if not (self.auth_username or "").strip():
|
||||
raise ValueError("GEOINTEL_AUTH_USERNAME is required when authentication is enabled")
|
||||
if not (self.auth_password_hash or "").startswith("pbkdf2_sha256$"):
|
||||
raise ValueError("GEOINTEL_AUTH_PASSWORD_HASH must be a PBKDF2-SHA256 hash")
|
||||
if len(self.auth_session_secret or "") < 32:
|
||||
raise ValueError("GEOINTEL_AUTH_SESSION_SECRET must contain at least 32 characters")
|
||||
return self
|
||||
|
||||
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,15 @@
|
||||
class AppError(Exception):
|
||||
"""Domain error used by services to return canonical API errors."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
code: str,
|
||||
message: str,
|
||||
details: dict | list | None = None,
|
||||
status_code: int = 400,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.details = details or {}
|
||||
self.status_code = status_code
|
||||
@@ -0,0 +1,14 @@
|
||||
import logging
|
||||
import sys
|
||||
|
||||
|
||||
def configure_logging(level: str = "INFO", sql_level: str = "WARNING") -> None:
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
stream=sys.stdout,
|
||||
force=True,
|
||||
)
|
||||
for name in ["uvicorn", "uvicorn.error", "uvicorn.access"]:
|
||||
logging.getLogger(name).setLevel(level)
|
||||
logging.getLogger("sqlalchemy.engine").setLevel(sql_level)
|
||||
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
# Stable server-owned identity: a public session must never attach itself to an
|
||||
# operator project merely because the display names happen to match.
|
||||
PUBLIC_DEMO_PROJECT_ID = UUID("6f7e6f12-9b62-4a3f-a5a0-4b3bb6b2c901")
|
||||
PUBLIC_DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
|
||||
PUBLIC_DEMO_PROJECT_MARKER = "geointel:public-demo:v1"
|
||||
|
||||
|
||||
def is_public_demo_project(project_id: UUID) -> bool:
|
||||
return project_id == PUBLIC_DEMO_PROJECT_ID
|
||||
@@ -0,0 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextvars import ContextVar, Token
|
||||
|
||||
|
||||
_request_id: ContextVar[str] = ContextVar("geointel_request_id", default="-")
|
||||
|
||||
|
||||
def get_request_id() -> str:
|
||||
return _request_id.get()
|
||||
|
||||
|
||||
def set_request_id(value: str) -> Token:
|
||||
return _request_id.set(value)
|
||||
|
||||
|
||||
def reset_request_id(token: Token) -> None:
|
||||
_request_id.reset(token)
|
||||
@@ -0,0 +1,4 @@
|
||||
from .base import Base
|
||||
from .session import get_db, get_engine
|
||||
|
||||
__all__ = ["Base", "get_db", "get_engine"]
|
||||
@@ -0,0 +1,5 @@
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
@@ -0,0 +1,20 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
|
||||
from app.core.config import get_settings
|
||||
|
||||
|
||||
engine = create_engine(get_settings().database_url, pool_pre_ping=True, future=True)
|
||||
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False, future=True)
|
||||
|
||||
|
||||
def get_db():
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def get_engine():
|
||||
return engine
|
||||
@@ -0,0 +1,467 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.core.logging import configure_logging
|
||||
from app.core.request_context import reset_request_id, set_request_id
|
||||
from app.db.session import SessionLocal
|
||||
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||
from app.services.auth_service import AuthService
|
||||
from app.services.analysis_job_worker import AnalysisJobWorker
|
||||
from app.services.aoi_operation_worker import AoiOperationWorker
|
||||
|
||||
|
||||
logger = logging.getLogger("geointel")
|
||||
SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
UNSAFE_HOST = re.compile(r"[/\\@?#\s\x00-\x1f\x7f]")
|
||||
|
||||
|
||||
def _to_error_payload(
|
||||
code: str,
|
||||
message: str,
|
||||
details: dict | list | None = None,
|
||||
request_id: str | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"error": code,
|
||||
"message": message,
|
||||
"details": details or {},
|
||||
"request_id": request_id,
|
||||
}
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
configure_logging(settings.log_level, settings.sql_log_level)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
worker_stop = asyncio.Event()
|
||||
worker_task = None
|
||||
analysis_worker_task = None
|
||||
if settings.reconcile_interrupted_runs_on_startup:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
result = RuntimeReconciliationService.reconcile(db)
|
||||
logger.info(
|
||||
"Runtime reconciliation completed: jobs=%s analysis_runs=%s resumed_aoi_partitions=%s exhausted_aoi_partitions=%s",
|
||||
result.interrupted_jobs,
|
||||
result.interrupted_analysis_runs,
|
||||
result.resumed_aoi_partitions,
|
||||
result.exhausted_aoi_partitions,
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("Runtime reconciliation failed")
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
if settings.aoi_worker_enabled:
|
||||
worker_task = asyncio.create_task(AoiOperationWorker.run(worker_stop, settings.aoi_worker_poll_seconds))
|
||||
if settings.analysis_worker_enabled:
|
||||
analysis_worker_task = asyncio.create_task(
|
||||
AnalysisJobWorker.run(worker_stop, settings.analysis_worker_poll_seconds)
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
worker_stop.set()
|
||||
for task in (worker_task, analysis_worker_task):
|
||||
if task is not None:
|
||||
await task
|
||||
|
||||
app = FastAPI(
|
||||
title="GeoIntel",
|
||||
version=settings.app_version,
|
||||
docs_url="/docs",
|
||||
redoc_url="/redoc",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
allow_credentials=True,
|
||||
)
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth.router, prefix=settings.api_prefix)
|
||||
app.include_router(analysis.router, prefix=settings.api_prefix)
|
||||
app.include_router(aoi_operations.router, prefix=settings.api_prefix)
|
||||
app.include_router(projects.router, prefix=settings.api_prefix)
|
||||
app.include_router(areas.router, prefix=settings.api_prefix)
|
||||
app.include_router(datasets.router, prefix=settings.api_prefix)
|
||||
app.include_router(jobs.router, prefix=settings.api_prefix)
|
||||
app.include_router(quality_checks.router, prefix=settings.api_prefix)
|
||||
app.include_router(exports.router, prefix=settings.api_prefix)
|
||||
app.include_router(external.router, prefix=settings.api_prefix)
|
||||
app.include_router(source_registry.router, prefix=settings.api_prefix)
|
||||
app.include_router(demo.router, prefix=settings.api_prefix)
|
||||
app.include_router(qa.router, prefix=settings.api_prefix)
|
||||
app.include_router(detection.router, prefix=settings.api_prefix)
|
||||
app.include_router(segmentation.router, prefix=settings.api_prefix)
|
||||
app.include_router(selection_partitions.router, prefix=settings.api_prefix)
|
||||
app.include_router(temporal.router, prefix=settings.api_prefix)
|
||||
app.include_router(assistant.router, prefix=settings.api_prefix)
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_identity(request: Request, call_next):
|
||||
supplied_request_id = request.headers.get("x-request-id", "")
|
||||
request_id = supplied_request_id if SAFE_REQUEST_ID.fullmatch(supplied_request_id) else str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
token = set_request_id(request_id)
|
||||
started_at = time.perf_counter()
|
||||
raw_path = str(request.scope.get("path") or "")
|
||||
guest_compute_acquired = False
|
||||
try:
|
||||
host = request.headers.get("host", "")
|
||||
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if not raw_path.startswith("/") or not host or UNSAFE_HOST.search(host):
|
||||
response = JSONResponse(
|
||||
status_code=400,
|
||||
content=_to_error_payload(
|
||||
"INVALID_REQUEST_TARGET",
|
||||
"The request target or Host header is invalid",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
if content_type == "application/x-www-form-urlencoded":
|
||||
response = JSONResponse(
|
||||
status_code=415,
|
||||
content=_to_error_payload(
|
||||
"UNSUPPORTED_CONTENT_TYPE",
|
||||
"URL-encoded form bodies are not supported",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
public_auth_paths = {
|
||||
f"{settings.api_prefix}/auth/session",
|
||||
f"{settings.api_prefix}/auth/login",
|
||||
f"{settings.api_prefix}/auth/guest",
|
||||
f"{settings.api_prefix}/auth/logout",
|
||||
f"{settings.api_prefix}/auth/authentik/start",
|
||||
f"{settings.api_prefix}/auth/authentik/callback",
|
||||
}
|
||||
direct_loopback_request = (
|
||||
request.client is not None
|
||||
and request.client.host in {"127.0.0.1", "::1"}
|
||||
and not request.headers.get("x-real-ip")
|
||||
and not request.headers.get("x-forwarded-for")
|
||||
)
|
||||
if (
|
||||
settings.auth_enabled
|
||||
and raw_path.startswith(f"{settings.api_prefix}/")
|
||||
and raw_path not in public_auth_paths
|
||||
and not direct_loopback_request
|
||||
):
|
||||
principal = AuthService.verify_session_token(
|
||||
request.cookies.get(auth.COOKIE_NAME),
|
||||
settings,
|
||||
)
|
||||
if principal is None:
|
||||
response = JSONResponse(
|
||||
status_code=401,
|
||||
content=_to_error_payload(
|
||||
"AUTHENTICATION_REQUIRED",
|
||||
"Meld u aan om de GeoIntel API te gebruiken.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
request.state.auth_principal = principal
|
||||
if principal.role == "guest":
|
||||
project_path_prefix = f"{settings.api_prefix}/projects/"
|
||||
guest_project_root = f"{project_path_prefix}{principal.project_id}"
|
||||
if raw_path.startswith(project_path_prefix):
|
||||
scoped_path = raw_path[len(project_path_prefix):]
|
||||
requested_project_id = scoped_path.split("/", 1)[0]
|
||||
if str(principal.project_id) != requested_project_id:
|
||||
response = JSONResponse(
|
||||
status_code=403,
|
||||
content=_to_error_payload(
|
||||
"GUEST_PROJECT_SCOPE_REQUIRED",
|
||||
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
query_project_id = request.query_params.get("project_id")
|
||||
if query_project_id and query_project_id != str(principal.project_id):
|
||||
response = JSONResponse(
|
||||
status_code=403,
|
||||
content=_to_error_payload(
|
||||
"GUEST_PROJECT_SCOPE_REQUIRED",
|
||||
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
guest_safe_read_paths = {
|
||||
f"{settings.api_prefix}/projects",
|
||||
f"{settings.api_prefix}/external/providers",
|
||||
f"{settings.api_prefix}/assistant/status",
|
||||
f"{settings.api_prefix}/assistant/models",
|
||||
f"{settings.api_prefix}/detection/models",
|
||||
f"{settings.api_prefix}/detection/model-assets",
|
||||
f"{settings.api_prefix}/detection/yolo/preflight",
|
||||
f"{settings.api_prefix}/segmentation/models",
|
||||
}
|
||||
normalized_path = raw_path.rstrip("/") or "/"
|
||||
guest_project_read = (
|
||||
normalized_path == guest_project_root
|
||||
or normalized_path.startswith(f"{guest_project_root}/")
|
||||
)
|
||||
is_read_request = request.method in {"GET", "HEAD", "OPTIONS"}
|
||||
if is_read_request:
|
||||
if (
|
||||
normalized_path == f"{settings.api_prefix}/detection/yolo/preflight"
|
||||
and request.query_params.get("check_model_load", "").lower() in {"1", "true", "yes", "on"}
|
||||
):
|
||||
response = JSONResponse(
|
||||
status_code=403,
|
||||
content=_to_error_payload(
|
||||
"GUEST_MODEL_LOAD_FORBIDDEN",
|
||||
"Model loading is available to authenticated operators only.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
guest_scoped_analysis_read = (
|
||||
query_project_id == str(principal.project_id)
|
||||
and normalized_path.startswith(
|
||||
(
|
||||
f"{settings.api_prefix}/detection/",
|
||||
f"{settings.api_prefix}/segmentation/",
|
||||
f"{settings.api_prefix}/exports/",
|
||||
)
|
||||
)
|
||||
)
|
||||
if (
|
||||
normalized_path not in guest_safe_read_paths
|
||||
and not guest_project_read
|
||||
and not guest_scoped_analysis_read
|
||||
):
|
||||
response = JSONResponse(
|
||||
status_code=403,
|
||||
content=_to_error_payload(
|
||||
"GUEST_ROUTE_NOT_AVAILABLE",
|
||||
"Deze API-route maakt geen deel uit van de afgeschermde GeoIntel-demo.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
else:
|
||||
guest_safe_post_paths = {
|
||||
f"{settings.api_prefix}/demo/workflow",
|
||||
f"{settings.api_prefix}/external/coverage/resolve",
|
||||
f"{settings.api_prefix}/analysis/change-detection",
|
||||
}
|
||||
guest_scoped_analysis_post_paths = {
|
||||
f"{settings.api_prefix}/detection/run",
|
||||
f"{settings.api_prefix}/detection/run-async",
|
||||
f"{settings.api_prefix}/segmentation/run",
|
||||
f"{settings.api_prefix}/segmentation/run-async",
|
||||
f"{settings.api_prefix}/qa/detections-vs-reference",
|
||||
f"{settings.api_prefix}/exports/geojson",
|
||||
f"{settings.api_prefix}/exports/metadata",
|
||||
f"{settings.api_prefix}/exports/report",
|
||||
f"{settings.api_prefix}/exports/map-result",
|
||||
}
|
||||
guest_safe_post_suffixes = (
|
||||
"/acquire",
|
||||
"/vector/select",
|
||||
"/vector/select/derive",
|
||||
"/raster/tile",
|
||||
"/raster/bathymetry/select",
|
||||
"/raster/terrain/select",
|
||||
"/raster/flood-hazard/select",
|
||||
"/raster/thematic/select",
|
||||
"/raster/walous/select",
|
||||
"/temporal/compare",
|
||||
"/datasets/vector/partitions/select",
|
||||
"/datasets/bathymetry/profiles/partitions/select",
|
||||
)
|
||||
is_guest_safe_post = request.method == "POST" and (
|
||||
raw_path in guest_safe_post_paths
|
||||
or (
|
||||
raw_path in guest_scoped_analysis_post_paths
|
||||
and query_project_id == str(principal.project_id)
|
||||
)
|
||||
or (
|
||||
query_project_id == str(principal.project_id)
|
||||
and raw_path.startswith(
|
||||
(
|
||||
f"{settings.api_prefix}/detection/runs/",
|
||||
f"{settings.api_prefix}/segmentation/runs/",
|
||||
)
|
||||
)
|
||||
and raw_path.endswith("/qa/reference")
|
||||
)
|
||||
or (
|
||||
raw_path.startswith(project_path_prefix)
|
||||
and (
|
||||
raw_path.endswith(guest_safe_post_suffixes)
|
||||
or raw_path.endswith("/assistant/query")
|
||||
)
|
||||
)
|
||||
)
|
||||
if not is_guest_safe_post:
|
||||
response = JSONResponse(
|
||||
status_code=403,
|
||||
content=_to_error_payload(
|
||||
"GUEST_READ_ONLY",
|
||||
"Gasttoegang laat alleen projectgebonden demo-analyses toe. Meld u aan als operator voor beheerwijzigingen.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
retry_after = AuthService.consume_guest_request(
|
||||
f"guest-compute:{principal.session_id}",
|
||||
max_requests=settings.guest_compute_requests_per_minute,
|
||||
)
|
||||
if retry_after:
|
||||
response = JSONResponse(
|
||||
status_code=429,
|
||||
content=_to_error_payload(
|
||||
"GUEST_COMPUTE_RATE_LIMITED",
|
||||
"The public demo compute budget is temporarily exhausted.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["retry-after"] = str(retry_after)
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
guest_compute_acquired = AuthService.try_acquire_guest_compute(
|
||||
max_concurrency=settings.guest_compute_max_concurrency,
|
||||
)
|
||||
if not guest_compute_acquired:
|
||||
response = JSONResponse(
|
||||
status_code=429,
|
||||
content=_to_error_payload(
|
||||
"GUEST_COMPUTE_BUSY",
|
||||
"The public demo is already processing its maximum number of jobs.",
|
||||
request_id=request_id,
|
||||
),
|
||||
)
|
||||
response.headers["retry-after"] = "10"
|
||||
response.headers["x-request-id"] = request_id
|
||||
return response
|
||||
response = await call_next(request)
|
||||
response.headers["x-request-id"] = request_id
|
||||
logger.info(
|
||||
"request_complete request_id=%s method=%s path=%s status=%s duration_ms=%.1f",
|
||||
request_id,
|
||||
request.method,
|
||||
raw_path,
|
||||
response.status_code,
|
||||
(time.perf_counter() - started_at) * 1000,
|
||||
)
|
||||
return response
|
||||
finally:
|
||||
if guest_compute_acquired:
|
||||
AuthService.release_guest_compute()
|
||||
reset_request_id(token)
|
||||
|
||||
@app.exception_handler(AppError)
|
||||
async def app_error(request: Request, exc: AppError): # noqa: ARG001
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content=_to_error_payload(
|
||||
exc.code,
|
||||
exc.message,
|
||||
exc.details,
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request: Request, exc: HTTPException): # noqa: ARG001
|
||||
code = "HTTP_ERROR"
|
||||
message = str(exc.detail)
|
||||
details = {}
|
||||
if isinstance(exc.detail, dict):
|
||||
code = str(exc.detail.get("error") or exc.detail.get("code") or code)
|
||||
message = str(exc.detail.get("message") or message)
|
||||
raw_details = exc.detail.get("details")
|
||||
details = raw_details if isinstance(raw_details, (dict, list)) else {}
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content=_to_error_payload(
|
||||
code,
|
||||
message,
|
||||
details,
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request: Request, exc: RequestValidationError): # noqa: ARG001
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content=_to_error_payload(
|
||||
"VALIDATION_ERROR",
|
||||
"Validation failed",
|
||||
exc.errors(),
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error(request: Request, exc: Exception):
|
||||
logger.exception(
|
||||
"Unhandled request error request_id=%s method=%s path=%s",
|
||||
request.state.request_id,
|
||||
request.method,
|
||||
str(request.scope.get("path") or ""),
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content=_to_error_payload(
|
||||
"INTERNAL_ERROR",
|
||||
"Unexpected server error",
|
||||
{"type": exc.__class__.__name__},
|
||||
request_id=request.state.request_id,
|
||||
),
|
||||
)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import uvicorn
|
||||
|
||||
settings = get_settings()
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host="0.0.0.0",
|
||||
port=8000,
|
||||
reload=settings.app_env == "development",
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API
|
||||
@@ -0,0 +1,43 @@
|
||||
from .entities import (
|
||||
AoiOperation,
|
||||
AoiOperationPartition,
|
||||
AnalysisRun,
|
||||
Area,
|
||||
Dataset,
|
||||
DatasetLineageEdge,
|
||||
DatasetQuarantine,
|
||||
DatasetVersion,
|
||||
Detection,
|
||||
DetectionReview,
|
||||
Export,
|
||||
Job,
|
||||
Metric,
|
||||
Project,
|
||||
QualityCheck,
|
||||
Segmentation,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
VectorFeature,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AnalysisRun",
|
||||
"AoiOperation",
|
||||
"AoiOperationPartition",
|
||||
"Area",
|
||||
"Dataset",
|
||||
"DatasetLineageEdge",
|
||||
"DatasetQuarantine",
|
||||
"DatasetVersion",
|
||||
"Detection",
|
||||
"DetectionReview",
|
||||
"Export",
|
||||
"Job",
|
||||
"Metric",
|
||||
"Project",
|
||||
"QualityCheck",
|
||||
"Segmentation",
|
||||
"SourceRegistry",
|
||||
"SourceSnapshot",
|
||||
"VectorFeature",
|
||||
]
|
||||
@@ -0,0 +1,788 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from geoalchemy2 import Geometry
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, UniqueConstraint, func, text
|
||||
from sqlalchemy.sql.sqltypes import Integer
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.db.base import Base
|
||||
|
||||
|
||||
SOURCE_CLASSIFICATIONS = (
|
||||
"authoritative",
|
||||
"corroborative",
|
||||
"contextual",
|
||||
"derived",
|
||||
"experimental",
|
||||
)
|
||||
SOURCE_FRESHNESS_STATUSES = (
|
||||
"unknown",
|
||||
"current",
|
||||
"due",
|
||||
"stale",
|
||||
"not_applicable",
|
||||
"review_required",
|
||||
)
|
||||
SOURCE_INGEST_STATUSES = (
|
||||
"registered",
|
||||
"configured",
|
||||
"not_configured",
|
||||
"available",
|
||||
"ingested",
|
||||
"failed",
|
||||
"quarantined",
|
||||
"legacy_unverified",
|
||||
)
|
||||
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
|
||||
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
|
||||
|
||||
|
||||
class Project(Base):
|
||||
__tablename__ = "projects"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
region: Mapped[str] = mapped_column(String(120), default="Belgium and Belgian North Sea")
|
||||
status: Mapped[str] = mapped_column(String(32), default="active")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
areas: Mapped[list["Area"]] = relationship("Area", back_populates="project", cascade="all, delete-orphan")
|
||||
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="project", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
class Area(Base):
|
||||
__tablename__ = "areas"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326), nullable=False)
|
||||
original_crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
area_m2: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
bbox: Mapped[str | None] = mapped_column(Geometry("Polygon", srid=4326), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
project: Mapped[Project] = relationship("Project", back_populates="areas")
|
||||
|
||||
|
||||
class SourceRegistry(Base):
|
||||
"""Server-owned source identity and authority contract.
|
||||
|
||||
Dataset metadata remains descriptive until a governed importer binds a
|
||||
dataset to both this registry entry and an immutable SourceSnapshot.
|
||||
"""
|
||||
|
||||
__tablename__ = "source_registry"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
|
||||
CheckConstraint(
|
||||
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
|
||||
name="ck_source_registry_classification",
|
||||
),
|
||||
CheckConstraint(
|
||||
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||
name="ck_source_registry_freshness_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||
"'failed', 'quarantined', 'legacy_unverified')",
|
||||
name="ck_source_registry_ingest_status",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
classification: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
|
||||
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
|
||||
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
|
||||
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
freshness_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||
)
|
||||
ingest_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="registered", server_default="registered"
|
||||
)
|
||||
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
|
||||
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
|
||||
)
|
||||
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
|
||||
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
|
||||
|
||||
|
||||
class SourceSnapshot(Base):
|
||||
"""Immutable source-version evidence recorded by governed ingestion."""
|
||||
|
||||
__tablename__ = "source_snapshots"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
|
||||
CheckConstraint(
|
||||
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||
name="ck_source_snapshots_freshness_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||
"'failed', 'quarantined', 'legacy_unverified')",
|
||||
name="ck_source_snapshots_ingest_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
|
||||
name="ck_source_snapshots_checksum_sha256",
|
||||
),
|
||||
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
|
||||
Index("ix_source_snapshots_checksum", "checksum_sha256"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source_registry_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
freshness_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||
)
|
||||
ingest_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="registered", server_default="registered"
|
||||
)
|
||||
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
|
||||
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
|
||||
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
|
||||
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
|
||||
|
||||
|
||||
class Dataset(Base):
|
||||
__tablename__ = "datasets"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_datasets_temporal_valid_range",
|
||||
),
|
||||
CheckConstraint(
|
||||
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||
name="ck_datasets_validation_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_datasets_provenance_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_datasets_lineage_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"quarantine_status IN ('not_quarantined', 'quarantined')",
|
||||
name="ck_datasets_quarantine_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||
name="ck_datasets_ingest_key_not_blank",
|
||||
),
|
||||
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
|
||||
Index(
|
||||
"ix_datasets_project_temporal_series_observed",
|
||||
"project_id",
|
||||
"temporal_series_key",
|
||||
"observed_at",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
dataset_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
source: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
original_filename: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
stored_filename: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("datasets.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
bounds_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
resolution_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
bands_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
dataset_role: Mapped[str] = mapped_column(String(32), nullable=False, default="source", server_default="source")
|
||||
source_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
validation_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="not_validated",
|
||||
server_default="not_validated",
|
||||
comment="not_validated | passed | failed",
|
||||
)
|
||||
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
lineage_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
quarantine_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="not_quarantined",
|
||||
server_default="not_quarantined",
|
||||
comment="not_quarantined | quarantined",
|
||||
)
|
||||
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
temporal_granularity: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(32), default="uploaded")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
project: Mapped[Project] = relationship("Project", back_populates="datasets")
|
||||
versions: Mapped[list["DatasetVersion"]] = relationship(
|
||||
"DatasetVersion",
|
||||
back_populates="dataset",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
vector_features: Mapped[list["VectorFeature"]] = relationship(
|
||||
"VectorFeature",
|
||||
back_populates="dataset",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
|
||||
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
|
||||
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||
"DatasetLineageEdge",
|
||||
foreign_keys="DatasetLineageEdge.parent_dataset_id",
|
||||
back_populates="parent_dataset",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||
"DatasetLineageEdge",
|
||||
foreign_keys="DatasetLineageEdge.child_dataset_id",
|
||||
back_populates="child_dataset",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
|
||||
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
class DatasetVersion(Base):
|
||||
__tablename__ = "dataset_versions"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_dataset_versions_temporal_valid_range",
|
||||
),
|
||||
CheckConstraint(
|
||||
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||
name="ck_dataset_versions_validation_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_dataset_versions_provenance_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_dataset_versions_lineage_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||
name="ck_dataset_versions_ingest_key_not_blank",
|
||||
),
|
||||
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
||||
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
|
||||
version: Mapped[int] = mapped_column(Integer, default=1)
|
||||
storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
validation_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="not_validated",
|
||||
server_default="not_validated",
|
||||
comment="not_validated | passed | failed",
|
||||
)
|
||||
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
lineage_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
||||
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
|
||||
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
|
||||
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
|
||||
|
||||
|
||||
class DatasetLineageEdge(Base):
|
||||
"""Immutable relationship between input/output datasets and transforms."""
|
||||
|
||||
__tablename__ = "dataset_lineage_edges"
|
||||
__table_args__ = (
|
||||
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
|
||||
UniqueConstraint(
|
||||
"parent_dataset_id",
|
||||
"child_dataset_id",
|
||||
"relation_type",
|
||||
"transformation_name",
|
||||
name="uq_dataset_lineage_edges_relation",
|
||||
),
|
||||
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
|
||||
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
parent_dataset: Mapped[Dataset] = relationship(
|
||||
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
|
||||
)
|
||||
child_dataset: Mapped[Dataset] = relationship(
|
||||
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
|
||||
)
|
||||
|
||||
|
||||
class DatasetQuarantine(Base):
|
||||
"""Durable fail-closed record for rejected or doubtful source artifacts."""
|
||||
|
||||
__tablename__ = "dataset_quarantines"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
|
||||
name="ck_dataset_quarantines_target_present",
|
||||
),
|
||||
CheckConstraint(
|
||||
"status IN ('quarantined', 'released', 'rejected')",
|
||||
name="ck_dataset_quarantines_status",
|
||||
),
|
||||
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
|
||||
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
stage: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="quarantined", server_default="quarantined"
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
|
||||
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
|
||||
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
|
||||
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
|
||||
|
||||
|
||||
class VectorFeature(Base):
|
||||
__tablename__ = "vector_features"
|
||||
__table_args__ = (
|
||||
Index("ix_vector_features_dataset_id", "dataset_id"),
|
||||
Index("ix_vector_features_geometry", "geometry", postgresql_using="gist"),
|
||||
Index("ix_vector_features_dataset_source_feature", "dataset_id", "source_feature_id"),
|
||||
Index(
|
||||
"ix_vector_features_dataset_municipality",
|
||||
"dataset_id",
|
||||
text("(properties_json ->> 'municipality')"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
|
||||
feature_class: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
source_feature_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
geometry: Mapped[str] = mapped_column(Geometry("Geometry", srid=4326, spatial_index=False), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="vector_features")
|
||||
|
||||
|
||||
class AnalysisRun(Base):
|
||||
__tablename__ = "analysis_runs"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
|
||||
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
|
||||
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||
analysis_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
model_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
model_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
parameters_json: Mapped[dict] = mapped_column(JSON, nullable=False)
|
||||
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
class Detection(Base):
|
||||
__tablename__ = "detections"
|
||||
__table_args__ = (
|
||||
Index("ix_detections_project_id", "project_id"),
|
||||
Index("ix_detections_dataset_id", "dataset_id"),
|
||||
Index("ix_detections_analysis_run_id", "analysis_run_id"),
|
||||
Index("ix_detections_class_name", "class_name"),
|
||||
Index("ix_detections_geometry", "geometry", postgresql_using="gist"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
|
||||
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
|
||||
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
model_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
class_name: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
confidence: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
geometry: Mapped[str] = mapped_column(Geometry("Geometry", srid=4326, spatial_index=False), nullable=False)
|
||||
bbox_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
source_tile_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class Segmentation(Base):
|
||||
__tablename__ = "segmentations"
|
||||
__table_args__ = (
|
||||
Index("ix_segmentations_project_id", "project_id"),
|
||||
Index("ix_segmentations_dataset_id", "dataset_id"),
|
||||
Index("ix_segmentations_analysis_run_id", "analysis_run_id"),
|
||||
Index("ix_segmentations_job_id", "job_id"),
|
||||
Index("ix_segmentations_class_name", "class_name"),
|
||||
Index("ix_segmentations_geometry", "geometry", postgresql_using="gist"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
|
||||
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
|
||||
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
model_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
class_name: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
confidence: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
|
||||
bbox_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
area_m2: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
mask_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
source_tile_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
|
||||
tile_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class QualityCheck(Base):
|
||||
__tablename__ = "quality_checks"
|
||||
__table_args__ = (
|
||||
Index("ix_quality_checks_project_id", "project_id"),
|
||||
Index("ix_quality_checks_reference_dataset_id", "reference_dataset_id"),
|
||||
Index("ix_quality_checks_candidate_dataset_id", "candidate_dataset_id"),
|
||||
Index("ix_quality_checks_analysis_run_id", "analysis_run_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
|
||||
candidate_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
|
||||
reference_dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
|
||||
check_type: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
findings_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class Metric(Base):
|
||||
__tablename__ = "metrics"
|
||||
__table_args__ = (
|
||||
Index("ix_metrics_quality_check_id", "quality_check_id"),
|
||||
Index("ix_metrics_analysis_run_id", "analysis_run_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
quality_check_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("quality_checks.id", ondelete="CASCADE"), nullable=True)
|
||||
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
|
||||
metric_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
metric_value: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
metric_unit: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
label: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class DetectionReview(Base):
|
||||
__tablename__ = "detection_reviews"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"evidence_role IN ('false_positive', 'false_negative')",
|
||||
name="ck_detection_reviews_evidence_role",
|
||||
),
|
||||
CheckConstraint(
|
||||
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
|
||||
"'reference_gap_or_change', 'qa_alignment_mismatch', "
|
||||
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
|
||||
name="ck_detection_reviews_decision",
|
||||
),
|
||||
UniqueConstraint(
|
||||
"quality_check_id",
|
||||
"evidence_role",
|
||||
"evidence_feature_id",
|
||||
name="uq_detection_reviews_evidence",
|
||||
),
|
||||
Index("ix_detection_reviews_project_id", "project_id"),
|
||||
Index("ix_detection_reviews_quality_check_id", "quality_check_id"),
|
||||
Index("ix_detection_reviews_analysis_run_id", "analysis_run_id"),
|
||||
Index("ix_detection_reviews_decision", "decision"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
quality_check_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("quality_checks.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("analysis_runs.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
evidence_role: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
evidence_feature_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
detection_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("detections.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
reference_feature_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("vector_features.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
decision: Mapped[str] = mapped_column(String(64), nullable=False, default="unreviewed", server_default="unreviewed")
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
reviewed_by: Mapped[str] = mapped_column(String(120), nullable=False, default="operator", server_default="operator")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class Export(Base):
|
||||
__tablename__ = "exports"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
|
||||
export_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
storage_path: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
job_type: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
|
||||
input_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
|
||||
output_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
|
||||
parameters_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class AoiOperation(Base):
|
||||
__tablename__ = "aoi_operations"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'running', 'partial', 'success', 'failed', 'cancelled')",
|
||||
name="ck_aoi_operations_status",
|
||||
),
|
||||
Index("ix_aoi_operations_project_status", "project_id", "status"),
|
||||
Index("ix_aoi_operations_geometry", "geometry", postgresql_using="gist"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
|
||||
parent_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||
operation_type: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
|
||||
request_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
plan_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class AoiOperationPartition(Base):
|
||||
__tablename__ = "aoi_operation_partitions"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'running', 'success', 'failed', 'skipped')",
|
||||
name="ck_aoi_operation_partitions_status",
|
||||
),
|
||||
UniqueConstraint("operation_id", "partition_key", name="uq_aoi_operation_partition_key"),
|
||||
Index("ix_aoi_operation_partitions_operation_status", "operation_id", "status"),
|
||||
Index("ix_aoi_operation_partitions_geometry", "geometry", postgresql_using="gist"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
operation_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("aoi_operations.id", ondelete="CASCADE"), nullable=False)
|
||||
child_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||
partition_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
provider_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
product_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
ordinal: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)
|
||||
checkpoint_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
@@ -0,0 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.providers import base, fixture, grb, manual, osm, registry
|
||||
|
||||
__all__ = ["base", "fixture", "grb", "manual", "osm", "registry"]
|
||||
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderCapability:
|
||||
provider_name: str
|
||||
display_name: str
|
||||
authority_level: str
|
||||
supported_layers: list[str]
|
||||
supported_geometry_types: list[str]
|
||||
supported_query_modes: list[str]
|
||||
fetch_signature: str
|
||||
configured: bool
|
||||
status: str
|
||||
limitation_message: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
not_configured_reason: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"provider_name": self.provider_name,
|
||||
"display_name": self.display_name,
|
||||
"authority_level": self.authority_level,
|
||||
"supported_layers": self.supported_layers,
|
||||
"supported_geometry_types": self.supported_geometry_types,
|
||||
"supported_query_modes": self.supported_query_modes,
|
||||
"fetch_signature": self.fetch_signature,
|
||||
"configured": self.configured,
|
||||
"status": self.status,
|
||||
"limitation_message": self.limitation_message,
|
||||
"attribution": self.attribution,
|
||||
"license_note": self.license_note,
|
||||
"not_configured_reason": self.not_configured_reason,
|
||||
}
|
||||
|
||||
|
||||
class BaseReferenceProvider:
|
||||
def __init__(
|
||||
self,
|
||||
provider_name: str,
|
||||
display_name: str,
|
||||
authority_level: str,
|
||||
supported_layers: list[str],
|
||||
supported_geometry_types: list[str],
|
||||
supported_query_modes: list[str],
|
||||
fetch_signature: str,
|
||||
limitation_message: str,
|
||||
attribution: str,
|
||||
license_note: str,
|
||||
configured: bool = False,
|
||||
) -> None:
|
||||
self.provider_name = provider_name
|
||||
self.display_name = display_name
|
||||
self.authority_level = authority_level
|
||||
self.supported_layers = supported_layers
|
||||
self.supported_geometry_types = supported_geometry_types
|
||||
self.supported_query_modes = supported_query_modes
|
||||
self.fetch_signature = fetch_signature
|
||||
self.limitation_message = limitation_message
|
||||
self.attribution = attribution
|
||||
self.license_note = license_note
|
||||
self._configured = configured
|
||||
|
||||
@property
|
||||
def capability(self) -> ProviderCapability:
|
||||
return ProviderCapability(
|
||||
provider_name=self.provider_name,
|
||||
display_name=self.display_name,
|
||||
authority_level=self.authority_level,
|
||||
supported_layers=self.supported_layers,
|
||||
supported_geometry_types=self.supported_geometry_types,
|
||||
supported_query_modes=self.supported_query_modes,
|
||||
fetch_signature=self.fetch_signature,
|
||||
configured=self.is_configured,
|
||||
status="configured" if self.is_configured else "not_configured",
|
||||
limitation_message=self.limitation_message,
|
||||
attribution=self.attribution,
|
||||
license_note=self.license_note,
|
||||
not_configured_reason=None if self.is_configured else "Provider integration is not configured yet",
|
||||
)
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
return self._configured
|
||||
|
||||
def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict[str, Any]:
|
||||
del project_id, area_id, layers
|
||||
return {
|
||||
"provider": self.provider_name,
|
||||
"status": "not_configured",
|
||||
"message": "Provider integration is not configured yet",
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.providers.base import BaseReferenceProvider
|
||||
|
||||
|
||||
class FixtureProvider(BaseReferenceProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
provider_name="fixture",
|
||||
display_name="Fixture data",
|
||||
authority_level="fixture",
|
||||
supported_layers=["buildings", "roads", "water", "landuse", "custom"],
|
||||
supported_geometry_types=["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon"],
|
||||
supported_query_modes=["fixture"],
|
||||
fetch_signature="tests/fixtures and demo fixture upload flow",
|
||||
limitation_message="Fixture provider represents local test/demo fixtures only.",
|
||||
attribution="GeoIntel local fixtures",
|
||||
license_note="Fixtures are for local development and tests; do not present them as official data.",
|
||||
configured=True,
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.providers.base import BaseReferenceProvider
|
||||
|
||||
|
||||
class GRBProvider(BaseReferenceProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
provider_name="grb",
|
||||
display_name="GRB",
|
||||
authority_level="authoritative",
|
||||
supported_layers=["buildings", "roads", "water", "parcels"],
|
||||
supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
|
||||
supported_query_modes=["bbox", "persisted_area"],
|
||||
fetch_signature="POST /api/v1/projects/{project_id}/datasets/grb/acquire",
|
||||
limitation_message=(
|
||||
"Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald. "
|
||||
"Volledige providerdownloads en onbeperkte queries zijn niet toegestaan."
|
||||
),
|
||||
attribution="Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
|
||||
license_note="Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.",
|
||||
configured=True,
|
||||
)
|
||||
|
||||
def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict:
|
||||
del project_id, area_id, layers
|
||||
return {
|
||||
"provider": self.provider_name,
|
||||
"status": "bounded_request_required",
|
||||
"message": (
|
||||
"Use POST /api/v1/projects/{project_id}/datasets/grb/acquire with an EPSG:4326 "
|
||||
"bounding box and one governed product key."
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.providers.base import BaseReferenceProvider
|
||||
|
||||
|
||||
class ManualProvider(BaseReferenceProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
provider_name="manual",
|
||||
display_name="Manual upload",
|
||||
authority_level="manual",
|
||||
supported_layers=["buildings", "roads", "water", "landuse", "custom"],
|
||||
supported_geometry_types=["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon"],
|
||||
supported_query_modes=["upload"],
|
||||
fetch_signature="POST /api/v1/projects/{project_id}/datasets/upload",
|
||||
limitation_message="Manual provider data is supplied through the existing dataset upload flow.",
|
||||
attribution="User supplied",
|
||||
license_note="License and attribution must be supplied by the uploader in source metadata.",
|
||||
configured=True,
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.providers.base import BaseReferenceProvider
|
||||
|
||||
|
||||
class OSMProvider(BaseReferenceProvider):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
provider_name="osm",
|
||||
display_name="OpenStreetMap",
|
||||
authority_level="contextual",
|
||||
supported_layers=["buildings", "roads", "water", "landuse"],
|
||||
supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
|
||||
supported_query_modes=["area"],
|
||||
fetch_signature="POST /api/v1/external/osm/fetch",
|
||||
limitation_message="OSM live Overpass/download integration is not configured in Sprint 7B.",
|
||||
attribution="OpenStreetMap contributors",
|
||||
license_note="OpenStreetMap data is available under ODbL; attribution is required.",
|
||||
configured=False,
|
||||
)
|
||||
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.providers.base import ProviderCapability
|
||||
from app.providers.fixture import FixtureProvider
|
||||
from app.providers.grb import GRBProvider
|
||||
from app.providers.manual import ManualProvider
|
||||
from app.providers.osm import OSMProvider
|
||||
|
||||
|
||||
class ProviderDatasetMapping(BaseModel):
|
||||
provider_name: str
|
||||
dataset_role: str
|
||||
source_name: str
|
||||
reference_required: bool
|
||||
write_path: str = "DatasetService"
|
||||
|
||||
|
||||
class ProviderImportResult(BaseModel):
|
||||
provider_name: str
|
||||
status: str
|
||||
message: str
|
||||
requested_layers: list[str]
|
||||
dataset_id: str | None = None
|
||||
dataset_role: str | None = None
|
||||
source_name: str | None = None
|
||||
|
||||
|
||||
class ExternalProviderRegistry:
|
||||
def __init__(self) -> None:
|
||||
self.providers = {
|
||||
"grb": GRBProvider(),
|
||||
"osm": OSMProvider(),
|
||||
"manual": ManualProvider(),
|
||||
"fixture": FixtureProvider(),
|
||||
}
|
||||
|
||||
def list_capabilities(self) -> list[ProviderCapability]:
|
||||
return [provider.capability for provider in self.providers.values()]
|
||||
|
||||
def get(self, provider_name: str):
|
||||
normalized = provider_name.strip().lower()
|
||||
if normalized not in self.providers:
|
||||
raise AppError(code="PROVIDER_NOT_FOUND", message="Provider not found", status_code=404)
|
||||
return self.providers[normalized]
|
||||
|
||||
def fetch(self, provider_name: str, project_id: str, area_id: str | None, layers: list[str]) -> dict:
|
||||
provider = self.get(provider_name)
|
||||
return provider.fetch(project_id=project_id, area_id=area_id, layers=layers)
|
||||
|
||||
def dataset_mapping(self, provider_name: str, requested_dataset_role: str | None = None) -> ProviderDatasetMapping:
|
||||
provider = self.get(provider_name)
|
||||
if provider.provider_name == "osm":
|
||||
dataset_role = "reference" if requested_dataset_role == "reference" else "source"
|
||||
return ProviderDatasetMapping(
|
||||
provider_name="osm",
|
||||
dataset_role=dataset_role,
|
||||
source_name="osm",
|
||||
reference_required=requested_dataset_role == "reference",
|
||||
)
|
||||
return ProviderDatasetMapping(
|
||||
provider_name=provider.provider_name,
|
||||
dataset_role="reference",
|
||||
source_name=provider.provider_name,
|
||||
reference_required=True,
|
||||
)
|
||||
|
||||
def import_contract(
|
||||
self,
|
||||
provider_name: str,
|
||||
project_id: str,
|
||||
area_id: str | None,
|
||||
layers: list[str],
|
||||
requested_dataset_role: str | None = None,
|
||||
) -> ProviderImportResult:
|
||||
del project_id, area_id
|
||||
provider = self.get(provider_name)
|
||||
mapping = self.dataset_mapping(provider.provider_name, requested_dataset_role=requested_dataset_role)
|
||||
if provider.provider_name == "grb":
|
||||
return ProviderImportResult(
|
||||
provider_name="grb",
|
||||
status="bounded_request_required",
|
||||
message=(
|
||||
"Use the governed project GRB acquisition endpoint with an EPSG:4326 bounding box "
|
||||
"and one supported layer."
|
||||
),
|
||||
requested_layers=layers,
|
||||
dataset_role=mapping.dataset_role,
|
||||
source_name=mapping.source_name,
|
||||
)
|
||||
if provider.provider_name == "osm":
|
||||
return ProviderImportResult(
|
||||
provider_name=provider.provider_name,
|
||||
status="not_configured",
|
||||
message=f"No live {provider.display_name} import is configured.",
|
||||
requested_layers=layers,
|
||||
dataset_role=mapping.dataset_role,
|
||||
source_name=mapping.source_name,
|
||||
)
|
||||
if provider.provider_name == "manual":
|
||||
return ProviderImportResult(
|
||||
provider_name="manual",
|
||||
status="upload_flow_required",
|
||||
message="Manual provider data must use the existing dataset upload/reference flow.",
|
||||
requested_layers=layers,
|
||||
dataset_role=mapping.dataset_role,
|
||||
source_name=mapping.source_name,
|
||||
)
|
||||
return ProviderImportResult(
|
||||
provider_name="fixture",
|
||||
status="fixture_flow_required",
|
||||
message="Fixture provider data must use checked-in demo/test fixture flows.",
|
||||
requested_layers=layers,
|
||||
dataset_role=mapping.dataset_role,
|
||||
source_name=mapping.source_name,
|
||||
)
|
||||
|
||||
|
||||
_registry = ExternalProviderRegistry()
|
||||
|
||||
|
||||
def list_provider_capabilities() -> list[ProviderCapability]:
|
||||
return _registry.list_capabilities()
|
||||
|
||||
|
||||
def get_provider(provider_name: str):
|
||||
return _registry.get(provider_name)
|
||||
|
||||
|
||||
def fetch_provider_data(provider_name: str, project_id: str, area_id: str | None, layers: list[str]) -> dict:
|
||||
return _registry.fetch(provider_name, project_id, area_id, layers)
|
||||
|
||||
|
||||
def get_provider_dataset_mapping(provider_name: str, requested_dataset_role: str | None = None) -> ProviderDatasetMapping:
|
||||
return _registry.dataset_mapping(provider_name, requested_dataset_role=requested_dataset_role)
|
||||
|
||||
|
||||
def import_provider_dataset(
|
||||
provider_name: str,
|
||||
project_id: str,
|
||||
area_id: str | None,
|
||||
layers: list[str],
|
||||
requested_dataset_role: str | None = None,
|
||||
) -> ProviderImportResult:
|
||||
return _registry.import_contract(
|
||||
provider_name=provider_name,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
layers=layers,
|
||||
requested_dataset_role=requested_dataset_role,
|
||||
)
|
||||
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .common import (
|
||||
ApiErrorEnvelope,
|
||||
ApiErrorItem,
|
||||
Envelope,
|
||||
GeoJsonFeature,
|
||||
GeoJsonFeatureCollection,
|
||||
ItemList,
|
||||
PaginationEnvelope,
|
||||
)
|
||||
from .coverage import (
|
||||
CoverageBBox,
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolutionItem,
|
||||
CoverageResolveRequest,
|
||||
CoverageResolveResponse,
|
||||
CoverageSourceContract,
|
||||
)
|
||||
from .project import ProjectCreate, ProjectDeleteResult, ProjectList, ProjectRead, ProjectUpdate
|
||||
from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
|
||||
from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
|
||||
from .dataset import DatasetCreateResponse, DatasetList
|
||||
from .source_freshness import (
|
||||
SourceFreshnessItem,
|
||||
SourceFreshnessReport,
|
||||
SourceFreshnessSummary,
|
||||
SourceIntegritySummary,
|
||||
)
|
||||
from .source_catalog import (
|
||||
SourceCatalogProbeItem,
|
||||
SourceCatalogProbeReport,
|
||||
SourceCatalogProbeSummary,
|
||||
)
|
||||
from .source_registry import (
|
||||
DatasetProvenanceRead,
|
||||
DatasetLineageEdgeRead,
|
||||
DatasetQuarantineRead,
|
||||
SourceRegistryDetailRead,
|
||||
SourceRegistryRead,
|
||||
SourceSnapshotRead,
|
||||
)
|
||||
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
||||
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
||||
from .official_vector import (
|
||||
OfficialVectorAcquireRequest,
|
||||
OfficialVectorAcquisitionResult,
|
||||
OfficialVectorProductRead,
|
||||
)
|
||||
from .detection import (
|
||||
DetectionListResponse,
|
||||
DetectionModelCapability,
|
||||
DetectionModelsResponse,
|
||||
DetectionQaRequest,
|
||||
DetectionRead,
|
||||
DetectionRunListResponse,
|
||||
DetectionRunRead,
|
||||
DetectionRunRequest,
|
||||
DetectionComparisonRequest,
|
||||
DetectionComparisonResponse,
|
||||
DetectionRunResponse,
|
||||
ModelAssetListResponse,
|
||||
ModelAssetRead,
|
||||
YoloPreflightResponse,
|
||||
)
|
||||
from .detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewSummary, DetectionReviewUpsert
|
||||
from .segmentation import (
|
||||
SegmentationListResponse,
|
||||
SegmentationModelCapability,
|
||||
SegmentationModelsResponse,
|
||||
SegmentationQaRequest,
|
||||
SegmentationRead,
|
||||
SegmentationRunListResponse,
|
||||
SegmentationRunRead,
|
||||
SegmentationRunRequest,
|
||||
SegmentationRunResponse,
|
||||
)
|
||||
from .health import HealthResponse, SystemCapabilities
|
||||
from .job import JobCreate, JobList, JobRead, JobStatus
|
||||
from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult, OrthophotoProductRead
|
||||
from .dhmv import (
|
||||
DhmvAcquireRequest,
|
||||
DhmvAcquisitionResult,
|
||||
DhmvProductRead,
|
||||
TerrainMetric,
|
||||
TerrainPartitionSelectionRequest,
|
||||
TerrainSelectionRequest,
|
||||
TerrainSelectionResponse,
|
||||
TerrainSelectionSummary,
|
||||
)
|
||||
from .spw_terrain import (
|
||||
SpwTerrainAcquireRequest,
|
||||
SpwTerrainAcquisitionResult,
|
||||
SpwTerrainProductRead,
|
||||
)
|
||||
from .flood_hazard import (
|
||||
FloodHazardAcquireRequest,
|
||||
FloodHazardAcquisitionResult,
|
||||
FloodHazardMetric,
|
||||
FloodHazardPartitionSelectionRequest,
|
||||
FloodHazardProductRead,
|
||||
FloodHazardSelectionRequest,
|
||||
FloodHazardSelectionResponse,
|
||||
FloodHazardSelectionSummary,
|
||||
)
|
||||
from .bathymetry import (
|
||||
BathymetryPartitionFinalizeRequest,
|
||||
BathymetryPartitionFinalizationResult,
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetryProfileAcquisitionResult,
|
||||
BathymetryRasterMetric,
|
||||
BathymetryRasterSelectionRequest,
|
||||
BathymetryRasterSelectionResponse,
|
||||
BathymetryRasterSelectionSummary,
|
||||
BathymetrySourceProbeRead,
|
||||
BathymetrySourceRead,
|
||||
MdkBathymetryAcquireRequest,
|
||||
MdkBathymetryAcquisitionResult,
|
||||
)
|
||||
from .thematic_raster import (
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterAcquisitionResult,
|
||||
ThematicRasterMetric,
|
||||
ThematicRasterProductRead,
|
||||
ThematicRasterSelectionRequest,
|
||||
ThematicRasterSelectionResponse,
|
||||
ThematicRasterSelectionSummary,
|
||||
)
|
||||
from .external import (
|
||||
ExternalFetchRequest,
|
||||
ExternalFetchResponse,
|
||||
ProviderCapabilitiesResponse,
|
||||
ProviderCapabilityResponse,
|
||||
ProviderImportRequest,
|
||||
ProviderImportResponse,
|
||||
ProviderLayersResponse,
|
||||
ProviderStatusResponse,
|
||||
)
|
||||
from .export import (
|
||||
ExportContentResponse,
|
||||
ExportCreateResponse,
|
||||
ExportListResponse,
|
||||
ExportRead,
|
||||
GeoJsonExportRequest,
|
||||
MetadataExportRequest,
|
||||
ReportExportRequest,
|
||||
)
|
||||
from .qa import (
|
||||
AnalysisQaResponse,
|
||||
QaProviderComparisonRequest,
|
||||
QaProviderComparisonResult,
|
||||
QualityEvidenceResponse,
|
||||
)
|
||||
from .operations import (
|
||||
RasterClipRequest,
|
||||
RasterIndexBaseRequest,
|
||||
RasterMetadataResponse,
|
||||
RasterNdviRequest,
|
||||
RasterNdwiRequest,
|
||||
RasterNdbiRequest,
|
||||
RasterOperationResult,
|
||||
RasterPreviewResponse,
|
||||
RasterReprojectRequest,
|
||||
RasterReprojectResponse,
|
||||
RasterStatsResponse,
|
||||
RasterTileManifest,
|
||||
RasterTileManifestTile,
|
||||
RasterTileRequest,
|
||||
RasterTileResponse,
|
||||
VectorBBoxResponse,
|
||||
VectorBufferRequest,
|
||||
VectorClipRequest,
|
||||
VectorIntersectRequest,
|
||||
VectorOperationRequest,
|
||||
VectorOperationResult,
|
||||
VectorSelectionBBox,
|
||||
VectorSelectionDeriveRequest,
|
||||
VectorSelectionRequest,
|
||||
VectorSelectionResponse,
|
||||
VectorSelectionMetric,
|
||||
VectorSelectionSummary,
|
||||
VectorStatsRequest,
|
||||
VectorStatsResponse,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Envelope",
|
||||
"ItemList",
|
||||
"GeoJsonFeature",
|
||||
"GeoJsonFeatureCollection",
|
||||
"ApiErrorEnvelope",
|
||||
"ApiErrorItem",
|
||||
"PaginationEnvelope",
|
||||
"CoverageBBox",
|
||||
"CoverageCatalogResponse",
|
||||
"CoverageResolutionItem",
|
||||
"CoverageResolveRequest",
|
||||
"CoverageResolveResponse",
|
||||
"CoverageSourceContract",
|
||||
"ProjectCreate",
|
||||
"ProjectRead",
|
||||
"ProjectUpdate",
|
||||
"ProjectList",
|
||||
"ProjectDeleteResult",
|
||||
"AreaCreate",
|
||||
"AreaRead",
|
||||
"AreaUpdate",
|
||||
"AreaList",
|
||||
"ChangeDetectionRequest",
|
||||
"ChangeDetectionSummary",
|
||||
"DatasetCreateResponse",
|
||||
"DatasetList",
|
||||
"SourceFreshnessItem",
|
||||
"SourceFreshnessReport",
|
||||
"SourceFreshnessSummary",
|
||||
"SourceIntegritySummary",
|
||||
"SourceCatalogProbeItem",
|
||||
"SourceCatalogProbeReport",
|
||||
"SourceCatalogProbeSummary",
|
||||
"SourceRegistryRead",
|
||||
"SourceRegistryDetailRead",
|
||||
"SourceSnapshotRead",
|
||||
"DatasetLineageEdgeRead",
|
||||
"DatasetQuarantineRead",
|
||||
"DatasetProvenanceRead",
|
||||
"GrbRefreshLayerPlan",
|
||||
"GrbRefreshPlan",
|
||||
"GrbRefreshPlanSummary",
|
||||
"GrbAcquireRequest",
|
||||
"GrbAcquisitionResult",
|
||||
"GrbProductRead",
|
||||
"OfficialVectorAcquireRequest",
|
||||
"OfficialVectorAcquisitionResult",
|
||||
"OfficialVectorProductRead",
|
||||
"DetectionListResponse",
|
||||
"DetectionModelCapability",
|
||||
"DetectionModelsResponse",
|
||||
"DetectionQaRequest",
|
||||
"DetectionRead",
|
||||
"DetectionRunListResponse",
|
||||
"DetectionRunRead",
|
||||
"DetectionRunRequest",
|
||||
"DetectionComparisonRequest",
|
||||
"DetectionComparisonResponse",
|
||||
"DetectionRunResponse",
|
||||
"ModelAssetListResponse",
|
||||
"ModelAssetRead",
|
||||
"YoloPreflightResponse",
|
||||
"DetectionReviewList",
|
||||
"DetectionReviewRead",
|
||||
"DetectionReviewSummary",
|
||||
"DetectionReviewUpsert",
|
||||
"SegmentationListResponse",
|
||||
"SegmentationModelCapability",
|
||||
"SegmentationModelsResponse",
|
||||
"SegmentationQaRequest",
|
||||
"SegmentationRead",
|
||||
"SegmentationRunListResponse",
|
||||
"SegmentationRunRead",
|
||||
"SegmentationRunRequest",
|
||||
"SegmentationRunResponse",
|
||||
"HealthResponse",
|
||||
"SystemCapabilities",
|
||||
"JobCreate",
|
||||
"JobList",
|
||||
"JobRead",
|
||||
"JobStatus",
|
||||
"OrthophotoAcquireRequest",
|
||||
"OrthophotoAcquisitionResult",
|
||||
"OrthophotoProductRead",
|
||||
"DhmvAcquireRequest",
|
||||
"DhmvAcquisitionResult",
|
||||
"DhmvProductRead",
|
||||
"SpwTerrainAcquireRequest",
|
||||
"SpwTerrainAcquisitionResult",
|
||||
"SpwTerrainProductRead",
|
||||
"TerrainMetric",
|
||||
"TerrainPartitionSelectionRequest",
|
||||
"TerrainSelectionRequest",
|
||||
"TerrainSelectionResponse",
|
||||
"TerrainSelectionSummary",
|
||||
"FloodHazardAcquireRequest",
|
||||
"FloodHazardAcquisitionResult",
|
||||
"FloodHazardMetric",
|
||||
"FloodHazardPartitionSelectionRequest",
|
||||
"FloodHazardProductRead",
|
||||
"FloodHazardSelectionRequest",
|
||||
"FloodHazardSelectionResponse",
|
||||
"FloodHazardSelectionSummary",
|
||||
"BathymetryProfileAcquireRequest",
|
||||
"BathymetryProfileAcquisitionResult",
|
||||
"BathymetryRasterMetric",
|
||||
"BathymetryRasterSelectionRequest",
|
||||
"BathymetryRasterSelectionResponse",
|
||||
"BathymetryRasterSelectionSummary",
|
||||
"BathymetryPartitionFinalizeRequest",
|
||||
"BathymetryPartitionFinalizationResult",
|
||||
"BathymetrySourceProbeRead",
|
||||
"BathymetrySourceRead",
|
||||
"MdkBathymetryAcquireRequest",
|
||||
"MdkBathymetryAcquisitionResult",
|
||||
"ThematicRasterAcquireRequest",
|
||||
"ThematicRasterAcquisitionResult",
|
||||
"ThematicRasterMetric",
|
||||
"ThematicRasterProductRead",
|
||||
"ThematicRasterSelectionRequest",
|
||||
"ThematicRasterSelectionResponse",
|
||||
"ThematicRasterSelectionSummary",
|
||||
"VectorBBoxResponse",
|
||||
"VectorClipRequest",
|
||||
"VectorBufferRequest",
|
||||
"VectorIntersectRequest",
|
||||
"VectorOperationRequest",
|
||||
"VectorOperationResult",
|
||||
"VectorSelectionBBox",
|
||||
"VectorSelectionDeriveRequest",
|
||||
"VectorSelectionRequest",
|
||||
"VectorSelectionResponse",
|
||||
"VectorSelectionMetric",
|
||||
"VectorSelectionSummary",
|
||||
"RasterClipRequest",
|
||||
"RasterStatsResponse",
|
||||
"RasterReprojectRequest",
|
||||
"RasterReprojectResponse",
|
||||
"RasterTileRequest",
|
||||
"RasterMetadataResponse",
|
||||
"RasterOperationResult",
|
||||
"RasterPreviewResponse",
|
||||
"RasterTileManifestTile",
|
||||
"RasterTileManifest",
|
||||
"RasterTileResponse",
|
||||
"RasterIndexBaseRequest",
|
||||
"RasterNdviRequest",
|
||||
"RasterNdwiRequest",
|
||||
"RasterNdbiRequest",
|
||||
"VectorStatsRequest",
|
||||
"VectorStatsResponse",
|
||||
"ExternalFetchRequest",
|
||||
"ExternalFetchResponse",
|
||||
"ProviderCapabilitiesResponse",
|
||||
"ProviderCapabilityResponse",
|
||||
"ProviderImportRequest",
|
||||
"ProviderImportResponse",
|
||||
"ProviderLayersResponse",
|
||||
"ProviderStatusResponse",
|
||||
"GeoJsonExportRequest",
|
||||
"MetadataExportRequest",
|
||||
"ReportExportRequest",
|
||||
"ExportRead",
|
||||
"ExportCreateResponse",
|
||||
"ExportListResponse",
|
||||
"ExportContentResponse",
|
||||
"QaProviderComparisonRequest",
|
||||
"QaProviderComparisonResult",
|
||||
"AnalysisQaResponse",
|
||||
"QualityEvidenceResponse",
|
||||
]
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class ChangeDetectionRequest(BaseModel):
|
||||
source_dataset_id: UUID
|
||||
target_dataset_id: UUID
|
||||
iou_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
|
||||
# Below this the two footprints are separate objects rather than one that
|
||||
# was redrawn; between the two thresholds the change class is "modified".
|
||||
modified_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
|
||||
include_unchanged: bool = True
|
||||
# Without a selection the comparison covers both datasets in full, which is
|
||||
# rarely the question and never a response a map can draw.
|
||||
bbox: VectorSelectionBBox | None = None
|
||||
area_id: UUID | None = None
|
||||
preview_limit: int = Field(default=2_000, ge=1, le=20_000)
|
||||
|
||||
|
||||
class ChangeDetectionSummary(BaseModel):
|
||||
source_dataset_id: UUID
|
||||
target_dataset_id: UUID
|
||||
source_feature_count: int
|
||||
target_feature_count: int
|
||||
added_count: int
|
||||
removed_count: int
|
||||
# A footprint that was redrawn rather than demolished and rebuilt. Without
|
||||
# this class it appeared as one removal plus one addition.
|
||||
modified_count: int = 0
|
||||
unchanged_count: int
|
||||
iou_threshold: float
|
||||
modified_iou_threshold: float | None = None
|
||||
selection_area_id: UUID | None = None
|
||||
# Counts describe the whole selection; the GeoJSON is capped so a regional
|
||||
# comparison does not return both datasets in one response.
|
||||
preview_limit: int | None = None
|
||||
preview_truncated: bool = False
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
generated_at: datetime
|
||||
geojson: dict
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class AoiOperationCreate(BaseModel):
|
||||
area_id: UUID | None = None
|
||||
bbox: VectorSelectionBBox | None = None
|
||||
operation_type: str = Field(min_length=1, max_length=128)
|
||||
provider_key: str = Field(min_length=1, max_length=120)
|
||||
product_key: str = Field(min_length=1, max_length=120)
|
||||
coverage_zone: str | None = Field(default=None, max_length=64)
|
||||
max_partition_side_m: float | None = Field(default=None, gt=0, le=60_000)
|
||||
max_attempts: int = Field(default=3, ge=1, le=10)
|
||||
parameters_json: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AoiPartitionRead(BaseModel):
|
||||
id: UUID
|
||||
partition_key: str
|
||||
provider_key: str
|
||||
product_key: str
|
||||
ordinal: int
|
||||
status: str
|
||||
attempt_count: int
|
||||
max_attempts: int
|
||||
checkpoint_json: dict | None = None
|
||||
result_json: dict | None = None
|
||||
error_message: str | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class AoiOperationRead(BaseModel):
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
area_id: UUID | None = None
|
||||
parent_job_id: UUID | None = None
|
||||
operation_type: str
|
||||
status: str
|
||||
request_json: dict
|
||||
plan_json: dict
|
||||
result_json: dict | None = None
|
||||
error_message: str | None = None
|
||||
progress: float
|
||||
partition_counts: dict[str, int]
|
||||
partitions: list[AoiPartitionRead] = Field(default_factory=list)
|
||||
created_at: datetime | None = None
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
|
||||
|
||||
class AoiOperationList(BaseModel):
|
||||
items: list[AoiOperationRead]
|
||||
total: int
|
||||
|
||||
|
||||
class AoiPartitionCheckpoint(BaseModel):
|
||||
checkpoint_json: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AoiPartitionComplete(BaseModel):
|
||||
result_json: dict = Field(default_factory=dict)
|
||||
skipped: bool = False
|
||||
|
||||
|
||||
class AoiPartitionFail(BaseModel):
|
||||
error_message: str = Field(min_length=1, max_length=4000)
|
||||
retryable: bool = True
|
||||
details: dict = Field(default_factory=dict)
|
||||
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class AreaCreate(BaseModel):
|
||||
name: str
|
||||
geometry: dict
|
||||
crs: str | None = "EPSG:4326"
|
||||
|
||||
|
||||
class AreaUpdate(BaseModel):
|
||||
name: str | None = None
|
||||
geometry: dict | None = None
|
||||
crs: str | None = None
|
||||
|
||||
|
||||
class AreaRead(BaseModel):
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
name: str
|
||||
original_crs: str | None
|
||||
area_m2: float | None
|
||||
created_at: datetime | None = None
|
||||
geometry_type: str | None = None
|
||||
geometry: dict | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class AreaListItem(AreaRead):
|
||||
pass
|
||||
|
||||
|
||||
class AreaList(BaseModel):
|
||||
items: list[AreaRead]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
class MunicipalitySearchItem(BaseModel):
|
||||
niscode: str
|
||||
name: str
|
||||
name_nl: str | None = None
|
||||
name_fr: str | None = None
|
||||
name_de: str | None = None
|
||||
|
||||
|
||||
class MunicipalitySearchList(BaseModel):
|
||||
items: list[MunicipalitySearchItem]
|
||||
total: int
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class AssistantChatMessage(BaseModel):
|
||||
role: Literal["user", "assistant"]
|
||||
content: str = Field(min_length=1, max_length=4_000)
|
||||
|
||||
|
||||
class AssistantQueryRequest(BaseModel):
|
||||
question: str = Field(min_length=2, max_length=2_000)
|
||||
model: str | None = Field(default=None, max_length=255)
|
||||
bbox: VectorSelectionBBox | None = None
|
||||
area_id: UUID | None = None
|
||||
history: list[AssistantChatMessage] = Field(default_factory=list, max_length=8)
|
||||
|
||||
|
||||
class AssistantModelRead(BaseModel):
|
||||
name: str
|
||||
size_bytes: int | None = None
|
||||
parameter_size: str | None = None
|
||||
quantization_level: str | None = None
|
||||
capabilities: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AssistantModelList(BaseModel):
|
||||
items: list[AssistantModelRead]
|
||||
total: int
|
||||
default_model: str | None = None
|
||||
|
||||
|
||||
class AssistantStatus(BaseModel):
|
||||
enabled: bool
|
||||
reachable: bool
|
||||
status: str
|
||||
base_url: str
|
||||
default_model: str | None = None
|
||||
model_count: int = 0
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class AssistantContextMetric(BaseModel):
|
||||
theme: str
|
||||
label: str
|
||||
value: float
|
||||
unit: str
|
||||
source: str
|
||||
dataset_id: UUID
|
||||
observed_at: datetime | None = None
|
||||
is_estimate: bool = False
|
||||
|
||||
|
||||
class AssistantTemporalSeries(BaseModel):
|
||||
temporal_series_key: str
|
||||
label: str
|
||||
source: str
|
||||
first_year: int
|
||||
last_year: int
|
||||
observation_count: int
|
||||
|
||||
|
||||
class AssistantEstimateDisclosure(BaseModel):
|
||||
"""A value in the answer that the source itself calls an estimate.
|
||||
|
||||
Derived from metric metadata rather than from the generated sentences, so
|
||||
the disclosure is present whatever wording the model chose.
|
||||
"""
|
||||
|
||||
theme: str
|
||||
label: str
|
||||
unit: str
|
||||
source: str
|
||||
dataset_id: UUID
|
||||
reason: str
|
||||
|
||||
|
||||
class AssistantQueryResponse(BaseModel):
|
||||
answer: str
|
||||
model: str
|
||||
scope_label: str
|
||||
context_metrics: list[AssistantContextMetric]
|
||||
temporal_series: list[AssistantTemporalSeries]
|
||||
estimate_disclosures: list[AssistantEstimateDisclosure] = Field(default_factory=list)
|
||||
source_dataset_ids: list[UUID]
|
||||
warnings: list[str]
|
||||
generated_at: datetime
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.common import Envelope
|
||||
|
||||
|
||||
class AuthLoginRequest(BaseModel):
|
||||
username: str = Field(min_length=1, max_length=128)
|
||||
password: str = Field(min_length=1, max_length=1024)
|
||||
|
||||
|
||||
class AuthSession(BaseModel):
|
||||
authentication_required: bool
|
||||
authenticated: bool
|
||||
username: str | None = None
|
||||
expires_at: datetime | None = None
|
||||
role: Literal["operator", "guest"] | None = None
|
||||
guest_access_enabled: bool = False
|
||||
authentik_enabled: bool = False
|
||||
guest_project_id: UUID | None = None
|
||||
|
||||
|
||||
class AuthSessionEnvelope(Envelope[AuthSession]):
|
||||
pass
|
||||
@@ -0,0 +1,173 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class BathymetryProfileAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class BathymetrySourceRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
owner: str
|
||||
authority_level: Literal["authoritative", "contextual"]
|
||||
geographic_coverage: str
|
||||
data_kind: str
|
||||
query_modes: list[str]
|
||||
vertical_reference: str
|
||||
horizontal_crs: str
|
||||
native_resolution: str | None = None
|
||||
integration_status: Literal["operational", "probe_only", "available_not_integrated", "catalog_only"]
|
||||
acquisition_supported: bool
|
||||
configured: bool
|
||||
service_url: str | None = None
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class BathymetryProfileAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
profile_count: int = Field(ge=0)
|
||||
document_count: int = Field(ge=0)
|
||||
structured_depth_count: int = Field(ge=0)
|
||||
structured_width_count: int = Field(ge=0)
|
||||
watercourse_count: int = Field(ge=0)
|
||||
bbox_epsg4326: list[float]
|
||||
clipped_to_area_id: UUID | None = None
|
||||
measurement_date_min: str | None = None
|
||||
measurement_date_max: str | None = None
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class BathymetryPartitionFinalizeRequest(BaseModel):
|
||||
partition_scope_key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9][a-z0-9_-]*$")
|
||||
expected_area_ids: list[UUID] = Field(min_length=1, max_length=500)
|
||||
dataset_ids: list[UUID] = Field(default_factory=list, max_length=500)
|
||||
no_profile_area_ids: list[UUID] = Field(default_factory=list, max_length=500)
|
||||
manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
observed_at: datetime
|
||||
|
||||
@field_validator("expected_area_ids", "dataset_ids", "no_profile_area_ids")
|
||||
@classmethod
|
||||
def require_unique_ids(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("Partition identifiers must be unique")
|
||||
return value
|
||||
|
||||
|
||||
class BathymetryPartitionFinalizationResult(BaseModel):
|
||||
partition_scope_key: str
|
||||
regional_partitions_complete: bool
|
||||
partition_count: int = Field(ge=1)
|
||||
data_partition_count: int = Field(ge=0)
|
||||
no_profile_partition_count: int = Field(ge=0)
|
||||
profile_count: int = Field(ge=0)
|
||||
document_count: int = Field(ge=0)
|
||||
structured_depth_count: int = Field(ge=0)
|
||||
measurement_date_min: str | None = None
|
||||
measurement_date_max: str | None = None
|
||||
dataset_ids: list[UUID]
|
||||
manifest_sha256: str
|
||||
observed_at: datetime
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class BathymetrySourceProbeRead(BaseModel):
|
||||
source_key: str
|
||||
status: Literal[
|
||||
"disabled",
|
||||
"invalid_configuration",
|
||||
"tls_error",
|
||||
"endpoint_unavailable",
|
||||
"invalid_capabilities",
|
||||
"reachable",
|
||||
]
|
||||
configured_url: str
|
||||
capabilities_url: str | None = None
|
||||
tls_verified: bool
|
||||
capabilities_reachable: bool
|
||||
acquisition_supported: bool = False
|
||||
wcs_version: str | None = None
|
||||
coverage_identifiers: list[str] = Field(default_factory=list)
|
||||
advertised_formats: list[str] = Field(default_factory=list)
|
||||
advertised_crs: list[str] = Field(default_factory=list)
|
||||
response_sha256: str | None = None
|
||||
checked_at: datetime
|
||||
message: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class MdkBathymetryAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class MdkBathymetryAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
coverage_id: str
|
||||
bbox_epsg4326: list[float]
|
||||
vertical_reference: str
|
||||
resolution_m: float = Field(gt=0)
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class BathymetryRasterSelectionRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
|
||||
|
||||
class BathymetryRasterMetric(BaseModel):
|
||||
metric_key: str
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
is_estimate: bool = False
|
||||
|
||||
|
||||
class BathymetryRasterSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
primary_metric_key: str
|
||||
metrics: list[BathymetryRasterMetric]
|
||||
|
||||
|
||||
class BathymetryRasterSelectionResponse(BaseModel):
|
||||
dataset_id: UUID
|
||||
product_key: str
|
||||
selection_bbox: VectorSelectionBBox
|
||||
selection_area_id: UUID | None = None
|
||||
selected_cell_count: int = Field(ge=1)
|
||||
valid_cell_count: int = Field(ge=1)
|
||||
coverage_ratio: float = Field(ge=0, le=1)
|
||||
# Set when the drawn selection is smaller than one source cell and the
|
||||
# analysis was widened to the cells it touches, so the value covers more
|
||||
# ground than was requested.
|
||||
cell_selection_warning: str | None = None
|
||||
resolution_m: float = Field(gt=0)
|
||||
vertical_reference: str
|
||||
survey_period: str
|
||||
summary: BathymetryRasterSelectionSummary
|
||||
unsupported_metrics: list[str]
|
||||
limitation_message: str
|
||||
generated_at: str
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Generic, Literal, TypeVar
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
DataT = TypeVar("DataT")
|
||||
|
||||
|
||||
class Envelope(BaseModel, Generic[DataT]):
|
||||
data: DataT
|
||||
|
||||
|
||||
class ItemList(BaseModel, Generic[DataT]):
|
||||
items: list[DataT]
|
||||
total: int
|
||||
|
||||
|
||||
class PaginatedEnvelope(ItemList[DataT], Generic[DataT]):
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
class PaginationEnvelope(BaseModel):
|
||||
items: list
|
||||
total: int
|
||||
limit: int = Field(default=50)
|
||||
offset: int = Field(default=0)
|
||||
|
||||
|
||||
class ApiErrorItem(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
details: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ApiErrorEnvelope(BaseModel):
|
||||
error: str
|
||||
message: str
|
||||
details: dict | list = Field(default_factory=dict)
|
||||
request_id: str | None = None
|
||||
|
||||
|
||||
class GeoJsonFeature(BaseModel):
|
||||
type: Literal["Feature"]
|
||||
id: str | int | None = None
|
||||
geometry: dict[str, Any] | None
|
||||
properties: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class GeoJsonFeatureCollection(BaseModel):
|
||||
type: Literal["FeatureCollection"]
|
||||
features: list[GeoJsonFeature]
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
|
||||
CoverageStatus = Literal["operational", "partial", "not_configured", "unsupported"]
|
||||
CoverageAuthority = Literal["authoritative", "official_context", "contextual"]
|
||||
CoverageAcquisitionMode = Literal[
|
||||
"operator_archive",
|
||||
"operator_wfs",
|
||||
"bounded_api",
|
||||
"bounded_raster",
|
||||
"catalog_only",
|
||||
]
|
||||
|
||||
|
||||
class CoverageBBox(BaseModel):
|
||||
minx: float = Field(ge=-180, le=180)
|
||||
miny: float = Field(ge=-90, le=90)
|
||||
maxx: float = Field(ge=-180, le=180)
|
||||
maxy: float = Field(ge=-90, le=90)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_extent(self) -> "CoverageBBox":
|
||||
if self.maxx <= self.minx or self.maxy <= self.miny:
|
||||
raise ValueError("bbox max values must be greater than min values")
|
||||
return self
|
||||
|
||||
|
||||
class CoverageSourceContract(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
authority_level: CoverageAuthority
|
||||
coverage_zones: list[str]
|
||||
themes: list[str]
|
||||
native_layers: list[str]
|
||||
supported_geometry_types: list[str]
|
||||
acquisition_mode: CoverageAcquisitionMode
|
||||
integration_status: CoverageStatus
|
||||
source_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class CoverageCatalogResponse(BaseModel):
|
||||
themes: list[str]
|
||||
zones: list[str]
|
||||
statuses: list[CoverageStatus]
|
||||
sources: list[CoverageSourceContract]
|
||||
|
||||
|
||||
class CoverageResolveRequest(BaseModel):
|
||||
project_id: UUID
|
||||
bbox: CoverageBBox
|
||||
themes: list[str] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class CoverageResolutionItem(BaseModel):
|
||||
zone: str
|
||||
theme: str
|
||||
status: CoverageStatus
|
||||
source_names: list[str]
|
||||
materialized_dataset_ids: list[UUID]
|
||||
evidence: list["CoverageEvidenceItem"] = Field(default_factory=list)
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class CoverageEvidenceItem(BaseModel):
|
||||
dataset_id: UUID
|
||||
source_name: str
|
||||
authority_level: CoverageAuthority
|
||||
source_version: str | None = None
|
||||
observed_at: str | None = None
|
||||
published_at: str | None = None
|
||||
crs: str | None = None
|
||||
resolution: dict | None = None
|
||||
coverage_bbox_epsg4326: list[float] | None = None
|
||||
attribution: str | None = None
|
||||
license_note: str | None = None
|
||||
checksum_sha256: str | None = None
|
||||
|
||||
|
||||
class CoverageResolveResponse(BaseModel):
|
||||
project_id: UUID
|
||||
bbox: CoverageBBox
|
||||
requested_themes: list[str]
|
||||
intersected_zones: list[str]
|
||||
outside_supported_scope: bool
|
||||
items: list[CoverageResolutionItem]
|
||||
warnings: list[str]
|
||||
@@ -0,0 +1,132 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class DatasetStorageResponse(BaseModel):
|
||||
original_filename: str | None = None
|
||||
stored_filename: str | None = None
|
||||
content_type: str | None = None
|
||||
size_bytes: int | None = None
|
||||
checksum_sha256: str | None = None
|
||||
|
||||
|
||||
class DatasetVectorSummary(BaseModel):
|
||||
feature_count: int | None = None
|
||||
geometry_types: list[str] | None = None
|
||||
bounds_json: dict | None = None
|
||||
approximate_area_m2: float | None = None
|
||||
crs: str | None = None
|
||||
feature_geometry_count: int | None = None
|
||||
invalid_features: int | None = None
|
||||
crs_assumed: bool | None = None
|
||||
|
||||
|
||||
class DatasetCreateResponse(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
dataset_type: str
|
||||
source: str
|
||||
dataset_role: str = "source"
|
||||
source_name: str | None = None
|
||||
reference_layer_name: str | None = None
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
ingest_key: str | None = None
|
||||
source_registry_id: UUID | None = None
|
||||
source_snapshot_id: UUID | None = None
|
||||
data_contract_key: str | None = None
|
||||
data_contract_version: str | None = None
|
||||
validation_status: str | None = None
|
||||
validation_report_json: dict | None = None
|
||||
provenance_status: str | None = None
|
||||
lineage_status: str | None = None
|
||||
quarantine_status: str | None = None
|
||||
imported_at: datetime | None = None
|
||||
temporal_series_key: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
temporal_granularity: str | None = None
|
||||
source_version: str | None = None
|
||||
project_id: UUID
|
||||
area_id: UUID | None = None
|
||||
storage_path: str | None = None
|
||||
original_filename: str | None = None
|
||||
stored_filename: str | None = None
|
||||
content_type: str | None = None
|
||||
size_bytes: int | None = None
|
||||
checksum_sha256: str | None = None
|
||||
crs: str | None = None
|
||||
bounds_json: dict | None = None
|
||||
metadata_json: dict | None = None
|
||||
vector_summary: DatasetVectorSummary | None = None
|
||||
status: str
|
||||
derived_from_dataset_id: UUID | None = None
|
||||
created_at: datetime | None = None
|
||||
feature_count: int | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class DatasetList(BaseModel):
|
||||
items: list[DatasetCreateResponse]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
class DatasetMetadataRefresh(BaseModel):
|
||||
feature_count: int | None = None
|
||||
geometry_types: list[str] | None = None
|
||||
bounds_json: dict | None = None
|
||||
crs: str | None = None
|
||||
|
||||
|
||||
class DatasetTemporalUpdate(BaseModel):
|
||||
temporal_series_key: str
|
||||
observed_at: datetime
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
temporal_granularity: str = "snapshot"
|
||||
source_version: str | None = None
|
||||
|
||||
|
||||
class DatasetVersionRead(BaseModel):
|
||||
id: UUID
|
||||
dataset_id: UUID
|
||||
version: int
|
||||
storage_path: str | None = None
|
||||
source_version: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
valid_from: datetime | None = None
|
||||
valid_to: datetime | None = None
|
||||
checksum_sha256: str | None = None
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
ingest_key: str | None = None
|
||||
source_registry_id: UUID | None = None
|
||||
source_snapshot_id: UUID | None = None
|
||||
data_contract_key: str | None = None
|
||||
data_contract_version: str | None = None
|
||||
validation_status: str | None = None
|
||||
validation_report_json: dict | None = None
|
||||
provenance_status: str | None = None
|
||||
lineage_status: str | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
dataset_id: UUID
|
||||
name: str | None = None
|
||||
|
||||
|
||||
class ExportRead(BaseModel):
|
||||
export_id: UUID
|
||||
path: str
|
||||
status: str
|
||||
@@ -0,0 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class DemoWorkflowResponse(BaseModel):
|
||||
project_id: UUID
|
||||
area_id: UUID
|
||||
reference_dataset_id: UUID
|
||||
candidate_dataset_id: UUID
|
||||
raster_dataset_id: UUID | None = None
|
||||
quality_check_id: UUID
|
||||
metric_count: int
|
||||
status: str
|
||||
message: str
|
||||
created: bool
|
||||
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class DetectionModelCapability(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
model_id: str
|
||||
display_name: str
|
||||
framework: str
|
||||
task_type: str
|
||||
supported_classes: list[str]
|
||||
configured: bool
|
||||
status: str
|
||||
limitation_message: str
|
||||
version: str | None = None
|
||||
training_scope: str | None = None
|
||||
validation_scope: str | None = None
|
||||
validated_regions: list[str] = Field(default_factory=list)
|
||||
nationally_validated: bool = False
|
||||
operator_review_required: bool = True
|
||||
|
||||
|
||||
class DetectionModelsResponse(BaseModel):
|
||||
models: list[DetectionModelCapability]
|
||||
|
||||
|
||||
class ModelAssetRead(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
model_asset_id: str
|
||||
filename: str
|
||||
display_name: str
|
||||
model_path: str
|
||||
suffix: str
|
||||
framework: str
|
||||
task_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
active: bool
|
||||
runtime_available: bool
|
||||
runtime_status: str
|
||||
governed_validation_status: str
|
||||
promotion_status: str
|
||||
status: str
|
||||
limitation_message: str
|
||||
will_download_models: bool = False
|
||||
|
||||
|
||||
class ModelAssetListResponse(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
items: list[ModelAssetRead]
|
||||
total: int
|
||||
model_directory: str
|
||||
|
||||
|
||||
class DetectionRunRequest(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
project_id: UUID
|
||||
dataset_id: UUID
|
||||
model_id: str
|
||||
model_asset_id: str | None = None
|
||||
confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
class_filter: list[str] | None = None
|
||||
tile_manifest_path: str | None = None
|
||||
parameters_json: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class DetectionQaRequest(BaseModel):
|
||||
reference_dataset_id: UUID
|
||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
class_name: str | None = None
|
||||
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
# Confidence cuts to report alongside the run's own operating point. They
|
||||
# are read off the one matching pass, so a sweep costs no extra inference.
|
||||
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class DetectionComparisonRequest(BaseModel):
|
||||
"""Place several runs side by side against one reference."""
|
||||
|
||||
analysis_run_ids: list[UUID] = Field(min_length=2, max_length=12)
|
||||
reference_dataset_id: UUID
|
||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class DetectionComparisonResponse(BaseModel):
|
||||
reference_dataset_id: UUID
|
||||
iou_threshold: float
|
||||
# Whether these runs answer the same question at all, and why not if they
|
||||
# do not. Numbers from incomparable runs are reported but never ranked as
|
||||
# if they were alternatives.
|
||||
comparability: dict
|
||||
ranking_metric: str
|
||||
rows: list[dict]
|
||||
|
||||
|
||||
class DetectionRunResponse(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
analysis_run_id: UUID
|
||||
job_id: UUID
|
||||
project_id: UUID
|
||||
dataset_id: UUID
|
||||
model_id: str
|
||||
status: str
|
||||
detection_count: int
|
||||
error_code: str | None = None
|
||||
message: str
|
||||
|
||||
|
||||
class DetectionRunRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
|
||||
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
job_id: UUID | None = None
|
||||
analysis_type: str
|
||||
status: str
|
||||
model_name: str | None = None
|
||||
model_version: str | None = None
|
||||
parameters_json: dict
|
||||
result_json: dict | None = None
|
||||
error_message: str | None = None
|
||||
created_at: datetime | None = None
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
|
||||
|
||||
class DetectionRunListResponse(BaseModel):
|
||||
items: list[DetectionRunRead]
|
||||
# ``total`` counts every run; ``items`` is the most recent page of them.
|
||||
total: int
|
||||
limit: int | None = None
|
||||
offset: int = 0
|
||||
truncated: bool = False
|
||||
|
||||
|
||||
class DetectionRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
|
||||
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
analysis_run_id: UUID | None = None
|
||||
job_id: UUID | None = None
|
||||
model_name: str
|
||||
model_version: str | None = None
|
||||
class_name: str
|
||||
confidence: float
|
||||
bbox_json: dict | None = None
|
||||
source_tile_path: str | None = None
|
||||
properties_json: dict | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
|
||||
class DetectionListResponse(BaseModel):
|
||||
items: list[DetectionRead]
|
||||
# ``total`` is the complete population; ``items`` is one page of it.
|
||||
total: int
|
||||
limit: int | None = None
|
||||
offset: int = 0
|
||||
truncated: bool = False
|
||||
|
||||
|
||||
class YoloPreflightChecks(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
enabled: bool
|
||||
dependencies_available: bool | None = None
|
||||
accelerator_ready: bool | None = None
|
||||
model_path_set: bool | None = None
|
||||
model_file_exists: bool | None = None
|
||||
model_provenance_manifest_path: str | None = None
|
||||
model_provenance_valid: bool | None = None
|
||||
model_load_requested: bool
|
||||
model_load_ok: bool | None = None
|
||||
manifest_path_set: bool | None = None
|
||||
manifest_valid: bool | None = None
|
||||
tile_paths_exist: bool | None = None
|
||||
tile_limit_ok: bool | None = None
|
||||
|
||||
|
||||
class YoloRuntimeDetails(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
dependencies_assumed: bool
|
||||
model_directory: str | None = None
|
||||
yolo_config_dir: str | None = None
|
||||
torch_version: str | None = None
|
||||
ultralytics_version: str | None = None
|
||||
cuda_available: bool | None = None
|
||||
configured_device: str
|
||||
cuda_required: bool
|
||||
|
||||
|
||||
class YoloPreflightResponse(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
model_id: str
|
||||
model_asset_id: str | None = None
|
||||
model_path: str | None = None
|
||||
tile_manifest_path: str | None = None
|
||||
status: str
|
||||
message: str
|
||||
checks: YoloPreflightChecks
|
||||
tile_count: int
|
||||
max_tiles: int
|
||||
will_download_models: bool
|
||||
will_run_inference: bool
|
||||
runtime: YoloRuntimeDetails
|
||||
error_code: str | None = None
|
||||
details: dict | None = None
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
DetectionEvidenceRole = Literal["false_positive", "false_negative"]
|
||||
DetectionReviewDecision = Literal[
|
||||
"confirmed_model_false_positive",
|
||||
"confirmed_model_false_negative",
|
||||
"reference_gap_or_change",
|
||||
"qa_alignment_mismatch",
|
||||
"imagery_obscured_or_uncertain",
|
||||
"uncertain",
|
||||
"unreviewed",
|
||||
]
|
||||
|
||||
|
||||
class DetectionReviewUpsert(BaseModel):
|
||||
evidence_role: DetectionEvidenceRole
|
||||
evidence_feature_id: str = Field(min_length=1, max_length=255)
|
||||
decision: DetectionReviewDecision
|
||||
notes: str | None = Field(default=None, max_length=2000)
|
||||
reviewed_by: str = Field(default="operator", min_length=1, max_length=120)
|
||||
|
||||
|
||||
class DetectionReviewRead(BaseModel):
|
||||
id: UUID | None = None
|
||||
project_id: UUID
|
||||
quality_check_id: UUID
|
||||
analysis_run_id: UUID | None = None
|
||||
evidence_role: DetectionEvidenceRole
|
||||
evidence_feature_id: str
|
||||
detection_id: UUID | None = None
|
||||
reference_feature_id: UUID | None = None
|
||||
decision: DetectionReviewDecision = "unreviewed"
|
||||
notes: str | None = None
|
||||
reviewed_by: str | None = None
|
||||
confidence: float | None = None
|
||||
class_name: str | None = None
|
||||
source_tile_path: str | None = None
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
|
||||
|
||||
class DetectionReviewSummary(BaseModel):
|
||||
total: int
|
||||
reviewed: int
|
||||
remaining: int
|
||||
false_positive_total: int
|
||||
false_negative_total: int
|
||||
decision_counts: dict[str, int]
|
||||
# The score with the operator's verdicts applied, next to the raw one. A
|
||||
# finding adjudicated as a reference gap is not the model's error, and an
|
||||
# interval covers what the unreviewed remainder could still turn out to be.
|
||||
reviewed_metrics: dict | None = None
|
||||
|
||||
|
||||
class DetectionReviewList(BaseModel):
|
||||
items: list[DetectionReviewRead]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
summary: DetectionReviewSummary
|
||||
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class DhmvAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str = "dtm_1m"
|
||||
resolution_m: float | None = Field(default=None, ge=1.0, le=10.0)
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class DhmvProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
surface_model: str
|
||||
coverage_id: str
|
||||
native_resolution_m: float
|
||||
source_crs: str
|
||||
vertical_reference: str
|
||||
acquisition_period: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class DhmvAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
surface_model: str
|
||||
coverage_id: str
|
||||
native_resolution_m: float
|
||||
resolution_m: float
|
||||
width: int
|
||||
height: int
|
||||
valid_pixel_count: int
|
||||
nodata_value: float
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg31370: list[float]
|
||||
vertical_reference: str
|
||||
acquisition_period: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class TerrainSelectionRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
|
||||
|
||||
class TerrainPartitionSelectionRequest(TerrainSelectionRequest):
|
||||
product_key: str = "dtm_1m"
|
||||
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
|
||||
|
||||
|
||||
class TerrainMetric(BaseModel):
|
||||
metric_key: str
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
derived: bool = True
|
||||
|
||||
|
||||
class TerrainSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
primary_metric_key: str
|
||||
metrics: list[TerrainMetric]
|
||||
|
||||
|
||||
class TerrainSelectionResponse(BaseModel):
|
||||
dataset_id: UUID
|
||||
dataset_ids: list[UUID] = Field(default_factory=list)
|
||||
partition_count: int = Field(default=1, ge=1)
|
||||
product_key: str
|
||||
surface_model: str
|
||||
selection_bbox: VectorSelectionBBox
|
||||
selection_area_id: UUID | None = None
|
||||
sample_count: int
|
||||
slope_sample_count: int
|
||||
coverage_ratio: float
|
||||
# Set when the drawn selection is smaller than one source cell and the
|
||||
# analysis was widened to the cells it touches, so the value covers more
|
||||
# ground than was requested.
|
||||
cell_selection_warning: str | None = None
|
||||
resolution_m: float
|
||||
vertical_reference: str
|
||||
summary: TerrainSelectionSummary
|
||||
unsupported_metrics: list[str]
|
||||
limitation_message: str
|
||||
generated_at: str
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, model_validator
|
||||
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
ExportKind = Literal["dataset", "detection_run", "segmentation_run", "vector_selection"]
|
||||
DetectionExportIntendedUse = Literal["review", "operational"]
|
||||
MapResultMode = Literal["current", "evolution"]
|
||||
|
||||
|
||||
class GeoJsonExportRequest(BaseModel):
|
||||
dataset_id: UUID | None = None
|
||||
analysis_run_id: UUID | None = None
|
||||
area_id: UUID | None = None
|
||||
export_kind: ExportKind = "dataset"
|
||||
name: str | None = None
|
||||
bbox: VectorSelectionBBox | None = None
|
||||
limit: int = 250
|
||||
intended_use: DetectionExportIntendedUse = "review"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_target(self) -> "GeoJsonExportRequest":
|
||||
if self.export_kind == "dataset" and self.dataset_id is None:
|
||||
raise ValueError("dataset_id is required for dataset GeoJSON exports")
|
||||
if self.export_kind == "vector_selection":
|
||||
if self.dataset_id is None:
|
||||
raise ValueError("dataset_id is required for vector selection GeoJSON exports")
|
||||
if self.bbox is None:
|
||||
raise ValueError("bbox is required for vector selection GeoJSON exports")
|
||||
if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None:
|
||||
raise ValueError("analysis_run_id is required for run GeoJSON exports")
|
||||
if self.intended_use == "operational" and self.export_kind != "detection_run":
|
||||
raise ValueError("operational intended_use is supported only for detection run exports")
|
||||
return self
|
||||
|
||||
|
||||
class MetadataExportRequest(BaseModel):
|
||||
project_id: UUID
|
||||
name: str | None = None
|
||||
|
||||
|
||||
class ReportExportRequest(BaseModel):
|
||||
project_id: UUID
|
||||
name: str | None = None
|
||||
|
||||
|
||||
class MapResultExportRequest(BaseModel):
|
||||
project_id: UUID
|
||||
mode: MapResultMode
|
||||
bbox: VectorSelectionBBox
|
||||
dataset_id: UUID | None = None
|
||||
earlier_dataset_id: UUID | None = None
|
||||
later_dataset_id: UUID | None = None
|
||||
area_id: UUID | None = None
|
||||
partitioned: bool = False
|
||||
product_key: str | None = None
|
||||
partition_scope_key: str | None = None
|
||||
theme_id: str | None = None
|
||||
name: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_map_target(self) -> "MapResultExportRequest":
|
||||
if self.mode == "current" and self.dataset_id is None:
|
||||
raise ValueError("dataset_id is required for current map-result exports")
|
||||
if self.mode == "evolution" and (
|
||||
self.earlier_dataset_id is None or self.later_dataset_id is None
|
||||
):
|
||||
raise ValueError("earlier_dataset_id and later_dataset_id are required for evolution exports")
|
||||
if self.partitioned and not self.product_key and not self.partition_scope_key:
|
||||
raise ValueError("product_key or partition_scope_key is required for partitioned exports")
|
||||
return self
|
||||
|
||||
|
||||
class ExportRead(BaseModel):
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
analysis_run_id: UUID | None = None
|
||||
export_type: str
|
||||
storage_path: str
|
||||
metadata_json: dict | None = None
|
||||
created_at: datetime | None = None
|
||||
status: str = "ready"
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ExportCreateResponse(BaseModel):
|
||||
export_id: UUID
|
||||
path: str
|
||||
status: str
|
||||
export_type: str
|
||||
metadata_json: dict | None = None
|
||||
|
||||
|
||||
class ExportListResponse(BaseModel):
|
||||
items: list[ExportRead]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
class ExportContentResponse(BaseModel):
|
||||
export_id: UUID
|
||||
export_type: str
|
||||
content: dict
|
||||
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ProviderCapabilityResponse(BaseModel):
|
||||
provider_name: str
|
||||
display_name: str
|
||||
authority_level: str
|
||||
supported_layers: list[str]
|
||||
supported_geometry_types: list[str]
|
||||
supported_query_modes: list[str]
|
||||
fetch_signature: str
|
||||
configured: bool
|
||||
status: str
|
||||
limitation_message: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
not_configured_reason: str | None = None
|
||||
|
||||
|
||||
class ProviderCapabilitiesResponse(BaseModel):
|
||||
providers: list[ProviderCapabilityResponse]
|
||||
|
||||
|
||||
class ProviderLayersResponse(BaseModel):
|
||||
provider_name: str
|
||||
layers: list[str]
|
||||
|
||||
|
||||
class ProviderStatusResponse(BaseModel):
|
||||
provider_name: str
|
||||
configured: bool
|
||||
status: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class ExternalFetchRequest(BaseModel):
|
||||
project_id: UUID
|
||||
area_id: UUID | None = None
|
||||
layers: list[str] = []
|
||||
|
||||
|
||||
class ExternalFetchResponse(BaseModel):
|
||||
provider: str
|
||||
status: str
|
||||
message: str
|
||||
requested_layers: list[str]
|
||||
project_id: UUID
|
||||
area_id: UUID | None = None
|
||||
|
||||
|
||||
class ProviderImportRequest(BaseModel):
|
||||
project_id: str
|
||||
area_id: str | None = None
|
||||
layers: list[str] = []
|
||||
dataset_role: str | None = None
|
||||
|
||||
|
||||
class ProviderImportResponse(BaseModel):
|
||||
provider_name: str
|
||||
status: str
|
||||
message: str
|
||||
requested_layers: list[str]
|
||||
dataset_id: str | None = None
|
||||
dataset_role: str | None = None
|
||||
source_name: str | None = None
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class FloodHazardAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str = "pluviaal_current_t100"
|
||||
resolution_m: float | None = Field(default=None, ge=2.0, le=20.0)
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class FloodHazardProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
mechanism: str
|
||||
climate_context: str
|
||||
probability_class: str
|
||||
return_period_years: int
|
||||
coverage_id: str
|
||||
native_resolution_m: float
|
||||
source_crs: str
|
||||
source_value_unit: str
|
||||
normalized_value_unit: str
|
||||
published_on: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class FloodHazardAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
mechanism: str
|
||||
climate_context: str
|
||||
probability_class: str
|
||||
return_period_years: int
|
||||
coverage_id: str
|
||||
resolution_m: float
|
||||
width: int
|
||||
height: int
|
||||
inundated_pixel_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg31370: list[float]
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class FloodHazardSelectionRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
|
||||
|
||||
class FloodHazardPartitionSelectionRequest(FloodHazardSelectionRequest):
|
||||
product_key: str = "pluviaal_current_t100"
|
||||
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
|
||||
|
||||
|
||||
class FloodHazardMetric(BaseModel):
|
||||
metric_key: str
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
derived: bool = True
|
||||
|
||||
|
||||
class FloodHazardSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
primary_metric_key: str
|
||||
metrics: list[FloodHazardMetric]
|
||||
|
||||
|
||||
class FloodHazardSelectionResponse(BaseModel):
|
||||
dataset_id: UUID
|
||||
dataset_ids: list[UUID] = Field(default_factory=list)
|
||||
partition_count: int = Field(default=1, ge=1)
|
||||
product_key: str
|
||||
mechanism: str
|
||||
climate_context: str
|
||||
probability_class: str
|
||||
return_period_years: int
|
||||
selection_bbox: VectorSelectionBBox
|
||||
selection_area_id: UUID | None = None
|
||||
# Three populations kept apart: cells drawn, cells the model covers, and
|
||||
# cells with a positive modelled depth. ``inundated_fraction`` is a share
|
||||
# of the modelled cells, and is null when nothing was modelled — absence
|
||||
# of a model is not evidence of zero risk.
|
||||
selected_cell_count: int
|
||||
valid_cell_count: int = 0
|
||||
no_data_cell_count: int = 0
|
||||
data_coverage_ratio: float = 1.0
|
||||
inundated_cell_count: int
|
||||
inundated_fraction: float | None = None
|
||||
coverage_warning: str | None = None
|
||||
resolution_m: float
|
||||
summary: FloodHazardSelectionSummary
|
||||
unsupported_metrics: list[str]
|
||||
limitation_message: str
|
||||
generated_at: str
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class GrbAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str = "buildings"
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class GrbProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
reference_layer_name: str
|
||||
collections: list[str]
|
||||
geometry_types: list[str]
|
||||
source_crs: str
|
||||
authority_level: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class GrbAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
reference_layer_name: str
|
||||
collections: list[str]
|
||||
feature_count: int
|
||||
candidate_feature_count: int
|
||||
page_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
source_version: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
GrbRefreshLayerStatus = Literal[
|
||||
"current",
|
||||
"update_available",
|
||||
"not_loaded",
|
||||
"review_required",
|
||||
"remote_unavailable",
|
||||
]
|
||||
|
||||
|
||||
class GrbRefreshLayerPlan(BaseModel):
|
||||
theme: Literal["buildings", "roads", "water", "parcels"]
|
||||
display_name: str
|
||||
collections: list[str]
|
||||
temporal_series_key: str
|
||||
status: GrbRefreshLayerStatus
|
||||
local_dataset_id: UUID | None = None
|
||||
local_source_version: str | None = None
|
||||
local_observed_at: datetime | None = None
|
||||
local_imported_at: datetime | None = None
|
||||
local_feature_count: int | None = None
|
||||
local_size_bytes: int | None = None
|
||||
retained_after_refresh: bool = True
|
||||
action_message: str
|
||||
|
||||
|
||||
class GrbRefreshPlanSummary(BaseModel):
|
||||
layer_count: int
|
||||
current_count: int
|
||||
update_available_count: int
|
||||
not_loaded_count: int
|
||||
review_required_count: int
|
||||
remote_unavailable_count: int
|
||||
new_dataset_count_if_applied: int
|
||||
retained_dataset_count: int
|
||||
current_feature_count: int
|
||||
current_size_bytes: int
|
||||
|
||||
|
||||
class GrbRefreshPlan(BaseModel):
|
||||
project_id: UUID
|
||||
scope: str
|
||||
generated_at: datetime
|
||||
remote_status: str
|
||||
remote_version: str | None = None
|
||||
remote_edition_date: date | None = None
|
||||
catalog_checked_at: datetime | None = None
|
||||
summary: GrbRefreshPlanSummary
|
||||
layers: list[GrbRefreshLayerPlan]
|
||||
execution_mode: Literal["operator_stage_then_apply"] = "operator_stage_then_apply"
|
||||
staging_required: bool = True
|
||||
automatic_import: bool = False
|
||||
destructive_replacement: bool = False
|
||||
message: str
|
||||
limitations: list[str]
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ProviderCapability(BaseModel):
|
||||
provider_name: str
|
||||
display_name: str
|
||||
authority_level: str
|
||||
supported_layers: list[str]
|
||||
supported_geometry_types: list[str]
|
||||
supported_query_modes: list[str]
|
||||
fetch_signature: str
|
||||
configured: bool
|
||||
status: str
|
||||
limitation_message: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
not_configured_reason: str | None = None
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
status: str
|
||||
service: str
|
||||
version: str
|
||||
build_sha: str | None = None
|
||||
build_time: str | None = None
|
||||
database: str | None = None
|
||||
postgis: str | None = None
|
||||
migration: str | None = None
|
||||
storage: str | None = None
|
||||
checks: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SystemCapabilities(BaseModel):
|
||||
postgis: bool
|
||||
rasterio: bool
|
||||
geopandas: bool
|
||||
yolo: bool | str
|
||||
yolo_status: str
|
||||
sam: bool | str
|
||||
grb: str
|
||||
sentinel: str
|
||||
version: str
|
||||
build_sha: str | None = None
|
||||
providers: list[ProviderCapability] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SystemCapabilitiesEnvelope(BaseModel):
|
||||
data: SystemCapabilities
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class JobCreate(BaseModel):
|
||||
job_type: str
|
||||
project_id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
input_dataset_id: UUID | None = None
|
||||
output_dataset_id: UUID | None = None
|
||||
parameters_json: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class JobRead(BaseModel):
|
||||
id: UUID
|
||||
job_type: str
|
||||
status: str
|
||||
project_id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
input_dataset_id: UUID | None = None
|
||||
output_dataset_id: UUID | None = None
|
||||
parameters_json: dict
|
||||
result_json: dict | None = None
|
||||
error_message: str | None = None
|
||||
created_at: datetime | None = None
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class JobStatus(BaseModel):
|
||||
id: UUID
|
||||
status: str
|
||||
error_message: str | None = None
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
result_json: dict | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class JobList(BaseModel):
|
||||
items: list[JobRead]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class OfficialVectorAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class OfficialVectorProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
provider: str
|
||||
source_name: str
|
||||
reference_layer_name: str
|
||||
service_type: str
|
||||
collection: str
|
||||
geometry_types: list[str]
|
||||
source_crs: str
|
||||
source_version: str
|
||||
observation_label: str
|
||||
authority_level: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
coverage_zones: list[str]
|
||||
|
||||
|
||||
class OfficialVectorAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
product_key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
provider: str
|
||||
source_name: str
|
||||
reference_layer_name: str
|
||||
service_type: str
|
||||
collection: str
|
||||
feature_count: int
|
||||
candidate_feature_count: int
|
||||
page_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
source_version: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
@@ -0,0 +1,264 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class VectorOperationResult(BaseModel):
|
||||
feature_count: int
|
||||
geometry_type_summary: dict[str, int]
|
||||
bounds_json: dict | None = None
|
||||
crs: str | None = None
|
||||
source_dataset_id: str
|
||||
|
||||
|
||||
class VectorOperationRequest(BaseModel):
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class VectorClipRequest(VectorOperationRequest):
|
||||
area_id: str
|
||||
|
||||
|
||||
class VectorBufferRequest(VectorOperationRequest):
|
||||
distance_m: float
|
||||
dissolve: bool = False
|
||||
|
||||
|
||||
class VectorIntersectRequest(VectorOperationRequest):
|
||||
other_dataset_id: str
|
||||
|
||||
|
||||
class VectorStatsRequest(BaseModel):
|
||||
pass
|
||||
|
||||
|
||||
class RasterReadyResponse(BaseModel):
|
||||
dataset_id: str
|
||||
ready: bool
|
||||
message: str | None = None
|
||||
|
||||
|
||||
class RasterOperationResult(BaseModel):
|
||||
dataset_id: str
|
||||
ready: bool
|
||||
metadata: dict | None = None
|
||||
output_dataset_id: str | None = None
|
||||
operation: str | None = None
|
||||
|
||||
|
||||
class RasterMetadataResponse(BaseModel):
|
||||
dataset_id: str
|
||||
driver: str | None = None
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
band_count: int | None = None
|
||||
crs: str | None = None
|
||||
bounds: list[float] | None = None
|
||||
resolution: list[float] | None = None
|
||||
dtype: list[str] | None = None
|
||||
nodata: list[float] | float | None = None
|
||||
transform: list[float] | None = None
|
||||
size_bytes: int | None = None
|
||||
checksum_sha256: str | None = None
|
||||
path: str | None = None
|
||||
|
||||
|
||||
class RasterPreviewResponse(BaseModel):
|
||||
dataset_id: str
|
||||
ready: bool
|
||||
preview: dict
|
||||
metadata: dict | None = None
|
||||
|
||||
|
||||
class RasterBandStats(BaseModel):
|
||||
band_index: int
|
||||
dtype: str | None = None
|
||||
min: float | None = None
|
||||
max: float | None = None
|
||||
mean: float | None = None
|
||||
std: float | None = None
|
||||
nodata_count: int
|
||||
nodata_ratio: float
|
||||
valid_pixel_count: int
|
||||
histogram: list[int] | None = None
|
||||
histogram_bins: list[float] | None = None
|
||||
|
||||
|
||||
class RasterStatsResponse(BaseModel):
|
||||
dataset_id: str
|
||||
source_dataset_id: str | None = None
|
||||
bands: list[RasterBandStats]
|
||||
generated_at: str | None = None
|
||||
metadata: dict | None = None
|
||||
|
||||
|
||||
class RasterReprojectRequest(BaseModel):
|
||||
target_crs: str | None = "EPSG:31370"
|
||||
resampling: str = "nearest"
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class RasterClipRequest(BaseModel):
|
||||
area_id: str
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class RasterTileRequest(BaseModel):
|
||||
tile_size: int = 512
|
||||
overlap: int = 64
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class RasterIndexBaseRequest(BaseModel):
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class RasterNdviRequest(RasterIndexBaseRequest):
|
||||
nir_band: int
|
||||
red_band: int
|
||||
|
||||
|
||||
class RasterNdwiRequest(RasterIndexBaseRequest):
|
||||
green_band: int
|
||||
nir_band: int
|
||||
|
||||
|
||||
class RasterNdbiRequest(RasterIndexBaseRequest):
|
||||
swir_band: int
|
||||
nir_band: int
|
||||
|
||||
|
||||
class RasterTileManifestTile(BaseModel):
|
||||
path: str
|
||||
pixel_window: list[int]
|
||||
bounds: list[float]
|
||||
transform: list[float]
|
||||
index: int
|
||||
|
||||
|
||||
class RasterTileManifest(BaseModel):
|
||||
tile_set_id: str
|
||||
source_dataset_id: str
|
||||
source_raster_id: str
|
||||
bounds: list[float]
|
||||
tile_size: int
|
||||
overlap: int
|
||||
parameters: dict[str, str | int | float | bool | None]
|
||||
created_at: str
|
||||
tile_paths: list[str]
|
||||
count: int
|
||||
tiles: list[RasterTileManifestTile]
|
||||
ai_inference: bool = False
|
||||
tile_server: str | None = None
|
||||
|
||||
|
||||
class RasterTileResponse(BaseModel):
|
||||
dataset_id: str
|
||||
ready: bool
|
||||
operation: str
|
||||
tile_set_id: str
|
||||
tile_size: int
|
||||
overlap: int
|
||||
manifest_path: str
|
||||
count: int
|
||||
manifest: RasterTileManifest
|
||||
|
||||
|
||||
class RasterReprojectResponse(BaseModel):
|
||||
dataset_id: str
|
||||
ready: bool
|
||||
operation: str
|
||||
output_dataset_id: str
|
||||
source_dataset_id: str
|
||||
target_dataset_id: str | None = None
|
||||
|
||||
|
||||
class RasterOperationUnavailable(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
|
||||
|
||||
class VectorBBoxResponse(BaseModel):
|
||||
dataset_id: str
|
||||
bounds_json: dict | None
|
||||
feature_count: int
|
||||
crs: str | None = None
|
||||
|
||||
|
||||
class VectorStatsResponse(BaseModel):
|
||||
dataset_id: str
|
||||
feature_count: int
|
||||
geometry_type_summary: dict[str, int]
|
||||
bounds_json: dict | None
|
||||
crs: str | None = None
|
||||
|
||||
|
||||
class VectorSelectionBBox(BaseModel):
|
||||
min_x: float
|
||||
min_y: float
|
||||
max_x: float
|
||||
max_y: float
|
||||
crs: str = "EPSG:4326"
|
||||
|
||||
@field_validator("crs")
|
||||
@classmethod
|
||||
def validate_crs(cls, value: str) -> str:
|
||||
if value.upper() != "EPSG:4326":
|
||||
raise ValueError("Only EPSG:4326 bbox selection is supported")
|
||||
return "EPSG:4326"
|
||||
|
||||
|
||||
class VectorSelectionRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
limit: int = Field(default=100, ge=1, le=1000)
|
||||
|
||||
|
||||
class VectorSelectionDeriveRequest(VectorSelectionRequest):
|
||||
output_name: str | None = None
|
||||
|
||||
|
||||
class VectorSelectionMetric(BaseModel):
|
||||
metric_key: str
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
is_estimate: bool = False
|
||||
warning: str | None = None
|
||||
|
||||
|
||||
class VectorSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
primary_metric_key: str | None = None
|
||||
# ``feature_count`` counts whole features that touch the selection, while
|
||||
# area and length metrics clip to it. These fields say how far the two
|
||||
# populations diverge, so the numbers on one panel can be read together.
|
||||
feature_count: int
|
||||
fully_covered_feature_count: int | None = None
|
||||
partially_covered_feature_count: int | None = None
|
||||
selection_edge_warning: str | None = None
|
||||
is_estimate: bool = False
|
||||
warning: str | None = None
|
||||
metrics: list[VectorSelectionMetric] = Field(default_factory=list)
|
||||
|
||||
|
||||
class VectorSelectionResponse(BaseModel):
|
||||
selection_bbox: VectorSelectionBBox
|
||||
selection_area_id: UUID | None = None
|
||||
feature_count: int
|
||||
total_feature_count: int | None = None
|
||||
limit: int
|
||||
truncated: bool
|
||||
geojson: dict
|
||||
summary: VectorSelectionSummary | None = None
|
||||
partition_count: int | None = None
|
||||
available_partition_count: int | None = None
|
||||
partition_scope_key: str | None = None
|
||||
source_name: str | None = None
|
||||
dataset_ids: list[UUID] | None = None
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class OrthophotoAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str = "most_recent"
|
||||
force_refresh: bool = False
|
||||
resolution_m: float | None = Field(default=None, ge=0.1, le=2.0)
|
||||
|
||||
|
||||
class OrthophotoProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
observation_label: str
|
||||
temporal_granularity: str
|
||||
native_resolution_m: float
|
||||
supports_detection: bool
|
||||
color_mode: str
|
||||
catalog_url: str
|
||||
limitation_message: str
|
||||
provider: str
|
||||
coverage_zone: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
|
||||
|
||||
class OrthophotoAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
observation_label: str
|
||||
temporal_granularity: str
|
||||
supports_detection: bool
|
||||
layer: str
|
||||
width: int
|
||||
height: int
|
||||
resolution_m: float
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg31370: list[float]
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ProjectCreate(BaseModel):
|
||||
name: str
|
||||
description: str | None = None
|
||||
region: str | None = "Belgium and Belgian North Sea"
|
||||
|
||||
|
||||
class ProjectUpdate(BaseModel):
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
region: str | None = None
|
||||
status: Literal["active", "archived"] | None = None
|
||||
|
||||
|
||||
class ProjectRead(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
description: str | None = None
|
||||
region: str
|
||||
status: str
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ProjectListItem(ProjectRead):
|
||||
pass
|
||||
|
||||
|
||||
class ProjectList(BaseModel):
|
||||
items: list[ProjectRead]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
class ProjectDeleteResult(BaseModel):
|
||||
deleted: bool
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.common import GeoJsonFeatureCollection
|
||||
|
||||
|
||||
class QaProviderComparisonRequest(BaseModel):
|
||||
candidate_dataset_id: UUID
|
||||
reference_dataset_id: UUID
|
||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
area_id: UUID | None = None
|
||||
|
||||
|
||||
class QaProviderComparisonResult(BaseModel):
|
||||
status: str
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
# Counts of the population that was actually matched, so that
|
||||
# ``matches + false_positives == candidate_feature_count`` holds even when
|
||||
# an area filter or an unparseable geometry removed features. The ``_raw``
|
||||
# fields keep the untouched dataset totals visible next to them.
|
||||
candidate_feature_count: int
|
||||
reference_feature_count: int
|
||||
candidate_feature_count_raw: int | None = None
|
||||
reference_feature_count_raw: int | None = None
|
||||
matches: int
|
||||
false_positives: int
|
||||
false_negatives: int
|
||||
precision: float | None
|
||||
recall: float | None
|
||||
f1_score: float | None
|
||||
mean_iou: float | None
|
||||
iou_threshold: float
|
||||
unsupported_geometry: bool = False
|
||||
unsupported_geometries: list[str] = Field(default_factory=list)
|
||||
match_evidence: list[dict] = Field(default_factory=list)
|
||||
false_positive_evidence: list[dict] = Field(default_factory=list)
|
||||
false_negative_evidence: list[dict] = Field(default_factory=list)
|
||||
generated_at: datetime
|
||||
|
||||
|
||||
class MetricRead(BaseModel):
|
||||
id: UUID
|
||||
quality_check_id: UUID | None = None
|
||||
analysis_run_id: UUID | None = None
|
||||
metric_key: str
|
||||
metric_value: float | None = None
|
||||
metric_unit: str | None = None
|
||||
label: str | None = None
|
||||
metadata_json: dict | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class QualityCheckRead(BaseModel):
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
job_id: UUID | None = None
|
||||
analysis_run_id: UUID | None = None
|
||||
candidate_dataset_id: UUID | None = None
|
||||
reference_dataset_id: UUID
|
||||
check_type: str
|
||||
status: str
|
||||
score: float | None = None
|
||||
parameters_json: dict | None = None
|
||||
findings_json: dict | None = None
|
||||
created_at: datetime | None = None
|
||||
completed_at: datetime | None = None
|
||||
metrics: list[MetricRead] = Field(default_factory=list)
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class QualityCheckList(BaseModel):
|
||||
items: list[QualityCheckRead]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
class QualityEvidenceResponse(BaseModel):
|
||||
quality_check_id: UUID
|
||||
project_id: UUID
|
||||
candidate_dataset_id: UUID | None = None
|
||||
reference_dataset_id: UUID
|
||||
analysis_run_id: UUID | None = None
|
||||
# The overlay is capped so a regional check stays reviewable; the counts in
|
||||
# the quality check itself are always complete.
|
||||
feature_count: int
|
||||
total_feature_count: int | None = None
|
||||
role_counts: dict[str, int] = Field(default_factory=dict)
|
||||
truncated: bool = False
|
||||
limit: int | None = None
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
geojson: GeoJsonFeatureCollection
|
||||
|
||||
|
||||
class AnalysisQaResponse(BaseModel):
|
||||
status: str
|
||||
quality_check_id: UUID
|
||||
analysis_run_id: UUID
|
||||
reference_dataset_id: UUID
|
||||
candidate_feature_count: int
|
||||
reference_feature_count: int
|
||||
candidate_feature_count_raw: int | None = None
|
||||
reference_feature_count_raw: int | None = None
|
||||
matches: int
|
||||
false_positives: int
|
||||
false_negatives: int
|
||||
precision: float | None = None
|
||||
recall: float | None = None
|
||||
f1_score: float | None = None
|
||||
mean_iou: float | None = None
|
||||
iou_threshold: float
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
coverage: dict[str, Any] | None = None
|
||||
temporal_compatibility: dict[str, Any] | None = None
|
||||
box_to_footprint_diagnostics: dict[str, Any] | None = None
|
||||
precision_recall_curve: dict[str, Any] | None = None
|
||||
calibration_sweep: list[dict[str, Any]] = Field(default_factory=list)
|
||||
match_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
||||
false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
||||
false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
||||
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.schemas.detection import DetectionModelCapability
|
||||
|
||||
|
||||
SegmentationModelCapability = DetectionModelCapability
|
||||
|
||||
|
||||
class SegmentationModelsResponse(BaseModel):
|
||||
models: list[SegmentationModelCapability]
|
||||
|
||||
|
||||
class SegmentationRunRequest(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
project_id: UUID
|
||||
dataset_id: UUID
|
||||
model_id: str
|
||||
confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
class_filter: list[str] | None = None
|
||||
tile_manifest_path: str | None = None
|
||||
parameters_json: dict = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SegmentationQaRequest(BaseModel):
|
||||
reference_dataset_id: UUID
|
||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
class_name: str | None = None
|
||||
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
# Read off the one matching pass, exactly as for detection.
|
||||
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class SegmentationRunResponse(BaseModel):
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
analysis_run_id: UUID
|
||||
job_id: UUID
|
||||
project_id: UUID
|
||||
dataset_id: UUID
|
||||
model_id: str
|
||||
status: str
|
||||
segmentation_count: int
|
||||
error_code: str | None = None
|
||||
message: str
|
||||
|
||||
|
||||
class SegmentationRunRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
|
||||
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
job_id: UUID | None = None
|
||||
analysis_type: str
|
||||
status: str
|
||||
model_name: str | None = None
|
||||
model_version: str | None = None
|
||||
parameters_json: dict
|
||||
result_json: dict | None = None
|
||||
error_message: str | None = None
|
||||
created_at: datetime | None = None
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
|
||||
|
||||
class SegmentationRunListResponse(BaseModel):
|
||||
items: list[SegmentationRunRead]
|
||||
# ``total`` counts every run; ``items`` is the most recent page of them.
|
||||
total: int
|
||||
limit: int | None = None
|
||||
offset: int = 0
|
||||
truncated: bool = False
|
||||
|
||||
|
||||
class SegmentationRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
|
||||
|
||||
id: UUID
|
||||
project_id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
analysis_run_id: UUID | None = None
|
||||
job_id: UUID | None = None
|
||||
model_name: str
|
||||
model_version: str | None = None
|
||||
class_name: str
|
||||
confidence: float | None = None
|
||||
bbox_json: dict | None = None
|
||||
area_m2: float | None = None
|
||||
mask_path: str | None = None
|
||||
source_tile_path: str | None = None
|
||||
tile_index: int | None = None
|
||||
properties_json: dict | None = None
|
||||
provenance_json: dict | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
|
||||
class SegmentationListResponse(BaseModel):
|
||||
items: list[SegmentationRead]
|
||||
# ``total`` describes the complete filtered population; ``items`` is one
|
||||
# stable confidence-ranked page of it.
|
||||
total: int
|
||||
limit: int | None = None
|
||||
offset: int = 0
|
||||
truncated: bool = False
|
||||
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class VectorPartitionSelectionRequest(BaseModel):
|
||||
dataset_ids: list[UUID] = Field(min_length=1, max_length=4096)
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
limit: int = Field(default=1000, ge=1, le=1000)
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
SourceCatalogProbeStatus = Literal["available", "degraded", "unavailable", "disabled"]
|
||||
SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "no_local_data", "unavailable"]
|
||||
|
||||
|
||||
class SourceCatalogProbeItem(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
service_type: Literal["WFS", "WMS", "HTML", "DCAT"]
|
||||
endpoint_url: str
|
||||
status: SourceCatalogProbeStatus
|
||||
reachable: bool
|
||||
checked_at: datetime
|
||||
cached: bool = False
|
||||
expected_layers: list[str]
|
||||
matched_layers: list[str]
|
||||
missing_layers: list[str]
|
||||
advertised_layer_count: int
|
||||
metadata_url: str | None = None
|
||||
metadata_identifier: str | None = None
|
||||
remote_title: str | None = None
|
||||
remote_version: str | None = None
|
||||
remote_modified_at: datetime | None = None
|
||||
remote_published_at: datetime | None = None
|
||||
local_source_version: str | None = None
|
||||
comparison_status: SourceCatalogComparisonStatus
|
||||
capabilities_sha256: str | None = None
|
||||
capabilities_etag: str | None = None
|
||||
capabilities_last_modified_at: datetime | None = None
|
||||
message: str
|
||||
error_code: str | None = None
|
||||
|
||||
|
||||
class SourceCatalogProbeSummary(BaseModel):
|
||||
provider_count: int
|
||||
available_count: int
|
||||
degraded_count: int
|
||||
unavailable_count: int
|
||||
disabled_count: int
|
||||
different_version_count: int
|
||||
|
||||
|
||||
class SourceCatalogProbeReport(BaseModel):
|
||||
project_id: UUID
|
||||
generated_at: datetime
|
||||
summary: SourceCatalogProbeSummary
|
||||
items: list[SourceCatalogProbeItem]
|
||||
limitations: list[str]
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
SourceFreshnessStatus = Literal["current", "due", "review_required", "local"]
|
||||
SourceRefreshPolicy = Literal["rolling_snapshot", "annual_release", "edition", "scenario", "archive", "local"]
|
||||
|
||||
|
||||
class SourceIntegritySummary(BaseModel):
|
||||
missing_version_count: int = 0
|
||||
checksum_mismatch_count: int = 0
|
||||
missing_storage_file_count: int = 0
|
||||
size_mismatch_count: int = 0
|
||||
|
||||
@property
|
||||
def issue_count(self) -> int:
|
||||
return (
|
||||
self.missing_version_count
|
||||
+ self.checksum_mismatch_count
|
||||
+ self.missing_storage_file_count
|
||||
+ self.size_mismatch_count
|
||||
)
|
||||
|
||||
|
||||
class SourceFreshnessItem(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
dataset_count: int
|
||||
ready_count: int
|
||||
version_count: int
|
||||
latest_imported_at: datetime | None = None
|
||||
latest_observed_at: datetime | None = None
|
||||
latest_source_version: str | None = None
|
||||
refresh_policy: SourceRefreshPolicy
|
||||
review_interval_days: int | None = None
|
||||
next_review_at: datetime | None = None
|
||||
status: SourceFreshnessStatus
|
||||
historical_series: bool
|
||||
auto_refresh_supported: bool = False
|
||||
reason: str
|
||||
recommended_action: str
|
||||
integrity: SourceIntegritySummary
|
||||
|
||||
|
||||
class SourceFreshnessSummary(BaseModel):
|
||||
source_count: int
|
||||
dataset_count: int
|
||||
current_count: int
|
||||
due_count: int
|
||||
review_required_count: int
|
||||
local_count: int
|
||||
sources_with_integrity_issues: int
|
||||
integrity_issue_count: int
|
||||
|
||||
|
||||
class SourceFreshnessReport(BaseModel):
|
||||
project_id: UUID
|
||||
generated_at: datetime
|
||||
summary: SourceFreshnessSummary
|
||||
items: list[SourceFreshnessItem]
|
||||
limitations: list[str]
|
||||
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SourceRegistryRead(BaseModel):
|
||||
"""Read-only, server-owned source-authority definition."""
|
||||
|
||||
id: UUID
|
||||
source_key: str
|
||||
display_name: str
|
||||
classification: str
|
||||
authority_name: str
|
||||
authority_scope_json: dict
|
||||
provider_adapter_key: str | None = None
|
||||
source_url: str | None = None
|
||||
license_name: str
|
||||
license_url: str | None = None
|
||||
usage_restrictions: str
|
||||
default_crs: str
|
||||
default_units: str
|
||||
spatial_resolution_json: dict
|
||||
temporal_coverage_json: dict
|
||||
geographic_coverage_json: dict
|
||||
expected_geometry_types_json: list
|
||||
expected_attributes_json: dict
|
||||
usage_policy_json: dict
|
||||
freshness_status: str
|
||||
ingest_status: str
|
||||
known_limitations_json: list
|
||||
registry_metadata_json: dict
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
snapshot_count: int = 0
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class SourceSnapshotRead(BaseModel):
|
||||
"""Immutable version/snapshot evidence attached to an imported dataset."""
|
||||
|
||||
id: UUID
|
||||
source_registry_id: UUID
|
||||
snapshot_key: str
|
||||
source_version: str | None = None
|
||||
snapshot_at: datetime | None = None
|
||||
fetched_at: datetime | None = None
|
||||
source_url: str | None = None
|
||||
checksum_sha256: str | None = None
|
||||
crs: str | None = None
|
||||
units: str | None = None
|
||||
spatial_resolution_json: dict
|
||||
temporal_coverage_json: dict
|
||||
geographic_coverage_json: dict
|
||||
observed_schema_json: dict
|
||||
freshness_status: str
|
||||
ingest_status: str
|
||||
known_limitations_json: list
|
||||
snapshot_metadata_json: dict
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class SourceRegistryDetailRead(BaseModel):
|
||||
source: SourceRegistryRead
|
||||
snapshots: list[SourceSnapshotRead]
|
||||
|
||||
|
||||
class DatasetLineageEdgeRead(BaseModel):
|
||||
id: UUID
|
||||
parent_dataset_id: UUID
|
||||
child_dataset_id: UUID
|
||||
parent_dataset_version_id: UUID | None = None
|
||||
child_dataset_version_id: UUID | None = None
|
||||
relation_type: str
|
||||
transformation_name: str
|
||||
transformation_version: str | None = None
|
||||
parameters_json: dict | None = None
|
||||
input_checksum_sha256: str | None = None
|
||||
output_checksum_sha256: str | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class DatasetQuarantineRead(BaseModel):
|
||||
id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
dataset_version_id: UUID | None = None
|
||||
source_snapshot_id: UUID | None = None
|
||||
stage: str
|
||||
reason_code: str
|
||||
details_json: dict | None = None
|
||||
artifact_path: str | None = None
|
||||
artifact_checksum_sha256: str | None = None
|
||||
status: str
|
||||
created_at: datetime | None = None
|
||||
resolved_at: datetime | None = None
|
||||
resolved_by: str | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class DatasetProvenanceRead(BaseModel):
|
||||
dataset_id: UUID
|
||||
source: SourceRegistryRead | None = None
|
||||
snapshot: SourceSnapshotRead | None = None
|
||||
data_contract_key: str | None = None
|
||||
data_contract_version: str | None = None
|
||||
validation_status: str | None = None
|
||||
validation_report_json: dict | None = None
|
||||
provenance_status: str | None = None
|
||||
lineage_status: str | None = None
|
||||
quarantine_status: str | None = None
|
||||
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
|
||||
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
|
||||
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class SpwTerrainAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str = "spw_mnt_1m_2021_2022"
|
||||
resolution_m: float | None = Field(default=None, ge=1.0, le=10.0)
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class SpwTerrainProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
surface_model: str
|
||||
source_filename: str
|
||||
native_resolution_m: float
|
||||
analysis_resolution_m: float
|
||||
source_crs: str
|
||||
vertical_reference: str
|
||||
acquisition_period: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
coverage_zones: list[str]
|
||||
configured: bool
|
||||
status: str
|
||||
|
||||
|
||||
class SpwTerrainAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
surface_model: str
|
||||
native_resolution_m: float
|
||||
resolution_m: float
|
||||
width: int
|
||||
height: int
|
||||
valid_pixel_count: int
|
||||
nodata_value: float
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg3812: list[float]
|
||||
vertical_reference: str
|
||||
acquisition_period: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class TemporalComparisonRequest(BaseModel):
|
||||
earlier_dataset_id: UUID
|
||||
later_dataset_id: UUID
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
preview_limit: int = Field(default=500, ge=1, le=1000)
|
||||
|
||||
|
||||
class TemporalDatasetRef(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
observed_at: datetime
|
||||
source_version: str | None = None
|
||||
|
||||
|
||||
class TemporalMetricComparison(BaseModel):
|
||||
metric_key: str = "primary"
|
||||
label: str
|
||||
unit: str
|
||||
aggregation_method: str
|
||||
earlier_value: float
|
||||
later_value: float
|
||||
absolute_change: float
|
||||
percent_change: float | None = None
|
||||
is_estimate: bool = False
|
||||
warning: str | None = None
|
||||
|
||||
|
||||
class TemporalObservationMetric(BaseModel):
|
||||
metric_key: str
|
||||
label: str
|
||||
value: float
|
||||
unit: str
|
||||
aggregation_method: str
|
||||
is_estimate: bool = False
|
||||
|
||||
|
||||
class TemporalObservation(BaseModel):
|
||||
dataset: TemporalDatasetRef
|
||||
metrics: list[TemporalObservationMetric]
|
||||
|
||||
|
||||
class TemporalObjectChanges(BaseModel):
|
||||
available: bool
|
||||
added_count: int | None = None
|
||||
removed_count: int | None = None
|
||||
modified_count: int | None = None
|
||||
unchanged_count: int | None = None
|
||||
|
||||
|
||||
class TemporalComparisonResponse(BaseModel):
|
||||
temporal_series_key: str
|
||||
earlier: TemporalDatasetRef
|
||||
later: TemporalDatasetRef
|
||||
selection_bbox: VectorSelectionBBox
|
||||
selection_area_id: UUID | None = None
|
||||
metric: TemporalMetricComparison
|
||||
metrics: list[TemporalMetricComparison] = Field(default_factory=list)
|
||||
timeline: list[TemporalObservation] = Field(default_factory=list)
|
||||
object_changes: TemporalObjectChanges
|
||||
geojson: dict
|
||||
warnings: list[str]
|
||||
generated_at: datetime
|
||||
|
||||
|
||||
class TemporalSeriesDataset(BaseModel):
|
||||
id: UUID
|
||||
name: str
|
||||
observed_at: datetime
|
||||
source_version: str | None = None
|
||||
feature_count: int | None = None
|
||||
|
||||
|
||||
class TemporalSeriesRead(BaseModel):
|
||||
temporal_series_key: str
|
||||
source_name: str | None = None
|
||||
reference_layer_name: str | None = None
|
||||
dataset_count: int
|
||||
first_observed_at: datetime
|
||||
last_observed_at: datetime
|
||||
datasets: list[TemporalSeriesDataset]
|
||||
@@ -0,0 +1,125 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class ThematicRasterAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class ThematicRasterProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
coverage_id: str
|
||||
native_resolution_m: float
|
||||
source_crs: str
|
||||
source_value_unit: str
|
||||
observation_year: int
|
||||
source_version: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
legend_min_label: str
|
||||
legend_max_label: str
|
||||
included_source_values: list[int]
|
||||
limitation_message: str
|
||||
analysis_resolution_m: float | None = None
|
||||
coverage_zones: list[str] = Field(default_factory=list)
|
||||
configured: bool = True
|
||||
status: str = "configured"
|
||||
|
||||
|
||||
class WalousAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
resolution_m: float
|
||||
width: int
|
||||
height: int
|
||||
valid_pixel_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg3812: list[float]
|
||||
observation_year: int
|
||||
source_value_unit: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class ThematicRasterAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
coverage_id: str
|
||||
resolution_m: float
|
||||
width: int
|
||||
height: int
|
||||
valid_pixel_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg31370: list[float]
|
||||
observation_year: int
|
||||
source_value_unit: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class ThematicRasterSelectionRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
|
||||
|
||||
class ThematicRasterMetric(BaseModel):
|
||||
metric_key: str
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
derived: bool = True
|
||||
is_estimate: bool = True
|
||||
|
||||
|
||||
class ThematicRasterSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
primary_metric_key: str
|
||||
metrics: list[ThematicRasterMetric]
|
||||
|
||||
|
||||
class ThematicRasterSelectionResponse(BaseModel):
|
||||
dataset_id: UUID
|
||||
product_key: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
selection_bbox: VectorSelectionBBox
|
||||
selection_area_id: UUID | None = None
|
||||
selected_cell_count: int
|
||||
valid_cell_count: int
|
||||
coverage_ratio: float
|
||||
# Set when the drawn selection is smaller than one source cell and the
|
||||
# analysis was widened to the cells it touches, so the value covers more
|
||||
# ground than was requested.
|
||||
cell_selection_warning: str | None = None
|
||||
resolution_m: float
|
||||
observation_year: int
|
||||
summary: ThematicRasterSelectionSummary
|
||||
unsupported_metrics: list[str]
|
||||
limitation_message: str
|
||||
generated_at: str
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Background execution for queued analysis runs.
|
||||
|
||||
Tiled GPU inference is minutes of work. Running it inside the HTTP request
|
||||
holds a worker thread for the whole duration, times the client out and leaves
|
||||
the operator without progress. Queued ``detection.run`` and
|
||||
``segmentation.run`` jobs are picked up here instead, mirroring the polling
|
||||
worker the AOI operations already use so the runtime keeps one job model.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import Job
|
||||
|
||||
logger = logging.getLogger("geointel.analysis_worker")
|
||||
|
||||
|
||||
class AnalysisJobWorker:
|
||||
HANDLED_JOB_TYPES = ("detection.run", "segmentation.run")
|
||||
BATCH_SIZE = 4
|
||||
|
||||
@staticmethod
|
||||
def _uuid(value: Any) -> UUID | None:
|
||||
if isinstance(value, UUID):
|
||||
return value
|
||||
try:
|
||||
return UUID(str(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _dispatch(db, job: Job) -> Any:
|
||||
# Imported lazily: both services import each other's helpers, and the
|
||||
# worker must not add a third edge to that cycle at module load.
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
|
||||
parameters = job.parameters_json if isinstance(job.parameters_json, dict) else {}
|
||||
project_id = AnalysisJobWorker._uuid(parameters.get("project_id"))
|
||||
dataset_id = AnalysisJobWorker._uuid(parameters.get("dataset_id"))
|
||||
if project_id is None or dataset_id is None:
|
||||
raise ValueError("Queued analysis job is missing project_id or dataset_id")
|
||||
|
||||
common = {
|
||||
"db": db,
|
||||
"project_id": project_id,
|
||||
"dataset_id": dataset_id,
|
||||
"model_id": parameters.get("model_id"),
|
||||
"confidence_threshold": float(parameters.get("confidence_threshold") or 0.0),
|
||||
"class_filter": parameters.get("class_filter") or [],
|
||||
"tile_manifest_path": parameters.get("tile_manifest_path"),
|
||||
"parameters_json": parameters.get("parameters_json") or {},
|
||||
"existing_job": job,
|
||||
}
|
||||
if job.job_type == "detection.run":
|
||||
return DetectionService.run_detection(
|
||||
model_asset_id=parameters.get("model_asset_id"),
|
||||
**common,
|
||||
)
|
||||
return SegmentationService.run_segmentation(**common)
|
||||
|
||||
@staticmethod
|
||||
def claim(db, job: Job) -> bool:
|
||||
"""Take the job out of the queue, atomically. Returns whether we won.
|
||||
|
||||
Selecting and then updating in a second statement lets two workers —
|
||||
a restarted process overlapping the previous one, or a second replica —
|
||||
both start tiled GPU inference on the same row. The conditional update
|
||||
makes exactly one caller see a row count of 1; the AOI worker beside
|
||||
this one already claims with FOR UPDATE SKIP LOCKED for the same reason.
|
||||
"""
|
||||
|
||||
claimed = (
|
||||
db.query(Job)
|
||||
.filter(Job.id == job.id, Job.status == "queued")
|
||||
.update({Job.status: "running"}, synchronize_session=False)
|
||||
)
|
||||
db.commit()
|
||||
if not claimed:
|
||||
return False
|
||||
job.status = "running"
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _finalize(db, job: Job, result: Any) -> None:
|
||||
"""Close a job the handler left open.
|
||||
|
||||
The analysis services normally set the terminal status themselves.
|
||||
If one returns without doing so, recording the outcome here is what
|
||||
keeps the job from sitting in "running" for ever.
|
||||
"""
|
||||
|
||||
if job.status != "running":
|
||||
return
|
||||
status = getattr(result, "status", None)
|
||||
if status == "success":
|
||||
job.status = "success"
|
||||
job.result_json = {
|
||||
"detection_count": getattr(result, "detection_count", None),
|
||||
"segmentation_count": getattr(result, "segmentation_count", None),
|
||||
}
|
||||
else:
|
||||
job.status = "failed"
|
||||
job.error_message = getattr(result, "message", None) or "Analysis run did not complete"
|
||||
job.result_json = {"error_code": getattr(result, "error_code", None) or "ANALYSIS_JOB_INCOMPLETE"}
|
||||
db.add(job)
|
||||
db.commit()
|
||||
|
||||
@staticmethod
|
||||
def _mark_failed(db, job: Job, *, code: str, message: str) -> None:
|
||||
try:
|
||||
db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
job.status = "failed"
|
||||
job.error_message = message
|
||||
job.result_json = {"error_code": code, "message": message}
|
||||
db.add(job)
|
||||
db.commit()
|
||||
|
||||
@staticmethod
|
||||
def run_once(db=None) -> int:
|
||||
"""Execute one batch of queued analysis jobs. Returns the batch size."""
|
||||
|
||||
owns_session = db is None
|
||||
session = db if db is not None else SessionLocal()
|
||||
try:
|
||||
rows = [
|
||||
job
|
||||
for job in (
|
||||
session.query(Job)
|
||||
.filter(Job.status == "queued")
|
||||
.filter(Job.job_type.in_(AnalysisJobWorker.HANDLED_JOB_TYPES))
|
||||
.order_by(Job.created_at)
|
||||
.limit(AnalysisJobWorker.BATCH_SIZE)
|
||||
.all()
|
||||
)
|
||||
if job.job_type in AnalysisJobWorker.HANDLED_JOB_TYPES and job.status == "queued"
|
||||
]
|
||||
claimed_count = 0
|
||||
for job in rows:
|
||||
if not AnalysisJobWorker.claim(session, job):
|
||||
# Another worker took it between the select and the claim.
|
||||
continue
|
||||
claimed_count += 1
|
||||
try:
|
||||
result = AnalysisJobWorker._dispatch(session, job)
|
||||
AnalysisJobWorker._finalize(session, job, result)
|
||||
except Exception as exc:
|
||||
code = getattr(exc, "code", None) or "ANALYSIS_JOB_INTERNAL_ERROR"
|
||||
message = getattr(exc, "message", None) or str(exc) or "Unexpected analysis job failure"
|
||||
AnalysisJobWorker._mark_failed(session, job, code=str(code), message=str(message))
|
||||
logger.exception("Analysis job failed job_id=%s job_type=%s", job.id, job.job_type)
|
||||
return claimed_count
|
||||
finally:
|
||||
if owns_session:
|
||||
session.close()
|
||||
|
||||
@staticmethod
|
||||
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
|
||||
while not stop_event.is_set():
|
||||
processed = await asyncio.to_thread(AnalysisJobWorker.run_once)
|
||||
if processed == 0:
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
@@ -0,0 +1,252 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import AoiOperation, AoiOperationPartition
|
||||
from app.schemas.grb import GrbAcquireRequest
|
||||
from app.schemas.dhmv import DhmvAcquireRequest
|
||||
from app.schemas.spw_terrain import SpwTerrainAcquireRequest
|
||||
from app.schemas.official_vector import OfficialVectorAcquireRequest
|
||||
from app.schemas.flood_hazard import FloodHazardAcquireRequest
|
||||
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
|
||||
from app.schemas.bathymetry import (
|
||||
BathymetryProfileAcquireRequest,
|
||||
MdkBathymetryAcquireRequest,
|
||||
)
|
||||
from app.schemas.job import JobCreate
|
||||
from app.schemas.operations import VectorSelectionBBox
|
||||
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||
from app.services.aoi_operation_service import AoiOperationService
|
||||
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
from app.services.spw_terrain_service import SpwTerrainService
|
||||
from app.services.official_vector_acquisition_service import (
|
||||
OfficialVectorAcquisitionService,
|
||||
)
|
||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||
from app.services.thematic_raster_acquisition_service import (
|
||||
ThematicRasterAcquisitionService,
|
||||
)
|
||||
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||
from app.services.bathymetry_profile_acquisition_service import (
|
||||
BathymetryProfileAcquisitionService,
|
||||
)
|
||||
from app.services.mdk_bathymetry_acquisition_service import (
|
||||
MdkBathymetryAcquisitionService,
|
||||
)
|
||||
from app.services.job_service import JobService
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
|
||||
|
||||
class AoiOperationExecutor:
|
||||
"""Execute one bounded partition through an existing governed provider."""
|
||||
|
||||
@staticmethod
|
||||
def execute_next(db, project_id: UUID, operation_id: UUID) -> dict:
|
||||
partition = AoiOperationService.claim_next(db, project_id, operation_id)
|
||||
if partition is None:
|
||||
AoiOperationService._refresh_parent(db, operation_id)
|
||||
return AoiOperationService.read(db, project_id, operation_id)
|
||||
operation = db.get(AoiOperation, operation_id)
|
||||
child = JobService.create_job(
|
||||
db,
|
||||
JobCreate(
|
||||
job_type=f"aoi.{operation.operation_type}.partition",
|
||||
project_id=project_id,
|
||||
parameters_json={
|
||||
"aoi_operation_id": str(operation_id),
|
||||
"partition_id": str(partition.id),
|
||||
"partition_key": partition.partition_key,
|
||||
"provider_key": partition.provider_key,
|
||||
"product_key": partition.product_key,
|
||||
},
|
||||
),
|
||||
)
|
||||
partition = db.get(AoiOperationPartition, partition.id)
|
||||
partition.child_job_id = child.id
|
||||
db.add(partition)
|
||||
db.commit()
|
||||
JobService.mark_running(db, child.id)
|
||||
try:
|
||||
result = AoiOperationExecutor._dispatch(
|
||||
db, project_id, operation, partition
|
||||
)
|
||||
output_id = (
|
||||
result.get("output_dataset_id") if isinstance(result, dict) else None
|
||||
)
|
||||
JobService.mark_success(
|
||||
db,
|
||||
child.id,
|
||||
result=result,
|
||||
output_dataset_id=UUID(str(output_id)) if output_id else None,
|
||||
)
|
||||
return AoiOperationService.complete(
|
||||
db, project_id, operation_id, partition.id, result
|
||||
)
|
||||
except AppError as exc:
|
||||
JobService.mark_failed(
|
||||
db, child.id, exc.message, {"code": exc.code, "details": exc.details}
|
||||
)
|
||||
return AoiOperationService.fail(
|
||||
db,
|
||||
project_id,
|
||||
operation_id,
|
||||
partition.id,
|
||||
exc.message,
|
||||
AoiOperationExecutor._retryable(exc),
|
||||
{"code": exc.code, "details": exc.details},
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
db.rollback()
|
||||
JobService.mark_failed(
|
||||
db,
|
||||
child.id,
|
||||
"Unexpected partition execution error",
|
||||
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
|
||||
)
|
||||
finally:
|
||||
AoiOperationService.fail(
|
||||
db,
|
||||
project_id,
|
||||
operation_id,
|
||||
partition.id,
|
||||
"Unexpected partition execution error",
|
||||
True,
|
||||
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
|
||||
)
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _dispatch(
|
||||
db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition
|
||||
) -> dict:
|
||||
geometry = to_shape(partition.geometry)
|
||||
min_x, min_y, max_x, max_y = geometry.bounds
|
||||
bbox = VectorSelectionBBox(
|
||||
min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326"
|
||||
)
|
||||
force_refresh = bool(
|
||||
(operation.request_json or {})
|
||||
.get("parameters_json", {})
|
||||
.get("force_refresh", False)
|
||||
)
|
||||
if partition.provider_key == "grb":
|
||||
return GrbAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
GrbAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "orthophoto":
|
||||
return OrthophotoAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
OrthophotoAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "dhmv":
|
||||
return DhmvAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
DhmvAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "spw_terrain":
|
||||
return SpwTerrainService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
SpwTerrainAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "official_vector":
|
||||
return OfficialVectorAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
OfficialVectorAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "flood_hazard":
|
||||
return FloodHazardAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
FloodHazardAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "thematic_raster":
|
||||
return ThematicRasterAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
ThematicRasterAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "walous":
|
||||
return WalousLandCoverService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
ThematicRasterAcquireRequest(
|
||||
bbox=bbox,
|
||||
area_id=operation.area_id,
|
||||
product_key=partition.product_key,
|
||||
force_refresh=force_refresh,
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "bathymetry_profiles":
|
||||
return BathymetryProfileAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
BathymetryProfileAcquireRequest(
|
||||
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
|
||||
),
|
||||
)
|
||||
if partition.provider_key == "mdk_bathymetry":
|
||||
return MdkBathymetryAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
MdkBathymetryAcquireRequest(
|
||||
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
|
||||
),
|
||||
)
|
||||
raise AppError(
|
||||
code="AOI_PROVIDER_UNSUPPORTED",
|
||||
message="No governed AOI executor is registered for this provider",
|
||||
details={"provider_key": partition.provider_key},
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _retryable(error: AppError) -> bool:
|
||||
return error.status_code >= 500 or error.code.endswith(
|
||||
("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")
|
||||
)
|
||||
@@ -0,0 +1,481 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import math
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from geoalchemy2.shape import from_shape, to_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import MultiPolygon, Polygon, box
|
||||
from shapely.ops import transform
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.core.config import get_settings
|
||||
from app.models import AoiOperation, AoiOperationPartition, Area, Project
|
||||
from app.schemas.aoi_operation import AoiOperationCreate
|
||||
|
||||
|
||||
class AoiOperationService:
|
||||
MAX_PARTITIONS = 4096
|
||||
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
SCOPE_AREA_NAMES = {
|
||||
"belgium": "Belgium land",
|
||||
"flanders": "Flanders",
|
||||
"wallonia": "Wallonia",
|
||||
"brussels": "Brussels-Capital Region",
|
||||
"belgian_north_sea": "Belgian part of the North Sea",
|
||||
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
||||
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
||||
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
|
||||
if db.get(Project, project_id) is None:
|
||||
raise AppError(
|
||||
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
|
||||
)
|
||||
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
|
||||
if payload.coverage_zone:
|
||||
geometry = AoiOperationService._clip_to_zone(
|
||||
db, project_id, geometry, payload.coverage_zone
|
||||
)
|
||||
geometry = AoiOperationService._as_multipolygon(geometry)
|
||||
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
|
||||
partition_side_m = AoiOperationService._partition_side(
|
||||
payload.provider_key, payload.max_partition_side_m
|
||||
)
|
||||
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
|
||||
operation_id = uuid4()
|
||||
now = datetime.now(timezone.utc)
|
||||
operation = AoiOperation(
|
||||
id=operation_id,
|
||||
project_id=project_id,
|
||||
area_id=payload.area_id,
|
||||
operation_type=payload.operation_type,
|
||||
status="queued",
|
||||
geometry=from_shape(geometry, srid=4326),
|
||||
request_json=payload.model_dump(mode="json", exclude_none=True),
|
||||
plan_json={
|
||||
"partition_strategy": "epsg31370_square_grid_intersection_v1",
|
||||
"max_partition_side_m": partition_side_m,
|
||||
"budget_source": "governed_provider_registry"
|
||||
if payload.max_partition_side_m is None
|
||||
else "stricter_operator_override",
|
||||
"partition_count": len(cells),
|
||||
"provider_key": payload.provider_key,
|
||||
"product_key": payload.product_key,
|
||||
},
|
||||
created_at=now,
|
||||
)
|
||||
db.add(operation)
|
||||
for ordinal, cell in enumerate(cells):
|
||||
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
|
||||
wgs84 = AoiOperationService._as_multipolygon(wgs84)
|
||||
digest = sha256(wgs84.wkb).hexdigest()[:20]
|
||||
db.add(
|
||||
AoiOperationPartition(
|
||||
id=uuid4(),
|
||||
operation_id=operation_id,
|
||||
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
|
||||
provider_key=payload.provider_key,
|
||||
product_key=payload.product_key,
|
||||
ordinal=ordinal,
|
||||
status="queued",
|
||||
geometry=from_shape(wgs84, srid=4326),
|
||||
attempt_count=0,
|
||||
max_attempts=payload.max_attempts,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return AoiOperationService.read(db, project_id, operation_id)
|
||||
|
||||
@staticmethod
|
||||
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
|
||||
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
|
||||
if area_name is None:
|
||||
raise AppError(
|
||||
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
|
||||
message="Unknown governed coverage zone",
|
||||
details={"coverage_zone": zone},
|
||||
status_code=422,
|
||||
)
|
||||
scope = (
|
||||
db.query(Area)
|
||||
.filter(Area.project_id == project_id, Area.name == area_name)
|
||||
.first()
|
||||
)
|
||||
if scope is None:
|
||||
raise AppError(
|
||||
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
|
||||
message="The governed coverage-zone geometry is not persisted in this project",
|
||||
details={"coverage_zone": zone},
|
||||
status_code=409,
|
||||
)
|
||||
clipped = geometry.intersection(to_shape(scope.geometry))
|
||||
if clipped.is_empty:
|
||||
raise AppError(
|
||||
code="AOI_OUTSIDE_PROVIDER_ZONE",
|
||||
message="The AOI does not intersect the provider coverage zone",
|
||||
details={"coverage_zone": zone},
|
||||
status_code=422,
|
||||
)
|
||||
return clipped
|
||||
|
||||
@staticmethod
|
||||
def _as_multipolygon(geometry) -> MultiPolygon:
|
||||
if isinstance(geometry, Polygon):
|
||||
return MultiPolygon([geometry])
|
||||
if isinstance(geometry, MultiPolygon):
|
||||
return geometry
|
||||
polygons = [
|
||||
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
|
||||
]
|
||||
if not polygons:
|
||||
raise AppError(
|
||||
code="AOI_GEOMETRY_EMPTY",
|
||||
message="AOI contains no polygonal area after clipping",
|
||||
status_code=422,
|
||||
)
|
||||
return MultiPolygon(polygons)
|
||||
|
||||
@staticmethod
|
||||
def _partition_side(provider_key: str, requested: float | None) -> float:
|
||||
settings = get_settings()
|
||||
|
||||
def raster_side(
|
||||
max_side_m: float, max_pixels: int, resolution_m: float
|
||||
) -> float:
|
||||
# Keep every square grid cell within both the provider's spatial
|
||||
# extent limit and its decoded-pixel budget. The small safety
|
||||
# margin absorbs ceil/edge rounding in the acquisition services.
|
||||
pixel_limited_side = (
|
||||
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
|
||||
)
|
||||
return min(float(max_side_m), pixel_limited_side)
|
||||
|
||||
budgets = {
|
||||
"orthophoto": float(settings.orthophoto_max_side_m),
|
||||
"grb": float(settings.grb_max_side_m),
|
||||
"dhmv": raster_side(
|
||||
settings.dhmv_max_side_m,
|
||||
settings.dhmv_max_pixels,
|
||||
settings.dhmv_resolution_m,
|
||||
),
|
||||
"spw_terrain": raster_side(
|
||||
settings.spw_terrain_max_side_m,
|
||||
settings.spw_terrain_max_pixels,
|
||||
settings.spw_terrain_analysis_resolution_m,
|
||||
),
|
||||
"official_vector": 20_000.0,
|
||||
"flood_hazard": raster_side(
|
||||
settings.flood_hazard_max_side_m,
|
||||
settings.flood_hazard_max_pixels,
|
||||
settings.flood_hazard_resolution_m,
|
||||
),
|
||||
"thematic_raster": raster_side(
|
||||
settings.thematic_raster_max_side_m,
|
||||
settings.thematic_raster_max_pixels,
|
||||
10.0,
|
||||
),
|
||||
"walous": raster_side(
|
||||
settings.walous_max_side_m,
|
||||
settings.walous_max_pixels,
|
||||
settings.walous_analysis_resolution_m,
|
||||
),
|
||||
"bathymetry_profiles": 20_000.0,
|
||||
"mdk_bathymetry": 20_000.0,
|
||||
}
|
||||
if provider_key not in budgets:
|
||||
raise AppError(
|
||||
code="AOI_PROVIDER_UNSUPPORTED",
|
||||
message="No governed partition budget is registered for this provider",
|
||||
details={"provider_key": provider_key},
|
||||
status_code=422,
|
||||
)
|
||||
governed = budgets[provider_key]
|
||||
return min(governed, float(requested)) if requested is not None else governed
|
||||
|
||||
@staticmethod
|
||||
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
|
||||
if (payload.area_id is None) == (payload.bbox is None):
|
||||
raise AppError(
|
||||
code="AOI_SELECTION_REQUIRED",
|
||||
message="Provide exactly one area_id or bbox",
|
||||
status_code=422,
|
||||
)
|
||||
if payload.area_id is not None:
|
||||
area = db.get(Area, payload.area_id)
|
||||
if area is None or area.project_id != project_id:
|
||||
raise AppError(
|
||||
code="AREA_NOT_FOUND", message="Area not found", status_code=404
|
||||
)
|
||||
return to_shape(area.geometry)
|
||||
bbox = payload.bbox
|
||||
if bbox is None or bbox.crs != "EPSG:4326":
|
||||
raise AppError(
|
||||
code="INVALID_AOI_CRS",
|
||||
message="AOI bbox must use EPSG:4326",
|
||||
status_code=422,
|
||||
)
|
||||
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
|
||||
|
||||
@staticmethod
|
||||
def _partition(geometry, side_m: float) -> list:
|
||||
min_x, min_y, max_x, max_y = geometry.bounds
|
||||
columns = max(1, math.ceil((max_x - min_x) / side_m))
|
||||
rows = max(1, math.ceil((max_y - min_y) / side_m))
|
||||
if columns * rows > AoiOperationService.MAX_PARTITIONS:
|
||||
raise AppError(
|
||||
code="AOI_PARTITION_LIMIT_EXCEEDED",
|
||||
message="AOI requires too many bounded partitions",
|
||||
details={
|
||||
"candidate_count": columns * rows,
|
||||
"max_partitions": AoiOperationService.MAX_PARTITIONS,
|
||||
},
|
||||
status_code=422,
|
||||
)
|
||||
partitions = []
|
||||
for row in range(rows):
|
||||
for column in range(columns):
|
||||
clipped = geometry.intersection(
|
||||
box(
|
||||
min_x + column * side_m,
|
||||
min_y + row * side_m,
|
||||
min(min_x + (column + 1) * side_m, max_x),
|
||||
min(min_y + (row + 1) * side_m, max_y),
|
||||
)
|
||||
)
|
||||
if not clipped.is_empty and clipped.area > 0:
|
||||
partitions.append(clipped)
|
||||
return partitions
|
||||
|
||||
@staticmethod
|
||||
def read(db, project_id: UUID, operation_id: UUID) -> dict:
|
||||
operation = db.get(AoiOperation, operation_id)
|
||||
if operation is None or operation.project_id != project_id:
|
||||
raise AppError(
|
||||
code="AOI_OPERATION_NOT_FOUND",
|
||||
message="AOI operation not found",
|
||||
status_code=404,
|
||||
)
|
||||
partitions = (
|
||||
db.query(AoiOperationPartition)
|
||||
.filter(AoiOperationPartition.operation_id == operation_id)
|
||||
.order_by(AoiOperationPartition.ordinal)
|
||||
.all()
|
||||
)
|
||||
counts = Counter(partition.status for partition in partitions)
|
||||
complete = counts["success"] + counts["skipped"]
|
||||
return {
|
||||
"id": operation.id,
|
||||
"project_id": operation.project_id,
|
||||
"area_id": operation.area_id,
|
||||
"parent_job_id": operation.parent_job_id,
|
||||
"operation_type": operation.operation_type,
|
||||
"status": operation.status,
|
||||
"request_json": operation.request_json,
|
||||
"plan_json": operation.plan_json,
|
||||
"result_json": operation.result_json,
|
||||
"error_message": operation.error_message,
|
||||
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
|
||||
"partition_counts": dict(counts),
|
||||
"partitions": partitions,
|
||||
"created_at": operation.created_at,
|
||||
"started_at": operation.started_at,
|
||||
"finished_at": operation.finished_at,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def list(db, project_id: UUID, limit: int = 50) -> dict:
|
||||
rows = (
|
||||
db.query(AoiOperation)
|
||||
.filter(AoiOperation.project_id == project_id)
|
||||
.order_by(AoiOperation.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return {
|
||||
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
|
||||
"total": len(rows),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def claim_next(db, project_id: UUID, operation_id: UUID):
|
||||
operation = db.get(AoiOperation, operation_id)
|
||||
if operation is None or operation.project_id != project_id:
|
||||
raise AppError(
|
||||
code="AOI_OPERATION_NOT_FOUND",
|
||||
message="AOI operation not found",
|
||||
status_code=404,
|
||||
)
|
||||
partition = (
|
||||
db.query(AoiOperationPartition)
|
||||
.filter(
|
||||
AoiOperationPartition.operation_id == operation_id,
|
||||
AoiOperationPartition.status == "queued",
|
||||
)
|
||||
.order_by(AoiOperationPartition.ordinal)
|
||||
.with_for_update(skip_locked=True)
|
||||
.first()
|
||||
)
|
||||
if partition is None:
|
||||
return None
|
||||
now = datetime.now(timezone.utc)
|
||||
partition.status = "running"
|
||||
partition.started_at = now
|
||||
partition.attempt_count += 1
|
||||
partition.error_message = None
|
||||
operation.status = "running"
|
||||
operation.started_at = operation.started_at or now
|
||||
db.add(partition)
|
||||
db.add(operation)
|
||||
db.commit()
|
||||
db.refresh(partition)
|
||||
return partition
|
||||
|
||||
@staticmethod
|
||||
def checkpoint(
|
||||
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
|
||||
):
|
||||
partition = AoiOperationService._partition_row(
|
||||
db, project_id, operation_id, partition_id
|
||||
)
|
||||
if partition.status != "running":
|
||||
raise AppError(
|
||||
code="AOI_PARTITION_NOT_RUNNING",
|
||||
message="Only a running partition can be checkpointed",
|
||||
status_code=409,
|
||||
)
|
||||
partition.checkpoint_json = checkpoint
|
||||
db.add(partition)
|
||||
db.commit()
|
||||
db.refresh(partition)
|
||||
return partition
|
||||
|
||||
@staticmethod
|
||||
def complete(
|
||||
db,
|
||||
project_id: UUID,
|
||||
operation_id: UUID,
|
||||
partition_id: UUID,
|
||||
result: dict,
|
||||
skipped: bool = False,
|
||||
):
|
||||
partition = AoiOperationService._partition_row(
|
||||
db, project_id, operation_id, partition_id
|
||||
)
|
||||
if partition.status == "success" or partition.status == "skipped":
|
||||
return AoiOperationService.read(db, project_id, operation_id)
|
||||
if partition.status != "running":
|
||||
raise AppError(
|
||||
code="AOI_PARTITION_NOT_RUNNING",
|
||||
message="Only a running partition can complete",
|
||||
status_code=409,
|
||||
)
|
||||
partition.status = "skipped" if skipped else "success"
|
||||
partition.result_json = result
|
||||
partition.finished_at = datetime.now(timezone.utc)
|
||||
db.add(partition)
|
||||
db.commit()
|
||||
AoiOperationService._refresh_parent(db, operation_id)
|
||||
return AoiOperationService.read(db, project_id, operation_id)
|
||||
|
||||
@staticmethod
|
||||
def fail(
|
||||
db,
|
||||
project_id: UUID,
|
||||
operation_id: UUID,
|
||||
partition_id: UUID,
|
||||
message: str,
|
||||
retryable: bool,
|
||||
details: dict,
|
||||
):
|
||||
partition = AoiOperationService._partition_row(
|
||||
db, project_id, operation_id, partition_id
|
||||
)
|
||||
partition.error_message = message
|
||||
partition.result_json = {"details": details}
|
||||
partition.status = (
|
||||
"queued"
|
||||
if retryable and partition.attempt_count < partition.max_attempts
|
||||
else "failed"
|
||||
)
|
||||
partition.finished_at = (
|
||||
None if partition.status == "queued" else datetime.now(timezone.utc)
|
||||
)
|
||||
db.add(partition)
|
||||
db.commit()
|
||||
AoiOperationService._refresh_parent(db, operation_id)
|
||||
return AoiOperationService.read(db, project_id, operation_id)
|
||||
|
||||
@staticmethod
|
||||
def _partition_row(db, project_id, operation_id, partition_id):
|
||||
operation = db.get(AoiOperation, operation_id)
|
||||
partition = db.get(AoiOperationPartition, partition_id)
|
||||
if (
|
||||
operation is None
|
||||
or operation.project_id != project_id
|
||||
or partition is None
|
||||
or partition.operation_id != operation_id
|
||||
):
|
||||
raise AppError(
|
||||
code="AOI_PARTITION_NOT_FOUND",
|
||||
message="AOI partition not found",
|
||||
status_code=404,
|
||||
)
|
||||
return partition
|
||||
|
||||
@staticmethod
|
||||
def _refresh_parent(db, operation_id):
|
||||
operation = db.get(AoiOperation, operation_id)
|
||||
partitions = (
|
||||
db.query(AoiOperationPartition)
|
||||
.filter(AoiOperationPartition.operation_id == operation_id)
|
||||
.order_by(AoiOperationPartition.ordinal)
|
||||
.all()
|
||||
)
|
||||
statuses = [partition.status for partition in partitions]
|
||||
output_dataset_ids = []
|
||||
for partition in partitions:
|
||||
output_id = (
|
||||
(partition.result_json or {}).get("output_dataset_id")
|
||||
if isinstance(partition.result_json, dict)
|
||||
else None
|
||||
)
|
||||
if output_id and str(output_id) not in output_dataset_ids:
|
||||
output_dataset_ids.append(str(output_id))
|
||||
operation.result_json = {
|
||||
"partition_count": len(partitions),
|
||||
"completed_partition_count": sum(
|
||||
status in {"success", "skipped"} for status in statuses
|
||||
),
|
||||
"failed_partition_count": statuses.count("failed"),
|
||||
"output_dataset_ids": output_dataset_ids,
|
||||
"merge_contract": "source_aware_spatial_union",
|
||||
"vector_deduplication": "source_feature_id_then_geometry",
|
||||
"raster_deduplication": "governed_mosaic_grid",
|
||||
"complete_coverage": bool(statuses)
|
||||
and all(status in {"success", "skipped"} for status in statuses),
|
||||
}
|
||||
now = datetime.now(timezone.utc)
|
||||
if statuses and all(status in {"success", "skipped"} for status in statuses):
|
||||
operation.status = "success"
|
||||
operation.finished_at = now
|
||||
operation.error_message = None
|
||||
elif "failed" in statuses and not any(
|
||||
status in {"queued", "running"} for status in statuses
|
||||
):
|
||||
operation.status = (
|
||||
"partial"
|
||||
if any(status in {"success", "skipped"} for status in statuses)
|
||||
else "failed"
|
||||
)
|
||||
operation.finished_at = now
|
||||
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
|
||||
db.add(operation)
|
||||
db.commit()
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import AoiOperation
|
||||
from app.services.aoi_operation_executor import AoiOperationExecutor
|
||||
|
||||
|
||||
logger = logging.getLogger("geointel.aoi_worker")
|
||||
|
||||
|
||||
class AoiOperationWorker:
|
||||
@staticmethod
|
||||
def run_once() -> int:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
rows = db.query(AoiOperation).filter(AoiOperation.status.in_(("queued", "running"))).order_by(AoiOperation.created_at).limit(10).all()
|
||||
for operation in rows:
|
||||
try:
|
||||
AoiOperationExecutor.execute_next(db, operation.project_id, operation.id)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("AOI partition execution failed operation_id=%s", operation.id)
|
||||
return len(rows)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@staticmethod
|
||||
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
|
||||
while not stop_event.is_set():
|
||||
processed = await asyncio.to_thread(AoiOperationWorker.run_once)
|
||||
if processed == 0:
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
@@ -0,0 +1,193 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from geoalchemy2.shape import from_shape, to_shape
|
||||
from shapely.geometry import mapping
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project, VectorFeature
|
||||
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
|
||||
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326
|
||||
|
||||
|
||||
class AreaService:
|
||||
@staticmethod
|
||||
def _municipality_dataset(db: Session, project_id: uuid.UUID) -> Dataset | None:
|
||||
return (
|
||||
db.query(Dataset)
|
||||
.filter(
|
||||
Dataset.project_id == project_id,
|
||||
Dataset.reference_layer_name == "belgium_municipalities",
|
||||
Dataset.status == "ready",
|
||||
)
|
||||
.order_by(Dataset.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _filter_municipality_properties(properties_items: list[dict], query: str, limit: int) -> tuple[list[dict], int]:
|
||||
normalized = query.strip().casefold()
|
||||
matches: list[dict] = []
|
||||
for properties in properties_items:
|
||||
names = [str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger")]
|
||||
niscode = str(properties.get("niscode") or "").strip()
|
||||
if normalized and normalized not in " ".join([niscode, *names]).casefold():
|
||||
continue
|
||||
display_name = next((name for name in names if name), niscode)
|
||||
matches.append({
|
||||
"niscode": niscode,
|
||||
"name": display_name,
|
||||
"name_nl": names[0] or None,
|
||||
"name_fr": names[1] or None,
|
||||
"name_de": names[2] or None,
|
||||
})
|
||||
matches.sort(key=lambda item: (item["name"].casefold(), item["niscode"]))
|
||||
return matches[:limit], len(matches)
|
||||
|
||||
@staticmethod
|
||||
def search_municipalities(db: Session, project_id: uuid.UUID, query: str, limit: int = 20) -> tuple[list[dict], int]:
|
||||
dataset = AreaService._municipality_dataset(db, project_id)
|
||||
if dataset is None:
|
||||
return [], 0
|
||||
property_rows = (
|
||||
db.query(VectorFeature.properties_json)
|
||||
.filter(VectorFeature.dataset_id == dataset.id)
|
||||
.all()
|
||||
)
|
||||
properties_items = [row[0] for row in property_rows if isinstance(row[0], dict)]
|
||||
return AreaService._filter_municipality_properties(properties_items, query, limit)
|
||||
|
||||
@staticmethod
|
||||
def activate_municipality(db: Session, project_id: uuid.UUID, niscode: str) -> Area:
|
||||
normalized_code = niscode.strip()
|
||||
dataset = AreaService._municipality_dataset(db, project_id)
|
||||
if dataset is None:
|
||||
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
|
||||
feature = (
|
||||
db.query(VectorFeature)
|
||||
.filter(
|
||||
VectorFeature.dataset_id == dataset.id,
|
||||
VectorFeature.properties_json["niscode"].as_string() == normalized_code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if feature is not None:
|
||||
properties = feature.properties_json if isinstance(feature.properties_json, dict) else {}
|
||||
display_name = next(
|
||||
(str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger") if str(properties.get(key) or "").strip()),
|
||||
normalized_code,
|
||||
)
|
||||
area_name = f"Gemeente {display_name} - NIS {normalized_code}"
|
||||
existing = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
|
||||
if existing is not None:
|
||||
return existing
|
||||
geometry = to_shape(feature.geometry)
|
||||
return AreaService.create_area(
|
||||
db,
|
||||
project_id,
|
||||
AreaCreate(name=area_name, geometry=mapping(geometry), crs="EPSG:4326"),
|
||||
)
|
||||
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
|
||||
|
||||
@staticmethod
|
||||
def serialize_area(area: Area) -> dict:
|
||||
geometry = to_shape(area.geometry) if area.geometry else None
|
||||
return AreaRead.model_validate(
|
||||
{
|
||||
"id": area.id,
|
||||
"project_id": area.project_id,
|
||||
"name": area.name,
|
||||
"original_crs": area.original_crs,
|
||||
"area_m2": area.area_m2,
|
||||
"created_at": area.created_at,
|
||||
"geometry_type": geometry.geom_type if geometry else None,
|
||||
"geometry": mapping(geometry) if geometry else None,
|
||||
}
|
||||
).model_dump()
|
||||
|
||||
@staticmethod
|
||||
def list_areas(db: Session, project_id: uuid.UUID, limit: int = 50, offset: int = 0) -> tuple[list[Area], int]:
|
||||
total = db.query(Area).filter(Area.project_id == project_id).count()
|
||||
areas = (
|
||||
db.query(Area)
|
||||
.filter(Area.project_id == project_id)
|
||||
.order_by(Area.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return areas, total
|
||||
|
||||
@staticmethod
|
||||
def create_area(db: Session, project_id: uuid.UUID, payload: AreaCreate) -> Area:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
try:
|
||||
multipolygon, original_crs = normalize_area_to_epsg4326(
|
||||
payload.geometry,
|
||||
payload.crs or "EPSG:4326",
|
||||
)
|
||||
metric_area = area_m2(multipolygon)
|
||||
except ValueError as exc:
|
||||
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
||||
|
||||
area = Area(
|
||||
project_id=project_id,
|
||||
name=payload.name.strip() or "Unnamed area",
|
||||
geometry=from_shape(multipolygon, srid=4326),
|
||||
original_crs=original_crs,
|
||||
area_m2=metric_area,
|
||||
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
|
||||
)
|
||||
db.add(area)
|
||||
db.commit()
|
||||
db.refresh(area)
|
||||
return area
|
||||
|
||||
@staticmethod
|
||||
def get_area(db: Session, area_id: uuid.UUID) -> Area:
|
||||
area = db.get(Area, area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
return area
|
||||
|
||||
@staticmethod
|
||||
def update_area(db: Session, area_id: uuid.UUID, payload: AreaUpdate) -> Area:
|
||||
area = db.get(Area, area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
|
||||
changed = False
|
||||
if payload.name is not None and payload.name.strip():
|
||||
area.name = payload.name.strip() or area.name
|
||||
changed = True
|
||||
if payload.crs is not None and payload.geometry is None:
|
||||
raise AppError(
|
||||
code="INVALID_AREA_CRS_UPDATE",
|
||||
message="crs can only be supplied together with replacement geometry",
|
||||
status_code=422,
|
||||
)
|
||||
if payload.geometry is not None:
|
||||
try:
|
||||
multipolygon, original_crs = normalize_area_to_epsg4326(
|
||||
payload.geometry,
|
||||
payload.crs or "EPSG:4326",
|
||||
)
|
||||
metric_area = area_m2(multipolygon)
|
||||
except ValueError as exc:
|
||||
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
||||
area.geometry = from_shape(multipolygon, srid=4326)
|
||||
area.original_crs = original_crs
|
||||
area.area_m2 = metric_area
|
||||
area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
|
||||
changed = True
|
||||
if not changed:
|
||||
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
|
||||
|
||||
db.add(area)
|
||||
db.commit()
|
||||
db.refresh(area)
|
||||
return area
|
||||
@@ -0,0 +1,260 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Literal, cast
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AuthPrincipal:
|
||||
username: str
|
||||
expires_at: int
|
||||
session_id: str = field(default_factory=lambda: secrets.token_urlsafe(12))
|
||||
role: Literal["operator", "guest"] = "operator"
|
||||
project_id: UUID | None = None
|
||||
|
||||
|
||||
class AuthService:
|
||||
HASH_NAME = "pbkdf2_sha256"
|
||||
HASH_ITERATIONS = 600_000
|
||||
MAX_FAILURES = 5
|
||||
FAILURE_WINDOW_SECONDS = 300
|
||||
_failures: dict[str, deque[float]] = {}
|
||||
_failure_lock = threading.Lock()
|
||||
_guest_requests: dict[str, deque[float]] = {}
|
||||
_guest_request_lock = threading.Lock()
|
||||
_active_guest_compute = 0
|
||||
_guest_compute_lock = threading.Lock()
|
||||
|
||||
@staticmethod
|
||||
def _b64_encode(value: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=")
|
||||
|
||||
@staticmethod
|
||||
def _b64_decode(value: str) -> bytes:
|
||||
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
||||
|
||||
@classmethod
|
||||
def hash_password(
|
||||
cls,
|
||||
password: str,
|
||||
*,
|
||||
salt: bytes | None = None,
|
||||
iterations: int | None = None,
|
||||
) -> str:
|
||||
resolved_salt = salt or secrets.token_bytes(18)
|
||||
resolved_iterations = iterations or cls.HASH_ITERATIONS
|
||||
digest = hashlib.pbkdf2_hmac(
|
||||
"sha256",
|
||||
password.encode("utf-8"),
|
||||
resolved_salt,
|
||||
resolved_iterations,
|
||||
)
|
||||
return "$".join(
|
||||
(
|
||||
cls.HASH_NAME,
|
||||
str(resolved_iterations),
|
||||
cls._b64_encode(resolved_salt),
|
||||
cls._b64_encode(digest),
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def verify_password(cls, password: str, encoded: str) -> bool:
|
||||
try:
|
||||
algorithm, iterations_raw, salt_raw, expected_raw = encoded.split("$", 3)
|
||||
if algorithm != cls.HASH_NAME:
|
||||
return False
|
||||
iterations = int(iterations_raw)
|
||||
if iterations < 100_000 or iterations > 2_000_000:
|
||||
return False
|
||||
salt = cls._b64_decode(salt_raw)
|
||||
expected = cls._b64_decode(expected_raw)
|
||||
actual = hashlib.pbkdf2_hmac(
|
||||
"sha256",
|
||||
password.encode("utf-8"),
|
||||
salt,
|
||||
iterations,
|
||||
)
|
||||
return hmac.compare_digest(actual, expected)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def credentials_match(cls, username: str, password: str, settings: Settings) -> bool:
|
||||
expected_username = settings.auth_username or ""
|
||||
expected_password_hash = settings.auth_password_hash or ""
|
||||
username_matches = hmac.compare_digest(
|
||||
username.encode("utf-8"),
|
||||
expected_username.encode("utf-8"),
|
||||
)
|
||||
password_matches = cls.verify_password(password, expected_password_hash)
|
||||
return username_matches and password_matches
|
||||
|
||||
@classmethod
|
||||
def create_session_token(
|
||||
cls,
|
||||
username: str,
|
||||
settings: Settings,
|
||||
*,
|
||||
role: Literal["operator", "guest"] = "operator",
|
||||
project_id: UUID | None = None,
|
||||
ttl_seconds: int | None = None,
|
||||
now: int | None = None,
|
||||
) -> str:
|
||||
issued_at = int(time.time() if now is None else now)
|
||||
if role == "guest" and project_id is None:
|
||||
raise ValueError("Guest sessions must be scoped to a demo project")
|
||||
resolved_ttl = ttl_seconds if ttl_seconds is not None else (
|
||||
settings.guest_session_ttl_seconds if role == "guest" else settings.auth_session_ttl_seconds
|
||||
)
|
||||
payload = {
|
||||
"exp": issued_at + resolved_ttl,
|
||||
"iat": issued_at,
|
||||
"jti": secrets.token_urlsafe(12),
|
||||
"role": role,
|
||||
"sub": username,
|
||||
"v": 2,
|
||||
}
|
||||
if project_id is not None:
|
||||
payload["project_id"] = str(project_id)
|
||||
encoded_payload = cls._b64_encode(
|
||||
json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||
)
|
||||
signature = hmac.new(
|
||||
(settings.auth_session_secret or "").encode("utf-8"),
|
||||
encoded_payload.encode("ascii"),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
return f"{encoded_payload}.{cls._b64_encode(signature)}"
|
||||
|
||||
@classmethod
|
||||
def verify_session_token(
|
||||
cls,
|
||||
token: str | None,
|
||||
settings: Settings,
|
||||
*,
|
||||
now: int | None = None,
|
||||
) -> AuthPrincipal | None:
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
encoded_payload, encoded_signature = token.split(".", 1)
|
||||
expected_signature = hmac.new(
|
||||
(settings.auth_session_secret or "").encode("utf-8"),
|
||||
encoded_payload.encode("ascii"),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
supplied_signature = cls._b64_decode(encoded_signature)
|
||||
if not hmac.compare_digest(expected_signature, supplied_signature):
|
||||
return None
|
||||
payload = json.loads(cls._b64_decode(encoded_payload))
|
||||
username = str(payload.get("sub") or "")
|
||||
expires_at = int(payload.get("exp") or 0)
|
||||
issued_at = int(payload.get("iat") or 0)
|
||||
version = int(payload.get("v") or 0)
|
||||
role_value = str(payload.get("role") or "operator")
|
||||
session_id = str(payload.get("jti") or "")
|
||||
current = int(time.time() if now is None else now)
|
||||
if version not in {1, 2} or role_value not in {"operator", "guest"} or not session_id:
|
||||
return None
|
||||
role = cast(Literal["operator", "guest"], role_value)
|
||||
if issued_at <= 0 or issued_at > current + 60 or expires_at <= current:
|
||||
return None
|
||||
if role == "operator":
|
||||
if username != settings.auth_username:
|
||||
return None
|
||||
max_ttl = settings.auth_session_ttl_seconds
|
||||
project_id = None
|
||||
else:
|
||||
if not settings.guest_access_enabled or username != settings.guest_display_name:
|
||||
return None
|
||||
max_ttl = settings.guest_session_ttl_seconds
|
||||
raw_project_id = payload.get("project_id")
|
||||
if not raw_project_id:
|
||||
return None
|
||||
project_id = UUID(str(raw_project_id))
|
||||
if project_id != PUBLIC_DEMO_PROJECT_ID:
|
||||
return None
|
||||
if expires_at - issued_at > max_ttl:
|
||||
return None
|
||||
return AuthPrincipal(
|
||||
username=username,
|
||||
expires_at=expires_at,
|
||||
session_id=session_id,
|
||||
role=role,
|
||||
project_id=project_id,
|
||||
)
|
||||
except (TypeError, ValueError, json.JSONDecodeError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def retry_after_seconds(cls, key: str, *, now: float | None = None) -> int:
|
||||
current = time.monotonic() if now is None else now
|
||||
with cls._failure_lock:
|
||||
attempts = cls._failures.setdefault(key, deque())
|
||||
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
|
||||
attempts.popleft()
|
||||
if len(attempts) < cls.MAX_FAILURES:
|
||||
if not attempts:
|
||||
cls._failures.pop(key, None)
|
||||
return 0
|
||||
return max(1, int(cls.FAILURE_WINDOW_SECONDS - (current - attempts[0])))
|
||||
|
||||
@classmethod
|
||||
def record_failure(cls, key: str, *, now: float | None = None) -> None:
|
||||
current = time.monotonic() if now is None else now
|
||||
with cls._failure_lock:
|
||||
attempts = cls._failures.setdefault(key, deque())
|
||||
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
|
||||
attempts.popleft()
|
||||
attempts.append(current)
|
||||
|
||||
@classmethod
|
||||
def clear_failures(cls, key: str) -> None:
|
||||
with cls._failure_lock:
|
||||
cls._failures.pop(key, None)
|
||||
|
||||
@classmethod
|
||||
def consume_guest_request(
|
||||
cls,
|
||||
key: str,
|
||||
*,
|
||||
max_requests: int,
|
||||
window_seconds: int = 60,
|
||||
now: float | None = None,
|
||||
) -> int:
|
||||
"""Record a guest action and return Retry-After seconds when limited."""
|
||||
current = time.monotonic() if now is None else now
|
||||
with cls._guest_request_lock:
|
||||
attempts = cls._guest_requests.setdefault(key, deque())
|
||||
while attempts and current - attempts[0] >= window_seconds:
|
||||
attempts.popleft()
|
||||
if len(attempts) >= max_requests:
|
||||
return max(1, int(window_seconds - (current - attempts[0])))
|
||||
attempts.append(current)
|
||||
return 0
|
||||
|
||||
@classmethod
|
||||
def try_acquire_guest_compute(cls, *, max_concurrency: int) -> bool:
|
||||
with cls._guest_compute_lock:
|
||||
if cls._active_guest_compute >= max_concurrency:
|
||||
return False
|
||||
cls._active_guest_compute += 1
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def release_guest_compute(cls) -> None:
|
||||
with cls._guest_compute_lock:
|
||||
cls._active_guest_compute = max(0, cls._active_guest_compute - 1)
|
||||
@@ -0,0 +1,207 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
import jwt
|
||||
from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
MAX_OIDC_JSON_BYTES = 1_048_576
|
||||
|
||||
|
||||
class _RejectRedirects(HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ANN201
|
||||
return None
|
||||
|
||||
|
||||
class AuthentikOidcService:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.issuer = (settings.authentik_issuer or "").rstrip("/")
|
||||
self.serializer = URLSafeTimedSerializer(
|
||||
settings.auth_session_secret or "",
|
||||
salt="geointel-authentik-v1",
|
||||
)
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(
|
||||
self.issuer
|
||||
and self.settings.authentik_client_id
|
||||
and self.settings.authentik_client_secret
|
||||
and self.settings.authentik_allowed_email
|
||||
)
|
||||
|
||||
@property
|
||||
def redirect_uri(self) -> str:
|
||||
return (
|
||||
f"{self.settings.public_base_url.rstrip('/')}"
|
||||
f"{self.settings.api_prefix}/auth/authentik/callback"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _origin(url: str) -> tuple[str, str, int]:
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme != "https" or not parsed.hostname:
|
||||
raise ValueError("OIDC URLs must use absolute HTTPS URLs")
|
||||
return parsed.scheme, parsed.hostname.casefold(), parsed.port or 443
|
||||
|
||||
def _validate_endpoint(self, url: str) -> str:
|
||||
parsed = urlsplit(url)
|
||||
if (
|
||||
self._origin(url) != self._origin(self.issuer)
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.fragment
|
||||
):
|
||||
raise ValueError("OIDC endpoint is outside the configured issuer origin")
|
||||
return url
|
||||
|
||||
def _fetch_json(
|
||||
self,
|
||||
url: str,
|
||||
data: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self._validate_endpoint(url)
|
||||
encoded = urlencode(data).encode("utf-8") if data is not None else None
|
||||
headers = {"Accept": "application/json"}
|
||||
if encoded is not None:
|
||||
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
request = Request(url, data=encoded, headers=headers)
|
||||
try:
|
||||
with build_opener(_RejectRedirects()).open(request, timeout=10) as response:
|
||||
declared_length = response.headers.get("Content-Length")
|
||||
if declared_length and int(declared_length) > MAX_OIDC_JSON_BYTES:
|
||||
raise ValueError("OIDC response exceeds the configured size limit")
|
||||
raw = response.read(MAX_OIDC_JSON_BYTES + 1)
|
||||
except HTTPError as exc:
|
||||
raise ValueError("OIDC endpoint returned an HTTP error or redirect") from exc
|
||||
if len(raw) > MAX_OIDC_JSON_BYTES:
|
||||
raise ValueError("OIDC response exceeds the configured size limit")
|
||||
payload = json.loads(raw)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("OIDC endpoint did not return a JSON object")
|
||||
return payload
|
||||
|
||||
def _discovery(self) -> dict[str, Any]:
|
||||
document = self._fetch_json(
|
||||
f"{self.issuer}/.well-known/openid-configuration"
|
||||
)
|
||||
if str(document.get("issuer", "")).rstrip("/") != self.issuer:
|
||||
raise ValueError("OIDC issuer mismatch")
|
||||
for key in ("authorization_endpoint", "token_endpoint", "jwks_uri"):
|
||||
endpoint = document.get(key)
|
||||
if not isinstance(endpoint, str):
|
||||
raise ValueError(f"OIDC discovery is missing {key}")
|
||||
self._validate_endpoint(endpoint)
|
||||
return document
|
||||
|
||||
def start(self) -> tuple[str, str]:
|
||||
if not self.enabled:
|
||||
raise ValueError("Authentik is not configured")
|
||||
state = secrets.token_urlsafe(32)
|
||||
nonce = secrets.token_urlsafe(32)
|
||||
verifier = secrets.token_urlsafe(48)
|
||||
flow = self.serializer.dumps(
|
||||
{"state": state, "nonce": nonce, "verifier": verifier}
|
||||
)
|
||||
challenge = (
|
||||
base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
|
||||
.rstrip(b"=")
|
||||
.decode()
|
||||
)
|
||||
discovery = self._discovery()
|
||||
query = urlencode(
|
||||
{
|
||||
"client_id": self.settings.authentik_client_id,
|
||||
"redirect_uri": self.redirect_uri,
|
||||
"response_type": "code",
|
||||
"scope": "openid email profile",
|
||||
"state": state,
|
||||
"nonce": nonce,
|
||||
"code_challenge": challenge,
|
||||
"code_challenge_method": "S256",
|
||||
}
|
||||
)
|
||||
return f"{discovery['authorization_endpoint']}?{query}", flow
|
||||
|
||||
def finish(self, *, code: str, state: str, flow_cookie: str) -> dict[str, Any]:
|
||||
if not self.enabled or not code:
|
||||
raise ValueError("OIDC flow is incomplete")
|
||||
try:
|
||||
flow = self.serializer.loads(flow_cookie, max_age=600)
|
||||
except (BadSignature, SignatureExpired) as exc:
|
||||
raise ValueError("Invalid OIDC flow") from exc
|
||||
if not isinstance(flow, dict):
|
||||
raise ValueError("Invalid OIDC flow payload")
|
||||
if not state or not secrets.compare_digest(state, str(flow.get("state", ""))):
|
||||
raise ValueError("OIDC state mismatch")
|
||||
verifier = str(flow.get("verifier", ""))
|
||||
nonce = str(flow.get("nonce", ""))
|
||||
if not verifier or not nonce:
|
||||
raise ValueError("OIDC flow payload is incomplete")
|
||||
|
||||
discovery = self._discovery()
|
||||
token_response = self._fetch_json(
|
||||
str(discovery["token_endpoint"]),
|
||||
{
|
||||
"grant_type": "authorization_code",
|
||||
"code": code,
|
||||
"redirect_uri": self.redirect_uri,
|
||||
"client_id": self.settings.authentik_client_id or "",
|
||||
"client_secret": self.settings.authentik_client_secret or "",
|
||||
"code_verifier": verifier,
|
||||
},
|
||||
)
|
||||
token = str(token_response.get("id_token", ""))
|
||||
if not token:
|
||||
raise ValueError("OIDC token response has no ID token")
|
||||
header = jwt.get_unverified_header(token)
|
||||
if header.get("alg") != "RS256" or not header.get("kid"):
|
||||
raise ValueError("OIDC ID token uses an unsupported signing header")
|
||||
jwks = self._fetch_json(str(discovery["jwks_uri"]))
|
||||
matching_keys = [
|
||||
key
|
||||
for key in jwks.get("keys", [])
|
||||
if isinstance(key, dict) and key.get("kid") == header["kid"]
|
||||
]
|
||||
if len(matching_keys) != 1:
|
||||
raise ValueError("OIDC signing key is missing or ambiguous")
|
||||
signing_key = jwt.PyJWK.from_dict(matching_keys[0]).key
|
||||
claims = jwt.decode(
|
||||
token,
|
||||
signing_key,
|
||||
algorithms=["RS256"],
|
||||
audience=self.settings.authentik_client_id,
|
||||
issuer=discovery["issuer"],
|
||||
options={
|
||||
"require": [
|
||||
"exp",
|
||||
"iat",
|
||||
"iss",
|
||||
"aud",
|
||||
"sub",
|
||||
"nonce",
|
||||
"email",
|
||||
"email_verified",
|
||||
]
|
||||
},
|
||||
)
|
||||
if not secrets.compare_digest(str(claims.get("nonce", "")), nonce):
|
||||
raise ValueError("OIDC nonce mismatch")
|
||||
email = str(claims.get("email", "")).strip().casefold()
|
||||
allowed = str(self.settings.authentik_allowed_email or "").strip().casefold()
|
||||
if claims.get("email_verified") is not True or not secrets.compare_digest(
|
||||
email, allowed
|
||||
):
|
||||
raise ValueError("OIDC identity is not authorized")
|
||||
return claims
|
||||
@@ -0,0 +1,958 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from typing import Any, Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import Point, box, mapping
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.services.outbound_request_guard import guarded_opener
|
||||
from app.models import Area, Dataset, DatasetVersion, Project
|
||||
from app.schemas.bathymetry import (
|
||||
BathymetryPartitionFinalizeRequest,
|
||||
BathymetryPartitionFinalizationResult,
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetryProfileAcquisitionResult,
|
||||
BathymetrySourceRead,
|
||||
)
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
class BathymetryProfileAcquisitionService:
|
||||
PROVIDER = "vmm_vha_bathymetry_profiles"
|
||||
SOURCE_VERSION = "VHA digitale atlas ArcGIS MapServer"
|
||||
PROFILE_OUT_FIELDS = (
|
||||
"OBJECTID,vhag,atlaspunt,opg_kruinb,opg_vloerb,d_opmeti,"
|
||||
"hyperlink,bron,kunstwerkid,opg_diepte"
|
||||
)
|
||||
ATTRIBUTION = "Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas"
|
||||
LICENSE_NOTE = "Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata."
|
||||
LIMITATION = (
|
||||
"Dwarsprofielen zijn historische puntmetingen met bronafhankelijke meetdatum en verticale referentie. "
|
||||
"Ze vormen geen continue actuele bodemkaart en ondersteunen zonder gelijktijdig waterpeil geen "
|
||||
"gebiedsdekkend of actueel watervolume."
|
||||
)
|
||||
_SOURCES = (
|
||||
{
|
||||
"key": "vha_inland_profiles",
|
||||
"display_name": "VHA dwarsprofielen binnenwater",
|
||||
"owner": "Vlaamse Milieumaatschappij",
|
||||
"authority_level": "authoritative",
|
||||
"geographic_coverage": "Vlaanderen, puntlocaties op gekarteerde waterlopen",
|
||||
"data_kind": "dwarsprofielpunten met meetvelden en brondocumenten",
|
||||
"query_modes": ["bbox", "persisted_area"],
|
||||
"vertical_reference": "document-specific; niet uniform als één peilreferentie te behandelen",
|
||||
"horizontal_crs": "EPSG:4326",
|
||||
"native_resolution": None,
|
||||
"integration_status": "operational",
|
||||
"acquisition_supported": True,
|
||||
"configured": True,
|
||||
"service_url": "https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
|
||||
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/vlaamse-hydrografische-atlas-waterlopen",
|
||||
"attribution": ATTRIBUTION,
|
||||
"license_note": LICENSE_NOTE,
|
||||
"limitation_message": LIMITATION,
|
||||
},
|
||||
{
|
||||
"key": "mdk_bcp_bathymetry",
|
||||
"display_name": "Dieptemodel Belgisch Continentaal Plat",
|
||||
"owner": "Agentschap Maritieme Dienstverlening en Kust",
|
||||
"authority_level": "authoritative",
|
||||
"geographic_coverage": "Belgisch Continentaal Plat en Noordzee",
|
||||
"data_kind": "continu bathymetrisch raster",
|
||||
"query_modes": ["wcs", "wmts", "bounded_raster"],
|
||||
"vertical_reference": "LAT",
|
||||
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
|
||||
"native_resolution": "20 x 20 m",
|
||||
"integration_status": "probe_only",
|
||||
"acquisition_supported": False,
|
||||
"configured": False,
|
||||
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
||||
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
|
||||
"attribution": "Agentschap Maritieme Dienstverlening en Kust",
|
||||
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
|
||||
"limitation_message": (
|
||||
"Alleen een read-only GetCapabilities-probe is beschikbaar. Rasteracquisitie blijft uit totdat "
|
||||
"WCS, maritieme begrenzing, tegels, LAT-semantiek en servercertificaten live zijn gevalideerd."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "spw_walloon_waterway_bathymetry",
|
||||
"display_name": "Bathymétrie des voies navigables et lacs-réservoirs",
|
||||
"owner": "Service public de Wallonie",
|
||||
"authority_level": "authoritative",
|
||||
"geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen",
|
||||
"data_kind": "bodemhoogteraster en XYZ-puntenwolk",
|
||||
"query_modes": ["operator_archive", "bounded_raster", "arcgis_map_service"],
|
||||
"vertical_reference": "mDNG",
|
||||
"horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden",
|
||||
"native_resolution": "0,5 m",
|
||||
"integration_status": "operational",
|
||||
"acquisition_supported": True,
|
||||
"configured": True,
|
||||
"service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer",
|
||||
"catalog_url": "https://geoportail.wallonie.be/catalogue/0a544b42-0b30-4c8e-85e7-38149b99eae0.html",
|
||||
"attribution": "Service public de Wallonie",
|
||||
"license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.",
|
||||
"limitation_message": (
|
||||
"De gepinde officiële release kan begrensd als raster worden geïmporteerd via de operator. "
|
||||
"Dekking verschilt per vaarweg; de waarden zijn bodemhoogtes in mDNG uit 2019-2022, "
|
||||
"zonder stilzwijgende datumconversie of afleiding van actuele waterdiepte."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "port_antwerp_bathymetry",
|
||||
"display_name": "Havenbathymetrie Antwerpen-Brugge",
|
||||
"owner": "Port of Antwerp-Bruges",
|
||||
"authority_level": "contextual",
|
||||
"geographic_coverage": "Gepubliceerde havenzones en meetcampagnes",
|
||||
"data_kind": "periodieke peilingen",
|
||||
"query_modes": ["catalog"],
|
||||
"vertical_reference": "product-specific",
|
||||
"horizontal_crs": "product-specific",
|
||||
"native_resolution": None,
|
||||
"integration_status": "catalog_only",
|
||||
"acquisition_supported": False,
|
||||
"configured": False,
|
||||
"service_url": None,
|
||||
"catalog_url": "https://data.gov.be/nl/datasets",
|
||||
"attribution": "Port of Antwerp-Bruges",
|
||||
"license_note": "Per publicatie te verifiëren.",
|
||||
"limitation_message": (
|
||||
"Alleen als cataloguskandidaat geregistreerd; er is nog geen stabiel, publiek en machineleesbaar "
|
||||
"acquisitiecontract in GeoIntel gevalideerd."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def list_sources(settings=None) -> list[dict[str, Any]]:
|
||||
from app.core.config import get_settings
|
||||
|
||||
resolved_settings = settings or get_settings()
|
||||
items: list[dict[str, Any]] = []
|
||||
for source in BathymetryProfileAcquisitionService._SOURCES:
|
||||
item = dict(source)
|
||||
if item["key"] == "mdk_bcp_bathymetry":
|
||||
mdk_configured = bool(
|
||||
resolved_settings.mdk_bathymetry_acquisition_enabled
|
||||
and (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
|
||||
)
|
||||
item["acquisition_supported"] = True
|
||||
item["configured"] = mdk_configured
|
||||
if mdk_configured:
|
||||
item["integration_status"] = "operational"
|
||||
item["limitation_message"] = (
|
||||
"Begrensde WCS-acquisitie is expliciet ingeschakeld en draait alleen wanneer de "
|
||||
"live readiness-probe bereikbaar is en het geconfigureerde coverage-id door de "
|
||||
"capabilities wordt geadverteerd. Dieptes blijven LAT-gerefereerd; watervolume "
|
||||
"blijft zonder compatibel wateroppervlak niet ondersteund."
|
||||
)
|
||||
else:
|
||||
item["limitation_message"] = (
|
||||
"Begrensde WCS-acquisitie bestaat maar staat uit. Zet "
|
||||
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true en configureer MDK_BATHYMETRY_COVERAGE_ID "
|
||||
"pas nadat de readiness-probe live 'reachable' rapporteert. Er wordt nooit "
|
||||
"onbeveiligd of ongevalideerd gedownload."
|
||||
)
|
||||
items.append(item)
|
||||
return [BathymetrySourceRead(**item).model_dump() for item in items]
|
||||
|
||||
@staticmethod
|
||||
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
|
||||
bbox = payload.bbox
|
||||
if bbox.crs.upper() != "EPSG:4326":
|
||||
raise AppError(
|
||||
code="BATHYMETRY_INVALID_CRS",
|
||||
message="Bathymetry profile acquisition requires EPSG:4326",
|
||||
status_code=400,
|
||||
)
|
||||
values = (bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
|
||||
if not all(math.isfinite(value) for value in values):
|
||||
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box values must be finite", status_code=400)
|
||||
if bbox.min_x >= bbox.max_x or bbox.min_y >= bbox.max_y:
|
||||
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box has no area", status_code=400)
|
||||
if bbox.min_x < -180 or bbox.max_x > 180 or bbox.min_y < -90 or bbox.max_y > 90:
|
||||
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400)
|
||||
return values
|
||||
|
||||
@staticmethod
|
||||
def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_values: tuple[float, float, float, float]):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
selection = box(*bbox_values)
|
||||
if area_id is None:
|
||||
return selection
|
||||
area = db.get(Area, area_id)
|
||||
if area is None:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
area_geometry = area.geometry if hasattr(area.geometry, "__geo_interface__") else to_shape(area.geometry)
|
||||
intersection = area_geometry.intersection(selection)
|
||||
if intersection.is_empty:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SCOPE_EMPTY",
|
||||
message="The requested bounding box does not intersect the selected area",
|
||||
status_code=400,
|
||||
)
|
||||
return intersection
|
||||
|
||||
@staticmethod
|
||||
def _query_url(base_url: str, parameters: dict[str, Any]) -> str:
|
||||
return f"{base_url}?{urlencode(parameters)}"
|
||||
|
||||
@staticmethod
|
||||
def _fetch_json(
|
||||
url: str,
|
||||
settings: Settings,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
request = Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "GeoIntel/1.0 bathymetry-profile-acquisition",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with (opener or guarded_opener(url))(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response:
|
||||
limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024
|
||||
content = response.read(limit + 1)
|
||||
except HTTPError as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_HTTP_ERROR",
|
||||
message="VHA profile service returned an HTTP error",
|
||||
details={"status_code": exc.code},
|
||||
status_code=502,
|
||||
) from exc
|
||||
except (TimeoutError, URLError, OSError) as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_UNAVAILABLE",
|
||||
message="VHA profile service is unavailable",
|
||||
status_code=502,
|
||||
) from exc
|
||||
if len(content) > limit:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_RESPONSE_TOO_LARGE",
|
||||
message="VHA profile response exceeded the configured size limit",
|
||||
status_code=502,
|
||||
)
|
||||
try:
|
||||
payload = json.loads(content.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA profile service returned invalid JSON",
|
||||
status_code=502,
|
||||
) from exc
|
||||
if not isinstance(payload, dict) or payload.get("error"):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA profile service returned an ArcGIS error",
|
||||
details={"provider_error": payload.get("error") if isinstance(payload, dict) else None},
|
||||
status_code=502,
|
||||
)
|
||||
return payload, hashlib.sha256(content).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _base_spatial_parameters(bbox_values: tuple[float, float, float, float]) -> dict[str, str]:
|
||||
return {
|
||||
"f": "json",
|
||||
"where": "1=1",
|
||||
"geometry": ",".join(f"{value:.12g}" for value in bbox_values),
|
||||
"geometryType": "esriGeometryEnvelope",
|
||||
"inSR": "4326",
|
||||
"outSR": "4326",
|
||||
"spatialRel": "esriSpatialRelIntersects",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _unseen_records(
|
||||
page_features: list[Any],
|
||||
seen_object_ids: set[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Every page must bring records the earlier pages did not.
|
||||
|
||||
An ArcGIS layer without ``supportsPagination`` accepts ``resultOffset``
|
||||
and ignores it, answering every page with the first one. Advancing the
|
||||
offset by the page length still reaches the announced count, so the
|
||||
completeness check below passed while the dataset held N copies of page
|
||||
one — a silent substitution of the source data, which is the one thing
|
||||
bounded acquisition exists to prevent.
|
||||
"""
|
||||
|
||||
fresh: list[dict[str, Any]] = []
|
||||
for item in page_features:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
attributes = item.get("attributes")
|
||||
object_id = attributes.get("OBJECTID") if isinstance(attributes, dict) else None
|
||||
if object_id is None:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA profile record has no OBJECTID, so pagination cannot be verified",
|
||||
status_code=502,
|
||||
)
|
||||
key = str(object_id)
|
||||
if key in seen_object_ids:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
|
||||
message="VHA profile pagination repeated a record; the layer is not honouring resultOffset",
|
||||
details={"object_id": key},
|
||||
status_code=502,
|
||||
)
|
||||
seen_object_ids.add(key)
|
||||
fresh.append(item)
|
||||
return fresh
|
||||
|
||||
@staticmethod
|
||||
def _fetch_profiles(
|
||||
bbox_values: tuple[float, float, float, float],
|
||||
settings: Settings,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
base = settings.bathymetry_profiles_layer_url.rstrip("/") + "/query"
|
||||
count_url = BathymetryProfileAcquisitionService._query_url(
|
||||
base,
|
||||
{
|
||||
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
|
||||
"returnCountOnly": "true",
|
||||
"returnGeometry": "false",
|
||||
},
|
||||
)
|
||||
count_payload, count_sha = BathymetryProfileAcquisitionService._fetch_json(count_url, settings, opener)
|
||||
candidate_count = int(count_payload.get("count") or 0)
|
||||
if candidate_count > settings.bathymetry_profiles_max_features:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SCOPE_TOO_LARGE",
|
||||
message="The requested profile scope exceeds the configured feature limit; acquire smaller area partitions",
|
||||
details={
|
||||
"candidate_count": candidate_count,
|
||||
"max_features": settings.bathymetry_profiles_max_features,
|
||||
},
|
||||
status_code=413,
|
||||
)
|
||||
|
||||
features: list[dict[str, Any]] = []
|
||||
response_hashes: list[str] = []
|
||||
request_urls: list[str] = [count_url]
|
||||
seen_object_ids: set[str] = set()
|
||||
offset = 0
|
||||
while offset < candidate_count:
|
||||
if len(request_urls) > settings.bathymetry_profiles_max_pages:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SCOPE_TOO_LARGE",
|
||||
message="VHA profile pagination exceeded the configured page limit; acquire smaller area partitions",
|
||||
details={"max_pages": settings.bathymetry_profiles_max_pages},
|
||||
status_code=422,
|
||||
)
|
||||
page_url = BathymetryProfileAcquisitionService._query_url(
|
||||
base,
|
||||
{
|
||||
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
|
||||
"outFields": BathymetryProfileAcquisitionService.PROFILE_OUT_FIELDS,
|
||||
"returnGeometry": "true",
|
||||
"orderByFields": "OBJECTID",
|
||||
"resultOffset": str(offset),
|
||||
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
|
||||
},
|
||||
)
|
||||
page, page_sha = BathymetryProfileAcquisitionService._fetch_json(page_url, settings, opener)
|
||||
page_features = page.get("features")
|
||||
if not isinstance(page_features, list):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA profile response does not contain a feature list",
|
||||
status_code=502,
|
||||
)
|
||||
response_hashes.append(page_sha)
|
||||
request_urls.append(page_url)
|
||||
if not page_features:
|
||||
break
|
||||
features.extend(
|
||||
BathymetryProfileAcquisitionService._unseen_records(page_features, seen_object_ids)
|
||||
)
|
||||
offset += len(page_features)
|
||||
if len(features) != candidate_count:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INCOMPLETE_RESPONSE",
|
||||
message="VHA profile pagination did not return the announced number of records",
|
||||
details={"expected": candidate_count, "received": len(features)},
|
||||
status_code=502,
|
||||
)
|
||||
return features, {
|
||||
"candidate_count": candidate_count,
|
||||
"request_urls": request_urls,
|
||||
"response_sha256": [count_sha, *response_hashes],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _fetch_watercourse_names(
|
||||
vhag_codes: set[int],
|
||||
settings: Settings,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> tuple[dict[int, dict[str, str | None]], list[str], list[str]]:
|
||||
if not vhag_codes:
|
||||
return {}, [], []
|
||||
base = settings.bathymetry_watercourse_layer_url.rstrip("/") + "/query"
|
||||
names: dict[int, dict[str, str | None]] = {}
|
||||
urls: list[str] = []
|
||||
hashes: list[str] = []
|
||||
ordered_codes = sorted(vhag_codes)
|
||||
for start in range(0, len(ordered_codes), 100):
|
||||
chunk = ordered_codes[start : start + 100]
|
||||
offset = 0
|
||||
seen_page_hashes: set[str] = set()
|
||||
while True:
|
||||
if len(seen_page_hashes) >= settings.bathymetry_profiles_max_pages:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SCOPE_TOO_LARGE",
|
||||
message="VHA watercourse pagination exceeded the configured page limit",
|
||||
details={"max_pages": settings.bathymetry_profiles_max_pages},
|
||||
status_code=422,
|
||||
)
|
||||
url = BathymetryProfileAcquisitionService._query_url(
|
||||
base,
|
||||
{
|
||||
"f": "json",
|
||||
"where": f"\"wlasvl.vhag\" IN ({','.join(str(code) for code in chunk)})",
|
||||
"outFields": "wlasvl.vhag,VHAG_TABEL.naam,VHAG_TABEL.namen",
|
||||
"returnGeometry": "false",
|
||||
"orderByFields": "wlasvl.vhag",
|
||||
"resultOffset": str(offset),
|
||||
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
|
||||
},
|
||||
)
|
||||
payload, response_sha = BathymetryProfileAcquisitionService._fetch_json(url, settings, opener)
|
||||
urls.append(url)
|
||||
hashes.append(response_sha)
|
||||
page_features = payload.get("features")
|
||||
if not isinstance(page_features, list):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA watercourse response does not contain a feature list",
|
||||
status_code=502,
|
||||
)
|
||||
if response_sha in seen_page_hashes:
|
||||
# The names themselves deduplicate by code, so a stuck
|
||||
# provider produced no visible change while the loop, which
|
||||
# ended only on exceededTransferLimit, kept requesting.
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
|
||||
message="VHA watercourse pagination returned the same page again",
|
||||
status_code=502,
|
||||
)
|
||||
seen_page_hashes.add(response_sha)
|
||||
for feature in page_features:
|
||||
attributes = feature.get("attributes") if isinstance(feature, dict) else None
|
||||
if not isinstance(attributes, dict):
|
||||
continue
|
||||
raw_code = attributes.get("wlasvl.vhag")
|
||||
if raw_code is None:
|
||||
continue
|
||||
code = int(raw_code)
|
||||
if code not in names:
|
||||
names[code] = {
|
||||
"name": attributes.get("VHAG_TABEL.naam"),
|
||||
"alternative_names": attributes.get("VHAG_TABEL.namen"),
|
||||
}
|
||||
if not payload.get("exceededTransferLimit") or not page_features:
|
||||
break
|
||||
offset += len(page_features)
|
||||
return names, urls, hashes
|
||||
|
||||
@staticmethod
|
||||
def _date_from_arcgis(value: Any) -> str | None:
|
||||
if not isinstance(value, (int, float)) or not math.isfinite(float(value)):
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(float(value) / 1000.0, tz=UTC).date().isoformat()
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _numeric_or_none(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
normalized = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return normalized if math.isfinite(normalized) else None
|
||||
|
||||
@staticmethod
|
||||
def _document_url(value: Any) -> str | None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
normalized = value.strip()
|
||||
if normalized.startswith("http://vha.waterinfo.be/"):
|
||||
normalized = "https://" + normalized[len("http://") :]
|
||||
return normalized if normalized.startswith("https://vha.waterinfo.be/") else None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_features(
|
||||
raw_features: list[dict[str, Any]],
|
||||
scope_geometry,
|
||||
watercourse_names: dict[int, dict[str, str | None]],
|
||||
*,
|
||||
partition_properties: dict[str, Any] | None = None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
normalized: list[dict[str, Any]] = []
|
||||
dates: list[str] = []
|
||||
watercourse_codes: set[int] = set()
|
||||
document_count = 0
|
||||
depth_count = 0
|
||||
width_count = 0
|
||||
for raw_feature in raw_features:
|
||||
attributes = raw_feature.get("attributes")
|
||||
geometry = raw_feature.get("geometry")
|
||||
if not isinstance(attributes, dict) or not isinstance(geometry, dict):
|
||||
continue
|
||||
x = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("x"))
|
||||
y = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("y"))
|
||||
if x is None or y is None:
|
||||
continue
|
||||
point = Point(x, y)
|
||||
if not scope_geometry.covers(point):
|
||||
continue
|
||||
raw_vhag = attributes.get("vhag")
|
||||
vhag = int(raw_vhag) if isinstance(raw_vhag, (int, float)) else None
|
||||
if vhag is not None:
|
||||
watercourse_codes.add(vhag)
|
||||
names = watercourse_names.get(vhag or -1, {})
|
||||
measurement_date = BathymetryProfileAcquisitionService._date_from_arcgis(attributes.get("d_opmeti"))
|
||||
if measurement_date:
|
||||
dates.append(measurement_date)
|
||||
document_url = BathymetryProfileAcquisitionService._document_url(attributes.get("hyperlink"))
|
||||
depth = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_diepte"))
|
||||
crown_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_kruinb"))
|
||||
floor_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_vloerb"))
|
||||
if document_url:
|
||||
document_count += 1
|
||||
if depth is not None:
|
||||
depth_count += 1
|
||||
if crown_width is not None or floor_width is not None:
|
||||
width_count += 1
|
||||
object_id = str(attributes.get("OBJECTID"))
|
||||
normalized.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": object_id,
|
||||
"properties": {
|
||||
"source_feature_id": object_id,
|
||||
"provider_record_id": object_id,
|
||||
"watercourse_vhag": vhag,
|
||||
"watercourse_name": names.get("name") or (f"VHA-waterloop {vhag}" if vhag else "Onbekende waterloop"),
|
||||
"watercourse_alternative_names": names.get("alternative_names"),
|
||||
"profile_number": attributes.get("atlaspunt"),
|
||||
"measurement_date": measurement_date,
|
||||
"recorded_depth_m": depth,
|
||||
"recorded_crown_width_m": crown_width,
|
||||
"recorded_floor_width_m": floor_width,
|
||||
"source_document_url": document_url,
|
||||
"document_available": document_url is not None,
|
||||
"structured_depth_available": depth is not None,
|
||||
"source_code": attributes.get("bron"),
|
||||
"structure_id": attributes.get("kunstwerkid"),
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"measurement_semantics": "historical_cross_section_profile_point",
|
||||
"vertical_reference": "document-specific",
|
||||
**(partition_properties or {}),
|
||||
},
|
||||
"geometry": mapping(point),
|
||||
}
|
||||
)
|
||||
return (
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"name": "vha_bathymetry_profiles",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": normalized,
|
||||
},
|
||||
{
|
||||
"profile_count": len(normalized),
|
||||
"document_count": document_count,
|
||||
"structured_depth_count": depth_count,
|
||||
"structured_width_count": width_count,
|
||||
"watercourse_count": len(watercourse_codes),
|
||||
"measurement_date_min": min(dates) if dates else None,
|
||||
"measurement_date_max": max(dates) if dates else None,
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _municipality_name(area: Area | None) -> str | None:
|
||||
if area is None:
|
||||
return None
|
||||
normalized = str(area.name or "").strip()
|
||||
prefix = "Gemeente "
|
||||
if not normalized.casefold().startswith(prefix.casefold()):
|
||||
return None
|
||||
municipality = normalized[len(prefix) :].split(" - ", 1)[0].strip()
|
||||
return municipality or None
|
||||
|
||||
@staticmethod
|
||||
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
||||
return (
|
||||
db.query(Dataset)
|
||||
.filter(
|
||||
Dataset.project_id == project_id,
|
||||
Dataset.name == filename,
|
||||
Dataset.source_name == BathymetryProfileAcquisitionService.PROVIDER,
|
||||
Dataset.status == "ready",
|
||||
)
|
||||
.order_by(Dataset.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _result(dataset: Dataset, *, reused: bool) -> dict[str, Any]:
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
return BathymetryProfileAcquisitionResult(
|
||||
output_dataset_id=dataset.id,
|
||||
reused=reused,
|
||||
provider=BathymetryProfileAcquisitionService.PROVIDER,
|
||||
profile_count=int(metadata.get("profile_count") or 0),
|
||||
document_count=int(metadata.get("document_count") or 0),
|
||||
structured_depth_count=int(metadata.get("structured_depth_count") or 0),
|
||||
structured_width_count=int(metadata.get("structured_width_count") or 0),
|
||||
watercourse_count=int(metadata.get("watercourse_count") or 0),
|
||||
bbox_epsg4326=list(metadata.get("bbox_epsg4326") or []),
|
||||
clipped_to_area_id=dataset.area_id,
|
||||
measurement_date_min=metadata.get("measurement_date_min"),
|
||||
measurement_date_max=metadata.get("measurement_date_max"),
|
||||
attribution=BathymetryProfileAcquisitionService.ATTRIBUTION,
|
||||
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
|
||||
).model_dump(mode="json")
|
||||
|
||||
@staticmethod
|
||||
def acquire(
|
||||
db,
|
||||
project_id: UUID,
|
||||
payload: BathymetryProfileAcquireRequest,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
resolved_settings = settings or get_settings()
|
||||
if not resolved_settings.bathymetry_profiles_enabled:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_NOT_CONFIGURED",
|
||||
message="VHA bathymetry profile acquisition is disabled",
|
||||
status_code=503,
|
||||
)
|
||||
bbox_values = BathymetryProfileAcquisitionService._validate_bbox(payload)
|
||||
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
|
||||
db, project_id, payload.area_id, bbox_values
|
||||
)
|
||||
area = db.get(Area, payload.area_id) if payload.area_id else None
|
||||
municipality = BathymetryProfileAcquisitionService._municipality_name(area)
|
||||
exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
|
||||
request_identity = {
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"bbox_epsg4326": list(exact_bbox),
|
||||
"area_id": str(payload.area_id) if payload.area_id else None,
|
||||
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
|
||||
}
|
||||
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
|
||||
filename = f"vha_bathymetry_profiles_{request_hash[:12]}.geojson"
|
||||
if not payload.force_refresh:
|
||||
cached = BathymetryProfileAcquisitionService._cached_dataset(db, project_id, filename)
|
||||
if cached is not None:
|
||||
return BathymetryProfileAcquisitionService._result(cached, reused=True)
|
||||
|
||||
raw_features, fetch_provenance = BathymetryProfileAcquisitionService._fetch_profiles(
|
||||
exact_bbox, resolved_settings, opener
|
||||
)
|
||||
vhag_codes = {
|
||||
int(feature["attributes"]["vhag"])
|
||||
for feature in raw_features
|
||||
if isinstance(feature.get("attributes"), dict)
|
||||
and isinstance(feature["attributes"].get("vhag"), (int, float))
|
||||
}
|
||||
names, name_urls, name_hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names(
|
||||
vhag_codes, resolved_settings, opener
|
||||
)
|
||||
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
|
||||
raw_features,
|
||||
scope_geometry,
|
||||
names,
|
||||
partition_properties={
|
||||
"partition_area_id": str(area.id),
|
||||
"partition_area_name": area.name,
|
||||
**({"municipality": municipality} if municipality else {}),
|
||||
}
|
||||
if area
|
||||
else None,
|
||||
)
|
||||
if summary["profile_count"] == 0:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_NO_PROFILES",
|
||||
message="No VHA bathymetry profiles intersect the requested area",
|
||||
status_code=404,
|
||||
)
|
||||
artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
acquired_at = datetime.now(UTC)
|
||||
source_metadata = {
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"service": "ArcGIS MapServer",
|
||||
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
|
||||
"theme": "bathymetry",
|
||||
"layer_name": "bathymetry_profiles",
|
||||
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
|
||||
"partition_area_id": str(area.id) if area else None,
|
||||
"partition_area_name": area.name if area else None,
|
||||
"municipality": municipality,
|
||||
"partitioned_source_audit": False,
|
||||
"regional_partitions_complete": False,
|
||||
"bbox_epsg4326": list(exact_bbox),
|
||||
**summary,
|
||||
"selection_aggregation": {
|
||||
"metric_key": "profile_count",
|
||||
"method": "feature_count",
|
||||
"label": "Dwarsprofielen",
|
||||
"unit": "profielen",
|
||||
},
|
||||
"selection_metrics": [
|
||||
{
|
||||
"metric_key": "recorded_depth_mean_m",
|
||||
"method": "mean",
|
||||
"property": "recorded_depth_m",
|
||||
"label": "Gemiddelde geregistreerde diepte",
|
||||
"unit": "m",
|
||||
"warning": "Alleen profielen met een gestructureerde dieptewaarde; meetdata kunnen verschillen.",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_depth_min_m",
|
||||
"method": "min",
|
||||
"property": "recorded_depth_m",
|
||||
"label": "Kleinste geregistreerde diepte",
|
||||
"unit": "m",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_depth_max_m",
|
||||
"method": "max",
|
||||
"property": "recorded_depth_m",
|
||||
"label": "Grootste geregistreerde diepte",
|
||||
"unit": "m",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_crown_width_mean_m",
|
||||
"method": "mean",
|
||||
"property": "recorded_crown_width_m",
|
||||
"label": "Gemiddelde geregistreerde kruinbreedte",
|
||||
"unit": "m",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_floor_width_mean_m",
|
||||
"method": "mean",
|
||||
"property": "recorded_floor_width_m",
|
||||
"label": "Gemiddelde geregistreerde vloerbreedte",
|
||||
"unit": "m",
|
||||
},
|
||||
],
|
||||
"attribution": BathymetryProfileAcquisitionService.ATTRIBUTION,
|
||||
"license_note": BathymetryProfileAcquisitionService.LICENSE_NOTE,
|
||||
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
|
||||
"volume_supported": False,
|
||||
}
|
||||
dataset = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project_id,
|
||||
area_id=payload.area_id,
|
||||
filename=filename,
|
||||
content=artifact,
|
||||
source="VHA digitale atlas dwarsprofielen",
|
||||
source_name=BathymetryProfileAcquisitionService.PROVIDER,
|
||||
dataset_role="reference",
|
||||
reference_layer_name="bathymetry_profiles",
|
||||
source_version=BathymetryProfileAcquisitionService.SOURCE_VERSION,
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata={
|
||||
"acquisition": "explicit_bounded_arcgis_feature_query",
|
||||
"acquired_at": acquired_at.isoformat(),
|
||||
"request_hash": request_hash,
|
||||
"profile_query_urls": fetch_provenance["request_urls"],
|
||||
"watercourse_name_query_urls": name_urls,
|
||||
"response_sha256": [*fetch_provenance["response_sha256"], *name_hashes],
|
||||
"artifact_sha256": hashlib.sha256(artifact).hexdigest(),
|
||||
"candidate_count": fetch_provenance["candidate_count"],
|
||||
"exact_profile_count": summary["profile_count"],
|
||||
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
|
||||
"scope_geometry_type": scope_geometry.geom_type,
|
||||
"vertical_reference": "document-specific",
|
||||
"bathymetric_surface_available": False,
|
||||
"water_surface_level_available": False,
|
||||
"water_volume_available": False,
|
||||
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
|
||||
},
|
||||
)
|
||||
persisted = db.get(Dataset, dataset.id)
|
||||
return BathymetryProfileAcquisitionService._result(persisted, reused=False)
|
||||
|
||||
@staticmethod
|
||||
def finalize_partitions(
|
||||
db,
|
||||
project_id: UUID,
|
||||
payload: BathymetryPartitionFinalizeRequest,
|
||||
) -> dict[str, Any]:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
expected_area_ids = set(payload.expected_area_ids)
|
||||
dataset_ids = set(payload.dataset_ids)
|
||||
no_profile_area_ids = set(payload.no_profile_area_ids)
|
||||
|
||||
areas: dict[UUID, Area] = {}
|
||||
for area_id in expected_area_ids:
|
||||
area = db.get(Area, area_id)
|
||||
if area is None or area.project_id != project_id:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_AREA_INVALID",
|
||||
message="Every expected partition Area must belong to the project",
|
||||
details={"area_id": str(area_id)},
|
||||
status_code=400,
|
||||
)
|
||||
if BathymetryProfileAcquisitionService._municipality_name(area) is None:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_AREA_INVALID",
|
||||
message="Bathymetry partitions must use persisted municipality Areas",
|
||||
details={"area_id": str(area_id), "area_name": area.name},
|
||||
status_code=400,
|
||||
)
|
||||
areas[area_id] = area
|
||||
|
||||
if not no_profile_area_ids.issubset(expected_area_ids):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_MANIFEST_INVALID",
|
||||
message="No-profile partitions must be part of the expected Area set",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
datasets: list[Dataset] = []
|
||||
data_area_ids: set[UUID] = set()
|
||||
for dataset_id in payload.dataset_ids:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if (
|
||||
dataset is None
|
||||
or dataset.project_id != project_id
|
||||
or dataset.source_name != BathymetryProfileAcquisitionService.PROVIDER
|
||||
or dataset.status != "ready"
|
||||
or dataset.area_id not in expected_area_ids
|
||||
):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_DATASET_INVALID",
|
||||
message="Every partition Dataset must be a ready VHA profile Dataset scoped to an expected Area",
|
||||
details={"dataset_id": str(dataset_id)},
|
||||
status_code=400,
|
||||
)
|
||||
if dataset.area_id in data_area_ids:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_DATASET_DUPLICATE",
|
||||
message="A complete manifest may reference only one profile Dataset per Area",
|
||||
details={"area_id": str(dataset.area_id)},
|
||||
status_code=400,
|
||||
)
|
||||
data_area_ids.add(dataset.area_id)
|
||||
datasets.append(dataset)
|
||||
|
||||
accounted_area_ids = data_area_ids.union(no_profile_area_ids)
|
||||
if accounted_area_ids != expected_area_ids:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE",
|
||||
message="Every expected Area must have one ready Dataset or an explicit no-profile result",
|
||||
details={
|
||||
"missing_area_ids": sorted(str(value) for value in expected_area_ids - accounted_area_ids),
|
||||
"unexpected_area_ids": sorted(str(value) for value in accounted_area_ids - expected_area_ids),
|
||||
},
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
profile_count = 0
|
||||
document_count = 0
|
||||
structured_depth_count = 0
|
||||
dates_min: list[str] = []
|
||||
dates_max: list[str] = []
|
||||
shared_metadata = {
|
||||
"partition_scope_key": payload.partition_scope_key,
|
||||
"partition_count": len(expected_area_ids),
|
||||
"data_partition_count": len(datasets),
|
||||
"no_profile_partition_count": len(no_profile_area_ids),
|
||||
"partition_manifest_sha256": payload.manifest_sha256,
|
||||
"partition_manifest_observed_at": payload.observed_at.isoformat(),
|
||||
"partitioned_source_audit": True,
|
||||
"regional_partitions_complete": True,
|
||||
}
|
||||
shared_provenance = {
|
||||
"partition_manifest_sha256": payload.manifest_sha256,
|
||||
"partition_manifest_observed_at": payload.observed_at.isoformat(),
|
||||
"partition_scope_key": payload.partition_scope_key,
|
||||
"regional_partitions_complete": True,
|
||||
"no_profile_area_ids": sorted(str(value) for value in no_profile_area_ids),
|
||||
}
|
||||
|
||||
for dataset in datasets:
|
||||
source_metadata = dict(dataset.source_metadata or {})
|
||||
provenance_metadata = dict(dataset.provenance_metadata or {})
|
||||
profile_count += int(source_metadata.get("profile_count") or 0)
|
||||
document_count += int(source_metadata.get("document_count") or 0)
|
||||
structured_depth_count += int(source_metadata.get("structured_depth_count") or 0)
|
||||
if source_metadata.get("measurement_date_min"):
|
||||
dates_min.append(str(source_metadata["measurement_date_min"]))
|
||||
if source_metadata.get("measurement_date_max"):
|
||||
dates_max.append(str(source_metadata["measurement_date_max"]))
|
||||
area = areas[dataset.area_id]
|
||||
source_metadata.update(
|
||||
{
|
||||
**shared_metadata,
|
||||
"coverage_scope": payload.partition_scope_key,
|
||||
"partition_area_id": str(area.id),
|
||||
"partition_area_name": area.name,
|
||||
"municipality": BathymetryProfileAcquisitionService._municipality_name(area),
|
||||
}
|
||||
)
|
||||
provenance_metadata.update(shared_provenance)
|
||||
dataset.source_metadata = source_metadata
|
||||
dataset.provenance_metadata = provenance_metadata
|
||||
|
||||
if dataset_ids:
|
||||
versions = (
|
||||
db.query(DatasetVersion)
|
||||
.filter(DatasetVersion.dataset_id.in_(dataset_ids))
|
||||
.all()
|
||||
)
|
||||
for version in versions:
|
||||
version.source_metadata = dict(
|
||||
next(dataset.source_metadata for dataset in datasets if dataset.id == version.dataset_id)
|
||||
)
|
||||
version.provenance_metadata = dict(
|
||||
next(dataset.provenance_metadata for dataset in datasets if dataset.id == version.dataset_id)
|
||||
)
|
||||
|
||||
db.commit()
|
||||
return BathymetryPartitionFinalizationResult(
|
||||
partition_scope_key=payload.partition_scope_key,
|
||||
regional_partitions_complete=True,
|
||||
partition_count=len(expected_area_ids),
|
||||
data_partition_count=len(datasets),
|
||||
no_profile_partition_count=len(no_profile_area_ids),
|
||||
profile_count=profile_count,
|
||||
document_count=document_count,
|
||||
structured_depth_count=structured_depth_count,
|
||||
measurement_date_min=min(dates_min) if dates_min else None,
|
||||
measurement_date_max=max(dates_max) if dates_max else None,
|
||||
dataset_ids=payload.dataset_ids,
|
||||
manifest_sha256=payload.manifest_sha256,
|
||||
observed_at=payload.observed_at,
|
||||
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
|
||||
).model_dump(mode="json")
|
||||
@@ -0,0 +1,366 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import math
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import box, mapping
|
||||
from shapely.ops import transform as shapely_transform
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.services.raster_cell_selection import select_cells
|
||||
from app.models import Area, Dataset
|
||||
from app.schemas.bathymetry import (
|
||||
BathymetryRasterMetric,
|
||||
BathymetryRasterSelectionRequest,
|
||||
BathymetryRasterSelectionResponse,
|
||||
BathymetryRasterSelectionSummary,
|
||||
)
|
||||
|
||||
|
||||
class BathymetryRasterAnalysisService:
|
||||
SOURCE_NAME = "spw_bathymetry"
|
||||
PRODUCT_KEY = "spw_bathymetry_50cm_mdng"
|
||||
UNSUPPORTED_METRICS = [
|
||||
"current_water_depth_m",
|
||||
"water_volume_m3",
|
||||
"vertical_datum_conversion",
|
||||
]
|
||||
LIMITATION = (
|
||||
"De rasterwaarden zijn waterbodemhoogtes in mDNG uit een samengestelde SPW-opmeting "
|
||||
"(2019-2022). Zonder een gelijktijdig waterpeil zijn actuele waterdiepte en watervolume "
|
||||
"niet berekenbaar. mDNG wordt niet stilzwijgend naar TAW, LAT of een ander verticaal datum omgezet."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
if dataset.dataset_type != "raster" or dataset.source_name != BathymetryRasterAnalysisService.SOURCE_NAME:
|
||||
raise AppError(
|
||||
code="INVALID_BATHYMETRY_RASTER_DATASET",
|
||||
message="Bathymetry analysis requires a governed SPW bathymetry raster dataset",
|
||||
status_code=400,
|
||||
)
|
||||
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
|
||||
raise AppError(
|
||||
code="DATASET_FILE_MISSING",
|
||||
message="Persisted bathymetry raster file is unavailable",
|
||||
status_code=404,
|
||||
)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _metadata(dataset: Dataset) -> dict:
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
if (
|
||||
metadata.get("product_key") != BathymetryRasterAnalysisService.PRODUCT_KEY
|
||||
or metadata.get("theme") != "bathymetry"
|
||||
or metadata.get("value_semantics") != "bed_elevation"
|
||||
or metadata.get("vertical_reference") != "mDNG"
|
||||
or metadata.get("source_crs") != "EPSG:3812"
|
||||
):
|
||||
raise AppError(
|
||||
code="INVALID_BATHYMETRY_RASTER_METADATA",
|
||||
message="Bathymetry raster provenance or value semantics are incomplete",
|
||||
status_code=409,
|
||||
)
|
||||
return metadata
|
||||
|
||||
@staticmethod
|
||||
def _selection_geometry(db, project_id: UUID, payload: BathymetryRasterSelectionRequest):
|
||||
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
|
||||
if payload.area_id is None:
|
||||
return selection
|
||||
area = db.get(Area, payload.area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(
|
||||
code="INVALID_DATASET_SCOPE",
|
||||
message="Area does not belong to this project",
|
||||
status_code=400,
|
||||
)
|
||||
selection = selection.intersection(to_shape(area.geometry))
|
||||
if selection.is_empty or selection.area <= 0:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SELECTION_OUTSIDE_AREA",
|
||||
message="Selection does not overlap the selected work area",
|
||||
status_code=422,
|
||||
)
|
||||
return selection
|
||||
|
||||
@staticmethod
|
||||
def analyze(
|
||||
db,
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
payload: BathymetryRasterSelectionRequest,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
) -> dict:
|
||||
resolved_settings = settings or get_settings()
|
||||
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
|
||||
source_metadata = BathymetryRasterAnalysisService._metadata(dataset)
|
||||
selection_4326 = BathymetryRasterAnalysisService._selection_geometry(db, project_id, payload)
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.mask import mask
|
||||
except ImportError as exc:
|
||||
raise AppError(
|
||||
code="RASTER_PROCESSING_UNAVAILABLE",
|
||||
message="Rasterio and numpy are required for bathymetry analysis",
|
||||
status_code=503,
|
||||
) from exc
|
||||
|
||||
try:
|
||||
with rasterio.open(dataset.storage_path) as source:
|
||||
if source.crs is None or source.crs.to_epsg() != 3812:
|
||||
raise AppError(
|
||||
code="INVALID_DATASET_CRS",
|
||||
message="SPW bathymetry raster CRS must be EPSG:3812",
|
||||
status_code=409,
|
||||
)
|
||||
if source.count != 1:
|
||||
raise AppError(
|
||||
code="INVALID_BATHYMETRY_RASTER_BANDS",
|
||||
message="SPW bathymetry requires one bed-elevation band",
|
||||
status_code=409,
|
||||
)
|
||||
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
|
||||
selection_metric = shapely_transform(transformer.transform, selection_4326)
|
||||
analysis_geometry = selection_metric.intersection(box(*source.bounds))
|
||||
if analysis_geometry.is_empty or analysis_geometry.area <= 0:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SELECTION_OUTSIDE_DATASET",
|
||||
message="Selection does not overlap the persisted bathymetry raster",
|
||||
status_code=422,
|
||||
)
|
||||
min_x, min_y, max_x, max_y = analysis_geometry.bounds
|
||||
expected_cells = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil(
|
||||
(max_y - min_y) / abs(source.res[1])
|
||||
)
|
||||
if expected_cells > resolved_settings.bathymetry_raster_max_pixels:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SELECTION_TOO_LARGE",
|
||||
message="Bathymetry analysis exceeds the configured raster cell limit",
|
||||
details={
|
||||
"pixel_count": expected_cells,
|
||||
"max_pixels": resolved_settings.bathymetry_raster_max_pixels,
|
||||
},
|
||||
status_code=422,
|
||||
)
|
||||
# ``all_touched`` keeps the values of cells the selection only
|
||||
# clips, so a selection finer than one cell still has data to
|
||||
# read. Which of those cells actually count is decided by
|
||||
# ``select_cells`` below, so the normal result is unchanged.
|
||||
clipped, clipped_transform = mask(
|
||||
source,
|
||||
[mapping(analysis_geometry)],
|
||||
crop=True,
|
||||
filled=False,
|
||||
indexes=[1],
|
||||
all_touched=True,
|
||||
)
|
||||
band = np.ma.asarray(clipped[0], dtype="float64")
|
||||
raw = band.filled(np.nan)
|
||||
cell_selection = select_cells(
|
||||
analysis_geometry,
|
||||
out_shape=band.shape,
|
||||
transform=clipped_transform,
|
||||
cell_area_m2=abs(float(source.res[0])) * abs(float(source.res[1])),
|
||||
)
|
||||
selected_cells = cell_selection.mask
|
||||
valid_cells = selected_cells & ~np.ma.getmaskarray(band) & np.isfinite(raw)
|
||||
if source.nodata is not None:
|
||||
valid_cells &= ~np.isclose(raw, float(source.nodata))
|
||||
values = raw[valid_cells]
|
||||
if values.size == 0:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_NO_VALID_DATA",
|
||||
message="No surveyed waterbed cells occur in this selection",
|
||||
status_code=422,
|
||||
)
|
||||
resolution_x = abs(float(source.res[0]))
|
||||
resolution_y = abs(float(source.res[1]))
|
||||
cell_area_m2 = resolution_x * resolution_y
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_ANALYSIS_FAILED",
|
||||
message="The persisted bathymetry raster could not be analysed",
|
||||
details={"reason": str(exc)},
|
||||
status_code=500,
|
||||
) from exc
|
||||
|
||||
def metric(key: str, label: str, value: float, unit: str, method: str) -> BathymetryRasterMetric:
|
||||
return BathymetryRasterMetric(
|
||||
metric_key=key,
|
||||
metric_label=label,
|
||||
metric_value=round(float(value), 4),
|
||||
metric_unit=unit,
|
||||
aggregation_method=method,
|
||||
)
|
||||
|
||||
selected_cell_count = int(selected_cells.sum())
|
||||
valid_cell_count = int(values.size)
|
||||
vertical_unit = str(source_metadata["vertical_reference"])
|
||||
coverage_ratio = valid_cell_count / max(1, selected_cell_count)
|
||||
metrics = [
|
||||
metric(
|
||||
"bed_elevation_mean_m",
|
||||
"Gemiddelde waterbodemhoogte",
|
||||
values.mean(),
|
||||
f"m {vertical_unit}",
|
||||
"mean_valid_source_cells",
|
||||
),
|
||||
metric(
|
||||
"bed_elevation_min_m",
|
||||
"Laagste waterbodemhoogte",
|
||||
values.min(),
|
||||
f"m {vertical_unit}",
|
||||
"minimum_valid_source_cells",
|
||||
),
|
||||
metric(
|
||||
"bed_elevation_max_m",
|
||||
"Hoogste waterbodemhoogte",
|
||||
values.max(),
|
||||
f"m {vertical_unit}",
|
||||
"maximum_valid_source_cells",
|
||||
),
|
||||
metric(
|
||||
"bed_elevation_p10_m",
|
||||
"10e percentiel waterbodemhoogte",
|
||||
np.percentile(values, 10),
|
||||
f"m {vertical_unit}",
|
||||
"percentile_10_valid_source_cells",
|
||||
),
|
||||
metric(
|
||||
"bed_elevation_p90_m",
|
||||
"90e percentiel waterbodemhoogte",
|
||||
np.percentile(values, 90),
|
||||
f"m {vertical_unit}",
|
||||
"percentile_90_valid_source_cells",
|
||||
),
|
||||
metric(
|
||||
"surveyed_bed_surface_ha",
|
||||
"Oppervlakte met gemeten waterbodem",
|
||||
valid_cell_count * cell_area_m2 / 10_000.0,
|
||||
"ha",
|
||||
"valid_source_cells_times_cell_area",
|
||||
),
|
||||
metric(
|
||||
"bathymetry_coverage_pct",
|
||||
"Dekking waterbodemmeting",
|
||||
coverage_ratio * 100.0,
|
||||
"%",
|
||||
"valid_source_cells_divided_by_selected_cells",
|
||||
),
|
||||
]
|
||||
primary = metrics[0]
|
||||
response = BathymetryRasterSelectionResponse(
|
||||
dataset_id=dataset.id,
|
||||
product_key=BathymetryRasterAnalysisService.PRODUCT_KEY,
|
||||
selection_bbox=payload.bbox,
|
||||
selection_area_id=payload.area_id,
|
||||
selected_cell_count=selected_cell_count,
|
||||
valid_cell_count=valid_cell_count,
|
||||
coverage_ratio=round(coverage_ratio, 6),
|
||||
cell_selection_warning=cell_selection.warning,
|
||||
resolution_m=round(max(resolution_x, resolution_y), 4),
|
||||
vertical_reference=vertical_unit,
|
||||
survey_period=str(source_metadata.get("survey_period") or "2019-2022"),
|
||||
summary=BathymetryRasterSelectionSummary(
|
||||
metric_label=primary.metric_label,
|
||||
metric_value=primary.metric_value,
|
||||
metric_unit=primary.metric_unit,
|
||||
aggregation_method=primary.aggregation_method,
|
||||
primary_metric_key=primary.metric_key,
|
||||
metrics=metrics,
|
||||
),
|
||||
unsupported_metrics=BathymetryRasterAnalysisService.UNSUPPORTED_METRICS,
|
||||
limitation_message=BathymetryRasterAnalysisService.LIMITATION,
|
||||
generated_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
return response.model_dump(mode="json")
|
||||
|
||||
@staticmethod
|
||||
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
|
||||
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
|
||||
BathymetryRasterAnalysisService._metadata(dataset)
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
from rasterio.enums import Resampling
|
||||
except ImportError as exc:
|
||||
raise AppError(
|
||||
code="RASTER_PROCESSING_UNAVAILABLE",
|
||||
message="Rasterio, numpy and Pillow are required for bathymetry rendering",
|
||||
status_code=503,
|
||||
) from exc
|
||||
|
||||
try:
|
||||
with rasterio.open(dataset.storage_path) as source:
|
||||
scale = min(1.0, max_dimension / max(source.width, source.height))
|
||||
width = max(1, round(source.width * scale))
|
||||
height = max(1, round(source.height * scale))
|
||||
data = source.read(
|
||||
1,
|
||||
out_shape=(height, width),
|
||||
masked=True,
|
||||
resampling=Resampling.bilinear,
|
||||
)
|
||||
values = np.asarray(data.filled(np.nan), dtype="float64")
|
||||
valid = np.isfinite(values) & ~np.ma.getmaskarray(data)
|
||||
if source.nodata is not None:
|
||||
valid &= ~np.isclose(values, float(source.nodata))
|
||||
if not valid.any():
|
||||
raise AppError(
|
||||
code="BATHYMETRY_NO_VALID_DATA",
|
||||
message="Bathymetry raster contains no renderable cells",
|
||||
status_code=422,
|
||||
)
|
||||
low, high = np.percentile(values[valid], [2, 98])
|
||||
if high <= low:
|
||||
high = low + 1.0
|
||||
normalized = np.clip((values - low) / (high - low), 0.0, 1.0)
|
||||
normalized = np.where(valid, normalized, 0.0)
|
||||
stops = np.asarray([0.0, 0.35, 0.7, 1.0])
|
||||
colors = np.asarray(
|
||||
[
|
||||
[8, 47, 73],
|
||||
[15, 118, 140],
|
||||
[103, 190, 170],
|
||||
[236, 224, 163],
|
||||
],
|
||||
dtype="float64",
|
||||
)
|
||||
rgba = np.zeros((height, width, 4), dtype="uint8")
|
||||
for channel in range(3):
|
||||
rgba[:, :, channel] = np.interp(
|
||||
normalized,
|
||||
stops,
|
||||
colors[:, channel],
|
||||
).astype("uint8")
|
||||
rgba[:, :, 3] = np.where(valid, 220, 0).astype("uint8")
|
||||
output = io.BytesIO()
|
||||
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
|
||||
return output.getvalue()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PREVIEW_FAILED",
|
||||
message="The persisted bathymetry raster could not be rendered",
|
||||
details={"reason": str(exc)},
|
||||
status_code=500,
|
||||
) from exc
|
||||
@@ -0,0 +1,412 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import from_shape, to_shape
|
||||
from shapely.geometry import mapping
|
||||
from shapely.geometry.base import BaseGeometry
|
||||
from shapely.strtree import STRtree
|
||||
from shapely.validation import make_valid
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, VectorFeature
|
||||
from app.schemas.analysis import ChangeDetectionSummary
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
|
||||
|
||||
class ChangeDetectionService:
|
||||
SUPPORTED_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"}
|
||||
|
||||
@staticmethod
|
||||
def compare_vector_datasets(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
source_dataset_id: UUID,
|
||||
target_dataset_id: UUID,
|
||||
iou_threshold: float = 0.8,
|
||||
include_unchanged: bool = True,
|
||||
modified_threshold: float = 0.3,
|
||||
bbox: dict[str, Any] | None = None,
|
||||
area_id: UUID | None = None,
|
||||
preview_limit: int = 2_000,
|
||||
) -> ChangeDetectionSummary:
|
||||
if source_dataset_id == target_dataset_id:
|
||||
raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400)
|
||||
if iou_threshold < 0 or iou_threshold > 1:
|
||||
raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400)
|
||||
if modified_threshold < 0 or modified_threshold > iou_threshold:
|
||||
raise AppError(
|
||||
code="INVALID_PARAMETERS",
|
||||
message="modified_threshold must be between 0 and iou_threshold",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source")
|
||||
target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target")
|
||||
|
||||
selection_geometry = ChangeDetectionService._selection_geometry(db, project_id, bbox=bbox, area_id=area_id)
|
||||
|
||||
source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset, selection_geometry)
|
||||
target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset, selection_geometry)
|
||||
|
||||
if not source_features:
|
||||
raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422)
|
||||
if not target_features:
|
||||
raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422)
|
||||
|
||||
source_features = ChangeDetectionService.restrict_to_selection(source_features, selection_geometry, label="Source")
|
||||
target_features = ChangeDetectionService.restrict_to_selection(target_features, selection_geometry, label="Target")
|
||||
|
||||
classified = ChangeDetectionService._classify_features(
|
||||
source_features,
|
||||
target_features,
|
||||
iou_threshold=iou_threshold,
|
||||
modified_threshold=modified_threshold,
|
||||
)
|
||||
|
||||
buckets: dict[str, list[dict[str, Any]]] = {"added": [], "removed": [], "modified": [], "unchanged": []}
|
||||
for item in classified:
|
||||
buckets[item["change_type"]].append(
|
||||
ChangeDetectionService._feature(
|
||||
geometry=item["geometry"],
|
||||
change_type=item["change_type"],
|
||||
source_dataset_id=source_dataset_id,
|
||||
target_dataset_id=target_dataset_id,
|
||||
source_feature_id=item["source_feature_id"],
|
||||
target_feature_id=item["target_feature_id"],
|
||||
iou=item["iou"],
|
||||
properties=item["properties"],
|
||||
)
|
||||
)
|
||||
|
||||
unchanged_count = len(buckets["unchanged"])
|
||||
if not include_unchanged:
|
||||
buckets["unchanged"] = []
|
||||
|
||||
geojson_features, preview_truncated = ChangeDetectionService.limit_preview(
|
||||
buckets["added"] + buckets["removed"] + buckets["modified"] + buckets["unchanged"],
|
||||
limit=preview_limit,
|
||||
)
|
||||
warnings = source_warnings + target_warnings
|
||||
edge_count = sum(
|
||||
1 for feature in source_features + target_features if feature.get("partially_covered")
|
||||
)
|
||||
if edge_count:
|
||||
warnings.append(
|
||||
f"{edge_count} objecten liggen deels buiten de selectie. Ze zijn volledig vergeleken, zodat de "
|
||||
"selectierand zelf geen wijziging veroorzaakt."
|
||||
)
|
||||
if preview_truncated:
|
||||
warnings.append(
|
||||
f"De tellingen gelden voor de volledige selectie; de kaart toont maximaal {preview_limit} objecten, "
|
||||
"wijzigingen eerst."
|
||||
)
|
||||
return ChangeDetectionSummary(
|
||||
source_dataset_id=source_dataset_id,
|
||||
target_dataset_id=target_dataset_id,
|
||||
source_feature_count=len(source_features),
|
||||
target_feature_count=len(target_features),
|
||||
added_count=len(buckets["added"]),
|
||||
removed_count=len(buckets["removed"]),
|
||||
modified_count=len(buckets["modified"]),
|
||||
unchanged_count=unchanged_count,
|
||||
iou_threshold=iou_threshold,
|
||||
modified_iou_threshold=modified_threshold,
|
||||
selection_area_id=area_id,
|
||||
preview_limit=preview_limit,
|
||||
preview_truncated=preview_truncated,
|
||||
warnings=warnings,
|
||||
generated_at=datetime.now(timezone.utc),
|
||||
geojson={"type": "FeatureCollection", "features": geojson_features},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _selection_geometry(
|
||||
db: Session,
|
||||
project_id: UUID,
|
||||
*,
|
||||
bbox: dict[str, Any] | None,
|
||||
area_id: UUID | None,
|
||||
) -> BaseGeometry | None:
|
||||
"""Resolve the drawn rectangle against the named work area, if any."""
|
||||
|
||||
from app.models import Area
|
||||
from shapely.geometry import box as shapely_box
|
||||
|
||||
selection = None
|
||||
if bbox:
|
||||
selection = shapely_box(
|
||||
float(bbox["min_x"]), float(bbox["min_y"]), float(bbox["max_x"]), float(bbox["max_y"])
|
||||
)
|
||||
if area_id is None:
|
||||
return selection
|
||||
|
||||
area = db.get(Area, area_id)
|
||||
if area is None or area.project_id != project_id:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
area_geometry = to_shape(area.geometry)
|
||||
if selection is None:
|
||||
return area_geometry
|
||||
intersection = selection.intersection(area_geometry)
|
||||
if intersection.is_empty or intersection.area <= 0:
|
||||
raise AppError(
|
||||
code="CHANGE_DETECTION_SELECTION_OUTSIDE_AREA",
|
||||
message="Selection does not overlap the selected work area",
|
||||
status_code=422,
|
||||
)
|
||||
return intersection
|
||||
|
||||
# Order the preview spends its budget in. An operator asking what changed
|
||||
# is not helped by a cap filled with unchanged footprints.
|
||||
PREVIEW_PRIORITY = {"modified": 0, "added": 1, "removed": 2, "unchanged": 3}
|
||||
|
||||
@staticmethod
|
||||
def restrict_to_selection(
|
||||
features: list[dict[str, Any]],
|
||||
selection_geometry: BaseGeometry | None,
|
||||
*,
|
||||
label: str = "Dataset",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Keep the features a drawn selection reaches, and say which it cuts.
|
||||
|
||||
Geometry is deliberately *not* clipped. A change class describes a whole
|
||||
object: comparing a clipped 2020 footprint against an unclipped 2024 one
|
||||
would manufacture "modified" along the selection edge. Clipping is right
|
||||
for an area metric and wrong for an identity comparison.
|
||||
"""
|
||||
|
||||
if selection_geometry is None:
|
||||
return features
|
||||
|
||||
kept: list[dict[str, Any]] = []
|
||||
for feature in features:
|
||||
geometry = feature["geometry"]
|
||||
if not geometry.intersects(selection_geometry):
|
||||
continue
|
||||
kept.append({**feature, "partially_covered": not selection_geometry.covers(geometry)})
|
||||
|
||||
if not kept:
|
||||
raise AppError(
|
||||
code="CHANGE_DETECTION_SELECTION_EMPTY",
|
||||
message=f"{label} dataset has no features inside this selection",
|
||||
status_code=422,
|
||||
)
|
||||
return kept
|
||||
|
||||
@staticmethod
|
||||
def limit_preview(
|
||||
features: list[dict[str, Any]],
|
||||
*,
|
||||
limit: int,
|
||||
) -> tuple[list[dict[str, Any]], bool]:
|
||||
"""Cap the returned geometry without capping the counts.
|
||||
|
||||
``include_unchanged`` defaulted to true and nothing bounded the result,
|
||||
so a regional comparison returned a FeatureCollection holding both
|
||||
datasets in full. The counts describe the whole selection; the preview
|
||||
describes what a map can usefully draw.
|
||||
"""
|
||||
|
||||
if limit <= 0 or len(features) <= limit:
|
||||
return features, False
|
||||
ordered = sorted(
|
||||
features,
|
||||
key=lambda item: ChangeDetectionService.PREVIEW_PRIORITY.get(item["change_type"], 9),
|
||||
)
|
||||
return ordered[:limit], True
|
||||
|
||||
@staticmethod
|
||||
def _classify_features(
|
||||
source_features: list[dict[str, Any]],
|
||||
target_features: list[dict[str, Any]],
|
||||
*,
|
||||
iou_threshold: float,
|
||||
modified_threshold: float,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Pair source with target footprints and label how each one changed.
|
||||
|
||||
Matching is indexed rather than a full cross product: comparing two
|
||||
municipal building layers is otherwise hundreds of millions of geometry
|
||||
intersections. Sources are considered largest first so a big footprint
|
||||
is not left over after a small neighbour claimed its counterpart.
|
||||
"""
|
||||
|
||||
target_geometries = [feature["geometry"] for feature in target_features]
|
||||
tree = STRtree(target_geometries) if target_geometries else None
|
||||
claimed: set[int] = set()
|
||||
classified: list[dict[str, Any]] = []
|
||||
|
||||
order = sorted(
|
||||
range(len(source_features)),
|
||||
key=lambda index: (-source_features[index]["geometry"].area, str(source_features[index]["feature_id"])),
|
||||
)
|
||||
for source_index in order:
|
||||
source_feature = source_features[source_index]
|
||||
geometry = source_feature["geometry"]
|
||||
best_iou = 0.0
|
||||
best_index: int | None = None
|
||||
candidates = [] if tree is None else sorted(int(value) for value in tree.query(geometry))
|
||||
for target_index in candidates:
|
||||
if target_index in claimed:
|
||||
continue
|
||||
candidate_iou = ChangeDetectionService._iou(geometry, target_geometries[target_index])
|
||||
if candidate_iou > best_iou:
|
||||
best_iou = candidate_iou
|
||||
best_index = target_index
|
||||
|
||||
if best_index is not None and best_iou >= iou_threshold:
|
||||
claimed.add(best_index)
|
||||
change_type = "unchanged"
|
||||
elif best_index is not None and best_iou >= modified_threshold:
|
||||
# The same object, redrawn: an annexe, a demolition of one wing,
|
||||
# or a resurvey. Reporting it as removed + added would hide it.
|
||||
claimed.add(best_index)
|
||||
change_type = "modified"
|
||||
else:
|
||||
change_type = "removed"
|
||||
|
||||
classified.append(
|
||||
{
|
||||
"change_type": change_type,
|
||||
"geometry": geometry if change_type != "modified" else target_geometries[best_index],
|
||||
"source_feature_id": source_feature["feature_id"],
|
||||
"target_feature_id": target_features[best_index]["feature_id"] if change_type != "removed" else None,
|
||||
"iou": best_iou if best_iou > 0 else None,
|
||||
"properties": source_feature["properties"],
|
||||
}
|
||||
)
|
||||
|
||||
classified.extend(
|
||||
{
|
||||
"change_type": "added",
|
||||
"geometry": target_feature["geometry"],
|
||||
"source_feature_id": None,
|
||||
"target_feature_id": target_feature["feature_id"],
|
||||
"iou": None,
|
||||
"properties": target_feature["properties"],
|
||||
}
|
||||
for target_index, target_feature in enumerate(target_features)
|
||||
if target_index not in claimed
|
||||
)
|
||||
return classified
|
||||
|
||||
@staticmethod
|
||||
def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
|
||||
if dataset.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message=f"{label} dataset does not belong to this project", status_code=400)
|
||||
VectorOperationsService._require_vector_dataset(dataset)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _load_features(
|
||||
db: Session,
|
||||
dataset: Dataset,
|
||||
selection_geometry: BaseGeometry | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
query = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id)
|
||||
if selection_geometry is not None and hasattr(query, "filter"):
|
||||
# Bound the load in the database. Pulling a regional building layer
|
||||
# into Python to then discard most of it costs memory and time for
|
||||
# nothing, and the fallback below has no such option.
|
||||
try:
|
||||
query = query.filter(
|
||||
func.ST_Intersects(VectorFeature.geometry, from_shape(selection_geometry, srid=4326))
|
||||
)
|
||||
except Exception:
|
||||
# Lightweight unit-test sessions do not implement every spatial
|
||||
# predicate; restrict_to_selection still bounds the population.
|
||||
pass
|
||||
rows = query.all()
|
||||
warnings: list[str] = []
|
||||
if rows:
|
||||
return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings
|
||||
|
||||
warnings.append(f"Dataset {dataset.id} has no persisted vector_features; falling back to stored GeoJSON artifact")
|
||||
_payload, raw_features = VectorOperationsService._load_dataset_payload(dataset)
|
||||
extracted = VectorOperationsService._extract_geometries(raw_features)
|
||||
return [
|
||||
ChangeDetectionService._raw_feature_to_feature(index, raw_feature, geometry)
|
||||
for index, (raw_feature, geometry) in enumerate(extracted)
|
||||
], warnings
|
||||
|
||||
@staticmethod
|
||||
def _row_to_feature(row: VectorFeature) -> dict[str, Any]:
|
||||
geometry = ChangeDetectionService._valid_comparable_geometry(to_shape(row.geometry))
|
||||
return {
|
||||
"feature_id": str(row.source_feature_id or row.id),
|
||||
"properties": dict(row.properties_json or {}),
|
||||
"geometry": geometry,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _raw_feature_to_feature(index: int, raw_feature: dict[str, Any], geometry: BaseGeometry) -> dict[str, Any]:
|
||||
properties = raw_feature.get("properties") if isinstance(raw_feature.get("properties"), dict) else {}
|
||||
source_id = raw_feature.get("id") or properties.get("id") or properties.get("source_feature_id") or str(index)
|
||||
return {
|
||||
"feature_id": str(source_id),
|
||||
"properties": dict(properties),
|
||||
"geometry": ChangeDetectionService._valid_comparable_geometry(geometry),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _valid_comparable_geometry(geometry: BaseGeometry) -> BaseGeometry:
|
||||
if geometry.is_empty:
|
||||
raise AppError(code="INVALID_GEOMETRY", message="Empty geometry cannot be compared", status_code=400)
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
raise AppError(code="INVALID_GEOMETRY", message="Geometry cannot be repaired for comparison", status_code=400)
|
||||
if geometry.geom_type not in ChangeDetectionService.SUPPORTED_GEOMETRY_TYPES:
|
||||
raise AppError(
|
||||
code="UNSUPPORTED_GEOMETRY",
|
||||
message="Change detection supports Polygon and MultiPolygon geometries only",
|
||||
details={"geometry_type": geometry.geom_type},
|
||||
status_code=422,
|
||||
)
|
||||
return geometry
|
||||
|
||||
@staticmethod
|
||||
def _iou(left: BaseGeometry, right: BaseGeometry) -> float:
|
||||
if left.area <= 0 or right.area <= 0:
|
||||
return 0.0
|
||||
intersection = left.intersection(right)
|
||||
if intersection.is_empty:
|
||||
return 0.0
|
||||
union_area = left.area + right.area - intersection.area
|
||||
if union_area <= 0:
|
||||
return 0.0
|
||||
return float(intersection.area / union_area)
|
||||
|
||||
@staticmethod
|
||||
def _feature(
|
||||
*,
|
||||
geometry: BaseGeometry,
|
||||
change_type: str,
|
||||
source_dataset_id: UUID,
|
||||
target_dataset_id: UUID,
|
||||
source_feature_id: str | None,
|
||||
target_feature_id: str | None,
|
||||
iou: float | None,
|
||||
properties: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"type": "Feature",
|
||||
"geometry": mapping(geometry),
|
||||
"properties": {
|
||||
**properties,
|
||||
"change_type": change_type,
|
||||
"source_dataset_id": str(source_dataset_id),
|
||||
"target_dataset_id": str(target_dataset_id),
|
||||
"source_feature_id": source_feature_id,
|
||||
"target_feature_id": target_feature_id,
|
||||
"iou": iou,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,645 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import unary_union
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.schemas.coverage import (
|
||||
CoverageBBox,
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolutionItem,
|
||||
CoverageResolveResponse,
|
||||
CoverageSourceContract,
|
||||
)
|
||||
|
||||
|
||||
THEMES = (
|
||||
"admin",
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"population",
|
||||
"parcels",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
"maritime_planning",
|
||||
"marine_environment",
|
||||
"bathymetry",
|
||||
)
|
||||
|
||||
ZONES = (
|
||||
"belgium",
|
||||
"flanders",
|
||||
"wallonia",
|
||||
"brussels",
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
)
|
||||
|
||||
STATUS_ORDER = ("unsupported", "not_configured", "partial", "operational")
|
||||
STATUS_RANK = {status: index for index, status in enumerate(STATUS_ORDER)}
|
||||
|
||||
SCOPE_AREA_NAMES = {
|
||||
"belgium": "Belgium land",
|
||||
"flanders": "Flanders",
|
||||
"wallonia": "Wallonia",
|
||||
"brussels": "Brussels-Capital Region",
|
||||
"belgian_north_sea": "Belgian part of the North Sea",
|
||||
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
||||
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
||||
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
||||
}
|
||||
|
||||
DETAIL_ZONES = (
|
||||
"flanders",
|
||||
"wallonia",
|
||||
"brussels",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SourceDefinition:
|
||||
contract: CoverageSourceContract
|
||||
materialized_layer_names: tuple[str, ...] = ()
|
||||
materialized_source_names: tuple[str, ...] = ()
|
||||
operational_themes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _contract(
|
||||
*,
|
||||
source_name: str,
|
||||
display_name: str,
|
||||
authority_level: str,
|
||||
coverage_zones: tuple[str, ...],
|
||||
themes: tuple[str, ...],
|
||||
native_layers: tuple[str, ...],
|
||||
geometry_types: tuple[str, ...],
|
||||
acquisition_mode: str,
|
||||
integration_status: str,
|
||||
source_url: str,
|
||||
attribution: str,
|
||||
license_note: str,
|
||||
limitation_message: str,
|
||||
materialized_layer_names: tuple[str, ...] = (),
|
||||
materialized_source_names: tuple[str, ...] = (),
|
||||
operational_themes: tuple[str, ...] = (),
|
||||
) -> _SourceDefinition:
|
||||
return _SourceDefinition(
|
||||
contract=CoverageSourceContract(
|
||||
source_name=source_name,
|
||||
display_name=display_name,
|
||||
authority_level=authority_level,
|
||||
coverage_zones=list(coverage_zones),
|
||||
themes=list(themes),
|
||||
native_layers=list(native_layers),
|
||||
supported_geometry_types=list(geometry_types),
|
||||
acquisition_mode=acquisition_mode,
|
||||
integration_status=integration_status,
|
||||
source_url=source_url,
|
||||
attribution=attribution,
|
||||
license_note=license_note,
|
||||
limitation_message=limitation_message,
|
||||
),
|
||||
materialized_layer_names=materialized_layer_names,
|
||||
materialized_source_names=materialized_source_names or (source_name,),
|
||||
operational_themes=operational_themes,
|
||||
)
|
||||
|
||||
|
||||
SOURCE_DEFINITIONS = (
|
||||
_contract(
|
||||
source_name="ngi_adminvector",
|
||||
display_name="NGI AdminVector",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("belgium", "flanders", "wallonia", "brussels", "belgian_north_sea"),
|
||||
themes=("admin",),
|
||||
native_layers=(
|
||||
"belgianterritory",
|
||||
"belgianmaritimezone",
|
||||
"region",
|
||||
"province",
|
||||
"municipality",
|
||||
),
|
||||
geometry_types=("Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_archive",
|
||||
integration_status="operational",
|
||||
source_url="https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9",
|
||||
attribution="National Geographic Institute (NGI), AdminVector",
|
||||
license_note="CC BY 4.0",
|
||||
limitation_message="Administrative reference geometry; it does not provide thematic land content.",
|
||||
materialized_layer_names=(
|
||||
"belgium_land_boundary",
|
||||
"belgium_regions",
|
||||
"belgium_provinces",
|
||||
"belgium_municipalities",
|
||||
),
|
||||
),
|
||||
_contract(
|
||||
source_name="statbel",
|
||||
display_name="Statbel statistical sectors and population",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("belgium", "flanders", "wallonia", "brussels"),
|
||||
themes=("admin", "population"),
|
||||
native_layers=("statistical_sectors", "population_statistics"),
|
||||
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
|
||||
acquisition_mode="operator_archive",
|
||||
integration_status="operational",
|
||||
source_url="https://statbel.fgov.be/en/open-data",
|
||||
attribution="Statbel",
|
||||
license_note="Consult the license of the selected Statbel release.",
|
||||
limitation_message=(
|
||||
"National editions require the governed plan-stage-review-apply operator; "
|
||||
"population in partially selected sectors is area-weighted."
|
||||
),
|
||||
materialized_layer_names=("population",),
|
||||
operational_themes=("population",),
|
||||
),
|
||||
_contract(
|
||||
source_name="digitaal_vlaanderen",
|
||||
display_name="Flemish authoritative services",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("flanders",),
|
||||
themes=(
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"parcels",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
),
|
||||
native_layers=("GRB", "BWK", "DHMV", "OMWRGBMRVL", "OGRK", "Mercator"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://www.vlaanderen.be/datavindplaats",
|
||||
attribution="Digitaal Vlaanderen and the authoritative Flemish source owners",
|
||||
license_note="Consult the license and attribution stored with each acquired dataset.",
|
||||
limitation_message="Operational only for bounded products implemented by GeoIntel and materialized in the project.",
|
||||
materialized_source_names=(
|
||||
"grb",
|
||||
"digitaal_vlaanderen_buildings_addresses_register",
|
||||
"digitaal_vlaanderen_dhmv",
|
||||
"digitaal_vlaanderen_orthophoto",
|
||||
"vmm_flood_hazard",
|
||||
"department_omgeving_thematic_raster",
|
||||
"inbo_bwk_natura2000",
|
||||
"dov_soil_map",
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
||||
),
|
||||
),
|
||||
_contract(
|
||||
source_name="vmm_vha_bathymetry_profiles",
|
||||
display_name="VHA historische dwarsprofielen",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("flanders",),
|
||||
themes=("bathymetry",),
|
||||
native_layers=("digitale_atlas_profile_points",),
|
||||
geometry_types=("Point",),
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
|
||||
attribution="Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas",
|
||||
license_note="Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata.",
|
||||
limitation_message=(
|
||||
"Historische puntmetingen met bronafhankelijke meetdatum en verticale referentie; "
|
||||
"geen continue actuele bodemkaart en zonder gelijktijdig waterpeil geen watervolume."
|
||||
),
|
||||
materialized_source_names=("vmm_vha_bathymetry_profiles",),
|
||||
),
|
||||
_contract(
|
||||
source_name="spw_geoportail",
|
||||
display_name="SPW Geoportail Wallonie",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("wallonia",),
|
||||
themes=(
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
"bathymetry",
|
||||
),
|
||||
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://geoportail.wallonie.be/catalogue",
|
||||
attribution="Service public de Wallonie",
|
||||
license_note="Consult the license of each Geoportail Wallonie product.",
|
||||
limitation_message=(
|
||||
"Bounded PICC buildings, road axes and hydrography, the legally current flood-hazard polygons, "
|
||||
"operator-imported SPW bathymetry and bounded SPW MNT terrain are operational; other Walloon themes remain separately governed."
|
||||
),
|
||||
materialized_source_names=("spw_picc", "spw_flood_hazard", "spw_walous_land_cover", "spw_bathymetry", "spw_terrain"),
|
||||
operational_themes=("buildings", "roads", "surface_water", "land_cover_use", "elevation", "flood_climate", "bathymetry"),
|
||||
),
|
||||
_contract(
|
||||
source_name="urbis",
|
||||
display_name="UrbIS Brussels",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("brussels",),
|
||||
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
|
||||
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://datastore.brussels",
|
||||
attribution="Brussels UrbIS",
|
||||
license_note="Consult the license of the selected UrbIS dataset.",
|
||||
limitation_message=(
|
||||
"Bounded UrbIS buildings, cadastral parcels, street axes and Land Cover blocks are operational. "
|
||||
"Permanent water uses the official WB block class; no separate hydrography network is inferred."
|
||||
),
|
||||
materialized_source_names=("urbis",),
|
||||
operational_themes=("buildings", "parcels", "roads", "surface_water", "land_cover_use"),
|
||||
),
|
||||
_contract(
|
||||
source_name="rbins_marine_reporting_units",
|
||||
display_name="RBINS marine reporting units",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("admin", "marine_environment"),
|
||||
native_layers=("marine_reporting_units_2024",),
|
||||
geometry_types=("Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_wfs",
|
||||
integration_status="operational",
|
||||
source_url=(
|
||||
"https://metadata.naturalsciences.be/geonetwork/srv/api/records/"
|
||||
"29f40b0d-2a3e-49a8-870a-e9b4acd4d1e3"
|
||||
),
|
||||
attribution="Royal Belgian Institute of Natural Sciences (RBINS), BMDC",
|
||||
license_note="Reuse conditions are retained from the source metadata with every persisted artifact.",
|
||||
limitation_message="The EEZ and continental shelf can share geometry while retaining different legal semantics.",
|
||||
materialized_layer_names=("marine_legal_scopes",),
|
||||
),
|
||||
_contract(
|
||||
source_name="rbins_msp_2026",
|
||||
display_name="Belgian Marine Spatial Plan 2026-2034",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("maritime_planning", "marine_environment"),
|
||||
native_layers=("imsp26",),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_wfs",
|
||||
integration_status="operational",
|
||||
source_url="https://www.health.belgium.be/en/themes/environment/marine-environment/marine-spatial-plan",
|
||||
attribution="Belgian federal Marine Environment service and RBINS",
|
||||
license_note="Official source metadata and attribution are retained with the imported snapshot.",
|
||||
limitation_message="The dataset represents the legally current 2026-2034 plan, not live maritime activity.",
|
||||
materialized_layer_names=("marine_spatial_plan_2026",),
|
||||
),
|
||||
_contract(
|
||||
source_name="mdk_bathymetry",
|
||||
display_name="MDK Belgian North Sea depth model",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("bathymetry",),
|
||||
native_layers=("depth_model_20m_lat",),
|
||||
geometry_types=("Raster",),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://www.vlaanderen.be/datavindplaats",
|
||||
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
|
||||
license_note="Consult the official product license before acquisition.",
|
||||
limitation_message=(
|
||||
"Bounded strict-TLS WCS acquisition is implemented but stays disabled until the operator enables it "
|
||||
"with a live-validated coverage id; no depths are synthesized."
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
|
||||
"buildings": {
|
||||
"grb": ("buildings",),
|
||||
"digitaal_vlaanderen_buildings_addresses_register": (),
|
||||
},
|
||||
"roads": {"grb": ("roads",)},
|
||||
"surface_water": {"grb": ("water",)},
|
||||
"land_cover_use": {
|
||||
"department_omgeving_thematic_raster": (),
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
|
||||
},
|
||||
"nature": {"inbo_bwk_natura2000": ()},
|
||||
"parcels": {
|
||||
"grb": ("parcels",),
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
|
||||
},
|
||||
"soil": {"dov_soil_map": ()},
|
||||
"elevation": {"digitaal_vlaanderen_dhmv": ()},
|
||||
"orthophoto": {"digitaal_vlaanderen_orthophoto": ()},
|
||||
"flood_climate": {"vmm_flood_hazard": ()},
|
||||
}
|
||||
|
||||
REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = {
|
||||
"spw_geoportail": {
|
||||
"buildings": {"spw_picc": ("buildings",)},
|
||||
"roads": {"spw_picc": ("roads",)},
|
||||
"surface_water": {"spw_picc": ("water",)},
|
||||
"land_cover_use": {"spw_walous_land_cover": ()},
|
||||
"elevation": {"spw_terrain": ()},
|
||||
"flood_climate": {"spw_flood_hazard": ("flood_hazard",)},
|
||||
"bathymetry": {"spw_bathymetry": ()},
|
||||
"orthophoto": {"spw_orthophoto": ()},
|
||||
},
|
||||
"urbis": {
|
||||
"buildings": {"urbis": ("buildings",)},
|
||||
"parcels": {"urbis": ("parcels",)},
|
||||
"roads": {"urbis": ("roads",)},
|
||||
"surface_water": {"urbis": ("water",)},
|
||||
"land_cover_use": {"urbis": ("space_occupation", "forest")},
|
||||
"orthophoto": {"urbis_orthophoto": ()},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class CoverageRegistryService:
|
||||
@staticmethod
|
||||
def catalog() -> CoverageCatalogResponse:
|
||||
return CoverageCatalogResponse(
|
||||
themes=list(THEMES),
|
||||
zones=list(ZONES),
|
||||
statuses=list(STATUS_ORDER),
|
||||
sources=[definition.contract for definition in SOURCE_DEFINITIONS],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def normalize_themes(themes: Iterable[str]) -> list[str]:
|
||||
requested = list(dict.fromkeys(str(theme).strip().lower() for theme in themes if str(theme).strip()))
|
||||
invalid = sorted(set(requested) - set(THEMES))
|
||||
if invalid:
|
||||
raise AppError(
|
||||
code="COVERAGE_THEME_UNSUPPORTED",
|
||||
message="One or more coverage themes are unsupported",
|
||||
status_code=422,
|
||||
details={"unsupported_themes": invalid, "supported_themes": list(THEMES)},
|
||||
)
|
||||
return requested or list(THEMES)
|
||||
|
||||
@staticmethod
|
||||
def _geometry(value: Any):
|
||||
if value is None:
|
||||
return None
|
||||
return value if hasattr(value, "__geo_interface__") else to_shape(value)
|
||||
|
||||
@staticmethod
|
||||
def _intersected_zones(areas: list[Area], selection) -> tuple[list[str], bool]:
|
||||
geometries: dict[str, Any] = {}
|
||||
by_name = {area.name: area for area in areas}
|
||||
for zone, area_name in SCOPE_AREA_NAMES.items():
|
||||
area = by_name.get(area_name)
|
||||
geometry = CoverageRegistryService._geometry(area.geometry) if area else None
|
||||
if geometry is not None and not geometry.is_empty:
|
||||
geometries[zone] = geometry
|
||||
|
||||
detail_intersections = [
|
||||
zone for zone in DETAIL_ZONES if zone in geometries and geometries[zone].intersects(selection)
|
||||
]
|
||||
zones = detail_intersections
|
||||
if not any(zone in zones for zone in ("flanders", "wallonia", "brussels")):
|
||||
if "belgium" in geometries and geometries["belgium"].intersects(selection):
|
||||
zones = ["belgium", *zones]
|
||||
if not any(zone in zones for zone in ("territorial_sea", "exclusive_economic_zone", "continental_shelf")):
|
||||
if "belgian_north_sea" in geometries and geometries["belgian_north_sea"].intersects(selection):
|
||||
zones = [*zones, "belgian_north_sea"]
|
||||
|
||||
intersected_geometries = [geometries[zone].intersection(selection) for zone in zones if zone in geometries]
|
||||
covered = unary_union(intersected_geometries) if intersected_geometries else None
|
||||
outside = covered is None or covered.is_empty or not covered.covers(selection)
|
||||
return zones, outside
|
||||
|
||||
@staticmethod
|
||||
def _matching_datasets(
|
||||
datasets: list[Dataset],
|
||||
definition: _SourceDefinition,
|
||||
theme: str,
|
||||
zone: str,
|
||||
selection: Any,
|
||||
) -> tuple[list[Dataset], bool]:
|
||||
if definition.operational_themes and theme not in definition.operational_themes:
|
||||
return [], False
|
||||
matches: list[Dataset] = []
|
||||
bounded_scopes: list[Any] = []
|
||||
zone_scoped_materialization = False
|
||||
for dataset in datasets:
|
||||
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
||||
continue
|
||||
# A source-name claim alone must not cause an unsafe artifact to
|
||||
# appear as operational authoritative coverage.
|
||||
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
|
||||
continue
|
||||
layer_names = definition.materialized_layer_names
|
||||
if definition.contract.source_name == "digitaal_vlaanderen":
|
||||
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
||||
if dataset.source_name not in theme_sources:
|
||||
continue
|
||||
layer_names = theme_sources[dataset.source_name]
|
||||
elif definition.contract.source_name in REGIONAL_THEME_DATASETS:
|
||||
theme_sources = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme, {})
|
||||
if dataset.source_name not in theme_sources:
|
||||
continue
|
||||
layer_names = theme_sources[dataset.source_name]
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
|
||||
if isinstance(coverage_zones, str):
|
||||
coverage_zones = [coverage_zones]
|
||||
acquired_bbox = metadata.get("bbox_epsg4326")
|
||||
if (
|
||||
definition.contract.acquisition_mode == "bounded_api"
|
||||
and isinstance(acquired_bbox, list)
|
||||
and len(acquired_bbox) == 4
|
||||
):
|
||||
try:
|
||||
acquired_scope = box(*(float(value) for value in acquired_bbox))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not acquired_scope.is_valid or not acquired_scope.intersects(selection):
|
||||
continue
|
||||
bounded_scopes.append(acquired_scope)
|
||||
elif definition.contract.acquisition_mode == "bounded_api" and coverage_zones:
|
||||
zone_scoped_materialization = zone in coverage_zones or "belgium" in coverage_zones
|
||||
layer_matches = not layer_names or dataset.reference_layer_name in layer_names
|
||||
zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones
|
||||
if layer_matches and zone_matches:
|
||||
matches.append(dataset)
|
||||
if not matches:
|
||||
return [], False
|
||||
fully_covered = True
|
||||
if definition.contract.acquisition_mode == "bounded_api":
|
||||
fully_covered = zone_scoped_materialization or (bool(bounded_scopes) and unary_union(bounded_scopes).covers(selection))
|
||||
return matches, fully_covered
|
||||
|
||||
@staticmethod
|
||||
def _resolve_item(
|
||||
*,
|
||||
zone: str,
|
||||
theme: str,
|
||||
datasets: list[Dataset],
|
||||
selection: Any,
|
||||
) -> CoverageResolutionItem:
|
||||
definitions = [
|
||||
definition
|
||||
for definition in SOURCE_DEFINITIONS
|
||||
if zone in definition.contract.coverage_zones and theme in definition.contract.themes
|
||||
]
|
||||
if not definitions:
|
||||
return CoverageResolutionItem(
|
||||
zone=zone,
|
||||
theme=theme,
|
||||
status="unsupported",
|
||||
source_names=[],
|
||||
materialized_dataset_ids=[],
|
||||
limitation_message="No audited source contract supports this theme in the selected zone.",
|
||||
)
|
||||
|
||||
materialized: list[Dataset] = []
|
||||
evidence: list[dict[str, Any]] = []
|
||||
source_statuses: list[str] = []
|
||||
limitations: list[str] = []
|
||||
for definition in definitions:
|
||||
matches, fully_covered = CoverageRegistryService._matching_datasets(
|
||||
datasets,
|
||||
definition,
|
||||
theme,
|
||||
zone,
|
||||
selection,
|
||||
)
|
||||
materialized.extend(matches)
|
||||
for dataset in matches:
|
||||
metadata = dataset.source_metadata if isinstance(getattr(dataset, "source_metadata", None), dict) else {}
|
||||
observed_at = getattr(dataset, "observed_at", None)
|
||||
published_at = metadata.get("published_at") or metadata.get("publication_date") or metadata.get("published_on")
|
||||
evidence.append({
|
||||
"dataset_id": dataset.id,
|
||||
"source_name": str(dataset.source_name or definition.contract.source_name),
|
||||
"authority_level": definition.contract.authority_level,
|
||||
"source_version": getattr(dataset, "source_version", None),
|
||||
"observed_at": observed_at.isoformat() if hasattr(observed_at, "isoformat") else (str(observed_at) if observed_at else None),
|
||||
"published_at": str(published_at) if published_at else None,
|
||||
"crs": getattr(dataset, "crs", None) or metadata.get("source_crs"),
|
||||
"resolution": getattr(dataset, "resolution_json", None),
|
||||
"coverage_bbox_epsg4326": metadata.get("bbox_epsg4326"),
|
||||
"attribution": metadata.get("attribution") or definition.contract.attribution,
|
||||
"license_note": metadata.get("license_note") or definition.contract.license_note,
|
||||
"checksum_sha256": getattr(dataset, "checksum_sha256", None),
|
||||
})
|
||||
if matches and fully_covered:
|
||||
source_statuses.append("operational")
|
||||
elif matches:
|
||||
source_statuses.append("partial")
|
||||
elif (
|
||||
definition.contract.integration_status == "operational"
|
||||
and (not definition.operational_themes or theme in definition.operational_themes)
|
||||
):
|
||||
source_statuses.append("partial")
|
||||
else:
|
||||
source_statuses.append(
|
||||
"not_configured"
|
||||
if definition.contract.integration_status == "operational"
|
||||
else definition.contract.integration_status
|
||||
)
|
||||
limitations.append(definition.contract.limitation_message)
|
||||
|
||||
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
|
||||
return CoverageResolutionItem(
|
||||
zone=zone,
|
||||
theme=theme,
|
||||
status=best_status,
|
||||
source_names=[definition.contract.source_name for definition in definitions],
|
||||
materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)),
|
||||
evidence=list({str(item["dataset_id"]): item for item in evidence}.values()),
|
||||
limitation_message=" ".join(dict.fromkeys(limitations)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def resolve(
|
||||
db: Session,
|
||||
project_id: UUID,
|
||||
bbox: CoverageBBox,
|
||||
themes: Iterable[str],
|
||||
) -> CoverageResolveResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
requested_themes = CoverageRegistryService.normalize_themes(themes)
|
||||
selection = box(bbox.minx, bbox.miny, bbox.maxx, bbox.maxy)
|
||||
areas = db.query(Area).filter(Area.project_id == project_id).all()
|
||||
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
|
||||
zones, outside_supported_scope = CoverageRegistryService._intersected_zones(areas, selection)
|
||||
if not zones:
|
||||
return CoverageResolveResponse(
|
||||
project_id=project_id,
|
||||
bbox=bbox,
|
||||
requested_themes=requested_themes,
|
||||
intersected_zones=[],
|
||||
outside_supported_scope=True,
|
||||
items=[],
|
||||
warnings=["The selection does not intersect a persisted Belgium or Belgian North Sea scope."],
|
||||
)
|
||||
|
||||
items = [
|
||||
CoverageRegistryService._resolve_item(
|
||||
zone=zone,
|
||||
theme=theme,
|
||||
datasets=datasets,
|
||||
selection=selection,
|
||||
)
|
||||
for zone in zones
|
||||
for theme in requested_themes
|
||||
]
|
||||
warnings = []
|
||||
if outside_supported_scope:
|
||||
warnings.append("Part of the selection lies outside the persisted Belgium and Belgian North Sea scopes.")
|
||||
if len(zones) > 1:
|
||||
warnings.append(
|
||||
"The selection crosses coverage zones; results remain split and only semantically compatible metrics may be merged."
|
||||
)
|
||||
return CoverageResolveResponse(
|
||||
project_id=project_id,
|
||||
bbox=bbox,
|
||||
requested_themes=requested_themes,
|
||||
intersected_zones=zones,
|
||||
outside_supported_scope=outside_supported_scope,
|
||||
items=items,
|
||||
warnings=warnings,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user