Initial public release
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s

This commit is contained in:
Jens
2026-08-31 21:56:53 +02:00
commit faeb58ef6d
1386 changed files with 263203 additions and 0 deletions
View File
+3
View File
@@ -0,0 +1,3 @@
from app.models.entities import AnalysisRun, Area, Dataset, Export, Project
__all__ = ["AnalysisRun", "Area", "Dataset", "Export", "Project"]
View File
View File
View File
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
from uuid import UUID
from fastapi import Request
from app.core.errors import AppError
def guest_project_scope(request: Request) -> UUID | None:
principal = getattr(request.state, "auth_principal", None)
if getattr(principal, "role", None) != "guest":
return None
project_id = getattr(principal, "project_id", None)
if isinstance(project_id, UUID):
return project_id
raise AppError(
code="GUEST_PROJECT_SCOPE_REQUIRED",
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
status_code=403,
)
def assert_guest_project_scope(request: Request, project_id: UUID) -> None:
guest_project_id = guest_project_scope(request)
if guest_project_id is not None and project_id != guest_project_id:
raise AppError(
code="GUEST_PROJECT_SCOPE_REQUIRED",
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
status_code=403,
)
def guest_scoped_project_filter(
request: Request,
requested_project_id: UUID | None,
) -> UUID | None:
guest_project_id = guest_project_scope(request)
if guest_project_id is None:
return requested_project_id
if requested_project_id is not None:
assert_guest_project_scope(request, requested_project_id)
return guest_project_id
View File
+15
View File
@@ -0,0 +1,15 @@
__all__ = [
"analysis",
"areas",
"assistant",
"auth",
"datasets",
"exports",
"external",
"health",
"jobs",
"projects",
"qa",
"source_registry",
"temporal",
]
+50
View File
@@ -0,0 +1,50 @@
from __future__ import annotations
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from app.api.guest_scope import assert_guest_project_scope
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Dataset
from app.schemas import Envelope, JobRead
from app.schemas.analysis import ChangeDetectionRequest
from app.services.change_detection_service import ChangeDetectionService
from app.services.job_service import JobService
from app.utils.response import envelope
router = APIRouter(prefix="/analysis", tags=["analysis"])
@router.post("/change-detection", response_model=Envelope[JobRead])
def run_change_detection(
payload: ChangeDetectionRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
source_dataset = db.get(Dataset, payload.source_dataset_id)
if not source_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
assert_guest_project_scope(request, source_dataset.project_id)
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target")
job = JobService.run_sync_job(
db=db,
project_id=source_dataset.project_id,
job_type="analysis.change-detection",
parameters=payload.model_dump(mode="json"),
input_dataset_id=payload.source_dataset_id,
operation=lambda: ChangeDetectionService.compare_vector_datasets(
db=db,
project_id=source_dataset.project_id,
source_dataset_id=payload.source_dataset_id,
target_dataset_id=payload.target_dataset_id,
iou_threshold=payload.iou_threshold,
modified_threshold=payload.modified_threshold,
include_unchanged=payload.include_unchanged,
bbox=payload.bbox.model_dump() if payload.bbox is not None else None,
area_id=payload.area_id,
preview_limit=payload.preview_limit,
).model_dump(mode="json"),
)
return envelope(job)
+58
View File
@@ -0,0 +1,58 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas.aoi_operation import AoiOperationCreate, AoiOperationList, AoiOperationRead, AoiPartitionCheckpoint, AoiPartitionComplete, AoiPartitionFail, AoiPartitionRead
from app.schemas.common import Envelope
from app.services.aoi_operation_service import AoiOperationService
from app.services.aoi_operation_executor import AoiOperationExecutor
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}/aoi-operations", tags=["aoi-operations"])
@router.post("", status_code=201, response_model=Envelope[AoiOperationRead])
def create_operation(project_id: UUID, payload: AoiOperationCreate, db: Session = Depends(get_db)):
return envelope(AoiOperationService.create(db, project_id, payload))
@router.get("", response_model=Envelope[AoiOperationList])
def list_operations(project_id: UUID, limit: int = Query(default=50, ge=1, le=200), db: Session = Depends(get_db)):
return envelope(AoiOperationService.list(db, project_id, limit))
@router.get("/{operation_id}", response_model=Envelope[AoiOperationRead])
def read_operation(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
return envelope(AoiOperationService.read(db, project_id, operation_id))
@router.post("/{operation_id}/partitions/claim", response_model=Envelope[AoiPartitionRead | None])
def claim_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
partition = AoiOperationService.claim_next(db, project_id, operation_id)
return envelope(AoiPartitionRead.model_validate(partition).model_dump() if partition else None)
@router.post("/{operation_id}/execute-next", response_model=Envelope[AoiOperationRead])
def execute_next_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
return envelope(AoiOperationExecutor.execute_next(db, project_id, operation_id))
@router.put("/{operation_id}/partitions/{partition_id}/checkpoint", response_model=Envelope[AoiPartitionRead])
def checkpoint_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionCheckpoint, db: Session = Depends(get_db)):
partition = AoiOperationService.checkpoint(db, project_id, operation_id, partition_id, payload.checkpoint_json)
return envelope(AoiPartitionRead.model_validate(partition).model_dump())
@router.post("/{operation_id}/partitions/{partition_id}/complete", response_model=Envelope[AoiOperationRead])
def complete_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionComplete, db: Session = Depends(get_db)):
return envelope(AoiOperationService.complete(db, project_id, operation_id, partition_id, payload.result_json, payload.skipped))
@router.post("/{operation_id}/partitions/{partition_id}/fail", response_model=Envelope[AoiOperationRead])
def fail_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionFail, db: Session = Depends(get_db)):
return envelope(AoiOperationService.fail(db, project_id, operation_id, partition_id, payload.error_message, payload.retryable, payload.details))
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from fastapi import HTTPException
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.models import Area
from app.schemas import Envelope
from app.schemas.area import AreaCreate, AreaList, AreaRead, AreaUpdate, MunicipalitySearchList
from app.services.area_service import AreaService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}/areas", tags=["areas"])
@router.get("", response_model=Envelope[AreaList])
def list_areas(
project_id: UUID,
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
):
areas, total = AreaService.list_areas(db, project_id=project_id, limit=limit, offset=offset)
return envelope({"items": [AreaService.serialize_area(area) for area in areas], "total": total, "limit": limit, "offset": offset})
@router.post("", status_code=201, response_model=Envelope[AreaRead])
def create_area(project_id: UUID, payload: AreaCreate, db: Session = Depends(get_db)):
area = AreaService.create_area(db, project_id, payload)
return envelope(AreaService.serialize_area(area))
@router.get("/municipalities", response_model=Envelope[MunicipalitySearchList])
def search_municipalities(
project_id: UUID,
query: str = Query(default="", max_length=120),
limit: int = Query(default=20, ge=1, le=50),
db: Session = Depends(get_db),
):
items, total = AreaService.search_municipalities(db, project_id, query, limit)
return envelope({"items": items, "total": total})
@router.post("/municipalities/{niscode}/activate", response_model=Envelope[AreaRead])
def activate_municipality(project_id: UUID, niscode: str, db: Session = Depends(get_db)):
area = AreaService.activate_municipality(db, project_id, niscode)
return envelope(AreaService.serialize_area(area))
@router.get("/{area_id}", response_model=Envelope[AreaRead])
def get_area(
project_id: UUID,
area_id: UUID,
db: Session = Depends(get_db),
):
area = AreaService.get_area(db, area_id)
if area.project_id != project_id:
raise HTTPException(status_code=404, detail="Area not found")
return envelope(AreaService.serialize_area(area))
@router.patch("/{area_id}", response_model=Envelope[AreaRead])
def update_area(
project_id: UUID,
area_id: UUID,
payload: AreaUpdate,
db: Session = Depends(get_db),
):
existing = db.get(Area, area_id)
if not existing or existing.project_id != project_id:
raise HTTPException(status_code=404, detail="Area not found")
area = AreaService.update_area(db, area_id, payload)
return envelope(AreaService.serialize_area(area))
+50
View File
@@ -0,0 +1,50 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas import Envelope
from app.schemas.assistant import (
AssistantModelList,
AssistantQueryRequest,
AssistantQueryResponse,
AssistantStatus,
)
from app.services.geo_assistant_service import GeoAssistantService
from app.utils.response import envelope
router = APIRouter(tags=["assistant"])
@router.get("/assistant/status", response_model=Envelope[AssistantStatus])
def assistant_status() -> dict:
return envelope(GeoAssistantService().status().model_dump())
@router.get("/assistant/models", response_model=Envelope[AssistantModelList])
def assistant_models() -> dict:
service = GeoAssistantService()
models = service.list_models()
return envelope(
{
"items": [model.model_dump() for model in models],
"total": len(models),
"default_model": service.settings.ollama_default_model,
}
)
@router.post(
"/projects/{project_id}/assistant/query",
response_model=Envelope[AssistantQueryResponse],
)
def assistant_query(
project_id: UUID,
payload: AssistantQueryRequest,
db: Session = Depends(get_db),
) -> dict:
return envelope(GeoAssistantService().query(db, project_id=project_id, payload=payload).model_dump())
+326
View File
@@ -0,0 +1,326 @@
from __future__ import annotations
import logging
from datetime import UTC, datetime
from ipaddress import ip_address, ip_network
from fastapi import APIRouter, Depends, Request, Response, status
from fastapi.responses import RedirectResponse
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.errors import AppError
from app.db.session import get_db
from app.schemas.auth import AuthLoginRequest, AuthSession, AuthSessionEnvelope
from app.services.auth_service import AuthPrincipal, AuthService
from app.services.authentik_oidc_service import AuthentikOidcService
from app.services.demo_workflow_service import DemoWorkflowService
router = APIRouter(prefix="/auth", tags=["auth"])
COOKIE_NAME = "geointel_session"
OIDC_FLOW_COOKIE_NAME = "geointel_oidc_flow"
logger = logging.getLogger("geointel.auth")
_TRUSTED_PROXY_NETWORKS = (
ip_network("127.0.0.0/8"),
ip_network("::1/128"),
ip_network("172.16.0.0/12"),
)
def _peer_is_trusted_proxy(request: Request) -> bool:
if request.client is None:
return False
try:
peer_address = ip_address(request.client.host)
except ValueError:
return False
return any(peer_address in network for network in _TRUSTED_PROXY_NETWORKS)
def _request_is_https(request: Request) -> bool:
if request.url.scheme == "https":
return True
if not _peer_is_trusted_proxy(request):
return False
forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",", 1)[0].strip().lower()
return forwarded_proto == "https"
def _client_host(request: Request) -> str:
peer = request.client.host if request.client else "unknown"
if not _peer_is_trusted_proxy(request):
return peer
forwarded = request.headers.get("x-real-ip", "").strip()
if not forwarded:
return peer
try:
return str(ip_address(forwarded))
except ValueError:
return peer
def _session_from_principal(
principal: AuthPrincipal,
*,
guest_access_enabled: bool,
authentik_enabled: bool,
) -> AuthSession:
return AuthSession(
authentication_required=True,
authenticated=True,
username=principal.username,
expires_at=datetime.fromtimestamp(principal.expires_at, tz=UTC),
role=principal.role,
guest_access_enabled=guest_access_enabled,
authentik_enabled=authentik_enabled,
guest_project_id=principal.project_id,
)
def _session_payload(request: Request) -> AuthSession:
settings = get_settings()
guest_access_enabled = settings.auth_enabled and settings.guest_access_enabled
authentik_enabled = AuthentikOidcService(settings).enabled
if not settings.auth_enabled:
return AuthSession(
authentication_required=False,
authenticated=True,
guest_access_enabled=False,
authentik_enabled=False,
)
principal = AuthService.verify_session_token(request.cookies.get(COOKIE_NAME), settings)
if principal is None:
return AuthSession(
authentication_required=True,
authenticated=False,
guest_access_enabled=guest_access_enabled,
authentik_enabled=authentik_enabled,
)
return _session_from_principal(
principal,
guest_access_enabled=guest_access_enabled,
authentik_enabled=authentik_enabled,
)
def _set_session_cookie(
*,
request: Request,
response: Response,
token: str,
max_age: int,
) -> None:
response.set_cookie(
key=COOKIE_NAME,
value=token,
max_age=max_age,
httponly=True,
secure=_request_is_https(request),
samesite="strict",
path="/",
)
@router.get("/session", response_model=AuthSessionEnvelope)
def session(request: Request) -> AuthSessionEnvelope:
return AuthSessionEnvelope(data=_session_payload(request))
@router.post("/login", response_model=AuthSessionEnvelope)
def login(payload: AuthLoginRequest, request: Request, response: Response) -> AuthSessionEnvelope:
settings = get_settings()
if not settings.auth_enabled:
raise AppError(
code="AUTHENTICATION_DISABLED",
message="Operator authentication is not enabled on this runtime",
status_code=status.HTTP_409_CONFLICT,
)
if settings.auth_require_https and not _request_is_https(request):
raise AppError(
code="AUTH_HTTPS_REQUIRED",
message="Operator authentication requires HTTPS on this runtime",
status_code=status.HTTP_426_UPGRADE_REQUIRED,
)
client_host = _client_host(request)
throttle_key = f"{client_host}:{payload.username.casefold()}"
retry_after = AuthService.retry_after_seconds(throttle_key)
if retry_after:
raise AppError(
code="LOGIN_RATE_LIMITED",
message="Te veel mislukte aanmeldpogingen. Probeer later opnieuw.",
details={"retry_after_seconds": retry_after},
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
)
if not AuthService.credentials_match(payload.username, payload.password, settings):
AuthService.record_failure(throttle_key)
raise AppError(
code="INVALID_CREDENTIALS",
message="Gebruikersnaam of wachtwoord is onjuist.",
status_code=status.HTTP_401_UNAUTHORIZED,
)
AuthService.clear_failures(throttle_key)
token = AuthService.create_session_token(payload.username, settings)
principal = AuthService.verify_session_token(token, settings)
if principal is None: # pragma: no cover - defensive invariant
raise AppError(
code="SESSION_CREATION_FAILED",
message="De beveiligde sessie kon niet worden aangemaakt.",
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
_set_session_cookie(
request=request,
response=response,
token=token,
max_age=settings.auth_session_ttl_seconds,
)
return AuthSessionEnvelope(
data=_session_from_principal(
principal,
guest_access_enabled=settings.guest_access_enabled,
authentik_enabled=AuthentikOidcService(settings).enabled,
)
)
@router.get("/authentik/start")
def authentik_start(request: Request) -> RedirectResponse:
settings = get_settings()
service = AuthentikOidcService(settings)
try:
location, flow = service.start()
except Exception as exc:
logger.warning("Authentik authorization start failed: %s", type(exc).__name__)
raise AppError(
code="AUTHENTIK_UNAVAILABLE",
message="Authentik is momenteel niet beschikbaar.",
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
) from exc
response = RedirectResponse(location, status_code=status.HTTP_302_FOUND)
response.set_cookie(
OIDC_FLOW_COOKIE_NAME,
flow,
max_age=600,
httponly=True,
secure=True,
samesite="lax",
path=f"{settings.api_prefix}/auth/authentik",
)
return response
@router.get("/authentik/callback")
def authentik_callback(
request: Request,
code: str = "",
state: str = "",
) -> RedirectResponse:
settings = get_settings()
service = AuthentikOidcService(settings)
base_url = settings.public_base_url.rstrip("/")
try:
service.finish(
code=code,
state=state,
flow_cookie=request.cookies.get(OIDC_FLOW_COOKIE_NAME, ""),
)
token = AuthService.create_session_token(
settings.auth_username or "operator",
settings,
)
except Exception as exc:
logger.warning("Authentik callback rejected: %s", type(exc).__name__)
response = RedirectResponse(
f"{base_url}/?authentik=error",
status_code=status.HTTP_302_FOUND,
)
else:
response = RedirectResponse(
f"{base_url}/",
status_code=status.HTTP_302_FOUND,
)
_set_session_cookie(
request=request,
response=response,
token=token,
max_age=settings.auth_session_ttl_seconds,
)
response.delete_cookie(
OIDC_FLOW_COOKIE_NAME,
path=f"{settings.api_prefix}/auth/authentik",
secure=True,
httponly=True,
samesite="lax",
)
return response
@router.post("/guest", response_model=AuthSessionEnvelope)
def guest_login(
request: Request,
response: Response,
db: Session = Depends(get_db),
) -> AuthSessionEnvelope:
settings = get_settings()
if not settings.auth_enabled or not settings.guest_access_enabled:
raise AppError(
code="GUEST_ACCESS_DISABLED",
message="Gasttoegang is niet ingeschakeld op deze GeoIntel-installatie.",
status_code=status.HTTP_403_FORBIDDEN,
)
client_host = _client_host(request)
retry_after = AuthService.consume_guest_request(
f"guest-login:{client_host}",
max_requests=settings.guest_login_requests_per_minute,
)
if retry_after:
raise AppError(
code="GUEST_LOGIN_RATE_LIMITED",
message="Too many guest sessions were requested. Try again later.",
details={"retry_after_seconds": retry_after},
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
)
demo = DemoWorkflowService.seed(db)
token = AuthService.create_session_token(
settings.guest_display_name,
settings,
role="guest",
project_id=demo.project_id,
ttl_seconds=settings.guest_session_ttl_seconds,
)
principal = AuthService.verify_session_token(token, settings)
if principal is None: # pragma: no cover - defensive invariant
raise AppError(
code="SESSION_CREATION_FAILED",
message="De tijdelijke gastensessie kon niet worden aangemaakt.",
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
_set_session_cookie(
request=request,
response=response,
token=token,
max_age=settings.guest_session_ttl_seconds,
)
return AuthSessionEnvelope(
data=_session_from_principal(
principal,
guest_access_enabled=True,
authentik_enabled=AuthentikOidcService(settings).enabled,
)
)
@router.post("/logout", response_model=AuthSessionEnvelope)
def logout(response: Response) -> AuthSessionEnvelope:
settings = get_settings()
response.delete_cookie(key=COOKIE_NAME, path="/", httponly=True, samesite="strict")
return AuthSessionEnvelope(
data=AuthSession(
authentication_required=settings.auth_enabled,
authenticated=not settings.auth_enabled,
guest_access_enabled=settings.auth_enabled and settings.guest_access_enabled,
authentik_enabled=AuthentikOidcService(settings).enabled,
)
)
File diff suppressed because it is too large Load Diff
+22
View File
@@ -0,0 +1,22 @@
from __future__ import annotations
from fastapi import APIRouter, Depends, status
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas import Envelope
from app.schemas.demo import DemoWorkflowResponse
from app.services.demo_workflow_service import DemoWorkflowService
from app.utils.response import envelope
router = APIRouter(prefix="/demo", tags=["demo"])
@router.post(
"/workflow",
status_code=status.HTTP_201_CREATED,
response_model=Envelope[DemoWorkflowResponse],
)
def seed_demo_workflow(db: Session = Depends(get_db)) -> dict:
result: DemoWorkflowResponse = DemoWorkflowService.seed(db)
return envelope(result.model_dump())
+336
View File
@@ -0,0 +1,336 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.orm import Session
from app.api.guest_scope import (
assert_guest_project_scope,
guest_project_scope,
guest_scoped_project_filter,
)
from app.db.session import get_db
from app.schemas import (
AnalysisQaResponse,
DetectionListResponse,
DetectionModelsResponse,
DetectionComparisonRequest,
DetectionComparisonResponse,
DetectionQaRequest,
DetectionRead,
DetectionRunListResponse,
DetectionRunRead,
DetectionRunRequest,
DetectionRunResponse,
Envelope,
GeoJsonFeatureCollection,
JobRead,
ModelAssetListResponse,
YoloPreflightResponse,
)
from app.services.detection_comparison_service import DetectionComparisonService
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.model_registry_service import ModelRegistryService
from app.services.yolo_preflight_service import YoloPreflightService
from app.utils.response import envelope
router = APIRouter(prefix="/detection", tags=["detection"])
@router.get("/models", response_model=Envelope[DetectionModelsResponse])
def list_detection_models() -> dict:
return envelope({"models": [model.model_dump() for model in ModelRegistryService.list_model_capabilities()]})
@router.get("/model-assets", response_model=Envelope[ModelAssetListResponse])
def list_detection_model_assets() -> dict:
return envelope(ModelAssetCatalogService.list_assets().model_dump())
@router.get("/yolo/preflight", response_model=Envelope[YoloPreflightResponse])
def get_yolo_preflight(
tile_manifest_path: str | None = None,
check_model_load: bool = False,
model_asset_id: str | None = None,
db: Session = Depends(get_db),
) -> dict:
return envelope(
YoloPreflightService.run(
tile_manifest_path=tile_manifest_path,
check_model_load=check_model_load,
model_asset_id=model_asset_id,
db=db,
)
)
@router.post("/run", response_model=Envelope[DetectionRunResponse])
def run_detection(
payload: DetectionRunRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
assert_guest_project_scope(request, payload.project_id)
result = DetectionService.run_detection(
db=db,
project_id=payload.project_id,
dataset_id=payload.dataset_id,
model_id=payload.model_id,
model_asset_id=payload.model_asset_id,
confidence_threshold=payload.confidence_threshold,
class_filter=payload.class_filter,
tile_manifest_path=payload.tile_manifest_path,
parameters_json=payload.parameters_json,
)
return envelope(result.model_dump())
@router.post("/run-async", response_model=Envelope[JobRead])
def queue_detection(
payload: DetectionRunRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
"""Queue a detection run for the background worker.
Tiled GPU inference takes minutes; ``POST /detection/run`` performs it
inside the request and is only appropriate for a handful of tiles. Poll
``GET /jobs/{id}`` for the queued run instead.
"""
assert_guest_project_scope(request, payload.project_id)
job = DetectionService.enqueue_detection(
db=db,
project_id=payload.project_id,
dataset_id=payload.dataset_id,
model_id=payload.model_id,
model_asset_id=payload.model_asset_id,
confidence_threshold=payload.confidence_threshold,
class_filter=payload.class_filter,
tile_manifest_path=payload.tile_manifest_path,
parameters_json=payload.parameters_json,
)
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
@router.get("/runs", response_model=Envelope[DetectionRunListResponse])
def list_detection_runs(
request: Request,
project_id: UUID | None = None,
dataset_id: UUID | None = None,
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
project_id = guest_scoped_project_filter(request, project_id)
return envelope(
DetectionService.list_runs(
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
).model_dump()
)
@router.get("/runs/{analysis_run_id}", response_model=Envelope[DetectionRunRead])
def get_detection_run(
analysis_run_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(run.model_dump())
@router.get(
"/runs/{analysis_run_id}/detections",
response_model=Envelope[DetectionListResponse],
)
def list_detection_run_detections(
analysis_run_id: UUID,
request: Request,
dataset_id: UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(
DetectionService.list_detections(
db,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
limit=limit,
offset=offset,
).model_dump()
)
@router.get(
"/datasets/{dataset_id}/detections",
response_model=Envelope[DetectionListResponse],
)
def list_dataset_detections(
dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope(
DetectionService.list_detections(
db,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
limit=limit,
offset=offset,
).model_dump()
)
@router.get("/detections/{detection_id}", response_model=Envelope[DetectionRead])
def get_detection(
detection_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
detection = DetectionService.get_detection(db, detection_id)
assert_guest_project_scope(request, detection.project_id)
return envelope(detection.model_dump())
@router.get(
"/runs/{analysis_run_id}/geojson",
response_model=Envelope[GeoJsonFeatureCollection],
)
def get_detection_run_geojson(
analysis_run_id: UUID,
request: Request,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(
DetectionService.detections_to_geojson(
db,
limit=limit,
analysis_run_id=analysis_run_id,
class_name=class_name,
min_confidence=min_confidence,
)
)
@router.get(
"/datasets/{dataset_id}/geojson",
response_model=Envelope[GeoJsonFeatureCollection],
)
def get_dataset_detection_geojson(
dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope(
DetectionService.detections_to_geojson(
db,
limit=limit,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
)
@router.post("/runs/compare", response_model=Envelope[DetectionComparisonResponse])
def compare_detection_runs(payload: DetectionComparisonRequest, db: Session = Depends(get_db)) -> dict:
"""Rank several runs against one reference on average precision.
The workbench ranks model variants by a stored F1 measured at each
variant's own confidence threshold, which orders the thresholds as much as
the models. Average precision describes the whole ranking a model produced.
Comparability is reported first: runs over different rasters, different
references or different inference coverage are not alternatives.
"""
return envelope(
DetectionComparisonService.compare_runs(
db,
analysis_run_ids=payload.analysis_run_ids,
reference_dataset_id=payload.reference_dataset_id,
iou_threshold=payload.iou_threshold,
)
)
@router.post(
"/runs/{analysis_run_id}/qa/reference",
response_model=Envelope[AnalysisQaResponse],
)
def compare_detection_run_with_reference(
analysis_run_id: UUID,
payload: DetectionQaRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(
DetectionService.compare_detections_with_reference(
db=db,
analysis_run_id=analysis_run_id,
reference_dataset_id=payload.reference_dataset_id,
iou_threshold=payload.iou_threshold,
class_name=payload.class_name,
min_confidence=payload.min_confidence,
calibration_thresholds=payload.calibration_thresholds,
)
)
+148
View File
@@ -0,0 +1,148 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, Query, Request
from fastapi.responses import FileResponse
from sqlalchemy.orm import Session
from app.api.guest_scope import assert_guest_project_scope, guest_project_scope
from app.core.errors import AppError
from app.db.session import get_db
from app.schemas import Envelope
from app.schemas.export import (
ExportContentResponse,
ExportCreateResponse,
ExportListResponse,
ExportRead,
GeoJsonExportRequest,
MapResultExportRequest,
MetadataExportRequest,
ReportExportRequest,
)
from app.services.export_service import ExportService
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService
from app.utils.response import envelope
router = APIRouter(prefix="/exports", tags=["exports"])
@router.post("/geojson", response_model=Envelope[ExportCreateResponse])
def export_geojson(
payload: GeoJsonExportRequest,
request: Request,
db: Session = Depends(get_db),
):
if guest_project_scope(request) is not None:
if payload.export_kind in {"dataset", "vector_selection"} and payload.dataset_id is not None:
dataset = DatasetService.get_dataset(db, payload.dataset_id)
assert_guest_project_scope(request, dataset.project_id)
elif payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
run = DetectionService.get_run(db, payload.analysis_run_id)
assert_guest_project_scope(request, run.project_id)
elif payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
run = SegmentationService.get_run(db, payload.analysis_run_id)
assert_guest_project_scope(request, run.project_id)
if payload.export_kind == "vector_selection" and payload.dataset_id is not None and payload.bbox is not None:
return envelope(
ExportService.export_vector_selection_geojson(
db,
payload.dataset_id,
payload.bbox.model_dump(),
area_id=payload.area_id,
limit=payload.limit,
name=payload.name,
).model_dump(mode="json")
)
if payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
return envelope(
ExportService.export_detection_run_geojson(
db,
payload.analysis_run_id,
payload.name,
intended_use=payload.intended_use,
).model_dump(mode="json")
)
if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
return envelope(
ExportService.export_segmentation_run_geojson(db, payload.analysis_run_id, payload.name).model_dump(mode="json")
)
if payload.dataset_id is not None:
return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json"))
raise AppError(
code="INVALID_EXPORT_REQUEST",
message="GeoJSON export request does not match any supported export target",
status_code=422,
)
@router.post("/metadata", response_model=Envelope[ExportCreateResponse])
def export_project_metadata(
payload: MetadataExportRequest,
request: Request,
db: Session = Depends(get_db),
):
assert_guest_project_scope(request, payload.project_id)
return envelope(ExportService.export_project_metadata(db, payload.project_id, payload.name).model_dump(mode="json"))
@router.post("/report", response_model=Envelope[ExportCreateResponse])
def export_project_report(
payload: ReportExportRequest,
request: Request,
db: Session = Depends(get_db),
):
assert_guest_project_scope(request, payload.project_id)
return envelope(ExportService.export_project_report(db, payload.project_id, payload.name).model_dump(mode="json"))
@router.post("/map-result", response_model=Envelope[ExportCreateResponse])
def export_map_result(
payload: MapResultExportRequest,
request: Request,
db: Session = Depends(get_db),
):
assert_guest_project_scope(request, payload.project_id)
return envelope(ExportService.export_map_result(db, payload).model_dump(mode="json"))
@router.get(
"/projects/{project_id}/exports",
response_model=Envelope[ExportListResponse],
)
def list_project_exports(
project_id: UUID,
request: Request,
limit: int = Query(default=50, ge=1, le=100),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
):
assert_guest_project_scope(request, project_id)
return envelope(ExportService.list_project_exports(db, project_id, limit=limit, offset=offset).model_dump(mode="json"))
@router.get("/{export_id}", response_model=Envelope[ExportRead])
def get_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
export = ExportService.get_export(db, export_id)
assert_guest_project_scope(request, export.project_id)
return envelope(export.model_dump(mode="json"))
@router.get("/{export_id}/download")
def download_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
if guest_project_scope(request) is not None:
export = ExportService.get_export(db, export_id)
assert_guest_project_scope(request, export.project_id)
path = ExportService.get_export_download_path(db, export_id)
media_type = "text/html" if path.suffix.lower() in {".html", ".htm"} else "application/json"
return FileResponse(path, filename=path.name, media_type=media_type)
@router.get("/{export_id}/content", response_model=Envelope[ExportContentResponse])
def get_export_content(export_id: UUID, request: Request, db: Session = Depends(get_db)):
if guest_project_scope(request) is not None:
export = ExportService.get_export(db, export_id)
assert_guest_project_scope(request, export.project_id)
return envelope(ExportService.get_export_content(db, export_id).model_dump(mode="json"))
+185
View File
@@ -0,0 +1,185 @@
from __future__ import annotations
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Area, Project
from app.providers.registry import fetch_provider_data, get_provider, import_provider_dataset, list_provider_capabilities
from app.schemas import (
CoverageCatalogResponse,
CoverageResolveRequest,
CoverageResolveResponse,
Envelope,
ExternalFetchRequest,
ExternalFetchResponse,
ProviderCapabilitiesResponse,
ProviderCapabilityResponse,
ProviderImportRequest,
ProviderImportResponse,
ProviderLayersResponse,
ProviderStatusResponse,
)
from app.services.coverage_registry_service import CoverageRegistryService
from app.utils.response import envelope
router = APIRouter(prefix="/external", tags=["external"])
def _validate_area_in_project(db: Session, project_id, area_id: str | None) -> None:
if area_id is None:
return
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
def _assert_project_exists(db: Session, project_id):
project = db.get(Project, project_id)
if not project:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
def _assert_guest_project_scope(request: Request, project_id) -> None:
principal = getattr(request.state, "auth_principal", None)
if (
getattr(principal, "role", None) == "guest"
and getattr(principal, "project_id", None) != project_id
):
raise AppError(
code="GUEST_PROJECT_SCOPE_REQUIRED",
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
status_code=403,
)
def _normalize_layer_input(layers: list[str] | None) -> list[str]:
return [layer.strip() for layer in (layers or []) if isinstance(layer, str) and layer.strip()]
def _provider_payload(provider_name: str) -> dict:
return get_provider(provider_name).capability.to_dict()
@router.get("/providers", response_model=Envelope[ProviderCapabilitiesResponse])
def list_external_providers() -> dict:
return envelope({
"providers": [provider.to_dict() for provider in list_provider_capabilities()],
})
@router.get("/coverage/catalog", response_model=Envelope[CoverageCatalogResponse])
def get_coverage_catalog() -> dict:
return envelope(CoverageRegistryService.catalog().model_dump())
@router.post("/coverage/resolve", response_model=Envelope[CoverageResolveResponse])
def resolve_project_coverage(
payload: CoverageResolveRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
_assert_guest_project_scope(request, payload.project_id)
result = CoverageRegistryService.resolve(
db,
project_id=payload.project_id,
bbox=payload.bbox,
themes=payload.themes,
)
return envelope(result.model_dump())
@router.get(
"/providers/capabilities",
response_model=Envelope[ProviderCapabilitiesResponse],
)
def get_external_provider_capabilities() -> dict:
return envelope({
"providers": [provider.to_dict() for provider in list_provider_capabilities()],
})
@router.get(
"/providers/{provider_name}",
response_model=Envelope[ProviderCapabilityResponse],
)
def get_external_provider(provider_name: str) -> dict:
return envelope(_provider_payload(provider_name))
@router.get(
"/providers/{provider_name}/layers",
response_model=Envelope[ProviderLayersResponse],
)
def get_external_provider_layers(provider_name: str) -> dict:
provider = get_provider(provider_name)
return envelope({
"provider_name": provider.provider_name,
"layers": provider.supported_layers,
})
@router.get(
"/providers/{provider_name}/status",
response_model=Envelope[ProviderStatusResponse],
)
def get_external_provider_status(provider_name: str) -> dict:
provider = get_provider(provider_name)
return envelope({
"provider_name": provider.provider_name,
"configured": provider.is_configured,
"status": provider.capability.status,
"limitation_message": provider.limitation_message,
})
@router.post(
"/providers/{provider_name}/import",
response_model=Envelope[ProviderImportResponse],
)
def import_external_provider_dataset(provider_name: str, payload: ProviderImportRequest) -> dict:
result = import_provider_dataset(
provider_name=provider_name,
project_id=payload.project_id,
area_id=payload.area_id,
layers=_normalize_layer_input(payload.layers),
requested_dataset_role=payload.dataset_role,
)
return envelope(result.model_dump())
def _run_fetch(payload: ExternalFetchRequest, provider_name: str) -> ExternalFetchResponse:
area_id_str = str(payload.area_id) if payload.area_id else None
response = fetch_provider_data(
provider_name=provider_name,
project_id=str(payload.project_id),
area_id=area_id_str,
layers=_normalize_layer_input(payload.layers),
)
return ExternalFetchResponse(
provider=provider_name,
status=response.get("status", "not_configured"),
message=response.get("message", "Provider fetch executed."),
requested_layers=_normalize_layer_input(payload.layers),
project_id=payload.project_id,
area_id=payload.area_id,
)
@router.post("/osm/fetch", response_model=Envelope[ExternalFetchResponse])
def fetch_osm(payload: ExternalFetchRequest, db: Session = Depends(get_db)) -> dict:
_assert_project_exists(db, payload.project_id)
_validate_area_in_project(db, payload.project_id, payload.area_id)
return envelope(_run_fetch(payload, "osm").model_dump())
@router.post("/grb/fetch", response_model=Envelope[ExternalFetchResponse])
def fetch_grb(payload: ExternalFetchRequest, db: Session = Depends(get_db)) -> dict:
_assert_project_exists(db, payload.project_id)
_validate_area_in_project(db, payload.project_id, payload.area_id)
return envelope(_run_fetch(payload, "grb").model_dump())
+162
View File
@@ -0,0 +1,162 @@
from __future__ import annotations
from importlib import import_module
from pathlib import Path
from tempfile import NamedTemporaryFile
from alembic.config import Config
from alembic.script import ScriptDirectory
from fastapi import APIRouter, Response, status
from sqlalchemy import text
from app.core.config import get_settings
from app.db.session import get_engine
from app.providers.registry import list_provider_capabilities
from app.schemas.health import (
HealthResponse,
SystemCapabilities,
SystemCapabilitiesEnvelope,
)
from app.services.model_registry_service import ModelRegistryService
router = APIRouter()
def _dependency_enabled(module: str) -> bool:
try:
import_module(module)
return True
except Exception:
return False
def _expected_migration_heads() -> list[str]:
backend_root = Path(__file__).resolve().parents[3]
config = Config(str(backend_root / "alembic.ini"))
config.set_main_option("script_location", str(backend_root / "alembic"))
return list(ScriptDirectory.from_config(config).get_heads())
def _database_checks() -> dict[str, str]:
checks = {
"database": "degraded",
"postgis": "degraded",
"migration": "degraded",
}
try:
with get_engine().connect() as connection:
connection.execute(text("SELECT 1"))
checks["database"] = "ok"
connection.execute(
text("SELECT PostGIS_Version()")
).scalar_one()
checks["postgis"] = "ok"
database_head = connection.execute(
text("SELECT version_num FROM alembic_version")
).scalar_one()
expected_heads = _expected_migration_heads()
if len(expected_heads) == 1 and database_head == expected_heads[0]:
checks["migration"] = "ok"
else:
checks["migration"] = "degraded"
except Exception:
return checks
return checks
def _storage_check(storage_root: str) -> str:
root = Path(storage_root).expanduser()
try:
root.mkdir(parents=True, exist_ok=True)
with NamedTemporaryFile(
prefix=".geointel-readiness-",
dir=root,
delete=True,
) as handle:
handle.write(b"ok")
handle.flush()
return "ok"
except OSError:
return "degraded"
def _readiness_payload() -> HealthResponse:
settings = get_settings()
checks = _database_checks()
checks["storage"] = _storage_check(settings.storage_root)
ready = all(
value == "ok" or value.startswith("ok:")
for value in checks.values()
)
return HealthResponse(
status="ok" if ready else "degraded",
service="geointel-backend",
version="public",
database=checks["database"],
postgis=checks["postgis"],
migration=checks["migration"],
storage=checks["storage"],
checks=checks,
)
@router.get("/health/live", response_model=HealthResponse)
def liveness() -> HealthResponse:
return HealthResponse(
status="ok",
service="geointel-backend",
version="public",
)
def _readiness_response(response: Response) -> HealthResponse:
payload = _readiness_payload()
if payload.status != "ok":
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
return payload
@router.get("/health", response_model=HealthResponse)
def readiness(response: Response) -> HealthResponse:
return _readiness_response(response)
@router.get("/health/ready", response_model=HealthResponse)
def readiness_explicit(response: Response) -> HealthResponse:
return _readiness_response(response)
@router.get(
"/api/v1/system/capabilities",
response_model=SystemCapabilitiesEnvelope,
)
def capabilities() -> SystemCapabilitiesEnvelope:
settings = get_settings()
providers = [item.to_dict() for item in list_provider_capabilities()]
configured_yolo = ModelRegistryService.get_model_capability(
settings.yolo_model_id,
settings=settings,
)
yolo_configured = bool(configured_yolo and configured_yolo.configured)
yolo_status = configured_yolo.status if configured_yolo else "not_configured"
configured_sam = ModelRegistryService.get_model_capability(
settings.sam_model_id,
settings=settings,
task_type="segmentation",
)
postgis_ready = _database_checks()["postgis"].startswith("ok:")
return SystemCapabilitiesEnvelope(
data=SystemCapabilities(
postgis=postgis_ready,
rasterio=_dependency_enabled("rasterio"),
geopandas=_dependency_enabled("geopandas"),
yolo=yolo_configured,
yolo_status=yolo_status,
sam=bool(configured_sam and configured_sam.configured),
grb="bounded",
sentinel="planned",
version=settings.app_version,
build_sha=settings.build_sha,
providers=providers,
)
)
+67
View File
@@ -0,0 +1,67 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas import Envelope, JobCreate, JobList, JobRead, JobStatus
from app.services.job_service import JobService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}", tags=["jobs"])
@router.post("/jobs", status_code=201, response_model=Envelope[JobRead])
def create_job(
project_id: UUID,
payload: JobCreate,
db: Session = Depends(get_db),
):
if payload.project_id != project_id:
raise HTTPException(status_code=400, detail="project_id mismatch")
return envelope(JobService.create_job(db, payload).model_dump())
@router.get("/jobs", response_model=Envelope[JobList])
def list_jobs(
project_id: UUID,
dataset_id: UUID | None = Query(default=None),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
):
items, total = JobService.list_jobs(
db,
project_id=project_id,
dataset_id=dataset_id,
limit=limit,
offset=offset,
)
return envelope(JobList(items=items, total=total, limit=limit, offset=offset).model_dump())
@router.get("/jobs/{job_id}", response_model=Envelope[JobRead])
def read_job(
project_id: UUID,
job_id: UUID,
db: Session = Depends(get_db),
):
job = JobService.get_job(db, job_id)
if job.project_id != project_id:
raise HTTPException(status_code=404, detail="Job not found")
return envelope(job.model_dump())
@router.get("/jobs/{job_id}/status", response_model=Envelope[JobStatus])
def read_job_status(
project_id: UUID,
job_id: UUID,
db: Session = Depends(get_db),
):
status_row = JobService.get_job_status(db, job_id)
if status_row["project_id"] != str(project_id):
raise HTTPException(status_code=404, detail="Job not found")
return envelope(JobStatus(**status_row).model_dump())
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
from typing import Literal
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas import Envelope
from app.schemas.project import ProjectCreate, ProjectDeleteResult, ProjectList, ProjectRead, ProjectUpdate
from app.services.project_service import ProjectService
from app.utils.response import envelope
router = APIRouter(prefix="/projects", tags=["projects"])
@router.get("", response_model=Envelope[ProjectList])
def list_projects(
request: Request,
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
name: str | None = Query(default=None, min_length=1, max_length=255),
project_status: Literal["active", "archived", "all"] = Query(default="active", alias="status"),
db: Session = Depends(get_db),
):
principal = getattr(request.state, "auth_principal", None)
if principal is not None and principal.role == "guest":
project = ProjectService.get_project(db, principal.project_id)
status_matches = bool(
project is not None
and (project_status == "all" or project.status == project_status)
)
name_matches = bool(
project is not None
and (name is None or name.casefold() in project.name.casefold())
)
matches = project is not None and status_matches and name_matches
visible = [project] if matches and offset == 0 else []
return envelope(
{
"items": [ProjectRead.model_validate(item).model_dump() for item in visible[:limit]],
"total": 1 if matches else 0,
"limit": limit,
"offset": offset,
}
)
projects, total = ProjectService.list_projects(
db,
limit=limit,
offset=offset,
name=name,
project_status=project_status,
)
return envelope({"items": [ProjectRead.model_validate(item).model_dump() for item in projects], "total": total, "limit": limit, "offset": offset})
@router.post("", status_code=status.HTTP_201_CREATED, response_model=Envelope[ProjectRead])
def create_project(payload: ProjectCreate, db: Session = Depends(get_db)):
project = ProjectService.create_project(db, payload)
return envelope(ProjectRead.model_validate(project).model_dump())
@router.get("/{project_id}", response_model=Envelope[ProjectRead])
def get_project(project_id: UUID, db: Session = Depends(get_db)):
project = ProjectService.get_project(db, project_id)
if not project:
raise HTTPException(status_code=404, detail="Project not found")
return envelope(ProjectRead.model_validate(project).model_dump())
@router.patch("/{project_id}", response_model=Envelope[ProjectRead])
def update_project(project_id: UUID, payload: ProjectUpdate, db: Session = Depends(get_db)):
project = ProjectService.update_project(db, project_id, payload)
if not project:
raise HTTPException(status_code=404, detail="Project not found")
return envelope(ProjectRead.model_validate(project).model_dump())
@router.delete(
"/{project_id}",
status_code=status.HTTP_200_OK,
response_model=Envelope[ProjectDeleteResult],
)
def delete_project(project_id: UUID, db: Session = Depends(get_db)):
if not ProjectService.delete_project(db, project_id):
raise HTTPException(status_code=404, detail="Project not found")
return envelope({"deleted": True})
+83
View File
@@ -0,0 +1,83 @@
from __future__ import annotations
import uuid
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.core.errors import AppError
from app.models import Dataset, Job
from app.schemas import Envelope, JobRead, QaProviderComparisonRequest
from app.services.qa_service import QaService
from app.services.job_service import JobService
from app.services.quality_service import QualityService
from app.utils.response import envelope
router = APIRouter(prefix="/qa", tags=["qa"])
@router.post("/detections-vs-reference", response_model=Envelope[JobRead])
def compare_candidate_with_reference(
payload: QaProviderComparisonRequest,
db: Session = Depends(get_db),
) -> dict:
candidate_dataset = db.get(Dataset, payload.candidate_dataset_id)
if not candidate_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Candidate dataset not found", status_code=404)
job = JobService.run_sync_job(
db=db,
project_id=candidate_dataset.project_id,
job_type="qa.compare-candidate-with-reference",
parameters=payload.model_dump(mode="json"),
input_dataset_id=candidate_dataset.id,
operation=lambda: QaService.compare_candidate_with_reference(
db=db,
project_id=candidate_dataset.project_id,
candidate_dataset_id=payload.candidate_dataset_id,
reference_dataset_id=payload.reference_dataset_id,
iou_threshold=payload.iou_threshold,
area_id=payload.area_id,
).model_dump(mode="json"),
)
result_json = job.get("result_json") if isinstance(job, dict) else None
if isinstance(result_json, dict) and job.get("status") == "success":
quality_check = QualityService.persist_quality_check(
db=db,
project_id=candidate_dataset.project_id,
job_id=uuid.UUID(str(job["id"])),
candidate_dataset_id=payload.candidate_dataset_id,
reference_dataset_id=payload.reference_dataset_id,
check_type="candidate_vs_reference",
status=str(result_json.get("status", "ok")),
score=result_json.get("f1_score"),
parameters=payload.model_dump(mode="json"),
findings={
"matches": result_json.get("matches"),
"false_positives": result_json.get("false_positives"),
"false_negatives": result_json.get("false_negatives"),
"warnings": result_json.get("warnings", []),
"unsupported_geometry": result_json.get("unsupported_geometry", False),
"unsupported_geometries": result_json.get("unsupported_geometries", []),
"match_evidence": result_json.get("match_evidence", []),
"false_positive_evidence": result_json.get("false_positive_evidence", []),
"false_negative_evidence": result_json.get("false_negative_evidence", []),
},
metrics={
"precision": result_json.get("precision"),
"recall": result_json.get("recall"),
"f1": result_json.get("f1_score"),
"mean_iou": result_json.get("mean_iou"),
"false_positive_count": result_json.get("false_positives"),
"false_negative_count": result_json.get("false_negatives"),
},
)
result_json["quality_check_id"] = str(quality_check.id)
job_record = db.get(Job, uuid.UUID(str(job["id"])))
if job_record:
job_record.result_json = result_json
db.add(job_record)
db.commit()
return envelope(job)
+106
View File
@@ -0,0 +1,106 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas import Envelope, QualityEvidenceResponse
from app.schemas.detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewUpsert
from app.schemas.qa import QualityCheckList
from app.services.detection_review_service import DetectionReviewService
from app.services.quality_evidence_service import QualityEvidenceService
from app.services.quality_check_service import QualityCheckService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}", tags=["quality-checks"])
@router.get("/quality-checks", response_model=Envelope[QualityCheckList])
def list_quality_checks(
project_id: UUID,
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
items, total = QualityCheckService.list_quality_checks(
db,
project_id=project_id,
limit=limit,
offset=offset,
)
return envelope(QualityCheckList(items=items, total=total, limit=limit, offset=offset).model_dump())
@router.get(
"/quality-checks/{quality_check_id}/evidence/geojson",
response_model=Envelope[QualityEvidenceResponse],
)
def get_quality_check_evidence_geojson(
project_id: UUID,
quality_check_id: UUID,
limit: int = Query(
default=QualityEvidenceService.DEFAULT_EVIDENCE_LIMIT,
ge=0,
le=100_000,
description="Maximum evidence features to draw; 0 returns everything. Misses and false positives first.",
),
db: Session = Depends(get_db),
) -> dict:
return envelope(
QualityEvidenceService.evidence_geojson(
db,
project_id=project_id,
quality_check_id=quality_check_id,
limit=limit,
)
)
@router.get(
"/quality-checks/{quality_check_id}/reviews",
response_model=Envelope[DetectionReviewList],
)
def list_detection_reviews(
project_id: UUID,
quality_check_id: UUID,
evidence_role: str | None = Query(default=None, pattern="^(false_positive|false_negative)$"),
decision: str | None = Query(default=None, max_length=64),
reviewed: bool | None = Query(default=None),
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
return envelope(
DetectionReviewService.list_reviews(
db,
project_id=project_id,
quality_check_id=quality_check_id,
evidence_role=evidence_role,
decision=decision,
reviewed=reviewed,
limit=limit,
offset=offset,
).model_dump()
)
@router.post(
"/quality-checks/{quality_check_id}/reviews",
response_model=Envelope[DetectionReviewRead],
)
def upsert_detection_review(
project_id: UUID,
quality_check_id: UUID,
payload: DetectionReviewUpsert,
db: Session = Depends(get_db),
) -> dict:
return envelope(
DetectionReviewService.upsert_review(
db,
project_id=project_id,
quality_check_id=quality_check_id,
payload=payload,
).model_dump()
)
+284
View File
@@ -0,0 +1,284 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.orm import Session
from app.api.guest_scope import (
assert_guest_project_scope,
guest_project_scope,
guest_scoped_project_filter,
)
from app.db.session import get_db
from app.schemas import (
AnalysisQaResponse,
Envelope,
GeoJsonFeatureCollection,
JobRead,
SegmentationListResponse,
SegmentationModelsResponse,
SegmentationQaRequest,
SegmentationRead,
SegmentationRunListResponse,
SegmentationRunRead,
SegmentationRunRequest,
SegmentationRunResponse,
)
from app.services.model_registry_service import ModelRegistryService
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService
from app.utils.response import envelope
router = APIRouter(prefix="/segmentation", tags=["segmentation"])
@router.get("/models", response_model=Envelope[SegmentationModelsResponse])
def list_segmentation_models() -> dict:
return envelope({"models": [model.model_dump() for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")]})
@router.post("/run", response_model=Envelope[SegmentationRunResponse])
def run_segmentation(
payload: SegmentationRunRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
assert_guest_project_scope(request, payload.project_id)
result = SegmentationService.run_segmentation(
db=db,
project_id=payload.project_id,
dataset_id=payload.dataset_id,
model_id=payload.model_id,
confidence_threshold=payload.confidence_threshold,
class_filter=payload.class_filter,
tile_manifest_path=payload.tile_manifest_path,
parameters_json=payload.parameters_json,
)
return envelope(result.model_dump())
@router.post("/run-async", response_model=Envelope[JobRead])
def queue_segmentation(
payload: SegmentationRunRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
"""Queue a segmentation run for the background worker.
Configured segmentation walks the same tile manifest as detection and is
just as unsuited to running inside the request. Poll ``GET /jobs/{id}``.
"""
assert_guest_project_scope(request, payload.project_id)
job = SegmentationService.enqueue_segmentation(
db=db,
project_id=payload.project_id,
dataset_id=payload.dataset_id,
model_id=payload.model_id,
confidence_threshold=payload.confidence_threshold,
class_filter=payload.class_filter,
tile_manifest_path=payload.tile_manifest_path,
parameters_json=payload.parameters_json,
)
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
@router.get("/runs", response_model=Envelope[SegmentationRunListResponse])
def list_segmentation_runs(
request: Request,
project_id: UUID | None = None,
dataset_id: UUID | None = None,
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
project_id = guest_scoped_project_filter(request, project_id)
return envelope(
SegmentationService.list_runs(
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
).model_dump()
)
@router.get("/runs/{analysis_run_id}", response_model=Envelope[SegmentationRunRead])
def get_segmentation_run(
analysis_run_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(run.model_dump())
@router.get(
"/runs/{analysis_run_id}/segmentations",
response_model=Envelope[SegmentationListResponse],
)
def list_segmentation_run_outputs(
analysis_run_id: UUID,
request: Request,
dataset_id: UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(
SegmentationService.list_segmentations(
db,
limit=limit,
offset=offset,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
).model_dump()
)
@router.get(
"/datasets/{dataset_id}/segmentations",
response_model=Envelope[SegmentationListResponse],
)
def list_dataset_segmentations(
dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope(
SegmentationService.list_segmentations(
db,
limit=limit,
offset=offset,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
).model_dump()
)
@router.get("/segmentations/{segmentation_id}", response_model=Envelope[SegmentationRead])
def get_segmentation(
segmentation_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
segmentation = SegmentationService.get_segmentation(db, segmentation_id)
assert_guest_project_scope(request, segmentation.project_id)
return envelope(segmentation.model_dump())
@router.get(
"/runs/{analysis_run_id}/geojson",
response_model=Envelope[GeoJsonFeatureCollection],
)
def get_segmentation_run_geojson(
analysis_run_id: UUID,
request: Request,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(
SegmentationService.segmentations_to_geojson(
db,
limit=limit,
analysis_run_id=analysis_run_id,
class_name=class_name,
min_confidence=min_confidence,
)
)
@router.get(
"/datasets/{dataset_id}/geojson",
response_model=Envelope[GeoJsonFeatureCollection],
)
def get_dataset_segmentation_geojson(
dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None,
class_name: str | None = None,
min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope(
SegmentationService.segmentations_to_geojson(
db,
limit=limit,
analysis_run_id=analysis_run_id,
dataset_id=dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
)
@router.post(
"/runs/{analysis_run_id}/qa/reference",
response_model=Envelope[AnalysisQaResponse],
)
def compare_segmentation_run_with_reference(
analysis_run_id: UUID,
payload: SegmentationQaRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
if guest_project_scope(request) is not None:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(
SegmentationService.compare_segmentations_with_reference(
db=db,
analysis_run_id=analysis_run_id,
reference_dataset_id=payload.reference_dataset_id,
iou_threshold=payload.iou_threshold,
class_name=payload.class_name,
min_confidence=payload.min_confidence,
calibration_thresholds=payload.calibration_thresholds,
)
)
@@ -0,0 +1,87 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Area, Dataset
from app.schemas.common import Envelope
from app.schemas.operations import VectorSelectionResponse
from app.schemas.selection_partitions import VectorPartitionSelectionRequest
from app.services.vector_feature_service import VectorFeatureService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}", tags=["selection-partitions"])
def _product_identity(dataset: Dataset) -> str:
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
return str(metadata.get("product_key") or dataset.reference_layer_name or "")
@router.post(
"/datasets/vector/partitions/select",
response_model=Envelope[VectorSelectionResponse],
)
def select_vector_partitions(
project_id: UUID,
payload: VectorPartitionSelectionRequest,
db: Session = Depends(get_db),
):
datasets = db.query(Dataset).filter(Dataset.id.in_(payload.dataset_ids)).all()
by_id = {dataset.id: dataset for dataset in datasets}
ordered = [by_id.get(dataset_id) for dataset_id in payload.dataset_ids]
if any(dataset is None or dataset.project_id != project_id for dataset in ordered):
raise AppError(code="DATASET_NOT_FOUND", message="One or more selection partitions were not found", status_code=404)
typed_datasets = [dataset for dataset in ordered if dataset is not None]
if any(dataset.dataset_type not in {"vector", "geojson"} or dataset.status != "ready" for dataset in typed_datasets):
raise AppError(
code="INVALID_VECTOR_PARTITIONS",
message="Every selection partition must be a ready vector dataset",
status_code=409,
)
source_names = {dataset.source_name for dataset in typed_datasets}
product_keys = {_product_identity(dataset) for dataset in typed_datasets}
if len(source_names) != 1 or len(product_keys) != 1:
raise AppError(
code="VECTOR_PARTITION_SOURCE_MISMATCH",
message="Selection partitions must belong to one governed source product",
details={"source_names": sorted(str(value) for value in source_names), "product_keys": sorted(product_keys)},
status_code=409,
)
selection_geometry = None
selection_area_id = None
if payload.area_id is not None:
selection_area = db.get(Area, payload.area_id)
if selection_area is None or selection_area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
selection_geometry, _covers_full_area = VectorFeatureService.constrain_bbox_to_area(
payload.bbox.model_dump(),
selection_area.geometry,
)
selection_area_id = selection_area.id
representative = typed_datasets[0]
dataset_ids = [dataset.id for dataset in typed_datasets]
result = VectorFeatureService.select_features_by_bbox(
db,
dataset_id=representative.id,
dataset_ids=dataset_ids,
bbox=payload.bbox.model_dump(),
limit=payload.limit,
dataset=representative,
selection_geometry=selection_geometry,
selection_area_id=selection_area_id,
deduplicate_source_features=True,
)
result.update(
partition_count=len(dataset_ids),
source_name=representative.source_name,
dataset_ids=dataset_ids,
)
return envelope(VectorSelectionResponse(**result).model_dump(exclude_none=True))
+119
View File
@@ -0,0 +1,119 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
from app.schemas import (
DatasetLineageEdgeRead,
DatasetProvenanceRead,
DatasetQuarantineRead,
Envelope,
ItemList,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from app.utils.response import envelope
router = APIRouter(tags=["source-registry"])
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
def list_source_registry(
classification: str | None = None,
db: Session = Depends(get_db),
) -> dict:
query = (
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
)
if classification:
query = query.filter(SourceRegistry.classification == classification.strip().lower())
rows = (
query.group_by(SourceRegistry.id)
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
.all()
)
items = [_source_read(source, snapshot_count=count) for source, count in rows]
return envelope({"items": items, "total": len(items)})
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
normalized_key = source_key.strip().lower()
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
if source is None:
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
snapshots = (
db.query(SourceSnapshot)
.filter(SourceSnapshot.source_registry_id == source.id)
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
.all()
)
detail = SourceRegistryDetailRead(
source=_source_read(source, snapshot_count=len(snapshots)),
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
)
return envelope(detail)
@router.get(
"/projects/{project_id}/datasets/{dataset_id}/provenance",
response_model=Envelope[DatasetProvenanceRead],
)
def get_dataset_provenance(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
) -> dict:
if db.get(Project, project_id) is None:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if dataset is None or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
lineage = (
db.query(DatasetLineageEdge)
.filter(
or_(
DatasetLineageEdge.parent_dataset_id == dataset.id,
DatasetLineageEdge.child_dataset_id == dataset.id,
)
)
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
.all()
)
quarantines = (
db.query(DatasetQuarantine)
.filter(DatasetQuarantine.dataset_id == dataset.id)
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
.all()
)
result = DatasetProvenanceRead(
dataset_id=dataset.id,
source=_source_read(source) if source else None,
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
data_contract_key=dataset.data_contract_key,
data_contract_version=dataset.data_contract_version,
validation_status=dataset.validation_status,
validation_report_json=dataset.validation_report_json,
provenance_status=dataset.provenance_status,
lineage_status=dataset.lineage_status,
quarantine_status=dataset.quarantine_status,
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
)
return envelope(result)
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas import Envelope, ItemList
from app.schemas.temporal import (
TemporalComparisonRequest,
TemporalComparisonResponse,
TemporalSeriesRead,
)
from app.services.temporal_analysis_service import TemporalAnalysisService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}/temporal", tags=["temporal"])
@router.get("/series", response_model=Envelope[ItemList[TemporalSeriesRead]])
def list_temporal_series(project_id: UUID, db: Session = Depends(get_db)):
series = TemporalAnalysisService.list_series(db, project_id)
return envelope({"items": [item.model_dump() for item in series], "total": len(series)})
@router.post("/compare", response_model=Envelope[TemporalComparisonResponse])
def compare_temporal_snapshots(
project_id: UUID,
payload: TemporalComparisonRequest,
db: Session = Depends(get_db),
):
return envelope(TemporalAnalysisService.compare(db, project_id=project_id, payload=payload).model_dump())
View File
+584
View File
@@ -0,0 +1,584 @@
from urllib.parse import urlsplit
from pydantic import AliasChoices, Field, field_validator, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
populate_by_name=True,
)
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
app_version: str = Field(
default="1.0.0",
validation_alias="GEOINTEL_APP_VERSION",
)
build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
auth_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_ENABLED")
auth_require_https: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_REQUIRE_HTTPS")
auth_username: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_USERNAME")
auth_password_hash: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_PASSWORD_HASH")
auth_session_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_SESSION_SECRET")
authentik_issuer: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ISSUER")
authentik_client_id: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_ID")
authentik_client_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_SECRET")
authentik_allowed_email: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ALLOWED_EMAIL")
public_base_url: str = Field(
default="http://localhost:1202",
validation_alias="GEOINTEL_PUBLIC_BASE_URL",
)
auth_session_ttl_seconds: int = Field(
default=43_200,
ge=900,
le=604_800,
validation_alias="GEOINTEL_AUTH_SESSION_TTL_SECONDS",
)
guest_access_enabled: bool = Field(
default=False,
validation_alias="GEOINTEL_GUEST_ACCESS_ENABLED",
)
guest_display_name: str = Field(
default="Gast",
min_length=1,
max_length=64,
validation_alias="GEOINTEL_GUEST_DISPLAY_NAME",
)
guest_session_ttl_seconds: int = Field(
default=7_200,
ge=900,
le=86_400,
validation_alias="GEOINTEL_GUEST_SESSION_TTL_SECONDS",
)
guest_login_requests_per_minute: int = Field(
default=3,
ge=1,
le=60,
validation_alias="GEOINTEL_GUEST_LOGIN_REQUESTS_PER_MINUTE",
)
guest_compute_requests_per_minute: int = Field(
default=4,
ge=1,
le=120,
validation_alias="GEOINTEL_GUEST_COMPUTE_REQUESTS_PER_MINUTE",
)
guest_compute_max_concurrency: int = Field(
default=2,
ge=1,
le=16,
validation_alias="GEOINTEL_GUEST_COMPUTE_MAX_CONCURRENCY",
)
database_url: str = Field(
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
validation_alias="DATABASE_URL",
)
storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT")
# Analysis consumes only artifacts under storage_root. Provisioning
# workflows that stage tiles elsewhere before ingest can opt out.
allow_external_artifact_paths: bool = Field(
default=False, validation_alias="GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS"
)
max_upload_mb: int = Field(
default=500,
ge=1,
le=2_048,
validation_alias=AliasChoices("GEOINTEL_MAX_UPLOAD_MB", "MAX_UPLOAD_MB"),
)
max_in_memory_vector_mb: int = Field(
default=64,
ge=1,
le=256,
validation_alias="GEOINTEL_MAX_IN_MEMORY_VECTOR_MB",
)
max_raster_pixels: int = Field(
default=40_000_000,
ge=1,
le=500_000_000,
validation_alias="GEOINTEL_MAX_RASTER_PIXELS",
)
max_raster_bands: int = Field(
default=16,
ge=1,
le=256,
validation_alias="GEOINTEL_MAX_RASTER_BANDS",
)
max_decoded_raster_mb: int = Field(
default=1024,
ge=16,
le=8192,
validation_alias="GEOINTEL_MAX_DECODED_RASTER_MB",
)
orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED")
orthophoto_wms_url: str = Field(
default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
validation_alias="ORTHOPHOTO_WMS_URL",
)
orthophoto_wms_layer: str = Field(default="Ortho", validation_alias="ORTHOPHOTO_WMS_LAYER")
spw_orthophoto_wms_url: str = Field(
default="https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_LAST/MapServer/WMSServer",
validation_alias="SPW_ORTHOPHOTO_WMS_URL",
)
brussels_orthophoto_wms_url: str = Field(
default="https://geoservices-grid.irisnet.be/geoserver/urbisgrid/ows",
validation_alias="BRUSSELS_ORTHOPHOTO_WMS_URL",
)
orthophoto_resolution_m: float = Field(default=1.0, gt=0, validation_alias="ORTHOPHOTO_RESOLUTION_M")
orthophoto_min_side_m: float = Field(default=128.0, gt=0, validation_alias="ORTHOPHOTO_MIN_SIDE_M")
orthophoto_max_side_m: float = Field(default=1024.0, gt=0, validation_alias="ORTHOPHOTO_MAX_SIDE_M")
orthophoto_timeout_seconds: int = Field(default=120, ge=1, validation_alias="ORTHOPHOTO_TIMEOUT_SECONDS")
orthophoto_max_response_mb: int = Field(default=32, ge=1, validation_alias="ORTHOPHOTO_MAX_RESPONSE_MB")
orthophoto_cache_ttl_hours: int = Field(default=24, ge=0, validation_alias="ORTHOPHOTO_CACHE_TTL_HOURS")
source_catalog_probe_enabled: bool = Field(default=True, validation_alias="SOURCE_CATALOG_PROBE_ENABLED")
source_catalog_grb_wfs_url: str = Field(
default="https://geo.api.vlaanderen.be/GRB/wfs",
validation_alias="SOURCE_CATALOG_GRB_WFS_URL",
)
source_catalog_alz_release_url: str = Field(
default="https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen",
validation_alias="SOURCE_CATALOG_ALZ_RELEASE_URL",
)
source_catalog_statbel_dcat_url: str = Field(
default="https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl",
validation_alias="SOURCE_CATALOG_STATBEL_DCAT_URL",
)
source_catalog_statbel_max_response_mb: int = Field(
default=5,
ge=1,
le=10,
validation_alias="SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB",
)
source_catalog_probe_timeout_seconds: int = Field(
default=10,
ge=1,
le=60,
validation_alias="SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS",
)
source_catalog_probe_max_response_mb: int = Field(
default=2,
ge=1,
le=10,
validation_alias="SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB",
)
source_catalog_probe_cache_ttl_seconds: int = Field(
default=900,
ge=0,
le=86_400,
validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS",
)
grb_enabled: bool = Field(default=True, validation_alias="GRB_ENABLED")
grb_ogc_api_url: str = Field(
default="https://geo.api.vlaanderen.be/GRB/ogc/features/v1",
validation_alias="GRB_OGC_API_URL",
)
grb_min_side_m: float = Field(default=10.0, gt=0, validation_alias="GRB_MIN_SIDE_M")
grb_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="GRB_MAX_SIDE_M")
grb_page_size: int = Field(default=1000, ge=1, le=1000, validation_alias="GRB_PAGE_SIZE")
grb_max_pages: int = Field(default=200, ge=1, le=1000, validation_alias="GRB_MAX_PAGES")
grb_max_features: int = Field(default=150_000, ge=1, validation_alias="GRB_MAX_FEATURES")
grb_timeout_seconds: int = Field(default=180, ge=1, le=600, validation_alias="GRB_TIMEOUT_SECONDS")
grb_max_response_mb: int = Field(default=20, ge=1, le=100, validation_alias="GRB_MAX_RESPONSE_MB")
grb_max_total_response_mb: int = Field(
default=256,
ge=1,
le=2048,
validation_alias="GRB_MAX_TOTAL_RESPONSE_MB",
)
grb_cache_ttl_hours: int = Field(default=24, ge=0, le=8760, validation_alias="GRB_CACHE_TTL_HOURS")
official_vector_enabled: bool = Field(default=True, validation_alias="OFFICIAL_VECTOR_ENABLED")
bwk_wfs_url: str = Field(
default="https://geo.api.vlaanderen.be/BWK/wfs",
validation_alias="BWK_WFS_URL",
)
dov_soil_wfs_url: str = Field(
default="https://www.dov.vlaanderen.be/geoserver/wfs",
validation_alias="DOV_SOIL_WFS_URL",
)
official_vector_min_side_m: float = Field(
default=10.0,
gt=0,
validation_alias="OFFICIAL_VECTOR_MIN_SIDE_M",
)
official_vector_max_side_m: float = Field(
default=20_000.0,
gt=0,
validation_alias="OFFICIAL_VECTOR_MAX_SIDE_M",
)
official_vector_page_size: int = Field(
default=1000,
ge=1,
le=2000,
validation_alias="OFFICIAL_VECTOR_PAGE_SIZE",
)
official_vector_max_pages: int = Field(
default=200,
ge=1,
le=1000,
validation_alias="OFFICIAL_VECTOR_MAX_PAGES",
)
official_vector_max_features: int = Field(
default=100_000,
ge=1,
validation_alias="OFFICIAL_VECTOR_MAX_FEATURES",
)
official_vector_timeout_seconds: int = Field(
default=180,
ge=1,
le=600,
validation_alias="OFFICIAL_VECTOR_TIMEOUT_SECONDS",
)
official_vector_max_response_mb: int = Field(
default=20,
ge=1,
le=100,
validation_alias="OFFICIAL_VECTOR_MAX_RESPONSE_MB",
)
official_vector_max_total_response_mb: int = Field(
default=256,
ge=1,
le=2048,
validation_alias="OFFICIAL_VECTOR_MAX_TOTAL_RESPONSE_MB",
)
official_vector_cache_ttl_hours: int = Field(
default=24,
ge=0,
le=8760,
validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS",
)
spw_picc_enabled: bool = Field(default=True, validation_alias="SPW_PICC_ENABLED")
spw_picc_mapserver_url: str = Field(
default=(
"https://geoservices.wallonie.be/arcgis/rest/services/"
"TOPOGRAPHIE/PICC_VDIFF/MapServer"
),
validation_alias="SPW_PICC_MAPSERVER_URL",
)
spw_flood_hazard_enabled: bool = Field(default=True, validation_alias="SPW_FLOOD_HAZARD_ENABLED")
spw_flood_hazard_mapserver_url: str = Field(
default=(
"https://geoservices.wallonie.be/arcgis/rest/services/"
"EAU/ALEA_INOND/MapServer"
),
validation_alias="SPW_FLOOD_HAZARD_MAPSERVER_URL",
)
urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED")
urbis_wfs_url: str = Field(
default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows",
validation_alias="URBIS_WFS_URL",
)
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
dhmv_wcs_url: str = Field(
default="https://geo.api.vlaanderen.be/DHMV/wcs",
validation_alias="DHMV_WCS_URL",
)
dhmv_resolution_m: float = Field(default=5.0, ge=1.0, le=10.0, validation_alias="DHMV_RESOLUTION_M")
dhmv_min_side_m: float = Field(default=10.0, gt=0, validation_alias="DHMV_MIN_SIDE_M")
dhmv_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="DHMV_MAX_SIDE_M")
dhmv_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="DHMV_MAX_PIXELS")
dhmv_timeout_seconds: int = Field(default=300, ge=1, validation_alias="DHMV_TIMEOUT_SECONDS")
dhmv_max_response_mb: int = Field(default=160, ge=1, validation_alias="DHMV_MAX_RESPONSE_MB")
flood_hazard_enabled: bool = Field(default=True, validation_alias="FLOOD_HAZARD_ENABLED")
flood_hazard_wcs_url: str = Field(
default="https://geoservice.waterinfo.be/OGRK/wcs",
validation_alias="FLOOD_HAZARD_WCS_URL",
)
flood_hazard_resolution_m: float = Field(default=5.0, ge=2.0, le=20.0, validation_alias="FLOOD_HAZARD_RESOLUTION_M")
flood_hazard_min_side_m: float = Field(default=10.0, gt=0, validation_alias="FLOOD_HAZARD_MIN_SIDE_M")
flood_hazard_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="FLOOD_HAZARD_MAX_SIDE_M")
flood_hazard_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="FLOOD_HAZARD_MAX_PIXELS")
flood_hazard_timeout_seconds: int = Field(default=300, ge=1, validation_alias="FLOOD_HAZARD_TIMEOUT_SECONDS")
flood_hazard_max_response_mb: int = Field(default=160, ge=1, validation_alias="FLOOD_HAZARD_MAX_RESPONSE_MB")
bathymetry_profiles_enabled: bool = Field(default=True, validation_alias="BATHYMETRY_PROFILES_ENABLED")
bathymetry_profiles_layer_url: str = Field(
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
validation_alias="BATHYMETRY_PROFILES_LAYER_URL",
)
bathymetry_watercourse_layer_url: str = Field(
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1",
validation_alias="BATHYMETRY_WATERCOURSE_LAYER_URL",
)
bathymetry_profiles_page_size: int = Field(
default=1000,
ge=1,
le=2000,
validation_alias="BATHYMETRY_PROFILES_PAGE_SIZE",
)
bathymetry_profiles_max_features: int = Field(
default=50_000,
ge=1,
le=250_000,
validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES",
)
bathymetry_profiles_max_pages: int = Field(
default=200,
ge=1,
le=5_000,
validation_alias="BATHYMETRY_PROFILES_MAX_PAGES",
)
bathymetry_profiles_timeout_seconds: int = Field(
default=120,
ge=1,
le=600,
validation_alias="BATHYMETRY_PROFILES_TIMEOUT_SECONDS",
)
bathymetry_profiles_max_response_mb: int = Field(
default=32,
ge=1,
le=256,
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
)
bathymetry_raster_max_pixels: int = Field(
default=30_000_000,
ge=1,
validation_alias="BATHYMETRY_RASTER_MAX_PIXELS",
)
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
mdk_bathymetry_wcs_url: str = Field(
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
validation_alias="MDK_BATHYMETRY_WCS_URL",
)
mdk_bathymetry_probe_timeout_seconds: int = Field(
default=20,
ge=1,
le=120,
validation_alias="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS",
)
mdk_bathymetry_probe_max_response_mb: int = Field(
default=4,
ge=1,
le=16,
validation_alias="MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB",
)
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
thematic_raster_wcs_url: str = Field(
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
validation_alias="THEMATIC_RASTER_WCS_URL",
)
mdk_bathymetry_acquisition_enabled: bool = Field(
default=False,
validation_alias="MDK_BATHYMETRY_ACQUISITION_ENABLED",
)
mdk_bathymetry_coverage_id: str | None = Field(default=None, validation_alias="MDK_BATHYMETRY_COVERAGE_ID")
mdk_bathymetry_request_crs: str = Field(default="EPSG:4326", validation_alias="MDK_BATHYMETRY_REQUEST_CRS")
mdk_bathymetry_max_bbox_deg2: float = Field(
default=0.25,
gt=0,
validation_alias="MDK_BATHYMETRY_MAX_BBOX_DEG2",
)
mdk_bathymetry_acquisition_timeout_seconds: int = Field(
default=120,
ge=1,
validation_alias="MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS",
)
mdk_bathymetry_acquisition_max_response_mb: int = Field(
default=160,
ge=1,
validation_alias="MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB",
)
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
walous_enabled: bool = Field(default=True, validation_alias="WALOUS_ENABLED")
walous_source_dir: str = Field(
default="/app/storage/source-cache/walous",
validation_alias="WALOUS_SOURCE_DIR",
)
walous_analysis_resolution_m: float = Field(
default=10.0,
ge=1.0,
le=100.0,
validation_alias="WALOUS_ANALYSIS_RESOLUTION_M",
)
walous_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="WALOUS_MAX_SIDE_M")
walous_max_pixels: int = Field(default=36_000_000, ge=1, validation_alias="WALOUS_MAX_PIXELS")
spw_terrain_enabled: bool = Field(default=True, validation_alias="SPW_TERRAIN_ENABLED")
spw_terrain_source_dir: str = Field(
default="/app/storage/source-cache/spw-terrain",
validation_alias="SPW_TERRAIN_SOURCE_DIR",
)
spw_terrain_analysis_resolution_m: float = Field(
default=5.0,
ge=1.0,
le=10.0,
validation_alias="SPW_TERRAIN_ANALYSIS_RESOLUTION_M",
)
spw_terrain_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="SPW_TERRAIN_MAX_SIDE_M")
spw_terrain_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="SPW_TERRAIN_MAX_PIXELS")
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
reconcile_interrupted_runs_on_startup: bool = Field(
default=False,
validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
)
aoi_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AOI_WORKER_ENABLED")
aoi_worker_poll_seconds: float = Field(default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_AOI_WORKER_POLL_SECONDS")
# Executes queued detection.run / segmentation.run jobs so tiled GPU
# inference never blocks an HTTP request.
analysis_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_ANALYSIS_WORKER_ENABLED")
analysis_worker_poll_seconds: float = Field(
default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_ANALYSIS_WORKER_POLL_SECONDS"
)
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
yolo_model_path: str | None = Field(default=None, validation_alias="YOLO_MODEL_PATH")
yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID")
yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME")
yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION")
yolo_model_classes: str = Field(default="building", validation_alias="YOLO_MODEL_CLASSES")
yolo_enforce_validation_scope: bool = Field(default=False, validation_alias="YOLO_ENFORCE_VALIDATION_SCOPE")
yolo_validation_scope_manifest_path: str | None = Field(
default=None,
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_PATH",
)
yolo_validation_scope_manifest_sha256: str | None = Field(
default=None,
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_SHA256",
)
# Deprecated compatibility field. Mutable Area names are never an
# inference authorization boundary; deployments must use the immutable
# checksum-bound scope manifest above.
yolo_validated_area_names: str = Field(default="Mol,Kempen", validation_alias="YOLO_VALIDATED_AREA_NAMES")
yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE")
yolo_require_cuda: bool = Field(default=False, validation_alias="YOLO_REQUIRE_CUDA")
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
yolo_suppress_tile_edge_detections: bool = Field(
default=True, validation_alias="YOLO_SUPPRESS_TILE_EDGE_DETECTIONS"
)
# Intersection over the smaller box. The candidate evaluation freezes this
# during calibration; serving a promoted model at a different value means
# the runtime suppresses detections the gate counted.
yolo_containment_nms_threshold: float = Field(
default=0.85, ge=0.0, le=1.0, validation_alias="YOLO_CONTAINMENT_NMS_THRESHOLD"
)
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
yolo_seg_enabled: bool = Field(default=False, validation_alias="YOLO_SEG_ENABLED")
yolo_seg_model_path: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_PATH")
yolo_seg_model_id: str = Field(default="yolo-seg-configured", validation_alias="YOLO_SEG_MODEL_ID")
yolo_seg_model_display_name: str = Field(
default="Configured YOLO segmentation",
validation_alias="YOLO_SEG_MODEL_DISPLAY_NAME",
)
yolo_seg_model_version: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_VERSION")
sam_enabled: bool = Field(default=False, validation_alias="SAM_ENABLED")
sam_model_path: str | None = Field(default=None, validation_alias="SAM_MODEL_PATH")
sam_model_id: str = Field(default="sam-configured", validation_alias="SAM_MODEL_ID")
sam_model_display_name: str = Field(
default="Configured SAM segmentation",
validation_alias="SAM_MODEL_DISPLAY_NAME",
)
sam_model_version: str | None = Field(default=None, validation_alias="SAM_MODEL_VERSION")
segmentation_max_masks_per_tile: int = Field(default=300, ge=1, validation_alias="SEGMENTATION_MAX_MASKS_PER_TILE")
# Masks and boxes overlap differently, so segmentation carries its own
# containment value rather than borrowing the detector's.
segmentation_containment_nms_threshold: float = Field(
default=0.85,
ge=0.0,
le=1.0,
validation_alias="SEGMENTATION_CONTAINMENT_NMS_THRESHOLD",
)
segmentation_duplicate_iou_threshold: float = Field(
default=0.5,
ge=0.0,
le=1.0,
validation_alias="SEGMENTATION_DUPLICATE_IOU_THRESHOLD",
)
ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED")
ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL")
ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL")
ollama_timeout_seconds: int = Field(default=120, ge=5, le=600, validation_alias="OLLAMA_TIMEOUT_SECONDS")
ollama_max_output_tokens: int = Field(default=1_200, ge=100, le=4_000, validation_alias="OLLAMA_MAX_OUTPUT_TOKENS")
ollama_context_tokens: int = Field(default=16_384, ge=4_096, le=131_072, validation_alias="OLLAMA_CONTEXT_TOKENS")
cors_origins: list[str] | str = Field(
default=["http://localhost:5173", "http://127.0.0.1:5173"],
validation_alias="CORS_ORIGINS",
)
@field_validator("cors_origins", mode="before")
@classmethod
def parse_cors_origins(cls, value: object) -> list[str]:
if isinstance(value, str):
return [item.strip() for item in value.split(",") if item.strip()]
if isinstance(value, list):
return value
if value is None:
return ["http://localhost:5173", "http://127.0.0.1:5173"]
return [str(value)]
@field_validator("ollama_base_url")
@classmethod
def validate_ollama_base_url(cls, value: str) -> str:
normalized = value.strip().rstrip("/")
if not normalized.startswith(("http://", "https://")):
raise ValueError("OLLAMA_BASE_URL must use http or https")
return normalized
@model_validator(mode="after")
def validate_operator_auth(self) -> "Settings":
self.guest_display_name = self.guest_display_name.strip()
if not self.guest_display_name:
raise ValueError("GEOINTEL_GUEST_DISPLAY_NAME must not be blank")
for field_name in (
"authentik_issuer",
"authentik_client_id",
"authentik_client_secret",
"authentik_allowed_email",
):
value = getattr(self, field_name)
setattr(self, field_name, value.strip() if value else None)
self.public_base_url = self.public_base_url.strip().rstrip("/")
authentik_values = (
self.authentik_issuer,
self.authentik_client_id,
self.authentik_client_secret,
self.authentik_allowed_email,
)
if any(authentik_values) and not all(authentik_values):
raise ValueError("All GEOINTEL_AUTHENTIK_* values must be configured together")
if all(authentik_values):
if not self.auth_enabled:
raise ValueError("GEOINTEL_AUTH_ENABLED must be true when Authentik is configured")
for label, value in (
("GEOINTEL_AUTHENTIK_ISSUER", self.authentik_issuer),
("GEOINTEL_PUBLIC_BASE_URL", self.public_base_url),
):
parsed = urlsplit(str(value))
if (
parsed.scheme != "https"
or not parsed.hostname
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
):
raise ValueError(f"{label} must be an absolute HTTPS URL without credentials, query or fragment")
public_url = urlsplit(self.public_base_url)
if public_url.path not in ("", "/"):
raise ValueError("GEOINTEL_PUBLIC_BASE_URL must not contain a path")
if "@" not in str(self.authentik_allowed_email) or any(
character.isspace() for character in str(self.authentik_allowed_email)
):
raise ValueError("GEOINTEL_AUTHENTIK_ALLOWED_EMAIL must be one valid e-mail address")
if not self.auth_enabled:
return self
if not (self.auth_username or "").strip():
raise ValueError("GEOINTEL_AUTH_USERNAME is required when authentication is enabled")
if not (self.auth_password_hash or "").startswith("pbkdf2_sha256$"):
raise ValueError("GEOINTEL_AUTH_PASSWORD_HASH must be a PBKDF2-SHA256 hash")
if len(self.auth_session_secret or "") < 32:
raise ValueError("GEOINTEL_AUTH_SESSION_SECRET must contain at least 32 characters")
return self
def get_settings() -> Settings:
return Settings()
+15
View File
@@ -0,0 +1,15 @@
class AppError(Exception):
"""Domain error used by services to return canonical API errors."""
def __init__(
self,
code: str,
message: str,
details: dict | list | None = None,
status_code: int = 400,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.details = details or {}
self.status_code = status_code
+14
View File
@@ -0,0 +1,14 @@
import logging
import sys
def configure_logging(level: str = "INFO", sql_level: str = "WARNING") -> None:
logging.basicConfig(
level=level,
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
stream=sys.stdout,
force=True,
)
for name in ["uvicorn", "uvicorn.error", "uvicorn.access"]:
logging.getLogger(name).setLevel(level)
logging.getLogger("sqlalchemy.engine").setLevel(sql_level)
+14
View File
@@ -0,0 +1,14 @@
from __future__ import annotations
from uuid import UUID
# Stable server-owned identity: a public session must never attach itself to an
# operator project merely because the display names happen to match.
PUBLIC_DEMO_PROJECT_ID = UUID("6f7e6f12-9b62-4a3f-a5a0-4b3bb6b2c901")
PUBLIC_DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
PUBLIC_DEMO_PROJECT_MARKER = "geointel:public-demo:v1"
def is_public_demo_project(project_id: UUID) -> bool:
return project_id == PUBLIC_DEMO_PROJECT_ID
+18
View File
@@ -0,0 +1,18 @@
from __future__ import annotations
from contextvars import ContextVar, Token
_request_id: ContextVar[str] = ContextVar("geointel_request_id", default="-")
def get_request_id() -> str:
return _request_id.get()
def set_request_id(value: str) -> Token:
return _request_id.set(value)
def reset_request_id(token: Token) -> None:
_request_id.reset(token)
View File
+4
View File
@@ -0,0 +1,4 @@
from .base import Base
from .session import get_db, get_engine
__all__ = ["Base", "get_db", "get_engine"]
+5
View File
@@ -0,0 +1,5 @@
from sqlalchemy.orm import DeclarativeBase
class Base(DeclarativeBase):
pass
+20
View File
@@ -0,0 +1,20 @@
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from app.core.config import get_settings
engine = create_engine(get_settings().database_url, pool_pre_ping=True, future=True)
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False, future=True)
def get_db():
db: Session = SessionLocal()
try:
yield db
finally:
db.close()
def get_engine():
return engine
View File
+467
View File
@@ -0,0 +1,467 @@
from __future__ import annotations
import logging
import asyncio
import re
import time
import uuid
from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
from app.core.config import get_settings
from app.core.errors import AppError
from app.core.logging import configure_logging
from app.core.request_context import reset_request_id, set_request_id
from app.db.session import SessionLocal
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
from app.services.auth_service import AuthService
from app.services.analysis_job_worker import AnalysisJobWorker
from app.services.aoi_operation_worker import AoiOperationWorker
logger = logging.getLogger("geointel")
SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
UNSAFE_HOST = re.compile(r"[/\\@?#\s\x00-\x1f\x7f]")
def _to_error_payload(
code: str,
message: str,
details: dict | list | None = None,
request_id: str | None = None,
) -> dict:
return {
"error": code,
"message": message,
"details": details or {},
"request_id": request_id,
}
def create_app() -> FastAPI:
settings = get_settings()
configure_logging(settings.log_level, settings.sql_log_level)
@asynccontextmanager
async def lifespan(_: FastAPI):
worker_stop = asyncio.Event()
worker_task = None
analysis_worker_task = None
if settings.reconcile_interrupted_runs_on_startup:
db = SessionLocal()
try:
result = RuntimeReconciliationService.reconcile(db)
logger.info(
"Runtime reconciliation completed: jobs=%s analysis_runs=%s resumed_aoi_partitions=%s exhausted_aoi_partitions=%s",
result.interrupted_jobs,
result.interrupted_analysis_runs,
result.resumed_aoi_partitions,
result.exhausted_aoi_partitions,
)
except Exception:
db.rollback()
logger.exception("Runtime reconciliation failed")
raise
finally:
db.close()
if settings.aoi_worker_enabled:
worker_task = asyncio.create_task(AoiOperationWorker.run(worker_stop, settings.aoi_worker_poll_seconds))
if settings.analysis_worker_enabled:
analysis_worker_task = asyncio.create_task(
AnalysisJobWorker.run(worker_stop, settings.analysis_worker_poll_seconds)
)
try:
yield
finally:
worker_stop.set()
for task in (worker_task, analysis_worker_task):
if task is not None:
await task
app = FastAPI(
title="GeoIntel",
version=settings.app_version,
docs_url="/docs",
redoc_url="/redoc",
lifespan=lifespan,
)
app.add_middleware(
CORSMiddleware,
allow_origins=settings.cors_origins,
allow_methods=["*"],
allow_headers=["*"],
allow_credentials=True,
)
app.include_router(health.router)
app.include_router(auth.router, prefix=settings.api_prefix)
app.include_router(analysis.router, prefix=settings.api_prefix)
app.include_router(aoi_operations.router, prefix=settings.api_prefix)
app.include_router(projects.router, prefix=settings.api_prefix)
app.include_router(areas.router, prefix=settings.api_prefix)
app.include_router(datasets.router, prefix=settings.api_prefix)
app.include_router(jobs.router, prefix=settings.api_prefix)
app.include_router(quality_checks.router, prefix=settings.api_prefix)
app.include_router(exports.router, prefix=settings.api_prefix)
app.include_router(external.router, prefix=settings.api_prefix)
app.include_router(source_registry.router, prefix=settings.api_prefix)
app.include_router(demo.router, prefix=settings.api_prefix)
app.include_router(qa.router, prefix=settings.api_prefix)
app.include_router(detection.router, prefix=settings.api_prefix)
app.include_router(segmentation.router, prefix=settings.api_prefix)
app.include_router(selection_partitions.router, prefix=settings.api_prefix)
app.include_router(temporal.router, prefix=settings.api_prefix)
app.include_router(assistant.router, prefix=settings.api_prefix)
@app.middleware("http")
async def request_identity(request: Request, call_next):
supplied_request_id = request.headers.get("x-request-id", "")
request_id = supplied_request_id if SAFE_REQUEST_ID.fullmatch(supplied_request_id) else str(uuid.uuid4())
request.state.request_id = request_id
token = set_request_id(request_id)
started_at = time.perf_counter()
raw_path = str(request.scope.get("path") or "")
guest_compute_acquired = False
try:
host = request.headers.get("host", "")
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if not raw_path.startswith("/") or not host or UNSAFE_HOST.search(host):
response = JSONResponse(
status_code=400,
content=_to_error_payload(
"INVALID_REQUEST_TARGET",
"The request target or Host header is invalid",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
if content_type == "application/x-www-form-urlencoded":
response = JSONResponse(
status_code=415,
content=_to_error_payload(
"UNSUPPORTED_CONTENT_TYPE",
"URL-encoded form bodies are not supported",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
public_auth_paths = {
f"{settings.api_prefix}/auth/session",
f"{settings.api_prefix}/auth/login",
f"{settings.api_prefix}/auth/guest",
f"{settings.api_prefix}/auth/logout",
f"{settings.api_prefix}/auth/authentik/start",
f"{settings.api_prefix}/auth/authentik/callback",
}
direct_loopback_request = (
request.client is not None
and request.client.host in {"127.0.0.1", "::1"}
and not request.headers.get("x-real-ip")
and not request.headers.get("x-forwarded-for")
)
if (
settings.auth_enabled
and raw_path.startswith(f"{settings.api_prefix}/")
and raw_path not in public_auth_paths
and not direct_loopback_request
):
principal = AuthService.verify_session_token(
request.cookies.get(auth.COOKIE_NAME),
settings,
)
if principal is None:
response = JSONResponse(
status_code=401,
content=_to_error_payload(
"AUTHENTICATION_REQUIRED",
"Meld u aan om de GeoIntel API te gebruiken.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
request.state.auth_principal = principal
if principal.role == "guest":
project_path_prefix = f"{settings.api_prefix}/projects/"
guest_project_root = f"{project_path_prefix}{principal.project_id}"
if raw_path.startswith(project_path_prefix):
scoped_path = raw_path[len(project_path_prefix):]
requested_project_id = scoped_path.split("/", 1)[0]
if str(principal.project_id) != requested_project_id:
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_PROJECT_SCOPE_REQUIRED",
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
query_project_id = request.query_params.get("project_id")
if query_project_id and query_project_id != str(principal.project_id):
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_PROJECT_SCOPE_REQUIRED",
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
guest_safe_read_paths = {
f"{settings.api_prefix}/projects",
f"{settings.api_prefix}/external/providers",
f"{settings.api_prefix}/assistant/status",
f"{settings.api_prefix}/assistant/models",
f"{settings.api_prefix}/detection/models",
f"{settings.api_prefix}/detection/model-assets",
f"{settings.api_prefix}/detection/yolo/preflight",
f"{settings.api_prefix}/segmentation/models",
}
normalized_path = raw_path.rstrip("/") or "/"
guest_project_read = (
normalized_path == guest_project_root
or normalized_path.startswith(f"{guest_project_root}/")
)
is_read_request = request.method in {"GET", "HEAD", "OPTIONS"}
if is_read_request:
if (
normalized_path == f"{settings.api_prefix}/detection/yolo/preflight"
and request.query_params.get("check_model_load", "").lower() in {"1", "true", "yes", "on"}
):
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_MODEL_LOAD_FORBIDDEN",
"Model loading is available to authenticated operators only.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
guest_scoped_analysis_read = (
query_project_id == str(principal.project_id)
and normalized_path.startswith(
(
f"{settings.api_prefix}/detection/",
f"{settings.api_prefix}/segmentation/",
f"{settings.api_prefix}/exports/",
)
)
)
if (
normalized_path not in guest_safe_read_paths
and not guest_project_read
and not guest_scoped_analysis_read
):
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_ROUTE_NOT_AVAILABLE",
"Deze API-route maakt geen deel uit van de afgeschermde GeoIntel-demo.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
else:
guest_safe_post_paths = {
f"{settings.api_prefix}/demo/workflow",
f"{settings.api_prefix}/external/coverage/resolve",
f"{settings.api_prefix}/analysis/change-detection",
}
guest_scoped_analysis_post_paths = {
f"{settings.api_prefix}/detection/run",
f"{settings.api_prefix}/detection/run-async",
f"{settings.api_prefix}/segmentation/run",
f"{settings.api_prefix}/segmentation/run-async",
f"{settings.api_prefix}/qa/detections-vs-reference",
f"{settings.api_prefix}/exports/geojson",
f"{settings.api_prefix}/exports/metadata",
f"{settings.api_prefix}/exports/report",
f"{settings.api_prefix}/exports/map-result",
}
guest_safe_post_suffixes = (
"/acquire",
"/vector/select",
"/vector/select/derive",
"/raster/tile",
"/raster/bathymetry/select",
"/raster/terrain/select",
"/raster/flood-hazard/select",
"/raster/thematic/select",
"/raster/walous/select",
"/temporal/compare",
"/datasets/vector/partitions/select",
"/datasets/bathymetry/profiles/partitions/select",
)
is_guest_safe_post = request.method == "POST" and (
raw_path in guest_safe_post_paths
or (
raw_path in guest_scoped_analysis_post_paths
and query_project_id == str(principal.project_id)
)
or (
query_project_id == str(principal.project_id)
and raw_path.startswith(
(
f"{settings.api_prefix}/detection/runs/",
f"{settings.api_prefix}/segmentation/runs/",
)
)
and raw_path.endswith("/qa/reference")
)
or (
raw_path.startswith(project_path_prefix)
and (
raw_path.endswith(guest_safe_post_suffixes)
or raw_path.endswith("/assistant/query")
)
)
)
if not is_guest_safe_post:
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_READ_ONLY",
"Gasttoegang laat alleen projectgebonden demo-analyses toe. Meld u aan als operator voor beheerwijzigingen.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
retry_after = AuthService.consume_guest_request(
f"guest-compute:{principal.session_id}",
max_requests=settings.guest_compute_requests_per_minute,
)
if retry_after:
response = JSONResponse(
status_code=429,
content=_to_error_payload(
"GUEST_COMPUTE_RATE_LIMITED",
"The public demo compute budget is temporarily exhausted.",
request_id=request_id,
),
)
response.headers["retry-after"] = str(retry_after)
response.headers["x-request-id"] = request_id
return response
guest_compute_acquired = AuthService.try_acquire_guest_compute(
max_concurrency=settings.guest_compute_max_concurrency,
)
if not guest_compute_acquired:
response = JSONResponse(
status_code=429,
content=_to_error_payload(
"GUEST_COMPUTE_BUSY",
"The public demo is already processing its maximum number of jobs.",
request_id=request_id,
),
)
response.headers["retry-after"] = "10"
response.headers["x-request-id"] = request_id
return response
response = await call_next(request)
response.headers["x-request-id"] = request_id
logger.info(
"request_complete request_id=%s method=%s path=%s status=%s duration_ms=%.1f",
request_id,
request.method,
raw_path,
response.status_code,
(time.perf_counter() - started_at) * 1000,
)
return response
finally:
if guest_compute_acquired:
AuthService.release_guest_compute()
reset_request_id(token)
@app.exception_handler(AppError)
async def app_error(request: Request, exc: AppError): # noqa: ARG001
return JSONResponse(
status_code=exc.status_code,
content=_to_error_payload(
exc.code,
exc.message,
exc.details,
request_id=request.state.request_id,
),
)
@app.exception_handler(HTTPException)
async def http_error(request: Request, exc: HTTPException): # noqa: ARG001
code = "HTTP_ERROR"
message = str(exc.detail)
details = {}
if isinstance(exc.detail, dict):
code = str(exc.detail.get("error") or exc.detail.get("code") or code)
message = str(exc.detail.get("message") or message)
raw_details = exc.detail.get("details")
details = raw_details if isinstance(raw_details, (dict, list)) else {}
return JSONResponse(
status_code=exc.status_code,
content=_to_error_payload(
code,
message,
details,
request_id=request.state.request_id,
),
)
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError): # noqa: ARG001
return JSONResponse(
status_code=422,
content=_to_error_payload(
"VALIDATION_ERROR",
"Validation failed",
exc.errors(),
request_id=request.state.request_id,
),
)
@app.exception_handler(Exception)
async def unexpected_error(request: Request, exc: Exception):
logger.exception(
"Unhandled request error request_id=%s method=%s path=%s",
request.state.request_id,
request.method,
str(request.scope.get("path") or ""),
)
return JSONResponse(
status_code=500,
content=_to_error_payload(
"INTERNAL_ERROR",
"Unexpected server error",
{"type": exc.__class__.__name__},
request_id=request.state.request_id,
),
)
return app
app = create_app()
def main() -> None:
import uvicorn
settings = get_settings()
uvicorn.run(
"app.main:app",
host="0.0.0.0",
port=8000,
reload=settings.app_env == "development",
)
+1
View File
@@ -0,0 +1 @@
from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API
View File
+43
View File
@@ -0,0 +1,43 @@
from .entities import (
AoiOperation,
AoiOperationPartition,
AnalysisRun,
Area,
Dataset,
DatasetLineageEdge,
DatasetQuarantine,
DatasetVersion,
Detection,
DetectionReview,
Export,
Job,
Metric,
Project,
QualityCheck,
Segmentation,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
__all__ = [
"AnalysisRun",
"AoiOperation",
"AoiOperationPartition",
"Area",
"Dataset",
"DatasetLineageEdge",
"DatasetQuarantine",
"DatasetVersion",
"Detection",
"DetectionReview",
"Export",
"Job",
"Metric",
"Project",
"QualityCheck",
"Segmentation",
"SourceRegistry",
"SourceSnapshot",
"VectorFeature",
]
+788
View File
@@ -0,0 +1,788 @@
from __future__ import annotations
import uuid
from datetime import datetime
from geoalchemy2 import Geometry
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Float, Index, JSON, String, Text, UniqueConstraint, func, text
from sqlalchemy.sql.sqltypes import Integer
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.db.base import Base
SOURCE_CLASSIFICATIONS = (
"authoritative",
"corroborative",
"contextual",
"derived",
"experimental",
)
SOURCE_FRESHNESS_STATUSES = (
"unknown",
"current",
"due",
"stale",
"not_applicable",
"review_required",
)
SOURCE_INGEST_STATUSES = (
"registered",
"configured",
"not_configured",
"available",
"ingested",
"failed",
"quarantined",
"legacy_unverified",
)
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
class Project(Base):
__tablename__ = "projects"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
region: Mapped[str] = mapped_column(String(120), default="Belgium and Belgian North Sea")
status: Mapped[str] = mapped_column(String(32), default="active")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
areas: Mapped[list["Area"]] = relationship("Area", back_populates="project", cascade="all, delete-orphan")
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="project", cascade="all, delete-orphan")
class Area(Base):
__tablename__ = "areas"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326), nullable=False)
original_crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
area_m2: Mapped[float | None] = mapped_column(Float, nullable=True)
bbox: Mapped[str | None] = mapped_column(Geometry("Polygon", srid=4326), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
project: Mapped[Project] = relationship("Project", back_populates="areas")
class SourceRegistry(Base):
"""Server-owned source identity and authority contract.
Dataset metadata remains descriptive until a governed importer binds a
dataset to both this registry entry and an immutable SourceSnapshot.
"""
__tablename__ = "source_registry"
__table_args__ = (
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
CheckConstraint(
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
name="ck_source_registry_classification",
),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_registry_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_registry_ingest_status",
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
classification: Mapped[str] = mapped_column(String(32), nullable=False)
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
)
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
class SourceSnapshot(Base):
"""Immutable source-version evidence recorded by governed ingestion."""
__tablename__ = "source_snapshots"
__table_args__ = (
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_snapshots_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_snapshots_ingest_status",
),
CheckConstraint(
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_source_snapshots_checksum_sha256",
),
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
Index("ix_source_snapshots_checksum", "checksum_sha256"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_registry_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
)
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
class Dataset(Base):
__tablename__ = "datasets"
__table_args__ = (
CheckConstraint(
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_datasets_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_datasets_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_lineage_status",
),
CheckConstraint(
"quarantine_status IN ('not_quarantined', 'quarantined')",
name="ck_datasets_quarantine_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_datasets_ingest_key_not_blank",
),
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
Index(
"ix_datasets_project_temporal_series_observed",
"project_id",
"temporal_series_key",
"observed_at",
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
dataset_type: Mapped[str] = mapped_column(String(64), nullable=False)
source: Mapped[str] = mapped_column(String(120), nullable=False)
storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
original_filename: Mapped[str | None] = mapped_column(String(255), nullable=True)
stored_filename: Mapped[str | None] = mapped_column(String(255), nullable=True)
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("datasets.id", ondelete="SET NULL"),
nullable=True,
)
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
bounds_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
resolution_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
bands_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
dataset_role: Mapped[str] = mapped_column(String(32), nullable=False, default="source", server_default="source")
source_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
quarantine_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_quarantined",
server_default="not_quarantined",
comment="not_quarantined | quarantined",
)
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
temporal_granularity: Mapped[str | None] = mapped_column(String(32), nullable=True)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="uploaded")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
project: Mapped[Project] = relationship("Project", back_populates="datasets")
versions: Mapped[list["DatasetVersion"]] = relationship(
"DatasetVersion",
back_populates="dataset",
cascade="all, delete-orphan",
)
vector_features: Mapped[list["VectorFeature"]] = relationship(
"VectorFeature",
back_populates="dataset",
cascade="all, delete-orphan",
)
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.parent_dataset_id",
back_populates="parent_dataset",
cascade="all, delete-orphan",
)
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.child_dataset_id",
back_populates="child_dataset",
cascade="all, delete-orphan",
)
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
)
class DatasetVersion(Base):
__tablename__ = "dataset_versions"
__table_args__ = (
CheckConstraint(
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_dataset_versions_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_dataset_versions_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_lineage_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_dataset_versions_ingest_key_not_blank",
),
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
version: Mapped[int] = mapped_column(Integer, default=1)
storage_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
class DatasetLineageEdge(Base):
"""Immutable relationship between input/output datasets and transforms."""
__tablename__ = "dataset_lineage_edges"
__table_args__ = (
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
UniqueConstraint(
"parent_dataset_id",
"child_dataset_id",
"relation_type",
"transformation_name",
name="uq_dataset_lineage_edges_relation",
),
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
parent_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
)
child_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
)
class DatasetQuarantine(Base):
"""Durable fail-closed record for rejected or doubtful source artifacts."""
__tablename__ = "dataset_quarantines"
__table_args__ = (
CheckConstraint(
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
name="ck_dataset_quarantines_target_present",
),
CheckConstraint(
"status IN ('quarantined', 'released', 'rejected')",
name="ck_dataset_quarantines_status",
),
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
)
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
stage: Mapped[str] = mapped_column(String(64), nullable=False)
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
status: Mapped[str] = mapped_column(
String(32), nullable=False, default="quarantined", server_default="quarantined"
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
class VectorFeature(Base):
__tablename__ = "vector_features"
__table_args__ = (
Index("ix_vector_features_dataset_id", "dataset_id"),
Index("ix_vector_features_geometry", "geometry", postgresql_using="gist"),
Index("ix_vector_features_dataset_source_feature", "dataset_id", "source_feature_id"),
Index(
"ix_vector_features_dataset_municipality",
"dataset_id",
text("(properties_json ->> 'municipality')"),
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
feature_class: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_feature_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
geometry: Mapped[str] = mapped_column(Geometry("Geometry", srid=4326, spatial_index=False), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="vector_features")
class AnalysisRun(Base):
__tablename__ = "analysis_runs"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
analysis_type: Mapped[str] = mapped_column(String(64), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False)
model_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
model_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
parameters_json: Mapped[dict] = mapped_column(JSON, nullable=False)
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
class Detection(Base):
__tablename__ = "detections"
__table_args__ = (
Index("ix_detections_project_id", "project_id"),
Index("ix_detections_dataset_id", "dataset_id"),
Index("ix_detections_analysis_run_id", "analysis_run_id"),
Index("ix_detections_class_name", "class_name"),
Index("ix_detections_geometry", "geometry", postgresql_using="gist"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
class_name: Mapped[str] = mapped_column(String(120), nullable=False)
confidence: Mapped[float] = mapped_column(Float, nullable=False)
geometry: Mapped[str] = mapped_column(Geometry("Geometry", srid=4326, spatial_index=False), nullable=False)
bbox_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_tile_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class Segmentation(Base):
__tablename__ = "segmentations"
__table_args__ = (
Index("ix_segmentations_project_id", "project_id"),
Index("ix_segmentations_dataset_id", "dataset_id"),
Index("ix_segmentations_analysis_run_id", "analysis_run_id"),
Index("ix_segmentations_job_id", "job_id"),
Index("ix_segmentations_class_name", "class_name"),
Index("ix_segmentations_geometry", "geometry", postgresql_using="gist"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
model_name: Mapped[str] = mapped_column(String(255), nullable=False)
model_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
class_name: Mapped[str] = mapped_column(String(120), nullable=False)
confidence: Mapped[float | None] = mapped_column(Float, nullable=True)
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
bbox_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
area_m2: Mapped[float | None] = mapped_column(Float, nullable=True)
mask_path: Mapped[str | None] = mapped_column(Text, nullable=True)
source_tile_path: Mapped[str | None] = mapped_column(String(500), nullable=True)
tile_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
properties_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class QualityCheck(Base):
__tablename__ = "quality_checks"
__table_args__ = (
Index("ix_quality_checks_project_id", "project_id"),
Index("ix_quality_checks_reference_dataset_id", "reference_dataset_id"),
Index("ix_quality_checks_candidate_dataset_id", "candidate_dataset_id"),
Index("ix_quality_checks_analysis_run_id", "analysis_run_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
candidate_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
reference_dataset_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False)
check_type: Mapped[str] = mapped_column(String(120), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False)
score: Mapped[float | None] = mapped_column(Float, nullable=True)
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
findings_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class Metric(Base):
__tablename__ = "metrics"
__table_args__ = (
Index("ix_metrics_quality_check_id", "quality_check_id"),
Index("ix_metrics_analysis_run_id", "analysis_run_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
quality_check_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("quality_checks.id", ondelete="CASCADE"), nullable=True)
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
metric_key: Mapped[str] = mapped_column(String(120), nullable=False)
metric_value: Mapped[float | None] = mapped_column(Float, nullable=True)
metric_unit: Mapped[str | None] = mapped_column(String(64), nullable=True)
label: Mapped[str | None] = mapped_column(String(120), nullable=True)
metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class DetectionReview(Base):
__tablename__ = "detection_reviews"
__table_args__ = (
CheckConstraint(
"evidence_role IN ('false_positive', 'false_negative')",
name="ck_detection_reviews_evidence_role",
),
CheckConstraint(
"decision IN ('confirmed_model_false_positive', 'confirmed_model_false_negative', "
"'reference_gap_or_change', 'qa_alignment_mismatch', "
"'imagery_obscured_or_uncertain', 'uncertain', 'unreviewed')",
name="ck_detection_reviews_decision",
),
UniqueConstraint(
"quality_check_id",
"evidence_role",
"evidence_feature_id",
name="uq_detection_reviews_evidence",
),
Index("ix_detection_reviews_project_id", "project_id"),
Index("ix_detection_reviews_quality_check_id", "quality_check_id"),
Index("ix_detection_reviews_analysis_run_id", "analysis_run_id"),
Index("ix_detection_reviews_decision", "decision"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
quality_check_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("quality_checks.id", ondelete="CASCADE"),
nullable=False,
)
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("analysis_runs.id", ondelete="SET NULL"),
nullable=True,
)
evidence_role: Mapped[str] = mapped_column(String(32), nullable=False)
evidence_feature_id: Mapped[str] = mapped_column(String(255), nullable=False)
detection_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("detections.id", ondelete="SET NULL"),
nullable=True,
)
reference_feature_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("vector_features.id", ondelete="SET NULL"),
nullable=True,
)
decision: Mapped[str] = mapped_column(String(64), nullable=False, default="unreviewed", server_default="unreviewed")
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
reviewed_by: Mapped[str] = mapped_column(String(120), nullable=False, default="operator", server_default="operator")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class Export(Base):
__tablename__ = "exports"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
analysis_run_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("analysis_runs.id", ondelete="SET NULL"), nullable=True)
export_type: Mapped[str] = mapped_column(String(64), nullable=False)
storage_path: Mapped[str] = mapped_column(String(500), nullable=False)
metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class Job(Base):
__tablename__ = "jobs"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
job_type: Mapped[str] = mapped_column(String(128), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
input_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
output_dataset_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True)
parameters_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class AoiOperation(Base):
__tablename__ = "aoi_operations"
__table_args__ = (
CheckConstraint(
"status IN ('queued', 'running', 'partial', 'success', 'failed', 'cancelled')",
name="ck_aoi_operations_status",
),
Index("ix_aoi_operations_project_status", "project_id", "status"),
Index("ix_aoi_operations_geometry", "geometry", postgresql_using="gist"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
parent_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
operation_type: Mapped[str] = mapped_column(String(128), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
request_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
plan_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class AoiOperationPartition(Base):
__tablename__ = "aoi_operation_partitions"
__table_args__ = (
CheckConstraint(
"status IN ('queued', 'running', 'success', 'failed', 'skipped')",
name="ck_aoi_operation_partitions_status",
),
UniqueConstraint("operation_id", "partition_key", name="uq_aoi_operation_partition_key"),
Index("ix_aoi_operation_partitions_operation_status", "operation_id", "status"),
Index("ix_aoi_operation_partitions_geometry", "geometry", postgresql_using="gist"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
operation_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("aoi_operations.id", ondelete="CASCADE"), nullable=False)
child_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
partition_key: Mapped[str] = mapped_column(String(255), nullable=False)
provider_key: Mapped[str] = mapped_column(String(120), nullable=False)
product_key: Mapped[str] = mapped_column(String(120), nullable=False)
ordinal: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)
checkpoint_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
View File
+5
View File
@@ -0,0 +1,5 @@
from __future__ import annotations
from app.providers import base, fixture, grb, manual, osm, registry
__all__ = ["base", "fixture", "grb", "manual", "osm", "registry"]
+96
View File
@@ -0,0 +1,96 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class ProviderCapability:
provider_name: str
display_name: str
authority_level: str
supported_layers: list[str]
supported_geometry_types: list[str]
supported_query_modes: list[str]
fetch_signature: str
configured: bool
status: str
limitation_message: str
attribution: str
license_note: str
not_configured_reason: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"provider_name": self.provider_name,
"display_name": self.display_name,
"authority_level": self.authority_level,
"supported_layers": self.supported_layers,
"supported_geometry_types": self.supported_geometry_types,
"supported_query_modes": self.supported_query_modes,
"fetch_signature": self.fetch_signature,
"configured": self.configured,
"status": self.status,
"limitation_message": self.limitation_message,
"attribution": self.attribution,
"license_note": self.license_note,
"not_configured_reason": self.not_configured_reason,
}
class BaseReferenceProvider:
def __init__(
self,
provider_name: str,
display_name: str,
authority_level: str,
supported_layers: list[str],
supported_geometry_types: list[str],
supported_query_modes: list[str],
fetch_signature: str,
limitation_message: str,
attribution: str,
license_note: str,
configured: bool = False,
) -> None:
self.provider_name = provider_name
self.display_name = display_name
self.authority_level = authority_level
self.supported_layers = supported_layers
self.supported_geometry_types = supported_geometry_types
self.supported_query_modes = supported_query_modes
self.fetch_signature = fetch_signature
self.limitation_message = limitation_message
self.attribution = attribution
self.license_note = license_note
self._configured = configured
@property
def capability(self) -> ProviderCapability:
return ProviderCapability(
provider_name=self.provider_name,
display_name=self.display_name,
authority_level=self.authority_level,
supported_layers=self.supported_layers,
supported_geometry_types=self.supported_geometry_types,
supported_query_modes=self.supported_query_modes,
fetch_signature=self.fetch_signature,
configured=self.is_configured,
status="configured" if self.is_configured else "not_configured",
limitation_message=self.limitation_message,
attribution=self.attribution,
license_note=self.license_note,
not_configured_reason=None if self.is_configured else "Provider integration is not configured yet",
)
@property
def is_configured(self) -> bool:
return self._configured
def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict[str, Any]:
del project_id, area_id, layers
return {
"provider": self.provider_name,
"status": "not_configured",
"message": "Provider integration is not configured yet",
}
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
from app.providers.base import BaseReferenceProvider
class FixtureProvider(BaseReferenceProvider):
def __init__(self) -> None:
super().__init__(
provider_name="fixture",
display_name="Fixture data",
authority_level="fixture",
supported_layers=["buildings", "roads", "water", "landuse", "custom"],
supported_geometry_types=["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon"],
supported_query_modes=["fixture"],
fetch_signature="tests/fixtures and demo fixture upload flow",
limitation_message="Fixture provider represents local test/demo fixtures only.",
attribution="GeoIntel local fixtures",
license_note="Fixtures are for local development and tests; do not present them as official data.",
configured=True,
)
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
from app.providers.base import BaseReferenceProvider
class GRBProvider(BaseReferenceProvider):
def __init__(self) -> None:
super().__init__(
provider_name="grb",
display_name="GRB",
authority_level="authoritative",
supported_layers=["buildings", "roads", "water", "parcels"],
supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
supported_query_modes=["bbox", "persisted_area"],
fetch_signature="POST /api/v1/projects/{project_id}/datasets/grb/acquire",
limitation_message=(
"Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald. "
"Volledige providerdownloads en onbeperkte queries zijn niet toegestaan."
),
attribution="Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
license_note="Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.",
configured=True,
)
def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict:
del project_id, area_id, layers
return {
"provider": self.provider_name,
"status": "bounded_request_required",
"message": (
"Use POST /api/v1/projects/{project_id}/datasets/grb/acquire with an EPSG:4326 "
"bounding box and one governed product key."
),
}
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
from app.providers.base import BaseReferenceProvider
class ManualProvider(BaseReferenceProvider):
def __init__(self) -> None:
super().__init__(
provider_name="manual",
display_name="Manual upload",
authority_level="manual",
supported_layers=["buildings", "roads", "water", "landuse", "custom"],
supported_geometry_types=["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon"],
supported_query_modes=["upload"],
fetch_signature="POST /api/v1/projects/{project_id}/datasets/upload",
limitation_message="Manual provider data is supplied through the existing dataset upload flow.",
attribution="User supplied",
license_note="License and attribution must be supplied by the uploader in source metadata.",
configured=True,
)
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
from app.providers.base import BaseReferenceProvider
class OSMProvider(BaseReferenceProvider):
def __init__(self) -> None:
super().__init__(
provider_name="osm",
display_name="OpenStreetMap",
authority_level="contextual",
supported_layers=["buildings", "roads", "water", "landuse"],
supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
supported_query_modes=["area"],
fetch_signature="POST /api/v1/external/osm/fetch",
limitation_message="OSM live Overpass/download integration is not configured in Sprint 7B.",
attribution="OpenStreetMap contributors",
license_note="OpenStreetMap data is available under ODbL; attribution is required.",
configured=False,
)
+153
View File
@@ -0,0 +1,153 @@
from __future__ import annotations
from pydantic import BaseModel
from app.core.errors import AppError
from app.providers.base import ProviderCapability
from app.providers.fixture import FixtureProvider
from app.providers.grb import GRBProvider
from app.providers.manual import ManualProvider
from app.providers.osm import OSMProvider
class ProviderDatasetMapping(BaseModel):
provider_name: str
dataset_role: str
source_name: str
reference_required: bool
write_path: str = "DatasetService"
class ProviderImportResult(BaseModel):
provider_name: str
status: str
message: str
requested_layers: list[str]
dataset_id: str | None = None
dataset_role: str | None = None
source_name: str | None = None
class ExternalProviderRegistry:
def __init__(self) -> None:
self.providers = {
"grb": GRBProvider(),
"osm": OSMProvider(),
"manual": ManualProvider(),
"fixture": FixtureProvider(),
}
def list_capabilities(self) -> list[ProviderCapability]:
return [provider.capability for provider in self.providers.values()]
def get(self, provider_name: str):
normalized = provider_name.strip().lower()
if normalized not in self.providers:
raise AppError(code="PROVIDER_NOT_FOUND", message="Provider not found", status_code=404)
return self.providers[normalized]
def fetch(self, provider_name: str, project_id: str, area_id: str | None, layers: list[str]) -> dict:
provider = self.get(provider_name)
return provider.fetch(project_id=project_id, area_id=area_id, layers=layers)
def dataset_mapping(self, provider_name: str, requested_dataset_role: str | None = None) -> ProviderDatasetMapping:
provider = self.get(provider_name)
if provider.provider_name == "osm":
dataset_role = "reference" if requested_dataset_role == "reference" else "source"
return ProviderDatasetMapping(
provider_name="osm",
dataset_role=dataset_role,
source_name="osm",
reference_required=requested_dataset_role == "reference",
)
return ProviderDatasetMapping(
provider_name=provider.provider_name,
dataset_role="reference",
source_name=provider.provider_name,
reference_required=True,
)
def import_contract(
self,
provider_name: str,
project_id: str,
area_id: str | None,
layers: list[str],
requested_dataset_role: str | None = None,
) -> ProviderImportResult:
del project_id, area_id
provider = self.get(provider_name)
mapping = self.dataset_mapping(provider.provider_name, requested_dataset_role=requested_dataset_role)
if provider.provider_name == "grb":
return ProviderImportResult(
provider_name="grb",
status="bounded_request_required",
message=(
"Use the governed project GRB acquisition endpoint with an EPSG:4326 bounding box "
"and one supported layer."
),
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
)
if provider.provider_name == "osm":
return ProviderImportResult(
provider_name=provider.provider_name,
status="not_configured",
message=f"No live {provider.display_name} import is configured.",
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
)
if provider.provider_name == "manual":
return ProviderImportResult(
provider_name="manual",
status="upload_flow_required",
message="Manual provider data must use the existing dataset upload/reference flow.",
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
)
return ProviderImportResult(
provider_name="fixture",
status="fixture_flow_required",
message="Fixture provider data must use checked-in demo/test fixture flows.",
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
)
_registry = ExternalProviderRegistry()
def list_provider_capabilities() -> list[ProviderCapability]:
return _registry.list_capabilities()
def get_provider(provider_name: str):
return _registry.get(provider_name)
def fetch_provider_data(provider_name: str, project_id: str, area_id: str | None, layers: list[str]) -> dict:
return _registry.fetch(provider_name, project_id, area_id, layers)
def get_provider_dataset_mapping(provider_name: str, requested_dataset_role: str | None = None) -> ProviderDatasetMapping:
return _registry.dataset_mapping(provider_name, requested_dataset_role=requested_dataset_role)
def import_provider_dataset(
provider_name: str,
project_id: str,
area_id: str | None,
layers: list[str],
requested_dataset_role: str | None = None,
) -> ProviderImportResult:
return _registry.import_contract(
provider_name=provider_name,
project_id=project_id,
area_id=area_id,
layers=layers,
requested_dataset_role=requested_dataset_role,
)
View File
View File
+357
View File
@@ -0,0 +1,357 @@
from __future__ import annotations
from .common import (
ApiErrorEnvelope,
ApiErrorItem,
Envelope,
GeoJsonFeature,
GeoJsonFeatureCollection,
ItemList,
PaginationEnvelope,
)
from .coverage import (
CoverageBBox,
CoverageCatalogResponse,
CoverageResolutionItem,
CoverageResolveRequest,
CoverageResolveResponse,
CoverageSourceContract,
)
from .project import ProjectCreate, ProjectDeleteResult, ProjectList, ProjectRead, ProjectUpdate
from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
from .dataset import DatasetCreateResponse, DatasetList
from .source_freshness import (
SourceFreshnessItem,
SourceFreshnessReport,
SourceFreshnessSummary,
SourceIntegritySummary,
)
from .source_catalog import (
SourceCatalogProbeItem,
SourceCatalogProbeReport,
SourceCatalogProbeSummary,
)
from .source_registry import (
DatasetProvenanceRead,
DatasetLineageEdgeRead,
DatasetQuarantineRead,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from .official_vector import (
OfficialVectorAcquireRequest,
OfficialVectorAcquisitionResult,
OfficialVectorProductRead,
)
from .detection import (
DetectionListResponse,
DetectionModelCapability,
DetectionModelsResponse,
DetectionQaRequest,
DetectionRead,
DetectionRunListResponse,
DetectionRunRead,
DetectionRunRequest,
DetectionComparisonRequest,
DetectionComparisonResponse,
DetectionRunResponse,
ModelAssetListResponse,
ModelAssetRead,
YoloPreflightResponse,
)
from .detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewSummary, DetectionReviewUpsert
from .segmentation import (
SegmentationListResponse,
SegmentationModelCapability,
SegmentationModelsResponse,
SegmentationQaRequest,
SegmentationRead,
SegmentationRunListResponse,
SegmentationRunRead,
SegmentationRunRequest,
SegmentationRunResponse,
)
from .health import HealthResponse, SystemCapabilities
from .job import JobCreate, JobList, JobRead, JobStatus
from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult, OrthophotoProductRead
from .dhmv import (
DhmvAcquireRequest,
DhmvAcquisitionResult,
DhmvProductRead,
TerrainMetric,
TerrainPartitionSelectionRequest,
TerrainSelectionRequest,
TerrainSelectionResponse,
TerrainSelectionSummary,
)
from .spw_terrain import (
SpwTerrainAcquireRequest,
SpwTerrainAcquisitionResult,
SpwTerrainProductRead,
)
from .flood_hazard import (
FloodHazardAcquireRequest,
FloodHazardAcquisitionResult,
FloodHazardMetric,
FloodHazardPartitionSelectionRequest,
FloodHazardProductRead,
FloodHazardSelectionRequest,
FloodHazardSelectionResponse,
FloodHazardSelectionSummary,
)
from .bathymetry import (
BathymetryPartitionFinalizeRequest,
BathymetryPartitionFinalizationResult,
BathymetryProfileAcquireRequest,
BathymetryProfileAcquisitionResult,
BathymetryRasterMetric,
BathymetryRasterSelectionRequest,
BathymetryRasterSelectionResponse,
BathymetryRasterSelectionSummary,
BathymetrySourceProbeRead,
BathymetrySourceRead,
MdkBathymetryAcquireRequest,
MdkBathymetryAcquisitionResult,
)
from .thematic_raster import (
ThematicRasterAcquireRequest,
ThematicRasterAcquisitionResult,
ThematicRasterMetric,
ThematicRasterProductRead,
ThematicRasterSelectionRequest,
ThematicRasterSelectionResponse,
ThematicRasterSelectionSummary,
)
from .external import (
ExternalFetchRequest,
ExternalFetchResponse,
ProviderCapabilitiesResponse,
ProviderCapabilityResponse,
ProviderImportRequest,
ProviderImportResponse,
ProviderLayersResponse,
ProviderStatusResponse,
)
from .export import (
ExportContentResponse,
ExportCreateResponse,
ExportListResponse,
ExportRead,
GeoJsonExportRequest,
MetadataExportRequest,
ReportExportRequest,
)
from .qa import (
AnalysisQaResponse,
QaProviderComparisonRequest,
QaProviderComparisonResult,
QualityEvidenceResponse,
)
from .operations import (
RasterClipRequest,
RasterIndexBaseRequest,
RasterMetadataResponse,
RasterNdviRequest,
RasterNdwiRequest,
RasterNdbiRequest,
RasterOperationResult,
RasterPreviewResponse,
RasterReprojectRequest,
RasterReprojectResponse,
RasterStatsResponse,
RasterTileManifest,
RasterTileManifestTile,
RasterTileRequest,
RasterTileResponse,
VectorBBoxResponse,
VectorBufferRequest,
VectorClipRequest,
VectorIntersectRequest,
VectorOperationRequest,
VectorOperationResult,
VectorSelectionBBox,
VectorSelectionDeriveRequest,
VectorSelectionRequest,
VectorSelectionResponse,
VectorSelectionMetric,
VectorSelectionSummary,
VectorStatsRequest,
VectorStatsResponse,
)
__all__ = [
"Envelope",
"ItemList",
"GeoJsonFeature",
"GeoJsonFeatureCollection",
"ApiErrorEnvelope",
"ApiErrorItem",
"PaginationEnvelope",
"CoverageBBox",
"CoverageCatalogResponse",
"CoverageResolutionItem",
"CoverageResolveRequest",
"CoverageResolveResponse",
"CoverageSourceContract",
"ProjectCreate",
"ProjectRead",
"ProjectUpdate",
"ProjectList",
"ProjectDeleteResult",
"AreaCreate",
"AreaRead",
"AreaUpdate",
"AreaList",
"ChangeDetectionRequest",
"ChangeDetectionSummary",
"DatasetCreateResponse",
"DatasetList",
"SourceFreshnessItem",
"SourceFreshnessReport",
"SourceFreshnessSummary",
"SourceIntegritySummary",
"SourceCatalogProbeItem",
"SourceCatalogProbeReport",
"SourceCatalogProbeSummary",
"SourceRegistryRead",
"SourceRegistryDetailRead",
"SourceSnapshotRead",
"DatasetLineageEdgeRead",
"DatasetQuarantineRead",
"DatasetProvenanceRead",
"GrbRefreshLayerPlan",
"GrbRefreshPlan",
"GrbRefreshPlanSummary",
"GrbAcquireRequest",
"GrbAcquisitionResult",
"GrbProductRead",
"OfficialVectorAcquireRequest",
"OfficialVectorAcquisitionResult",
"OfficialVectorProductRead",
"DetectionListResponse",
"DetectionModelCapability",
"DetectionModelsResponse",
"DetectionQaRequest",
"DetectionRead",
"DetectionRunListResponse",
"DetectionRunRead",
"DetectionRunRequest",
"DetectionComparisonRequest",
"DetectionComparisonResponse",
"DetectionRunResponse",
"ModelAssetListResponse",
"ModelAssetRead",
"YoloPreflightResponse",
"DetectionReviewList",
"DetectionReviewRead",
"DetectionReviewSummary",
"DetectionReviewUpsert",
"SegmentationListResponse",
"SegmentationModelCapability",
"SegmentationModelsResponse",
"SegmentationQaRequest",
"SegmentationRead",
"SegmentationRunListResponse",
"SegmentationRunRead",
"SegmentationRunRequest",
"SegmentationRunResponse",
"HealthResponse",
"SystemCapabilities",
"JobCreate",
"JobList",
"JobRead",
"JobStatus",
"OrthophotoAcquireRequest",
"OrthophotoAcquisitionResult",
"OrthophotoProductRead",
"DhmvAcquireRequest",
"DhmvAcquisitionResult",
"DhmvProductRead",
"SpwTerrainAcquireRequest",
"SpwTerrainAcquisitionResult",
"SpwTerrainProductRead",
"TerrainMetric",
"TerrainPartitionSelectionRequest",
"TerrainSelectionRequest",
"TerrainSelectionResponse",
"TerrainSelectionSummary",
"FloodHazardAcquireRequest",
"FloodHazardAcquisitionResult",
"FloodHazardMetric",
"FloodHazardPartitionSelectionRequest",
"FloodHazardProductRead",
"FloodHazardSelectionRequest",
"FloodHazardSelectionResponse",
"FloodHazardSelectionSummary",
"BathymetryProfileAcquireRequest",
"BathymetryProfileAcquisitionResult",
"BathymetryRasterMetric",
"BathymetryRasterSelectionRequest",
"BathymetryRasterSelectionResponse",
"BathymetryRasterSelectionSummary",
"BathymetryPartitionFinalizeRequest",
"BathymetryPartitionFinalizationResult",
"BathymetrySourceProbeRead",
"BathymetrySourceRead",
"MdkBathymetryAcquireRequest",
"MdkBathymetryAcquisitionResult",
"ThematicRasterAcquireRequest",
"ThematicRasterAcquisitionResult",
"ThematicRasterMetric",
"ThematicRasterProductRead",
"ThematicRasterSelectionRequest",
"ThematicRasterSelectionResponse",
"ThematicRasterSelectionSummary",
"VectorBBoxResponse",
"VectorClipRequest",
"VectorBufferRequest",
"VectorIntersectRequest",
"VectorOperationRequest",
"VectorOperationResult",
"VectorSelectionBBox",
"VectorSelectionDeriveRequest",
"VectorSelectionRequest",
"VectorSelectionResponse",
"VectorSelectionMetric",
"VectorSelectionSummary",
"RasterClipRequest",
"RasterStatsResponse",
"RasterReprojectRequest",
"RasterReprojectResponse",
"RasterTileRequest",
"RasterMetadataResponse",
"RasterOperationResult",
"RasterPreviewResponse",
"RasterTileManifestTile",
"RasterTileManifest",
"RasterTileResponse",
"RasterIndexBaseRequest",
"RasterNdviRequest",
"RasterNdwiRequest",
"RasterNdbiRequest",
"VectorStatsRequest",
"VectorStatsResponse",
"ExternalFetchRequest",
"ExternalFetchResponse",
"ProviderCapabilitiesResponse",
"ProviderCapabilityResponse",
"ProviderImportRequest",
"ProviderImportResponse",
"ProviderLayersResponse",
"ProviderStatusResponse",
"GeoJsonExportRequest",
"MetadataExportRequest",
"ReportExportRequest",
"ExportRead",
"ExportCreateResponse",
"ExportListResponse",
"ExportContentResponse",
"QaProviderComparisonRequest",
"QaProviderComparisonResult",
"AnalysisQaResponse",
"QualityEvidenceResponse",
]
+46
View File
@@ -0,0 +1,46 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.operations import VectorSelectionBBox
class ChangeDetectionRequest(BaseModel):
source_dataset_id: UUID
target_dataset_id: UUID
iou_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
# Below this the two footprints are separate objects rather than one that
# was redrawn; between the two thresholds the change class is "modified".
modified_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
include_unchanged: bool = True
# Without a selection the comparison covers both datasets in full, which is
# rarely the question and never a response a map can draw.
bbox: VectorSelectionBBox | None = None
area_id: UUID | None = None
preview_limit: int = Field(default=2_000, ge=1, le=20_000)
class ChangeDetectionSummary(BaseModel):
source_dataset_id: UUID
target_dataset_id: UUID
source_feature_count: int
target_feature_count: int
added_count: int
removed_count: int
# A footprint that was redrawn rather than demolished and rebuilt. Without
# this class it appeared as one removal plus one addition.
modified_count: int = 0
unchanged_count: int
iou_threshold: float
modified_iou_threshold: float | None = None
selection_area_id: UUID | None = None
# Counts describe the whole selection; the GeoJSON is capped so a regional
# comparison does not return both datasets in one response.
preview_limit: int | None = None
preview_truncated: bool = False
warnings: list[str] = Field(default_factory=list)
generated_at: datetime
geojson: dict
+75
View File
@@ -0,0 +1,75 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.operations import VectorSelectionBBox
class AoiOperationCreate(BaseModel):
area_id: UUID | None = None
bbox: VectorSelectionBBox | None = None
operation_type: str = Field(min_length=1, max_length=128)
provider_key: str = Field(min_length=1, max_length=120)
product_key: str = Field(min_length=1, max_length=120)
coverage_zone: str | None = Field(default=None, max_length=64)
max_partition_side_m: float | None = Field(default=None, gt=0, le=60_000)
max_attempts: int = Field(default=3, ge=1, le=10)
parameters_json: dict = Field(default_factory=dict)
class AoiPartitionRead(BaseModel):
id: UUID
partition_key: str
provider_key: str
product_key: str
ordinal: int
status: str
attempt_count: int
max_attempts: int
checkpoint_json: dict | None = None
result_json: dict | None = None
error_message: str | None = None
model_config = {"from_attributes": True}
class AoiOperationRead(BaseModel):
id: UUID
project_id: UUID
area_id: UUID | None = None
parent_job_id: UUID | None = None
operation_type: str
status: str
request_json: dict
plan_json: dict
result_json: dict | None = None
error_message: str | None = None
progress: float
partition_counts: dict[str, int]
partitions: list[AoiPartitionRead] = Field(default_factory=list)
created_at: datetime | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
class AoiOperationList(BaseModel):
items: list[AoiOperationRead]
total: int
class AoiPartitionCheckpoint(BaseModel):
checkpoint_json: dict = Field(default_factory=dict)
class AoiPartitionComplete(BaseModel):
result_json: dict = Field(default_factory=dict)
skipped: bool = False
class AoiPartitionFail(BaseModel):
error_message: str = Field(min_length=1, max_length=4000)
retryable: bool = True
details: dict = Field(default_factory=dict)
+55
View File
@@ -0,0 +1,55 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel
class AreaCreate(BaseModel):
name: str
geometry: dict
crs: str | None = "EPSG:4326"
class AreaUpdate(BaseModel):
name: str | None = None
geometry: dict | None = None
crs: str | None = None
class AreaRead(BaseModel):
id: UUID
project_id: UUID
name: str
original_crs: str | None
area_m2: float | None
created_at: datetime | None = None
geometry_type: str | None = None
geometry: dict | None = None
model_config = {"from_attributes": True}
class AreaListItem(AreaRead):
pass
class AreaList(BaseModel):
items: list[AreaRead]
total: int
limit: int
offset: int
class MunicipalitySearchItem(BaseModel):
niscode: str
name: str
name_nl: str | None = None
name_fr: str | None = None
name_de: str | None = None
class MunicipalitySearchList(BaseModel):
items: list[MunicipalitySearchItem]
total: int
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.operations import VectorSelectionBBox
class AssistantChatMessage(BaseModel):
role: Literal["user", "assistant"]
content: str = Field(min_length=1, max_length=4_000)
class AssistantQueryRequest(BaseModel):
question: str = Field(min_length=2, max_length=2_000)
model: str | None = Field(default=None, max_length=255)
bbox: VectorSelectionBBox | None = None
area_id: UUID | None = None
history: list[AssistantChatMessage] = Field(default_factory=list, max_length=8)
class AssistantModelRead(BaseModel):
name: str
size_bytes: int | None = None
parameter_size: str | None = None
quantization_level: str | None = None
capabilities: list[str] = Field(default_factory=list)
class AssistantModelList(BaseModel):
items: list[AssistantModelRead]
total: int
default_model: str | None = None
class AssistantStatus(BaseModel):
enabled: bool
reachable: bool
status: str
base_url: str
default_model: str | None = None
model_count: int = 0
limitation_message: str
class AssistantContextMetric(BaseModel):
theme: str
label: str
value: float
unit: str
source: str
dataset_id: UUID
observed_at: datetime | None = None
is_estimate: bool = False
class AssistantTemporalSeries(BaseModel):
temporal_series_key: str
label: str
source: str
first_year: int
last_year: int
observation_count: int
class AssistantEstimateDisclosure(BaseModel):
"""A value in the answer that the source itself calls an estimate.
Derived from metric metadata rather than from the generated sentences, so
the disclosure is present whatever wording the model chose.
"""
theme: str
label: str
unit: str
source: str
dataset_id: UUID
reason: str
class AssistantQueryResponse(BaseModel):
answer: str
model: str
scope_label: str
context_metrics: list[AssistantContextMetric]
temporal_series: list[AssistantTemporalSeries]
estimate_disclosures: list[AssistantEstimateDisclosure] = Field(default_factory=list)
source_dataset_ids: list[UUID]
warnings: list[str]
generated_at: datetime
+29
View File
@@ -0,0 +1,29 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.common import Envelope
class AuthLoginRequest(BaseModel):
username: str = Field(min_length=1, max_length=128)
password: str = Field(min_length=1, max_length=1024)
class AuthSession(BaseModel):
authentication_required: bool
authenticated: bool
username: str | None = None
expires_at: datetime | None = None
role: Literal["operator", "guest"] | None = None
guest_access_enabled: bool = False
authentik_enabled: bool = False
guest_project_id: UUID | None = None
class AuthSessionEnvelope(Envelope[AuthSession]):
pass
+173
View File
@@ -0,0 +1,173 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field, field_validator
from .operations import VectorSelectionBBox
class BathymetryProfileAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
force_refresh: bool = False
class BathymetrySourceRead(BaseModel):
key: str
display_name: str
owner: str
authority_level: Literal["authoritative", "contextual"]
geographic_coverage: str
data_kind: str
query_modes: list[str]
vertical_reference: str
horizontal_crs: str
native_resolution: str | None = None
integration_status: Literal["operational", "probe_only", "available_not_integrated", "catalog_only"]
acquisition_supported: bool
configured: bool
service_url: str | None = None
catalog_url: str
attribution: str
license_note: str
limitation_message: str
class BathymetryProfileAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
profile_count: int = Field(ge=0)
document_count: int = Field(ge=0)
structured_depth_count: int = Field(ge=0)
structured_width_count: int = Field(ge=0)
watercourse_count: int = Field(ge=0)
bbox_epsg4326: list[float]
clipped_to_area_id: UUID | None = None
measurement_date_min: str | None = None
measurement_date_max: str | None = None
attribution: str
limitation_message: str
class BathymetryPartitionFinalizeRequest(BaseModel):
partition_scope_key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9][a-z0-9_-]*$")
expected_area_ids: list[UUID] = Field(min_length=1, max_length=500)
dataset_ids: list[UUID] = Field(default_factory=list, max_length=500)
no_profile_area_ids: list[UUID] = Field(default_factory=list, max_length=500)
manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
observed_at: datetime
@field_validator("expected_area_ids", "dataset_ids", "no_profile_area_ids")
@classmethod
def require_unique_ids(cls, value: list[UUID]) -> list[UUID]:
if len(value) != len(set(value)):
raise ValueError("Partition identifiers must be unique")
return value
class BathymetryPartitionFinalizationResult(BaseModel):
partition_scope_key: str
regional_partitions_complete: bool
partition_count: int = Field(ge=1)
data_partition_count: int = Field(ge=0)
no_profile_partition_count: int = Field(ge=0)
profile_count: int = Field(ge=0)
document_count: int = Field(ge=0)
structured_depth_count: int = Field(ge=0)
measurement_date_min: str | None = None
measurement_date_max: str | None = None
dataset_ids: list[UUID]
manifest_sha256: str
observed_at: datetime
limitation_message: str
class BathymetrySourceProbeRead(BaseModel):
source_key: str
status: Literal[
"disabled",
"invalid_configuration",
"tls_error",
"endpoint_unavailable",
"invalid_capabilities",
"reachable",
]
configured_url: str
capabilities_url: str | None = None
tls_verified: bool
capabilities_reachable: bool
acquisition_supported: bool = False
wcs_version: str | None = None
coverage_identifiers: list[str] = Field(default_factory=list)
advertised_formats: list[str] = Field(default_factory=list)
advertised_crs: list[str] = Field(default_factory=list)
response_sha256: str | None = None
checked_at: datetime
message: str
limitation_message: str
class MdkBathymetryAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
force_refresh: bool = False
class MdkBathymetryAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
coverage_id: str
bbox_epsg4326: list[float]
vertical_reference: str
resolution_m: float = Field(gt=0)
attribution: str
limitation_message: str
class BathymetryRasterSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
class BathymetryRasterMetric(BaseModel):
metric_key: str
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
is_estimate: bool = False
class BathymetryRasterSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
primary_metric_key: str
metrics: list[BathymetryRasterMetric]
class BathymetryRasterSelectionResponse(BaseModel):
dataset_id: UUID
product_key: str
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
selected_cell_count: int = Field(ge=1)
valid_cell_count: int = Field(ge=1)
coverage_ratio: float = Field(ge=0, le=1)
# Set when the drawn selection is smaller than one source cell and the
# analysis was widened to the cells it touches, so the value covers more
# ground than was requested.
cell_selection_warning: str | None = None
resolution_m: float = Field(gt=0)
vertical_reference: str
survey_period: str
summary: BathymetryRasterSelectionSummary
unsupported_metrics: list[str]
limitation_message: str
generated_at: str
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
from typing import Any, Generic, Literal, TypeVar
from pydantic import BaseModel, Field
DataT = TypeVar("DataT")
class Envelope(BaseModel, Generic[DataT]):
data: DataT
class ItemList(BaseModel, Generic[DataT]):
items: list[DataT]
total: int
class PaginatedEnvelope(ItemList[DataT], Generic[DataT]):
limit: int
offset: int
class PaginationEnvelope(BaseModel):
items: list
total: int
limit: int = Field(default=50)
offset: int = Field(default=0)
class ApiErrorItem(BaseModel):
code: str
message: str
details: dict = Field(default_factory=dict)
class ApiErrorEnvelope(BaseModel):
error: str
message: str
details: dict | list = Field(default_factory=dict)
request_id: str | None = None
class GeoJsonFeature(BaseModel):
type: Literal["Feature"]
id: str | int | None = None
geometry: dict[str, Any] | None
properties: dict[str, Any] = Field(default_factory=dict)
class GeoJsonFeatureCollection(BaseModel):
type: Literal["FeatureCollection"]
features: list[GeoJsonFeature]
+94
View File
@@ -0,0 +1,94 @@
from __future__ import annotations
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field, model_validator
CoverageStatus = Literal["operational", "partial", "not_configured", "unsupported"]
CoverageAuthority = Literal["authoritative", "official_context", "contextual"]
CoverageAcquisitionMode = Literal[
"operator_archive",
"operator_wfs",
"bounded_api",
"bounded_raster",
"catalog_only",
]
class CoverageBBox(BaseModel):
minx: float = Field(ge=-180, le=180)
miny: float = Field(ge=-90, le=90)
maxx: float = Field(ge=-180, le=180)
maxy: float = Field(ge=-90, le=90)
@model_validator(mode="after")
def validate_extent(self) -> "CoverageBBox":
if self.maxx <= self.minx or self.maxy <= self.miny:
raise ValueError("bbox max values must be greater than min values")
return self
class CoverageSourceContract(BaseModel):
source_name: str
display_name: str
authority_level: CoverageAuthority
coverage_zones: list[str]
themes: list[str]
native_layers: list[str]
supported_geometry_types: list[str]
acquisition_mode: CoverageAcquisitionMode
integration_status: CoverageStatus
source_url: str
attribution: str
license_note: str
limitation_message: str
class CoverageCatalogResponse(BaseModel):
themes: list[str]
zones: list[str]
statuses: list[CoverageStatus]
sources: list[CoverageSourceContract]
class CoverageResolveRequest(BaseModel):
project_id: UUID
bbox: CoverageBBox
themes: list[str] = Field(default_factory=list, max_length=32)
class CoverageResolutionItem(BaseModel):
zone: str
theme: str
status: CoverageStatus
source_names: list[str]
materialized_dataset_ids: list[UUID]
evidence: list["CoverageEvidenceItem"] = Field(default_factory=list)
limitation_message: str
class CoverageEvidenceItem(BaseModel):
dataset_id: UUID
source_name: str
authority_level: CoverageAuthority
source_version: str | None = None
observed_at: str | None = None
published_at: str | None = None
crs: str | None = None
resolution: dict | None = None
coverage_bbox_epsg4326: list[float] | None = None
attribution: str | None = None
license_note: str | None = None
checksum_sha256: str | None = None
class CoverageResolveResponse(BaseModel):
project_id: UUID
bbox: CoverageBBox
requested_themes: list[str]
intersected_zones: list[str]
outside_supported_scope: bool
items: list[CoverageResolutionItem]
warnings: list[str]
+132
View File
@@ -0,0 +1,132 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel
class DatasetStorageResponse(BaseModel):
original_filename: str | None = None
stored_filename: str | None = None
content_type: str | None = None
size_bytes: int | None = None
checksum_sha256: str | None = None
class DatasetVectorSummary(BaseModel):
feature_count: int | None = None
geometry_types: list[str] | None = None
bounds_json: dict | None = None
approximate_area_m2: float | None = None
crs: str | None = None
feature_geometry_count: int | None = None
invalid_features: int | None = None
crs_assumed: bool | None = None
class DatasetCreateResponse(BaseModel):
id: UUID
name: str
dataset_type: str
source: str
dataset_role: str = "source"
source_name: str | None = None
reference_layer_name: str | None = None
source_metadata: dict | None = None
provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
imported_at: datetime | None = None
temporal_series_key: str | None = None
observed_at: datetime | None = None
valid_from: datetime | None = None
valid_to: datetime | None = None
temporal_granularity: str | None = None
source_version: str | None = None
project_id: UUID
area_id: UUID | None = None
storage_path: str | None = None
original_filename: str | None = None
stored_filename: str | None = None
content_type: str | None = None
size_bytes: int | None = None
checksum_sha256: str | None = None
crs: str | None = None
bounds_json: dict | None = None
metadata_json: dict | None = None
vector_summary: DatasetVectorSummary | None = None
status: str
derived_from_dataset_id: UUID | None = None
created_at: datetime | None = None
feature_count: int | None = None
model_config = {"from_attributes": True}
class DatasetList(BaseModel):
items: list[DatasetCreateResponse]
total: int
limit: int
offset: int
class DatasetMetadataRefresh(BaseModel):
feature_count: int | None = None
geometry_types: list[str] | None = None
bounds_json: dict | None = None
crs: str | None = None
class DatasetTemporalUpdate(BaseModel):
temporal_series_key: str
observed_at: datetime
valid_from: datetime | None = None
valid_to: datetime | None = None
temporal_granularity: str = "snapshot"
source_version: str | None = None
class DatasetVersionRead(BaseModel):
id: UUID
dataset_id: UUID
version: int
storage_path: str | None = None
source_version: str | None = None
observed_at: datetime | None = None
valid_from: datetime | None = None
valid_to: datetime | None = None
checksum_sha256: str | None = None
source_metadata: dict | None = None
provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
class ExportRequest(BaseModel):
dataset_id: UUID
name: str | None = None
class ExportRead(BaseModel):
export_id: UUID
path: str
status: str
+18
View File
@@ -0,0 +1,18 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
class DemoWorkflowResponse(BaseModel):
project_id: UUID
area_id: UUID
reference_dataset_id: UUID
candidate_dataset_id: UUID
raster_dataset_id: UUID | None = None
quality_check_id: UUID
metric_count: int
status: str
message: str
created: bool
+220
View File
@@ -0,0 +1,220 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
class DetectionModelCapability(BaseModel):
model_config = ConfigDict(protected_namespaces=())
model_id: str
display_name: str
framework: str
task_type: str
supported_classes: list[str]
configured: bool
status: str
limitation_message: str
version: str | None = None
training_scope: str | None = None
validation_scope: str | None = None
validated_regions: list[str] = Field(default_factory=list)
nationally_validated: bool = False
operator_review_required: bool = True
class DetectionModelsResponse(BaseModel):
models: list[DetectionModelCapability]
class ModelAssetRead(BaseModel):
model_config = ConfigDict(protected_namespaces=())
model_asset_id: str
filename: str
display_name: str
model_path: str
suffix: str
framework: str
task_type: str
size_bytes: int
sha256: str
active: bool
runtime_available: bool
runtime_status: str
governed_validation_status: str
promotion_status: str
status: str
limitation_message: str
will_download_models: bool = False
class ModelAssetListResponse(BaseModel):
model_config = ConfigDict(protected_namespaces=())
items: list[ModelAssetRead]
total: int
model_directory: str
class DetectionRunRequest(BaseModel):
model_config = ConfigDict(protected_namespaces=())
project_id: UUID
dataset_id: UUID
model_id: str
model_asset_id: str | None = None
confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class_filter: list[str] | None = None
tile_manifest_path: str | None = None
parameters_json: dict = Field(default_factory=dict)
class DetectionQaRequest(BaseModel):
reference_dataset_id: UUID
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class_name: str | None = None
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
# Confidence cuts to report alongside the run's own operating point. They
# are read off the one matching pass, so a sweep costs no extra inference.
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
class DetectionComparisonRequest(BaseModel):
"""Place several runs side by side against one reference."""
analysis_run_ids: list[UUID] = Field(min_length=2, max_length=12)
reference_dataset_id: UUID
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class DetectionComparisonResponse(BaseModel):
reference_dataset_id: UUID
iou_threshold: float
# Whether these runs answer the same question at all, and why not if they
# do not. Numbers from incomparable runs are reported but never ranked as
# if they were alternatives.
comparability: dict
ranking_metric: str
rows: list[dict]
class DetectionRunResponse(BaseModel):
model_config = ConfigDict(protected_namespaces=())
analysis_run_id: UUID
job_id: UUID
project_id: UUID
dataset_id: UUID
model_id: str
status: str
detection_count: int
error_code: str | None = None
message: str
class DetectionRunRead(BaseModel):
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
id: UUID
project_id: UUID
dataset_id: UUID | None = None
job_id: UUID | None = None
analysis_type: str
status: str
model_name: str | None = None
model_version: str | None = None
parameters_json: dict
result_json: dict | None = None
error_message: str | None = None
created_at: datetime | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
class DetectionRunListResponse(BaseModel):
items: list[DetectionRunRead]
# ``total`` counts every run; ``items`` is the most recent page of them.
total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
class DetectionRead(BaseModel):
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
id: UUID
project_id: UUID
dataset_id: UUID | None = None
analysis_run_id: UUID | None = None
job_id: UUID | None = None
model_name: str
model_version: str | None = None
class_name: str
confidence: float
bbox_json: dict | None = None
source_tile_path: str | None = None
properties_json: dict | None = None
created_at: datetime | None = None
class DetectionListResponse(BaseModel):
items: list[DetectionRead]
# ``total`` is the complete population; ``items`` is one page of it.
total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
class YoloPreflightChecks(BaseModel):
model_config = ConfigDict(protected_namespaces=())
enabled: bool
dependencies_available: bool | None = None
accelerator_ready: bool | None = None
model_path_set: bool | None = None
model_file_exists: bool | None = None
model_provenance_manifest_path: str | None = None
model_provenance_valid: bool | None = None
model_load_requested: bool
model_load_ok: bool | None = None
manifest_path_set: bool | None = None
manifest_valid: bool | None = None
tile_paths_exist: bool | None = None
tile_limit_ok: bool | None = None
class YoloRuntimeDetails(BaseModel):
model_config = ConfigDict(protected_namespaces=())
dependencies_assumed: bool
model_directory: str | None = None
yolo_config_dir: str | None = None
torch_version: str | None = None
ultralytics_version: str | None = None
cuda_available: bool | None = None
configured_device: str
cuda_required: bool
class YoloPreflightResponse(BaseModel):
model_config = ConfigDict(protected_namespaces=())
model_id: str
model_asset_id: str | None = None
model_path: str | None = None
tile_manifest_path: str | None = None
status: str
message: str
checks: YoloPreflightChecks
tile_count: int
max_tiles: int
will_download_models: bool
will_run_inference: bool
runtime: YoloRuntimeDetails
error_code: str | None = None
details: dict | None = None
+67
View File
@@ -0,0 +1,67 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field
DetectionEvidenceRole = Literal["false_positive", "false_negative"]
DetectionReviewDecision = Literal[
"confirmed_model_false_positive",
"confirmed_model_false_negative",
"reference_gap_or_change",
"qa_alignment_mismatch",
"imagery_obscured_or_uncertain",
"uncertain",
"unreviewed",
]
class DetectionReviewUpsert(BaseModel):
evidence_role: DetectionEvidenceRole
evidence_feature_id: str = Field(min_length=1, max_length=255)
decision: DetectionReviewDecision
notes: str | None = Field(default=None, max_length=2000)
reviewed_by: str = Field(default="operator", min_length=1, max_length=120)
class DetectionReviewRead(BaseModel):
id: UUID | None = None
project_id: UUID
quality_check_id: UUID
analysis_run_id: UUID | None = None
evidence_role: DetectionEvidenceRole
evidence_feature_id: str
detection_id: UUID | None = None
reference_feature_id: UUID | None = None
decision: DetectionReviewDecision = "unreviewed"
notes: str | None = None
reviewed_by: str | None = None
confidence: float | None = None
class_name: str | None = None
source_tile_path: str | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
class DetectionReviewSummary(BaseModel):
total: int
reviewed: int
remaining: int
false_positive_total: int
false_negative_total: int
decision_counts: dict[str, int]
# The score with the operator's verdicts applied, next to the raw one. A
# finding adjudicated as a reference gap is not the model's error, and an
# interval covers what the unreviewed remainder could still turn out to be.
reviewed_metrics: dict | None = None
class DetectionReviewList(BaseModel):
items: list[DetectionReviewRead]
total: int
limit: int
offset: int
summary: DetectionReviewSummary
+102
View File
@@ -0,0 +1,102 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class DhmvAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "dtm_1m"
resolution_m: float | None = Field(default=None, ge=1.0, le=10.0)
force_refresh: bool = False
class DhmvProductRead(BaseModel):
key: str
display_name: str
surface_model: str
coverage_id: str
native_resolution_m: float
source_crs: str
vertical_reference: str
acquisition_period: str
catalog_url: str
attribution: str
limitation_message: str
class DhmvAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
surface_model: str
coverage_id: str
native_resolution_m: float
resolution_m: float
width: int
height: int
valid_pixel_count: int
nodata_value: float
bbox_epsg4326: list[float]
bbox_epsg31370: list[float]
vertical_reference: str
acquisition_period: str
attribution: str
limitation_message: str
class TerrainSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
class TerrainPartitionSelectionRequest(TerrainSelectionRequest):
product_key: str = "dtm_1m"
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
class TerrainMetric(BaseModel):
metric_key: str
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
derived: bool = True
class TerrainSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
primary_metric_key: str
metrics: list[TerrainMetric]
class TerrainSelectionResponse(BaseModel):
dataset_id: UUID
dataset_ids: list[UUID] = Field(default_factory=list)
partition_count: int = Field(default=1, ge=1)
product_key: str
surface_model: str
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
sample_count: int
slope_sample_count: int
coverage_ratio: float
# Set when the drawn selection is smaller than one source cell and the
# analysis was widened to the cells it touches, so the value covers more
# ground than was requested.
cell_selection_warning: str | None = None
resolution_m: float
vertical_reference: str
summary: TerrainSelectionSummary
unsupported_metrics: list[str]
limitation_message: str
generated_at: str
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, model_validator
from app.schemas.operations import VectorSelectionBBox
ExportKind = Literal["dataset", "detection_run", "segmentation_run", "vector_selection"]
DetectionExportIntendedUse = Literal["review", "operational"]
MapResultMode = Literal["current", "evolution"]
class GeoJsonExportRequest(BaseModel):
dataset_id: UUID | None = None
analysis_run_id: UUID | None = None
area_id: UUID | None = None
export_kind: ExportKind = "dataset"
name: str | None = None
bbox: VectorSelectionBBox | None = None
limit: int = 250
intended_use: DetectionExportIntendedUse = "review"
@model_validator(mode="after")
def validate_target(self) -> "GeoJsonExportRequest":
if self.export_kind == "dataset" and self.dataset_id is None:
raise ValueError("dataset_id is required for dataset GeoJSON exports")
if self.export_kind == "vector_selection":
if self.dataset_id is None:
raise ValueError("dataset_id is required for vector selection GeoJSON exports")
if self.bbox is None:
raise ValueError("bbox is required for vector selection GeoJSON exports")
if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None:
raise ValueError("analysis_run_id is required for run GeoJSON exports")
if self.intended_use == "operational" and self.export_kind != "detection_run":
raise ValueError("operational intended_use is supported only for detection run exports")
return self
class MetadataExportRequest(BaseModel):
project_id: UUID
name: str | None = None
class ReportExportRequest(BaseModel):
project_id: UUID
name: str | None = None
class MapResultExportRequest(BaseModel):
project_id: UUID
mode: MapResultMode
bbox: VectorSelectionBBox
dataset_id: UUID | None = None
earlier_dataset_id: UUID | None = None
later_dataset_id: UUID | None = None
area_id: UUID | None = None
partitioned: bool = False
product_key: str | None = None
partition_scope_key: str | None = None
theme_id: str | None = None
name: str | None = None
@model_validator(mode="after")
def validate_map_target(self) -> "MapResultExportRequest":
if self.mode == "current" and self.dataset_id is None:
raise ValueError("dataset_id is required for current map-result exports")
if self.mode == "evolution" and (
self.earlier_dataset_id is None or self.later_dataset_id is None
):
raise ValueError("earlier_dataset_id and later_dataset_id are required for evolution exports")
if self.partitioned and not self.product_key and not self.partition_scope_key:
raise ValueError("product_key or partition_scope_key is required for partitioned exports")
return self
class ExportRead(BaseModel):
id: UUID
project_id: UUID
analysis_run_id: UUID | None = None
export_type: str
storage_path: str
metadata_json: dict | None = None
created_at: datetime | None = None
status: str = "ready"
model_config = {"from_attributes": True}
class ExportCreateResponse(BaseModel):
export_id: UUID
path: str
status: str
export_type: str
metadata_json: dict | None = None
class ExportListResponse(BaseModel):
items: list[ExportRead]
total: int
limit: int
offset: int
class ExportContentResponse(BaseModel):
export_id: UUID
export_type: str
content: dict
+68
View File
@@ -0,0 +1,68 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
class ProviderCapabilityResponse(BaseModel):
provider_name: str
display_name: str
authority_level: str
supported_layers: list[str]
supported_geometry_types: list[str]
supported_query_modes: list[str]
fetch_signature: str
configured: bool
status: str
limitation_message: str
attribution: str
license_note: str
not_configured_reason: str | None = None
class ProviderCapabilitiesResponse(BaseModel):
providers: list[ProviderCapabilityResponse]
class ProviderLayersResponse(BaseModel):
provider_name: str
layers: list[str]
class ProviderStatusResponse(BaseModel):
provider_name: str
configured: bool
status: str
limitation_message: str
class ExternalFetchRequest(BaseModel):
project_id: UUID
area_id: UUID | None = None
layers: list[str] = []
class ExternalFetchResponse(BaseModel):
provider: str
status: str
message: str
requested_layers: list[str]
project_id: UUID
area_id: UUID | None = None
class ProviderImportRequest(BaseModel):
project_id: str
area_id: str | None = None
layers: list[str] = []
dataset_role: str | None = None
class ProviderImportResponse(BaseModel):
provider_name: str
status: str
message: str
requested_layers: list[str]
dataset_id: str | None = None
dataset_role: str | None = None
source_name: str | None = None
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class FloodHazardAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "pluviaal_current_t100"
resolution_m: float | None = Field(default=None, ge=2.0, le=20.0)
force_refresh: bool = False
class FloodHazardProductRead(BaseModel):
key: str
display_name: str
mechanism: str
climate_context: str
probability_class: str
return_period_years: int
coverage_id: str
native_resolution_m: float
source_crs: str
source_value_unit: str
normalized_value_unit: str
published_on: str
catalog_url: str
attribution: str
limitation_message: str
class FloodHazardAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
mechanism: str
climate_context: str
probability_class: str
return_period_years: int
coverage_id: str
resolution_m: float
width: int
height: int
inundated_pixel_count: int
bbox_epsg4326: list[float]
bbox_epsg31370: list[float]
attribution: str
limitation_message: str
class FloodHazardSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
class FloodHazardPartitionSelectionRequest(FloodHazardSelectionRequest):
product_key: str = "pluviaal_current_t100"
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
class FloodHazardMetric(BaseModel):
metric_key: str
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
derived: bool = True
class FloodHazardSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
primary_metric_key: str
metrics: list[FloodHazardMetric]
class FloodHazardSelectionResponse(BaseModel):
dataset_id: UUID
dataset_ids: list[UUID] = Field(default_factory=list)
partition_count: int = Field(default=1, ge=1)
product_key: str
mechanism: str
climate_context: str
probability_class: str
return_period_years: int
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
# Three populations kept apart: cells drawn, cells the model covers, and
# cells with a positive modelled depth. ``inundated_fraction`` is a share
# of the modelled cells, and is null when nothing was modelled — absence
# of a model is not evidence of zero risk.
selected_cell_count: int
valid_cell_count: int = 0
no_data_cell_count: int = 0
data_coverage_ratio: float = 1.0
inundated_cell_count: int
inundated_fraction: float | None = None
coverage_warning: str | None = None
resolution_m: float
summary: FloodHazardSelectionSummary
unsupported_metrics: list[str]
limitation_message: str
generated_at: str
+45
View File
@@ -0,0 +1,45 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
from .operations import VectorSelectionBBox
class GrbAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "buildings"
force_refresh: bool = False
class GrbProductRead(BaseModel):
key: str
display_name: str
reference_layer_name: str
collections: list[str]
geometry_types: list[str]
source_crs: str
authority_level: str
catalog_url: str
attribution: str
license_note: str
limitation_message: str
class GrbAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
reference_layer_name: str
collections: list[str]
feature_count: int
candidate_feature_count: int
page_count: int
bbox_epsg4326: list[float]
source_version: str
attribution: str
limitation_message: str
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
from datetime import date, datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
GrbRefreshLayerStatus = Literal[
"current",
"update_available",
"not_loaded",
"review_required",
"remote_unavailable",
]
class GrbRefreshLayerPlan(BaseModel):
theme: Literal["buildings", "roads", "water", "parcels"]
display_name: str
collections: list[str]
temporal_series_key: str
status: GrbRefreshLayerStatus
local_dataset_id: UUID | None = None
local_source_version: str | None = None
local_observed_at: datetime | None = None
local_imported_at: datetime | None = None
local_feature_count: int | None = None
local_size_bytes: int | None = None
retained_after_refresh: bool = True
action_message: str
class GrbRefreshPlanSummary(BaseModel):
layer_count: int
current_count: int
update_available_count: int
not_loaded_count: int
review_required_count: int
remote_unavailable_count: int
new_dataset_count_if_applied: int
retained_dataset_count: int
current_feature_count: int
current_size_bytes: int
class GrbRefreshPlan(BaseModel):
project_id: UUID
scope: str
generated_at: datetime
remote_status: str
remote_version: str | None = None
remote_edition_date: date | None = None
catalog_checked_at: datetime | None = None
summary: GrbRefreshPlanSummary
layers: list[GrbRefreshLayerPlan]
execution_mode: Literal["operator_stage_then_apply"] = "operator_stage_then_apply"
staging_required: bool = True
automatic_import: bool = False
destructive_replacement: bool = False
message: str
limitations: list[str]
+50
View File
@@ -0,0 +1,50 @@
from __future__ import annotations
from pydantic import BaseModel, Field
class ProviderCapability(BaseModel):
provider_name: str
display_name: str
authority_level: str
supported_layers: list[str]
supported_geometry_types: list[str]
supported_query_modes: list[str]
fetch_signature: str
configured: bool
status: str
limitation_message: str
attribution: str
license_note: str
not_configured_reason: str | None = None
class HealthResponse(BaseModel):
status: str
service: str
version: str
build_sha: str | None = None
build_time: str | None = None
database: str | None = None
postgis: str | None = None
migration: str | None = None
storage: str | None = None
checks: dict[str, str] = Field(default_factory=dict)
class SystemCapabilities(BaseModel):
postgis: bool
rasterio: bool
geopandas: bool
yolo: bool | str
yolo_status: str
sam: bool | str
grb: str
sentinel: str
version: str
build_sha: str | None = None
providers: list[ProviderCapability] = Field(default_factory=list)
class SystemCapabilitiesEnvelope(BaseModel):
data: SystemCapabilities
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
class JobCreate(BaseModel):
job_type: str
project_id: UUID
dataset_id: UUID | None = None
input_dataset_id: UUID | None = None
output_dataset_id: UUID | None = None
parameters_json: dict = Field(default_factory=dict)
class JobRead(BaseModel):
id: UUID
job_type: str
status: str
project_id: UUID
dataset_id: UUID | None = None
input_dataset_id: UUID | None = None
output_dataset_id: UUID | None = None
parameters_json: dict
result_json: dict | None = None
error_message: str | None = None
created_at: datetime | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
model_config = {"from_attributes": True}
class JobStatus(BaseModel):
id: UUID
status: str
error_message: str | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
result_json: dict | None = None
model_config = {"from_attributes": True}
class JobList(BaseModel):
items: list[JobRead]
total: int
limit: int
offset: int
+55
View File
@@ -0,0 +1,55 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
from .operations import VectorSelectionBBox
class OfficialVectorAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str
force_refresh: bool = False
class OfficialVectorProductRead(BaseModel):
key: str
display_name: str
theme: str
provider: str
source_name: str
reference_layer_name: str
service_type: str
collection: str
geometry_types: list[str]
source_crs: str
source_version: str
observation_label: str
authority_level: str
catalog_url: str
attribution: str
license_note: str
limitation_message: str
coverage_zones: list[str]
class OfficialVectorAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
product_key: str
display_name: str
theme: str
provider: str
source_name: str
reference_layer_name: str
service_type: str
collection: str
feature_count: int
candidate_feature_count: int
page_count: int
bbox_epsg4326: list[float]
source_version: str
attribution: str
limitation_message: str
+264
View File
@@ -0,0 +1,264 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field, field_validator
class VectorOperationResult(BaseModel):
feature_count: int
geometry_type_summary: dict[str, int]
bounds_json: dict | None = None
crs: str | None = None
source_dataset_id: str
class VectorOperationRequest(BaseModel):
output_name: str | None = None
class VectorClipRequest(VectorOperationRequest):
area_id: str
class VectorBufferRequest(VectorOperationRequest):
distance_m: float
dissolve: bool = False
class VectorIntersectRequest(VectorOperationRequest):
other_dataset_id: str
class VectorStatsRequest(BaseModel):
pass
class RasterReadyResponse(BaseModel):
dataset_id: str
ready: bool
message: str | None = None
class RasterOperationResult(BaseModel):
dataset_id: str
ready: bool
metadata: dict | None = None
output_dataset_id: str | None = None
operation: str | None = None
class RasterMetadataResponse(BaseModel):
dataset_id: str
driver: str | None = None
width: int | None = None
height: int | None = None
band_count: int | None = None
crs: str | None = None
bounds: list[float] | None = None
resolution: list[float] | None = None
dtype: list[str] | None = None
nodata: list[float] | float | None = None
transform: list[float] | None = None
size_bytes: int | None = None
checksum_sha256: str | None = None
path: str | None = None
class RasterPreviewResponse(BaseModel):
dataset_id: str
ready: bool
preview: dict
metadata: dict | None = None
class RasterBandStats(BaseModel):
band_index: int
dtype: str | None = None
min: float | None = None
max: float | None = None
mean: float | None = None
std: float | None = None
nodata_count: int
nodata_ratio: float
valid_pixel_count: int
histogram: list[int] | None = None
histogram_bins: list[float] | None = None
class RasterStatsResponse(BaseModel):
dataset_id: str
source_dataset_id: str | None = None
bands: list[RasterBandStats]
generated_at: str | None = None
metadata: dict | None = None
class RasterReprojectRequest(BaseModel):
target_crs: str | None = "EPSG:31370"
resampling: str = "nearest"
output_name: str | None = None
class RasterClipRequest(BaseModel):
area_id: str
output_name: str | None = None
class RasterTileRequest(BaseModel):
tile_size: int = 512
overlap: int = 64
output_name: str | None = None
class RasterIndexBaseRequest(BaseModel):
output_name: str | None = None
class RasterNdviRequest(RasterIndexBaseRequest):
nir_band: int
red_band: int
class RasterNdwiRequest(RasterIndexBaseRequest):
green_band: int
nir_band: int
class RasterNdbiRequest(RasterIndexBaseRequest):
swir_band: int
nir_band: int
class RasterTileManifestTile(BaseModel):
path: str
pixel_window: list[int]
bounds: list[float]
transform: list[float]
index: int
class RasterTileManifest(BaseModel):
tile_set_id: str
source_dataset_id: str
source_raster_id: str
bounds: list[float]
tile_size: int
overlap: int
parameters: dict[str, str | int | float | bool | None]
created_at: str
tile_paths: list[str]
count: int
tiles: list[RasterTileManifestTile]
ai_inference: bool = False
tile_server: str | None = None
class RasterTileResponse(BaseModel):
dataset_id: str
ready: bool
operation: str
tile_set_id: str
tile_size: int
overlap: int
manifest_path: str
count: int
manifest: RasterTileManifest
class RasterReprojectResponse(BaseModel):
dataset_id: str
ready: bool
operation: str
output_dataset_id: str
source_dataset_id: str
target_dataset_id: str | None = None
class RasterOperationUnavailable(BaseModel):
code: str
message: str
class VectorBBoxResponse(BaseModel):
dataset_id: str
bounds_json: dict | None
feature_count: int
crs: str | None = None
class VectorStatsResponse(BaseModel):
dataset_id: str
feature_count: int
geometry_type_summary: dict[str, int]
bounds_json: dict | None
crs: str | None = None
class VectorSelectionBBox(BaseModel):
min_x: float
min_y: float
max_x: float
max_y: float
crs: str = "EPSG:4326"
@field_validator("crs")
@classmethod
def validate_crs(cls, value: str) -> str:
if value.upper() != "EPSG:4326":
raise ValueError("Only EPSG:4326 bbox selection is supported")
return "EPSG:4326"
class VectorSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
limit: int = Field(default=100, ge=1, le=1000)
class VectorSelectionDeriveRequest(VectorSelectionRequest):
output_name: str | None = None
class VectorSelectionMetric(BaseModel):
metric_key: str
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
is_estimate: bool = False
warning: str | None = None
class VectorSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
primary_metric_key: str | None = None
# ``feature_count`` counts whole features that touch the selection, while
# area and length metrics clip to it. These fields say how far the two
# populations diverge, so the numbers on one panel can be read together.
feature_count: int
fully_covered_feature_count: int | None = None
partially_covered_feature_count: int | None = None
selection_edge_warning: str | None = None
is_estimate: bool = False
warning: str | None = None
metrics: list[VectorSelectionMetric] = Field(default_factory=list)
class VectorSelectionResponse(BaseModel):
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
feature_count: int
total_feature_count: int | None = None
limit: int
truncated: bool
geojson: dict
summary: VectorSelectionSummary | None = None
partition_count: int | None = None
available_partition_count: int | None = None
partition_scope_key: str | None = None
source_name: str | None = None
dataset_ids: list[UUID] | None = None
+50
View File
@@ -0,0 +1,50 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class OrthophotoAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "most_recent"
force_refresh: bool = False
resolution_m: float | None = Field(default=None, ge=0.1, le=2.0)
class OrthophotoProductRead(BaseModel):
key: str
display_name: str
observation_label: str
temporal_granularity: str
native_resolution_m: float
supports_detection: bool
color_mode: str
catalog_url: str
limitation_message: str
provider: str
coverage_zone: str
attribution: str
license_note: str
class OrthophotoAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
observation_label: str
temporal_granularity: str
supports_detection: bool
layer: str
width: int
height: int
resolution_m: float
bbox_epsg4326: list[float]
bbox_epsg31370: list[float]
attribution: str
limitation_message: str
+47
View File
@@ -0,0 +1,47 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
class ProjectCreate(BaseModel):
name: str
description: str | None = None
region: str | None = "Belgium and Belgian North Sea"
class ProjectUpdate(BaseModel):
name: str | None = None
description: str | None = None
region: str | None = None
status: Literal["active", "archived"] | None = None
class ProjectRead(BaseModel):
id: UUID
name: str
description: str | None = None
region: str
status: str
created_at: datetime | None = None
updated_at: datetime | None = None
model_config = {"from_attributes": True}
class ProjectListItem(ProjectRead):
pass
class ProjectList(BaseModel):
items: list[ProjectRead]
total: int
limit: int
offset: int
class ProjectDeleteResult(BaseModel):
deleted: bool
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.common import GeoJsonFeatureCollection
class QaProviderComparisonRequest(BaseModel):
candidate_dataset_id: UUID
reference_dataset_id: UUID
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
area_id: UUID | None = None
class QaProviderComparisonResult(BaseModel):
status: str
warnings: list[str] = Field(default_factory=list)
# Counts of the population that was actually matched, so that
# ``matches + false_positives == candidate_feature_count`` holds even when
# an area filter or an unparseable geometry removed features. The ``_raw``
# fields keep the untouched dataset totals visible next to them.
candidate_feature_count: int
reference_feature_count: int
candidate_feature_count_raw: int | None = None
reference_feature_count_raw: int | None = None
matches: int
false_positives: int
false_negatives: int
precision: float | None
recall: float | None
f1_score: float | None
mean_iou: float | None
iou_threshold: float
unsupported_geometry: bool = False
unsupported_geometries: list[str] = Field(default_factory=list)
match_evidence: list[dict] = Field(default_factory=list)
false_positive_evidence: list[dict] = Field(default_factory=list)
false_negative_evidence: list[dict] = Field(default_factory=list)
generated_at: datetime
class MetricRead(BaseModel):
id: UUID
quality_check_id: UUID | None = None
analysis_run_id: UUID | None = None
metric_key: str
metric_value: float | None = None
metric_unit: str | None = None
label: str | None = None
metadata_json: dict | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
class QualityCheckRead(BaseModel):
id: UUID
project_id: UUID
job_id: UUID | None = None
analysis_run_id: UUID | None = None
candidate_dataset_id: UUID | None = None
reference_dataset_id: UUID
check_type: str
status: str
score: float | None = None
parameters_json: dict | None = None
findings_json: dict | None = None
created_at: datetime | None = None
completed_at: datetime | None = None
metrics: list[MetricRead] = Field(default_factory=list)
model_config = {"from_attributes": True}
class QualityCheckList(BaseModel):
items: list[QualityCheckRead]
total: int
limit: int
offset: int
class QualityEvidenceResponse(BaseModel):
quality_check_id: UUID
project_id: UUID
candidate_dataset_id: UUID | None = None
reference_dataset_id: UUID
analysis_run_id: UUID | None = None
# The overlay is capped so a regional check stays reviewable; the counts in
# the quality check itself are always complete.
feature_count: int
total_feature_count: int | None = None
role_counts: dict[str, int] = Field(default_factory=dict)
truncated: bool = False
limit: int | None = None
warnings: list[str] = Field(default_factory=list)
geojson: GeoJsonFeatureCollection
class AnalysisQaResponse(BaseModel):
status: str
quality_check_id: UUID
analysis_run_id: UUID
reference_dataset_id: UUID
candidate_feature_count: int
reference_feature_count: int
candidate_feature_count_raw: int | None = None
reference_feature_count_raw: int | None = None
matches: int
false_positives: int
false_negatives: int
precision: float | None = None
recall: float | None = None
f1_score: float | None = None
mean_iou: float | None = None
iou_threshold: float
warnings: list[str] = Field(default_factory=list)
coverage: dict[str, Any] | None = None
temporal_compatibility: dict[str, Any] | None = None
box_to_footprint_diagnostics: dict[str, Any] | None = None
precision_recall_curve: dict[str, Any] | None = None
calibration_sweep: list[dict[str, Any]] = Field(default_factory=list)
match_evidence: list[dict[str, Any]] = Field(default_factory=list)
false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)
+110
View File
@@ -0,0 +1,110 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from app.schemas.detection import DetectionModelCapability
SegmentationModelCapability = DetectionModelCapability
class SegmentationModelsResponse(BaseModel):
models: list[SegmentationModelCapability]
class SegmentationRunRequest(BaseModel):
model_config = ConfigDict(protected_namespaces=())
project_id: UUID
dataset_id: UUID
model_id: str
confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class_filter: list[str] | None = None
tile_manifest_path: str | None = None
parameters_json: dict = Field(default_factory=dict)
class SegmentationQaRequest(BaseModel):
reference_dataset_id: UUID
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class_name: str | None = None
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
# Read off the one matching pass, exactly as for detection.
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
class SegmentationRunResponse(BaseModel):
model_config = ConfigDict(protected_namespaces=())
analysis_run_id: UUID
job_id: UUID
project_id: UUID
dataset_id: UUID
model_id: str
status: str
segmentation_count: int
error_code: str | None = None
message: str
class SegmentationRunRead(BaseModel):
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
id: UUID
project_id: UUID
dataset_id: UUID | None = None
job_id: UUID | None = None
analysis_type: str
status: str
model_name: str | None = None
model_version: str | None = None
parameters_json: dict
result_json: dict | None = None
error_message: str | None = None
created_at: datetime | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
class SegmentationRunListResponse(BaseModel):
items: list[SegmentationRunRead]
# ``total`` counts every run; ``items`` is the most recent page of them.
total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
class SegmentationRead(BaseModel):
model_config = ConfigDict(from_attributes=True, protected_namespaces=())
id: UUID
project_id: UUID
dataset_id: UUID | None = None
analysis_run_id: UUID | None = None
job_id: UUID | None = None
model_name: str
model_version: str | None = None
class_name: str
confidence: float | None = None
bbox_json: dict | None = None
area_m2: float | None = None
mask_path: str | None = None
source_tile_path: str | None = None
tile_index: int | None = None
properties_json: dict | None = None
provenance_json: dict | None = None
created_at: datetime | None = None
class SegmentationListResponse(BaseModel):
items: list[SegmentationRead]
# ``total`` describes the complete filtered population; ``items`` is one
# stable confidence-ranked page of it.
total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
@@ -0,0 +1,14 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class VectorPartitionSelectionRequest(BaseModel):
dataset_ids: list[UUID] = Field(min_length=1, max_length=4096)
bbox: VectorSelectionBBox
area_id: UUID | None = None
limit: int = Field(default=1000, ge=1, le=1000)
+56
View File
@@ -0,0 +1,56 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
SourceCatalogProbeStatus = Literal["available", "degraded", "unavailable", "disabled"]
SourceCatalogComparisonStatus = Literal["same", "different", "not_comparable", "no_local_data", "unavailable"]
class SourceCatalogProbeItem(BaseModel):
source_name: str
display_name: str
service_type: Literal["WFS", "WMS", "HTML", "DCAT"]
endpoint_url: str
status: SourceCatalogProbeStatus
reachable: bool
checked_at: datetime
cached: bool = False
expected_layers: list[str]
matched_layers: list[str]
missing_layers: list[str]
advertised_layer_count: int
metadata_url: str | None = None
metadata_identifier: str | None = None
remote_title: str | None = None
remote_version: str | None = None
remote_modified_at: datetime | None = None
remote_published_at: datetime | None = None
local_source_version: str | None = None
comparison_status: SourceCatalogComparisonStatus
capabilities_sha256: str | None = None
capabilities_etag: str | None = None
capabilities_last_modified_at: datetime | None = None
message: str
error_code: str | None = None
class SourceCatalogProbeSummary(BaseModel):
provider_count: int
available_count: int
degraded_count: int
unavailable_count: int
disabled_count: int
different_version_count: int
class SourceCatalogProbeReport(BaseModel):
project_id: UUID
generated_at: datetime
summary: SourceCatalogProbeSummary
items: list[SourceCatalogProbeItem]
limitations: list[str]
+66
View File
@@ -0,0 +1,66 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
SourceFreshnessStatus = Literal["current", "due", "review_required", "local"]
SourceRefreshPolicy = Literal["rolling_snapshot", "annual_release", "edition", "scenario", "archive", "local"]
class SourceIntegritySummary(BaseModel):
missing_version_count: int = 0
checksum_mismatch_count: int = 0
missing_storage_file_count: int = 0
size_mismatch_count: int = 0
@property
def issue_count(self) -> int:
return (
self.missing_version_count
+ self.checksum_mismatch_count
+ self.missing_storage_file_count
+ self.size_mismatch_count
)
class SourceFreshnessItem(BaseModel):
source_name: str
display_name: str
dataset_count: int
ready_count: int
version_count: int
latest_imported_at: datetime | None = None
latest_observed_at: datetime | None = None
latest_source_version: str | None = None
refresh_policy: SourceRefreshPolicy
review_interval_days: int | None = None
next_review_at: datetime | None = None
status: SourceFreshnessStatus
historical_series: bool
auto_refresh_supported: bool = False
reason: str
recommended_action: str
integrity: SourceIntegritySummary
class SourceFreshnessSummary(BaseModel):
source_count: int
dataset_count: int
current_count: int
due_count: int
review_required_count: int
local_count: int
sources_with_integrity_issues: int
integrity_issue_count: int
class SourceFreshnessReport(BaseModel):
project_id: UUID
generated_at: datetime
summary: SourceFreshnessSummary
items: list[SourceFreshnessItem]
limitations: list[str]
+120
View File
@@ -0,0 +1,120 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
class SourceRegistryRead(BaseModel):
"""Read-only, server-owned source-authority definition."""
id: UUID
source_key: str
display_name: str
classification: str
authority_name: str
authority_scope_json: dict
provider_adapter_key: str | None = None
source_url: str | None = None
license_name: str
license_url: str | None = None
usage_restrictions: str
default_crs: str
default_units: str
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
expected_geometry_types_json: list
expected_attributes_json: dict
usage_policy_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
registry_metadata_json: dict
created_at: datetime | None = None
updated_at: datetime | None = None
snapshot_count: int = 0
model_config = {"from_attributes": True}
class SourceSnapshotRead(BaseModel):
"""Immutable version/snapshot evidence attached to an imported dataset."""
id: UUID
source_registry_id: UUID
snapshot_key: str
source_version: str | None = None
snapshot_at: datetime | None = None
fetched_at: datetime | None = None
source_url: str | None = None
checksum_sha256: str | None = None
crs: str | None = None
units: str | None = None
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
observed_schema_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
snapshot_metadata_json: dict
created_at: datetime | None = None
model_config = {"from_attributes": True}
class SourceRegistryDetailRead(BaseModel):
source: SourceRegistryRead
snapshots: list[SourceSnapshotRead]
class DatasetLineageEdgeRead(BaseModel):
id: UUID
parent_dataset_id: UUID
child_dataset_id: UUID
parent_dataset_version_id: UUID | None = None
child_dataset_version_id: UUID | None = None
relation_type: str
transformation_name: str
transformation_version: str | None = None
parameters_json: dict | None = None
input_checksum_sha256: str | None = None
output_checksum_sha256: str | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
class DatasetQuarantineRead(BaseModel):
id: UUID
dataset_id: UUID | None = None
dataset_version_id: UUID | None = None
source_snapshot_id: UUID | None = None
stage: str
reason_code: str
details_json: dict | None = None
artifact_path: str | None = None
artifact_checksum_sha256: str | None = None
status: str
created_at: datetime | None = None
resolved_at: datetime | None = None
resolved_by: str | None = None
model_config = {"from_attributes": True}
class DatasetProvenanceRead(BaseModel):
dataset_id: UUID
source: SourceRegistryRead | None = None
snapshot: SourceSnapshotRead | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
+55
View File
@@ -0,0 +1,55 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class SpwTerrainAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "spw_mnt_1m_2021_2022"
resolution_m: float | None = Field(default=None, ge=1.0, le=10.0)
force_refresh: bool = False
class SpwTerrainProductRead(BaseModel):
key: str
display_name: str
surface_model: str
source_filename: str
native_resolution_m: float
analysis_resolution_m: float
source_crs: str
vertical_reference: str
acquisition_period: str
catalog_url: str
attribution: str
license_note: str
limitation_message: str
coverage_zones: list[str]
configured: bool
status: str
class SpwTerrainAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
surface_model: str
native_resolution_m: float
resolution_m: float
width: int
height: int
valid_pixel_count: int
nodata_value: float
bbox_epsg4326: list[float]
bbox_epsg3812: list[float]
vertical_reference: str
acquisition_period: str
attribution: str
limitation_message: str
+91
View File
@@ -0,0 +1,91 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.operations import VectorSelectionBBox
class TemporalComparisonRequest(BaseModel):
earlier_dataset_id: UUID
later_dataset_id: UUID
bbox: VectorSelectionBBox
area_id: UUID | None = None
preview_limit: int = Field(default=500, ge=1, le=1000)
class TemporalDatasetRef(BaseModel):
id: UUID
name: str
observed_at: datetime
source_version: str | None = None
class TemporalMetricComparison(BaseModel):
metric_key: str = "primary"
label: str
unit: str
aggregation_method: str
earlier_value: float
later_value: float
absolute_change: float
percent_change: float | None = None
is_estimate: bool = False
warning: str | None = None
class TemporalObservationMetric(BaseModel):
metric_key: str
label: str
value: float
unit: str
aggregation_method: str
is_estimate: bool = False
class TemporalObservation(BaseModel):
dataset: TemporalDatasetRef
metrics: list[TemporalObservationMetric]
class TemporalObjectChanges(BaseModel):
available: bool
added_count: int | None = None
removed_count: int | None = None
modified_count: int | None = None
unchanged_count: int | None = None
class TemporalComparisonResponse(BaseModel):
temporal_series_key: str
earlier: TemporalDatasetRef
later: TemporalDatasetRef
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
metric: TemporalMetricComparison
metrics: list[TemporalMetricComparison] = Field(default_factory=list)
timeline: list[TemporalObservation] = Field(default_factory=list)
object_changes: TemporalObjectChanges
geojson: dict
warnings: list[str]
generated_at: datetime
class TemporalSeriesDataset(BaseModel):
id: UUID
name: str
observed_at: datetime
source_version: str | None = None
feature_count: int | None = None
class TemporalSeriesRead(BaseModel):
temporal_series_key: str
source_name: str | None = None
reference_layer_name: str | None = None
dataset_count: int
first_observed_at: datetime
last_observed_at: datetime
datasets: list[TemporalSeriesDataset]
+125
View File
@@ -0,0 +1,125 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class ThematicRasterAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str
force_refresh: bool = False
class ThematicRasterProductRead(BaseModel):
key: str
display_name: str
theme: str
metric_kind: str
coverage_id: str
native_resolution_m: float
source_crs: str
source_value_unit: str
observation_year: int
source_version: str
catalog_url: str
attribution: str
license_note: str
legend_min_label: str
legend_max_label: str
included_source_values: list[int]
limitation_message: str
analysis_resolution_m: float | None = None
coverage_zones: list[str] = Field(default_factory=list)
configured: bool = True
status: str = "configured"
class WalousAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
theme: str
metric_kind: str
resolution_m: float
width: int
height: int
valid_pixel_count: int
bbox_epsg4326: list[float]
bbox_epsg3812: list[float]
observation_year: int
source_value_unit: str
attribution: str
limitation_message: str
class ThematicRasterAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
theme: str
metric_kind: str
coverage_id: str
resolution_m: float
width: int
height: int
valid_pixel_count: int
bbox_epsg4326: list[float]
bbox_epsg31370: list[float]
observation_year: int
source_value_unit: str
attribution: str
limitation_message: str
class ThematicRasterSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
class ThematicRasterMetric(BaseModel):
metric_key: str
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
derived: bool = True
is_estimate: bool = True
class ThematicRasterSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
primary_metric_key: str
metrics: list[ThematicRasterMetric]
class ThematicRasterSelectionResponse(BaseModel):
dataset_id: UUID
product_key: str
theme: str
metric_kind: str
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
selected_cell_count: int
valid_cell_count: int
coverage_ratio: float
# Set when the drawn selection is smaller than one source cell and the
# analysis was widened to the cells it touches, so the value covers more
# ground than was requested.
cell_selection_warning: str | None = None
resolution_m: float
observation_year: int
summary: ThematicRasterSelectionSummary
unsupported_metrics: list[str]
limitation_message: str
generated_at: str
View File
+172
View File
@@ -0,0 +1,172 @@
"""Background execution for queued analysis runs.
Tiled GPU inference is minutes of work. Running it inside the HTTP request
holds a worker thread for the whole duration, times the client out and leaves
the operator without progress. Queued ``detection.run`` and
``segmentation.run`` jobs are picked up here instead, mirroring the polling
worker the AOI operations already use so the runtime keeps one job model.
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any
from uuid import UUID
from app.db.session import SessionLocal
from app.models import Job
logger = logging.getLogger("geointel.analysis_worker")
class AnalysisJobWorker:
HANDLED_JOB_TYPES = ("detection.run", "segmentation.run")
BATCH_SIZE = 4
@staticmethod
def _uuid(value: Any) -> UUID | None:
if isinstance(value, UUID):
return value
try:
return UUID(str(value))
except (TypeError, ValueError):
return None
@staticmethod
def _dispatch(db, job: Job) -> Any:
# Imported lazily: both services import each other's helpers, and the
# worker must not add a third edge to that cycle at module load.
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService
parameters = job.parameters_json if isinstance(job.parameters_json, dict) else {}
project_id = AnalysisJobWorker._uuid(parameters.get("project_id"))
dataset_id = AnalysisJobWorker._uuid(parameters.get("dataset_id"))
if project_id is None or dataset_id is None:
raise ValueError("Queued analysis job is missing project_id or dataset_id")
common = {
"db": db,
"project_id": project_id,
"dataset_id": dataset_id,
"model_id": parameters.get("model_id"),
"confidence_threshold": float(parameters.get("confidence_threshold") or 0.0),
"class_filter": parameters.get("class_filter") or [],
"tile_manifest_path": parameters.get("tile_manifest_path"),
"parameters_json": parameters.get("parameters_json") or {},
"existing_job": job,
}
if job.job_type == "detection.run":
return DetectionService.run_detection(
model_asset_id=parameters.get("model_asset_id"),
**common,
)
return SegmentationService.run_segmentation(**common)
@staticmethod
def claim(db, job: Job) -> bool:
"""Take the job out of the queue, atomically. Returns whether we won.
Selecting and then updating in a second statement lets two workers —
a restarted process overlapping the previous one, or a second replica —
both start tiled GPU inference on the same row. The conditional update
makes exactly one caller see a row count of 1; the AOI worker beside
this one already claims with FOR UPDATE SKIP LOCKED for the same reason.
"""
claimed = (
db.query(Job)
.filter(Job.id == job.id, Job.status == "queued")
.update({Job.status: "running"}, synchronize_session=False)
)
db.commit()
if not claimed:
return False
job.status = "running"
return True
@staticmethod
def _finalize(db, job: Job, result: Any) -> None:
"""Close a job the handler left open.
The analysis services normally set the terminal status themselves.
If one returns without doing so, recording the outcome here is what
keeps the job from sitting in "running" for ever.
"""
if job.status != "running":
return
status = getattr(result, "status", None)
if status == "success":
job.status = "success"
job.result_json = {
"detection_count": getattr(result, "detection_count", None),
"segmentation_count": getattr(result, "segmentation_count", None),
}
else:
job.status = "failed"
job.error_message = getattr(result, "message", None) or "Analysis run did not complete"
job.result_json = {"error_code": getattr(result, "error_code", None) or "ANALYSIS_JOB_INCOMPLETE"}
db.add(job)
db.commit()
@staticmethod
def _mark_failed(db, job: Job, *, code: str, message: str) -> None:
try:
db.rollback()
except Exception:
pass
job.status = "failed"
job.error_message = message
job.result_json = {"error_code": code, "message": message}
db.add(job)
db.commit()
@staticmethod
def run_once(db=None) -> int:
"""Execute one batch of queued analysis jobs. Returns the batch size."""
owns_session = db is None
session = db if db is not None else SessionLocal()
try:
rows = [
job
for job in (
session.query(Job)
.filter(Job.status == "queued")
.filter(Job.job_type.in_(AnalysisJobWorker.HANDLED_JOB_TYPES))
.order_by(Job.created_at)
.limit(AnalysisJobWorker.BATCH_SIZE)
.all()
)
if job.job_type in AnalysisJobWorker.HANDLED_JOB_TYPES and job.status == "queued"
]
claimed_count = 0
for job in rows:
if not AnalysisJobWorker.claim(session, job):
# Another worker took it between the select and the claim.
continue
claimed_count += 1
try:
result = AnalysisJobWorker._dispatch(session, job)
AnalysisJobWorker._finalize(session, job, result)
except Exception as exc:
code = getattr(exc, "code", None) or "ANALYSIS_JOB_INTERNAL_ERROR"
message = getattr(exc, "message", None) or str(exc) or "Unexpected analysis job failure"
AnalysisJobWorker._mark_failed(session, job, code=str(code), message=str(message))
logger.exception("Analysis job failed job_id=%s job_type=%s", job.id, job.job_type)
return claimed_count
finally:
if owns_session:
session.close()
@staticmethod
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
while not stop_event.is_set():
processed = await asyncio.to_thread(AnalysisJobWorker.run_once)
if processed == 0:
try:
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
except TimeoutError:
pass
@@ -0,0 +1,252 @@
from __future__ import annotations
from uuid import UUID
from geoalchemy2.shape import to_shape
from app.core.errors import AppError
from app.models import AoiOperation, AoiOperationPartition
from app.schemas.grb import GrbAcquireRequest
from app.schemas.dhmv import DhmvAcquireRequest
from app.schemas.spw_terrain import SpwTerrainAcquireRequest
from app.schemas.official_vector import OfficialVectorAcquireRequest
from app.schemas.flood_hazard import FloodHazardAcquireRequest
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
from app.schemas.bathymetry import (
BathymetryProfileAcquireRequest,
MdkBathymetryAcquireRequest,
)
from app.schemas.job import JobCreate
from app.schemas.operations import VectorSelectionBBox
from app.schemas.orthophoto import OrthophotoAcquireRequest
from app.services.aoi_operation_service import AoiOperationService
from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.spw_terrain_service import SpwTerrainService
from app.services.official_vector_acquisition_service import (
OfficialVectorAcquisitionService,
)
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.thematic_raster_acquisition_service import (
ThematicRasterAcquisitionService,
)
from app.services.walous_land_cover_service import WalousLandCoverService
from app.services.bathymetry_profile_acquisition_service import (
BathymetryProfileAcquisitionService,
)
from app.services.mdk_bathymetry_acquisition_service import (
MdkBathymetryAcquisitionService,
)
from app.services.job_service import JobService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
class AoiOperationExecutor:
"""Execute one bounded partition through an existing governed provider."""
@staticmethod
def execute_next(db, project_id: UUID, operation_id: UUID) -> dict:
partition = AoiOperationService.claim_next(db, project_id, operation_id)
if partition is None:
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
operation = db.get(AoiOperation, operation_id)
child = JobService.create_job(
db,
JobCreate(
job_type=f"aoi.{operation.operation_type}.partition",
project_id=project_id,
parameters_json={
"aoi_operation_id": str(operation_id),
"partition_id": str(partition.id),
"partition_key": partition.partition_key,
"provider_key": partition.provider_key,
"product_key": partition.product_key,
},
),
)
partition = db.get(AoiOperationPartition, partition.id)
partition.child_job_id = child.id
db.add(partition)
db.commit()
JobService.mark_running(db, child.id)
try:
result = AoiOperationExecutor._dispatch(
db, project_id, operation, partition
)
output_id = (
result.get("output_dataset_id") if isinstance(result, dict) else None
)
JobService.mark_success(
db,
child.id,
result=result,
output_dataset_id=UUID(str(output_id)) if output_id else None,
)
return AoiOperationService.complete(
db, project_id, operation_id, partition.id, result
)
except AppError as exc:
JobService.mark_failed(
db, child.id, exc.message, {"code": exc.code, "details": exc.details}
)
return AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
exc.message,
AoiOperationExecutor._retryable(exc),
{"code": exc.code, "details": exc.details},
)
except Exception:
try:
db.rollback()
JobService.mark_failed(
db,
child.id,
"Unexpected partition execution error",
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
finally:
AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
"Unexpected partition execution error",
True,
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
raise
@staticmethod
def _dispatch(
db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition
) -> dict:
geometry = to_shape(partition.geometry)
min_x, min_y, max_x, max_y = geometry.bounds
bbox = VectorSelectionBBox(
min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326"
)
force_refresh = bool(
(operation.request_json or {})
.get("parameters_json", {})
.get("force_refresh", False)
)
if partition.provider_key == "grb":
return GrbAcquisitionService.acquire(
db,
project_id,
GrbAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "orthophoto":
return OrthophotoAcquisitionService.acquire(
db,
project_id,
OrthophotoAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "dhmv":
return DhmvAcquisitionService.acquire(
db,
project_id,
DhmvAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "spw_terrain":
return SpwTerrainService.acquire(
db,
project_id,
SpwTerrainAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "official_vector":
return OfficialVectorAcquisitionService.acquire(
db,
project_id,
OfficialVectorAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "flood_hazard":
return FloodHazardAcquisitionService.acquire(
db,
project_id,
FloodHazardAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "thematic_raster":
return ThematicRasterAcquisitionService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "walous":
return WalousLandCoverService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "bathymetry_profiles":
return BathymetryProfileAcquisitionService.acquire(
db,
project_id,
BathymetryProfileAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
if partition.provider_key == "mdk_bathymetry":
return MdkBathymetryAcquisitionService.acquire(
db,
project_id,
MdkBathymetryAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed AOI executor is registered for this provider",
details={"provider_key": partition.provider_key},
status_code=422,
)
@staticmethod
def _retryable(error: AppError) -> bool:
return error.status_code >= 500 or error.code.endswith(
("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")
)
@@ -0,0 +1,481 @@
from __future__ import annotations
from collections import Counter
from datetime import datetime, timezone
from hashlib import sha256
import math
from uuid import UUID, uuid4
from geoalchemy2.shape import from_shape, to_shape
from pyproj import Transformer
from shapely.geometry import MultiPolygon, Polygon, box
from shapely.ops import transform
from app.core.errors import AppError
from app.core.config import get_settings
from app.models import AoiOperation, AoiOperationPartition, Area, Project
from app.schemas.aoi_operation import AoiOperationCreate
class AoiOperationService:
MAX_PARTITIONS = 4096
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
SCOPE_AREA_NAMES = {
"belgium": "Belgium land",
"flanders": "Flanders",
"wallonia": "Wallonia",
"brussels": "Brussels-Capital Region",
"belgian_north_sea": "Belgian part of the North Sea",
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
"continental_shelf": "Belgian continental shelf beyond territorial sea",
}
@staticmethod
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
if db.get(Project, project_id) is None:
raise AppError(
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
)
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
if payload.coverage_zone:
geometry = AoiOperationService._clip_to_zone(
db, project_id, geometry, payload.coverage_zone
)
geometry = AoiOperationService._as_multipolygon(geometry)
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
partition_side_m = AoiOperationService._partition_side(
payload.provider_key, payload.max_partition_side_m
)
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
operation_id = uuid4()
now = datetime.now(timezone.utc)
operation = AoiOperation(
id=operation_id,
project_id=project_id,
area_id=payload.area_id,
operation_type=payload.operation_type,
status="queued",
geometry=from_shape(geometry, srid=4326),
request_json=payload.model_dump(mode="json", exclude_none=True),
plan_json={
"partition_strategy": "epsg31370_square_grid_intersection_v1",
"max_partition_side_m": partition_side_m,
"budget_source": "governed_provider_registry"
if payload.max_partition_side_m is None
else "stricter_operator_override",
"partition_count": len(cells),
"provider_key": payload.provider_key,
"product_key": payload.product_key,
},
created_at=now,
)
db.add(operation)
for ordinal, cell in enumerate(cells):
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
wgs84 = AoiOperationService._as_multipolygon(wgs84)
digest = sha256(wgs84.wkb).hexdigest()[:20]
db.add(
AoiOperationPartition(
id=uuid4(),
operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key,
product_key=payload.product_key,
ordinal=ordinal,
status="queued",
geometry=from_shape(wgs84, srid=4326),
attempt_count=0,
max_attempts=payload.max_attempts,
created_at=now,
)
)
db.commit()
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
if area_name is None:
raise AppError(
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
message="Unknown governed coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
scope = (
db.query(Area)
.filter(Area.project_id == project_id, Area.name == area_name)
.first()
)
if scope is None:
raise AppError(
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
message="The governed coverage-zone geometry is not persisted in this project",
details={"coverage_zone": zone},
status_code=409,
)
clipped = geometry.intersection(to_shape(scope.geometry))
if clipped.is_empty:
raise AppError(
code="AOI_OUTSIDE_PROVIDER_ZONE",
message="The AOI does not intersect the provider coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
return clipped
@staticmethod
def _as_multipolygon(geometry) -> MultiPolygon:
if isinstance(geometry, Polygon):
return MultiPolygon([geometry])
if isinstance(geometry, MultiPolygon):
return geometry
polygons = [
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
]
if not polygons:
raise AppError(
code="AOI_GEOMETRY_EMPTY",
message="AOI contains no polygonal area after clipping",
status_code=422,
)
return MultiPolygon(polygons)
@staticmethod
def _partition_side(provider_key: str, requested: float | None) -> float:
settings = get_settings()
def raster_side(
max_side_m: float, max_pixels: int, resolution_m: float
) -> float:
# Keep every square grid cell within both the provider's spatial
# extent limit and its decoded-pixel budget. The small safety
# margin absorbs ceil/edge rounding in the acquisition services.
pixel_limited_side = (
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
)
return min(float(max_side_m), pixel_limited_side)
budgets = {
"orthophoto": float(settings.orthophoto_max_side_m),
"grb": float(settings.grb_max_side_m),
"dhmv": raster_side(
settings.dhmv_max_side_m,
settings.dhmv_max_pixels,
settings.dhmv_resolution_m,
),
"spw_terrain": raster_side(
settings.spw_terrain_max_side_m,
settings.spw_terrain_max_pixels,
settings.spw_terrain_analysis_resolution_m,
),
"official_vector": 20_000.0,
"flood_hazard": raster_side(
settings.flood_hazard_max_side_m,
settings.flood_hazard_max_pixels,
settings.flood_hazard_resolution_m,
),
"thematic_raster": raster_side(
settings.thematic_raster_max_side_m,
settings.thematic_raster_max_pixels,
10.0,
),
"walous": raster_side(
settings.walous_max_side_m,
settings.walous_max_pixels,
settings.walous_analysis_resolution_m,
),
"bathymetry_profiles": 20_000.0,
"mdk_bathymetry": 20_000.0,
}
if provider_key not in budgets:
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed partition budget is registered for this provider",
details={"provider_key": provider_key},
status_code=422,
)
governed = budgets[provider_key]
return min(governed, float(requested)) if requested is not None else governed
@staticmethod
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
if (payload.area_id is None) == (payload.bbox is None):
raise AppError(
code="AOI_SELECTION_REQUIRED",
message="Provide exactly one area_id or bbox",
status_code=422,
)
if payload.area_id is not None:
area = db.get(Area, payload.area_id)
if area is None or area.project_id != project_id:
raise AppError(
code="AREA_NOT_FOUND", message="Area not found", status_code=404
)
return to_shape(area.geometry)
bbox = payload.bbox
if bbox is None or bbox.crs != "EPSG:4326":
raise AppError(
code="INVALID_AOI_CRS",
message="AOI bbox must use EPSG:4326",
status_code=422,
)
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
@staticmethod
def _partition(geometry, side_m: float) -> list:
min_x, min_y, max_x, max_y = geometry.bounds
columns = max(1, math.ceil((max_x - min_x) / side_m))
rows = max(1, math.ceil((max_y - min_y) / side_m))
if columns * rows > AoiOperationService.MAX_PARTITIONS:
raise AppError(
code="AOI_PARTITION_LIMIT_EXCEEDED",
message="AOI requires too many bounded partitions",
details={
"candidate_count": columns * rows,
"max_partitions": AoiOperationService.MAX_PARTITIONS,
},
status_code=422,
)
partitions = []
for row in range(rows):
for column in range(columns):
clipped = geometry.intersection(
box(
min_x + column * side_m,
min_y + row * side_m,
min(min_x + (column + 1) * side_m, max_x),
min(min_y + (row + 1) * side_m, max_y),
)
)
if not clipped.is_empty and clipped.area > 0:
partitions.append(clipped)
return partitions
@staticmethod
def read(db, project_id: UUID, operation_id: UUID) -> dict:
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
counts = Counter(partition.status for partition in partitions)
complete = counts["success"] + counts["skipped"]
return {
"id": operation.id,
"project_id": operation.project_id,
"area_id": operation.area_id,
"parent_job_id": operation.parent_job_id,
"operation_type": operation.operation_type,
"status": operation.status,
"request_json": operation.request_json,
"plan_json": operation.plan_json,
"result_json": operation.result_json,
"error_message": operation.error_message,
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
"partition_counts": dict(counts),
"partitions": partitions,
"created_at": operation.created_at,
"started_at": operation.started_at,
"finished_at": operation.finished_at,
}
@staticmethod
def list(db, project_id: UUID, limit: int = 50) -> dict:
rows = (
db.query(AoiOperation)
.filter(AoiOperation.project_id == project_id)
.order_by(AoiOperation.created_at.desc())
.limit(limit)
.all()
)
return {
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
"total": len(rows),
}
@staticmethod
def claim_next(db, project_id: UUID, operation_id: UUID):
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partition = (
db.query(AoiOperationPartition)
.filter(
AoiOperationPartition.operation_id == operation_id,
AoiOperationPartition.status == "queued",
)
.order_by(AoiOperationPartition.ordinal)
.with_for_update(skip_locked=True)
.first()
)
if partition is None:
return None
now = datetime.now(timezone.utc)
partition.status = "running"
partition.started_at = now
partition.attempt_count += 1
partition.error_message = None
operation.status = "running"
operation.started_at = operation.started_at or now
db.add(partition)
db.add(operation)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def checkpoint(
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status != "running":
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can be checkpointed",
status_code=409,
)
partition.checkpoint_json = checkpoint
db.add(partition)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def complete(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
result: dict,
skipped: bool = False,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status == "success" or partition.status == "skipped":
return AoiOperationService.read(db, project_id, operation_id)
if partition.status != "running":
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can complete",
status_code=409,
)
partition.status = "skipped" if skipped else "success"
partition.result_json = result
partition.finished_at = datetime.now(timezone.utc)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def fail(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
message: str,
retryable: bool,
details: dict,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
partition.error_message = message
partition.result_json = {"details": details}
partition.status = (
"queued"
if retryable and partition.attempt_count < partition.max_attempts
else "failed"
)
partition.finished_at = (
None if partition.status == "queued" else datetime.now(timezone.utc)
)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def _partition_row(db, project_id, operation_id, partition_id):
operation = db.get(AoiOperation, operation_id)
partition = db.get(AoiOperationPartition, partition_id)
if (
operation is None
or operation.project_id != project_id
or partition is None
or partition.operation_id != operation_id
):
raise AppError(
code="AOI_PARTITION_NOT_FOUND",
message="AOI partition not found",
status_code=404,
)
return partition
@staticmethod
def _refresh_parent(db, operation_id):
operation = db.get(AoiOperation, operation_id)
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
statuses = [partition.status for partition in partitions]
output_dataset_ids = []
for partition in partitions:
output_id = (
(partition.result_json or {}).get("output_dataset_id")
if isinstance(partition.result_json, dict)
else None
)
if output_id and str(output_id) not in output_dataset_ids:
output_dataset_ids.append(str(output_id))
operation.result_json = {
"partition_count": len(partitions),
"completed_partition_count": sum(
status in {"success", "skipped"} for status in statuses
),
"failed_partition_count": statuses.count("failed"),
"output_dataset_ids": output_dataset_ids,
"merge_contract": "source_aware_spatial_union",
"vector_deduplication": "source_feature_id_then_geometry",
"raster_deduplication": "governed_mosaic_grid",
"complete_coverage": bool(statuses)
and all(status in {"success", "skipped"} for status in statuses),
}
now = datetime.now(timezone.utc)
if statuses and all(status in {"success", "skipped"} for status in statuses):
operation.status = "success"
operation.finished_at = now
operation.error_message = None
elif "failed" in statuses and not any(
status in {"queued", "running"} for status in statuses
):
operation.status = (
"partial"
if any(status in {"success", "skipped"} for status in statuses)
else "failed"
)
operation.finished_at = now
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
db.add(operation)
db.commit()
@@ -0,0 +1,38 @@
from __future__ import annotations
import asyncio
import logging
from app.db.session import SessionLocal
from app.models import AoiOperation
from app.services.aoi_operation_executor import AoiOperationExecutor
logger = logging.getLogger("geointel.aoi_worker")
class AoiOperationWorker:
@staticmethod
def run_once() -> int:
db = SessionLocal()
try:
rows = db.query(AoiOperation).filter(AoiOperation.status.in_(("queued", "running"))).order_by(AoiOperation.created_at).limit(10).all()
for operation in rows:
try:
AoiOperationExecutor.execute_next(db, operation.project_id, operation.id)
except Exception:
db.rollback()
logger.exception("AOI partition execution failed operation_id=%s", operation.id)
return len(rows)
finally:
db.close()
@staticmethod
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
while not stop_event.is_set():
processed = await asyncio.to_thread(AoiOperationWorker.run_once)
if processed == 0:
try:
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
except TimeoutError:
pass
+193
View File
@@ -0,0 +1,193 @@
from __future__ import annotations
import uuid
from sqlalchemy.orm import Session
from geoalchemy2.shape import from_shape, to_shape
from shapely.geometry import mapping
from app.core.errors import AppError
from app.models import Area, Dataset, Project, VectorFeature
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326
class AreaService:
@staticmethod
def _municipality_dataset(db: Session, project_id: uuid.UUID) -> Dataset | None:
return (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.reference_layer_name == "belgium_municipalities",
Dataset.status == "ready",
)
.order_by(Dataset.created_at.desc())
.first()
)
@staticmethod
def _filter_municipality_properties(properties_items: list[dict], query: str, limit: int) -> tuple[list[dict], int]:
normalized = query.strip().casefold()
matches: list[dict] = []
for properties in properties_items:
names = [str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger")]
niscode = str(properties.get("niscode") or "").strip()
if normalized and normalized not in " ".join([niscode, *names]).casefold():
continue
display_name = next((name for name in names if name), niscode)
matches.append({
"niscode": niscode,
"name": display_name,
"name_nl": names[0] or None,
"name_fr": names[1] or None,
"name_de": names[2] or None,
})
matches.sort(key=lambda item: (item["name"].casefold(), item["niscode"]))
return matches[:limit], len(matches)
@staticmethod
def search_municipalities(db: Session, project_id: uuid.UUID, query: str, limit: int = 20) -> tuple[list[dict], int]:
dataset = AreaService._municipality_dataset(db, project_id)
if dataset is None:
return [], 0
property_rows = (
db.query(VectorFeature.properties_json)
.filter(VectorFeature.dataset_id == dataset.id)
.all()
)
properties_items = [row[0] for row in property_rows if isinstance(row[0], dict)]
return AreaService._filter_municipality_properties(properties_items, query, limit)
@staticmethod
def activate_municipality(db: Session, project_id: uuid.UUID, niscode: str) -> Area:
normalized_code = niscode.strip()
dataset = AreaService._municipality_dataset(db, project_id)
if dataset is None:
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
feature = (
db.query(VectorFeature)
.filter(
VectorFeature.dataset_id == dataset.id,
VectorFeature.properties_json["niscode"].as_string() == normalized_code,
)
.first()
)
if feature is not None:
properties = feature.properties_json if isinstance(feature.properties_json, dict) else {}
display_name = next(
(str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger") if str(properties.get(key) or "").strip()),
normalized_code,
)
area_name = f"Gemeente {display_name} - NIS {normalized_code}"
existing = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
if existing is not None:
return existing
geometry = to_shape(feature.geometry)
return AreaService.create_area(
db,
project_id,
AreaCreate(name=area_name, geometry=mapping(geometry), crs="EPSG:4326"),
)
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
@staticmethod
def serialize_area(area: Area) -> dict:
geometry = to_shape(area.geometry) if area.geometry else None
return AreaRead.model_validate(
{
"id": area.id,
"project_id": area.project_id,
"name": area.name,
"original_crs": area.original_crs,
"area_m2": area.area_m2,
"created_at": area.created_at,
"geometry_type": geometry.geom_type if geometry else None,
"geometry": mapping(geometry) if geometry else None,
}
).model_dump()
@staticmethod
def list_areas(db: Session, project_id: uuid.UUID, limit: int = 50, offset: int = 0) -> tuple[list[Area], int]:
total = db.query(Area).filter(Area.project_id == project_id).count()
areas = (
db.query(Area)
.filter(Area.project_id == project_id)
.order_by(Area.created_at.desc())
.offset(offset)
.limit(limit)
.all()
)
return areas, total
@staticmethod
def create_area(db: Session, project_id: uuid.UUID, payload: AreaCreate) -> Area:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
try:
multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
area = Area(
project_id=project_id,
name=payload.name.strip() or "Unnamed area",
geometry=from_shape(multipolygon, srid=4326),
original_crs=original_crs,
area_m2=metric_area,
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
)
db.add(area)
db.commit()
db.refresh(area)
return area
@staticmethod
def get_area(db: Session, area_id: uuid.UUID) -> Area:
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
return area
@staticmethod
def update_area(db: Session, area_id: uuid.UUID, payload: AreaUpdate) -> Area:
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
changed = False
if payload.name is not None and payload.name.strip():
area.name = payload.name.strip() or area.name
changed = True
if payload.crs is not None and payload.geometry is None:
raise AppError(
code="INVALID_AREA_CRS_UPDATE",
message="crs can only be supplied together with replacement geometry",
status_code=422,
)
if payload.geometry is not None:
try:
multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
area.geometry = from_shape(multipolygon, srid=4326)
area.original_crs = original_crs
area.area_m2 = metric_area
area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
changed = True
if not changed:
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
db.add(area)
db.commit()
db.refresh(area)
return area
+260
View File
@@ -0,0 +1,260 @@
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import secrets
import threading
import time
from collections import deque
from dataclasses import dataclass, field
from typing import Literal, cast
from uuid import UUID
from app.core.config import Settings
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
@dataclass(frozen=True)
class AuthPrincipal:
username: str
expires_at: int
session_id: str = field(default_factory=lambda: secrets.token_urlsafe(12))
role: Literal["operator", "guest"] = "operator"
project_id: UUID | None = None
class AuthService:
HASH_NAME = "pbkdf2_sha256"
HASH_ITERATIONS = 600_000
MAX_FAILURES = 5
FAILURE_WINDOW_SECONDS = 300
_failures: dict[str, deque[float]] = {}
_failure_lock = threading.Lock()
_guest_requests: dict[str, deque[float]] = {}
_guest_request_lock = threading.Lock()
_active_guest_compute = 0
_guest_compute_lock = threading.Lock()
@staticmethod
def _b64_encode(value: bytes) -> str:
return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=")
@staticmethod
def _b64_decode(value: str) -> bytes:
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
@classmethod
def hash_password(
cls,
password: str,
*,
salt: bytes | None = None,
iterations: int | None = None,
) -> str:
resolved_salt = salt or secrets.token_bytes(18)
resolved_iterations = iterations or cls.HASH_ITERATIONS
digest = hashlib.pbkdf2_hmac(
"sha256",
password.encode("utf-8"),
resolved_salt,
resolved_iterations,
)
return "$".join(
(
cls.HASH_NAME,
str(resolved_iterations),
cls._b64_encode(resolved_salt),
cls._b64_encode(digest),
)
)
@classmethod
def verify_password(cls, password: str, encoded: str) -> bool:
try:
algorithm, iterations_raw, salt_raw, expected_raw = encoded.split("$", 3)
if algorithm != cls.HASH_NAME:
return False
iterations = int(iterations_raw)
if iterations < 100_000 or iterations > 2_000_000:
return False
salt = cls._b64_decode(salt_raw)
expected = cls._b64_decode(expected_raw)
actual = hashlib.pbkdf2_hmac(
"sha256",
password.encode("utf-8"),
salt,
iterations,
)
return hmac.compare_digest(actual, expected)
except (TypeError, ValueError):
return False
@classmethod
def credentials_match(cls, username: str, password: str, settings: Settings) -> bool:
expected_username = settings.auth_username or ""
expected_password_hash = settings.auth_password_hash or ""
username_matches = hmac.compare_digest(
username.encode("utf-8"),
expected_username.encode("utf-8"),
)
password_matches = cls.verify_password(password, expected_password_hash)
return username_matches and password_matches
@classmethod
def create_session_token(
cls,
username: str,
settings: Settings,
*,
role: Literal["operator", "guest"] = "operator",
project_id: UUID | None = None,
ttl_seconds: int | None = None,
now: int | None = None,
) -> str:
issued_at = int(time.time() if now is None else now)
if role == "guest" and project_id is None:
raise ValueError("Guest sessions must be scoped to a demo project")
resolved_ttl = ttl_seconds if ttl_seconds is not None else (
settings.guest_session_ttl_seconds if role == "guest" else settings.auth_session_ttl_seconds
)
payload = {
"exp": issued_at + resolved_ttl,
"iat": issued_at,
"jti": secrets.token_urlsafe(12),
"role": role,
"sub": username,
"v": 2,
}
if project_id is not None:
payload["project_id"] = str(project_id)
encoded_payload = cls._b64_encode(
json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")
)
signature = hmac.new(
(settings.auth_session_secret or "").encode("utf-8"),
encoded_payload.encode("ascii"),
hashlib.sha256,
).digest()
return f"{encoded_payload}.{cls._b64_encode(signature)}"
@classmethod
def verify_session_token(
cls,
token: str | None,
settings: Settings,
*,
now: int | None = None,
) -> AuthPrincipal | None:
if not token:
return None
try:
encoded_payload, encoded_signature = token.split(".", 1)
expected_signature = hmac.new(
(settings.auth_session_secret or "").encode("utf-8"),
encoded_payload.encode("ascii"),
hashlib.sha256,
).digest()
supplied_signature = cls._b64_decode(encoded_signature)
if not hmac.compare_digest(expected_signature, supplied_signature):
return None
payload = json.loads(cls._b64_decode(encoded_payload))
username = str(payload.get("sub") or "")
expires_at = int(payload.get("exp") or 0)
issued_at = int(payload.get("iat") or 0)
version = int(payload.get("v") or 0)
role_value = str(payload.get("role") or "operator")
session_id = str(payload.get("jti") or "")
current = int(time.time() if now is None else now)
if version not in {1, 2} or role_value not in {"operator", "guest"} or not session_id:
return None
role = cast(Literal["operator", "guest"], role_value)
if issued_at <= 0 or issued_at > current + 60 or expires_at <= current:
return None
if role == "operator":
if username != settings.auth_username:
return None
max_ttl = settings.auth_session_ttl_seconds
project_id = None
else:
if not settings.guest_access_enabled or username != settings.guest_display_name:
return None
max_ttl = settings.guest_session_ttl_seconds
raw_project_id = payload.get("project_id")
if not raw_project_id:
return None
project_id = UUID(str(raw_project_id))
if project_id != PUBLIC_DEMO_PROJECT_ID:
return None
if expires_at - issued_at > max_ttl:
return None
return AuthPrincipal(
username=username,
expires_at=expires_at,
session_id=session_id,
role=role,
project_id=project_id,
)
except (TypeError, ValueError, json.JSONDecodeError, UnicodeDecodeError):
return None
@classmethod
def retry_after_seconds(cls, key: str, *, now: float | None = None) -> int:
current = time.monotonic() if now is None else now
with cls._failure_lock:
attempts = cls._failures.setdefault(key, deque())
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
attempts.popleft()
if len(attempts) < cls.MAX_FAILURES:
if not attempts:
cls._failures.pop(key, None)
return 0
return max(1, int(cls.FAILURE_WINDOW_SECONDS - (current - attempts[0])))
@classmethod
def record_failure(cls, key: str, *, now: float | None = None) -> None:
current = time.monotonic() if now is None else now
with cls._failure_lock:
attempts = cls._failures.setdefault(key, deque())
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
attempts.popleft()
attempts.append(current)
@classmethod
def clear_failures(cls, key: str) -> None:
with cls._failure_lock:
cls._failures.pop(key, None)
@classmethod
def consume_guest_request(
cls,
key: str,
*,
max_requests: int,
window_seconds: int = 60,
now: float | None = None,
) -> int:
"""Record a guest action and return Retry-After seconds when limited."""
current = time.monotonic() if now is None else now
with cls._guest_request_lock:
attempts = cls._guest_requests.setdefault(key, deque())
while attempts and current - attempts[0] >= window_seconds:
attempts.popleft()
if len(attempts) >= max_requests:
return max(1, int(window_seconds - (current - attempts[0])))
attempts.append(current)
return 0
@classmethod
def try_acquire_guest_compute(cls, *, max_concurrency: int) -> bool:
with cls._guest_compute_lock:
if cls._active_guest_compute >= max_concurrency:
return False
cls._active_guest_compute += 1
return True
@classmethod
def release_guest_compute(cls) -> None:
with cls._guest_compute_lock:
cls._active_guest_compute = max(0, cls._active_guest_compute - 1)
@@ -0,0 +1,207 @@
from __future__ import annotations
import base64
import hashlib
import json
import secrets
from typing import Any
from urllib.error import HTTPError
from urllib.parse import urlencode, urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
import jwt
from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
from app.core.config import Settings
MAX_OIDC_JSON_BYTES = 1_048_576
class _RejectRedirects(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ANN201
return None
class AuthentikOidcService:
def __init__(self, settings: Settings):
self.settings = settings
self.issuer = (settings.authentik_issuer or "").rstrip("/")
self.serializer = URLSafeTimedSerializer(
settings.auth_session_secret or "",
salt="geointel-authentik-v1",
)
@property
def enabled(self) -> bool:
return bool(
self.issuer
and self.settings.authentik_client_id
and self.settings.authentik_client_secret
and self.settings.authentik_allowed_email
)
@property
def redirect_uri(self) -> str:
return (
f"{self.settings.public_base_url.rstrip('/')}"
f"{self.settings.api_prefix}/auth/authentik/callback"
)
@staticmethod
def _origin(url: str) -> tuple[str, str, int]:
parsed = urlsplit(url)
if parsed.scheme != "https" or not parsed.hostname:
raise ValueError("OIDC URLs must use absolute HTTPS URLs")
return parsed.scheme, parsed.hostname.casefold(), parsed.port or 443
def _validate_endpoint(self, url: str) -> str:
parsed = urlsplit(url)
if (
self._origin(url) != self._origin(self.issuer)
or parsed.username
or parsed.password
or parsed.fragment
):
raise ValueError("OIDC endpoint is outside the configured issuer origin")
return url
def _fetch_json(
self,
url: str,
data: dict[str, str] | None = None,
) -> dict[str, Any]:
self._validate_endpoint(url)
encoded = urlencode(data).encode("utf-8") if data is not None else None
headers = {"Accept": "application/json"}
if encoded is not None:
headers["Content-Type"] = "application/x-www-form-urlencoded"
request = Request(url, data=encoded, headers=headers)
try:
with build_opener(_RejectRedirects()).open(request, timeout=10) as response:
declared_length = response.headers.get("Content-Length")
if declared_length and int(declared_length) > MAX_OIDC_JSON_BYTES:
raise ValueError("OIDC response exceeds the configured size limit")
raw = response.read(MAX_OIDC_JSON_BYTES + 1)
except HTTPError as exc:
raise ValueError("OIDC endpoint returned an HTTP error or redirect") from exc
if len(raw) > MAX_OIDC_JSON_BYTES:
raise ValueError("OIDC response exceeds the configured size limit")
payload = json.loads(raw)
if not isinstance(payload, dict):
raise ValueError("OIDC endpoint did not return a JSON object")
return payload
def _discovery(self) -> dict[str, Any]:
document = self._fetch_json(
f"{self.issuer}/.well-known/openid-configuration"
)
if str(document.get("issuer", "")).rstrip("/") != self.issuer:
raise ValueError("OIDC issuer mismatch")
for key in ("authorization_endpoint", "token_endpoint", "jwks_uri"):
endpoint = document.get(key)
if not isinstance(endpoint, str):
raise ValueError(f"OIDC discovery is missing {key}")
self._validate_endpoint(endpoint)
return document
def start(self) -> tuple[str, str]:
if not self.enabled:
raise ValueError("Authentik is not configured")
state = secrets.token_urlsafe(32)
nonce = secrets.token_urlsafe(32)
verifier = secrets.token_urlsafe(48)
flow = self.serializer.dumps(
{"state": state, "nonce": nonce, "verifier": verifier}
)
challenge = (
base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
.rstrip(b"=")
.decode()
)
discovery = self._discovery()
query = urlencode(
{
"client_id": self.settings.authentik_client_id,
"redirect_uri": self.redirect_uri,
"response_type": "code",
"scope": "openid email profile",
"state": state,
"nonce": nonce,
"code_challenge": challenge,
"code_challenge_method": "S256",
}
)
return f"{discovery['authorization_endpoint']}?{query}", flow
def finish(self, *, code: str, state: str, flow_cookie: str) -> dict[str, Any]:
if not self.enabled or not code:
raise ValueError("OIDC flow is incomplete")
try:
flow = self.serializer.loads(flow_cookie, max_age=600)
except (BadSignature, SignatureExpired) as exc:
raise ValueError("Invalid OIDC flow") from exc
if not isinstance(flow, dict):
raise ValueError("Invalid OIDC flow payload")
if not state or not secrets.compare_digest(state, str(flow.get("state", ""))):
raise ValueError("OIDC state mismatch")
verifier = str(flow.get("verifier", ""))
nonce = str(flow.get("nonce", ""))
if not verifier or not nonce:
raise ValueError("OIDC flow payload is incomplete")
discovery = self._discovery()
token_response = self._fetch_json(
str(discovery["token_endpoint"]),
{
"grant_type": "authorization_code",
"code": code,
"redirect_uri": self.redirect_uri,
"client_id": self.settings.authentik_client_id or "",
"client_secret": self.settings.authentik_client_secret or "",
"code_verifier": verifier,
},
)
token = str(token_response.get("id_token", ""))
if not token:
raise ValueError("OIDC token response has no ID token")
header = jwt.get_unverified_header(token)
if header.get("alg") != "RS256" or not header.get("kid"):
raise ValueError("OIDC ID token uses an unsupported signing header")
jwks = self._fetch_json(str(discovery["jwks_uri"]))
matching_keys = [
key
for key in jwks.get("keys", [])
if isinstance(key, dict) and key.get("kid") == header["kid"]
]
if len(matching_keys) != 1:
raise ValueError("OIDC signing key is missing or ambiguous")
signing_key = jwt.PyJWK.from_dict(matching_keys[0]).key
claims = jwt.decode(
token,
signing_key,
algorithms=["RS256"],
audience=self.settings.authentik_client_id,
issuer=discovery["issuer"],
options={
"require": [
"exp",
"iat",
"iss",
"aud",
"sub",
"nonce",
"email",
"email_verified",
]
},
)
if not secrets.compare_digest(str(claims.get("nonce", "")), nonce):
raise ValueError("OIDC nonce mismatch")
email = str(claims.get("email", "")).strip().casefold()
allowed = str(self.settings.authentik_allowed_email or "").strip().casefold()
if claims.get("email_verified") is not True or not secrets.compare_digest(
email, allowed
):
raise ValueError("OIDC identity is not authorized")
return claims
@@ -0,0 +1,958 @@
from __future__ import annotations
from datetime import UTC, datetime
import hashlib
import json
import math
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
from shapely.geometry import Point, box, mapping
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.outbound_request_guard import guarded_opener
from app.models import Area, Dataset, DatasetVersion, Project
from app.schemas.bathymetry import (
BathymetryPartitionFinalizeRequest,
BathymetryPartitionFinalizationResult,
BathymetryProfileAcquireRequest,
BathymetryProfileAcquisitionResult,
BathymetrySourceRead,
)
from app.services.dataset_service import DatasetService
class BathymetryProfileAcquisitionService:
PROVIDER = "vmm_vha_bathymetry_profiles"
SOURCE_VERSION = "VHA digitale atlas ArcGIS MapServer"
PROFILE_OUT_FIELDS = (
"OBJECTID,vhag,atlaspunt,opg_kruinb,opg_vloerb,d_opmeti,"
"hyperlink,bron,kunstwerkid,opg_diepte"
)
ATTRIBUTION = "Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas"
LICENSE_NOTE = "Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata."
LIMITATION = (
"Dwarsprofielen zijn historische puntmetingen met bronafhankelijke meetdatum en verticale referentie. "
"Ze vormen geen continue actuele bodemkaart en ondersteunen zonder gelijktijdig waterpeil geen "
"gebiedsdekkend of actueel watervolume."
)
_SOURCES = (
{
"key": "vha_inland_profiles",
"display_name": "VHA dwarsprofielen binnenwater",
"owner": "Vlaamse Milieumaatschappij",
"authority_level": "authoritative",
"geographic_coverage": "Vlaanderen, puntlocaties op gekarteerde waterlopen",
"data_kind": "dwarsprofielpunten met meetvelden en brondocumenten",
"query_modes": ["bbox", "persisted_area"],
"vertical_reference": "document-specific; niet uniform als één peilreferentie te behandelen",
"horizontal_crs": "EPSG:4326",
"native_resolution": None,
"integration_status": "operational",
"acquisition_supported": True,
"configured": True,
"service_url": "https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/vlaamse-hydrografische-atlas-waterlopen",
"attribution": ATTRIBUTION,
"license_note": LICENSE_NOTE,
"limitation_message": LIMITATION,
},
{
"key": "mdk_bcp_bathymetry",
"display_name": "Dieptemodel Belgisch Continentaal Plat",
"owner": "Agentschap Maritieme Dienstverlening en Kust",
"authority_level": "authoritative",
"geographic_coverage": "Belgisch Continentaal Plat en Noordzee",
"data_kind": "continu bathymetrisch raster",
"query_modes": ["wcs", "wmts", "bounded_raster"],
"vertical_reference": "LAT",
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
"native_resolution": "20 x 20 m",
"integration_status": "probe_only",
"acquisition_supported": False,
"configured": False,
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
"attribution": "Agentschap Maritieme Dienstverlening en Kust",
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
"limitation_message": (
"Alleen een read-only GetCapabilities-probe is beschikbaar. Rasteracquisitie blijft uit totdat "
"WCS, maritieme begrenzing, tegels, LAT-semantiek en servercertificaten live zijn gevalideerd."
),
},
{
"key": "spw_walloon_waterway_bathymetry",
"display_name": "Bathymétrie des voies navigables et lacs-réservoirs",
"owner": "Service public de Wallonie",
"authority_level": "authoritative",
"geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen",
"data_kind": "bodemhoogteraster en XYZ-puntenwolk",
"query_modes": ["operator_archive", "bounded_raster", "arcgis_map_service"],
"vertical_reference": "mDNG",
"horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden",
"native_resolution": "0,5 m",
"integration_status": "operational",
"acquisition_supported": True,
"configured": True,
"service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer",
"catalog_url": "https://geoportail.wallonie.be/catalogue/0a544b42-0b30-4c8e-85e7-38149b99eae0.html",
"attribution": "Service public de Wallonie",
"license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.",
"limitation_message": (
"De gepinde officiële release kan begrensd als raster worden geïmporteerd via de operator. "
"Dekking verschilt per vaarweg; de waarden zijn bodemhoogtes in mDNG uit 2019-2022, "
"zonder stilzwijgende datumconversie of afleiding van actuele waterdiepte."
),
},
{
"key": "port_antwerp_bathymetry",
"display_name": "Havenbathymetrie Antwerpen-Brugge",
"owner": "Port of Antwerp-Bruges",
"authority_level": "contextual",
"geographic_coverage": "Gepubliceerde havenzones en meetcampagnes",
"data_kind": "periodieke peilingen",
"query_modes": ["catalog"],
"vertical_reference": "product-specific",
"horizontal_crs": "product-specific",
"native_resolution": None,
"integration_status": "catalog_only",
"acquisition_supported": False,
"configured": False,
"service_url": None,
"catalog_url": "https://data.gov.be/nl/datasets",
"attribution": "Port of Antwerp-Bruges",
"license_note": "Per publicatie te verifiëren.",
"limitation_message": (
"Alleen als cataloguskandidaat geregistreerd; er is nog geen stabiel, publiek en machineleesbaar "
"acquisitiecontract in GeoIntel gevalideerd."
),
},
)
@staticmethod
def list_sources(settings=None) -> list[dict[str, Any]]:
from app.core.config import get_settings
resolved_settings = settings or get_settings()
items: list[dict[str, Any]] = []
for source in BathymetryProfileAcquisitionService._SOURCES:
item = dict(source)
if item["key"] == "mdk_bcp_bathymetry":
mdk_configured = bool(
resolved_settings.mdk_bathymetry_acquisition_enabled
and (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
)
item["acquisition_supported"] = True
item["configured"] = mdk_configured
if mdk_configured:
item["integration_status"] = "operational"
item["limitation_message"] = (
"Begrensde WCS-acquisitie is expliciet ingeschakeld en draait alleen wanneer de "
"live readiness-probe bereikbaar is en het geconfigureerde coverage-id door de "
"capabilities wordt geadverteerd. Dieptes blijven LAT-gerefereerd; watervolume "
"blijft zonder compatibel wateroppervlak niet ondersteund."
)
else:
item["limitation_message"] = (
"Begrensde WCS-acquisitie bestaat maar staat uit. Zet "
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true en configureer MDK_BATHYMETRY_COVERAGE_ID "
"pas nadat de readiness-probe live 'reachable' rapporteert. Er wordt nooit "
"onbeveiligd of ongevalideerd gedownload."
)
items.append(item)
return [BathymetrySourceRead(**item).model_dump() for item in items]
@staticmethod
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
bbox = payload.bbox
if bbox.crs.upper() != "EPSG:4326":
raise AppError(
code="BATHYMETRY_INVALID_CRS",
message="Bathymetry profile acquisition requires EPSG:4326",
status_code=400,
)
values = (bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
if not all(math.isfinite(value) for value in values):
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box values must be finite", status_code=400)
if bbox.min_x >= bbox.max_x or bbox.min_y >= bbox.max_y:
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box has no area", status_code=400)
if bbox.min_x < -180 or bbox.max_x > 180 or bbox.min_y < -90 or bbox.max_y > 90:
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400)
return values
@staticmethod
def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_values: tuple[float, float, float, float]):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
selection = box(*bbox_values)
if area_id is None:
return selection
area = db.get(Area, area_id)
if area is None:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
area_geometry = area.geometry if hasattr(area.geometry, "__geo_interface__") else to_shape(area.geometry)
intersection = area_geometry.intersection(selection)
if intersection.is_empty:
raise AppError(
code="BATHYMETRY_SCOPE_EMPTY",
message="The requested bounding box does not intersect the selected area",
status_code=400,
)
return intersection
@staticmethod
def _query_url(base_url: str, parameters: dict[str, Any]) -> str:
return f"{base_url}?{urlencode(parameters)}"
@staticmethod
def _fetch_json(
url: str,
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[dict[str, Any], str]:
request = Request(
url,
headers={
"Accept": "application/json",
"User-Agent": "GeoIntel/1.0 bathymetry-profile-acquisition",
},
)
try:
with (opener or guarded_opener(url))(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response:
limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024
content = response.read(limit + 1)
except HTTPError as exc:
raise AppError(
code="BATHYMETRY_PROVIDER_HTTP_ERROR",
message="VHA profile service returned an HTTP error",
details={"status_code": exc.code},
status_code=502,
) from exc
except (TimeoutError, URLError, OSError) as exc:
raise AppError(
code="BATHYMETRY_PROVIDER_UNAVAILABLE",
message="VHA profile service is unavailable",
status_code=502,
) from exc
if len(content) > limit:
raise AppError(
code="BATHYMETRY_PROVIDER_RESPONSE_TOO_LARGE",
message="VHA profile response exceeded the configured size limit",
status_code=502,
)
try:
payload = json.loads(content.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile service returned invalid JSON",
status_code=502,
) from exc
if not isinstance(payload, dict) or payload.get("error"):
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile service returned an ArcGIS error",
details={"provider_error": payload.get("error") if isinstance(payload, dict) else None},
status_code=502,
)
return payload, hashlib.sha256(content).hexdigest()
@staticmethod
def _base_spatial_parameters(bbox_values: tuple[float, float, float, float]) -> dict[str, str]:
return {
"f": "json",
"where": "1=1",
"geometry": ",".join(f"{value:.12g}" for value in bbox_values),
"geometryType": "esriGeometryEnvelope",
"inSR": "4326",
"outSR": "4326",
"spatialRel": "esriSpatialRelIntersects",
}
@staticmethod
def _unseen_records(
page_features: list[Any],
seen_object_ids: set[str],
) -> list[dict[str, Any]]:
"""Every page must bring records the earlier pages did not.
An ArcGIS layer without ``supportsPagination`` accepts ``resultOffset``
and ignores it, answering every page with the first one. Advancing the
offset by the page length still reaches the announced count, so the
completeness check below passed while the dataset held N copies of page
one — a silent substitution of the source data, which is the one thing
bounded acquisition exists to prevent.
"""
fresh: list[dict[str, Any]] = []
for item in page_features:
if not isinstance(item, dict):
continue
attributes = item.get("attributes")
object_id = attributes.get("OBJECTID") if isinstance(attributes, dict) else None
if object_id is None:
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile record has no OBJECTID, so pagination cannot be verified",
status_code=502,
)
key = str(object_id)
if key in seen_object_ids:
raise AppError(
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
message="VHA profile pagination repeated a record; the layer is not honouring resultOffset",
details={"object_id": key},
status_code=502,
)
seen_object_ids.add(key)
fresh.append(item)
return fresh
@staticmethod
def _fetch_profiles(
bbox_values: tuple[float, float, float, float],
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
base = settings.bathymetry_profiles_layer_url.rstrip("/") + "/query"
count_url = BathymetryProfileAcquisitionService._query_url(
base,
{
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
"returnCountOnly": "true",
"returnGeometry": "false",
},
)
count_payload, count_sha = BathymetryProfileAcquisitionService._fetch_json(count_url, settings, opener)
candidate_count = int(count_payload.get("count") or 0)
if candidate_count > settings.bathymetry_profiles_max_features:
raise AppError(
code="BATHYMETRY_SCOPE_TOO_LARGE",
message="The requested profile scope exceeds the configured feature limit; acquire smaller area partitions",
details={
"candidate_count": candidate_count,
"max_features": settings.bathymetry_profiles_max_features,
},
status_code=413,
)
features: list[dict[str, Any]] = []
response_hashes: list[str] = []
request_urls: list[str] = [count_url]
seen_object_ids: set[str] = set()
offset = 0
while offset < candidate_count:
if len(request_urls) > settings.bathymetry_profiles_max_pages:
raise AppError(
code="BATHYMETRY_SCOPE_TOO_LARGE",
message="VHA profile pagination exceeded the configured page limit; acquire smaller area partitions",
details={"max_pages": settings.bathymetry_profiles_max_pages},
status_code=422,
)
page_url = BathymetryProfileAcquisitionService._query_url(
base,
{
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
"outFields": BathymetryProfileAcquisitionService.PROFILE_OUT_FIELDS,
"returnGeometry": "true",
"orderByFields": "OBJECTID",
"resultOffset": str(offset),
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
},
)
page, page_sha = BathymetryProfileAcquisitionService._fetch_json(page_url, settings, opener)
page_features = page.get("features")
if not isinstance(page_features, list):
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile response does not contain a feature list",
status_code=502,
)
response_hashes.append(page_sha)
request_urls.append(page_url)
if not page_features:
break
features.extend(
BathymetryProfileAcquisitionService._unseen_records(page_features, seen_object_ids)
)
offset += len(page_features)
if len(features) != candidate_count:
raise AppError(
code="BATHYMETRY_PROVIDER_INCOMPLETE_RESPONSE",
message="VHA profile pagination did not return the announced number of records",
details={"expected": candidate_count, "received": len(features)},
status_code=502,
)
return features, {
"candidate_count": candidate_count,
"request_urls": request_urls,
"response_sha256": [count_sha, *response_hashes],
}
@staticmethod
def _fetch_watercourse_names(
vhag_codes: set[int],
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[dict[int, dict[str, str | None]], list[str], list[str]]:
if not vhag_codes:
return {}, [], []
base = settings.bathymetry_watercourse_layer_url.rstrip("/") + "/query"
names: dict[int, dict[str, str | None]] = {}
urls: list[str] = []
hashes: list[str] = []
ordered_codes = sorted(vhag_codes)
for start in range(0, len(ordered_codes), 100):
chunk = ordered_codes[start : start + 100]
offset = 0
seen_page_hashes: set[str] = set()
while True:
if len(seen_page_hashes) >= settings.bathymetry_profiles_max_pages:
raise AppError(
code="BATHYMETRY_SCOPE_TOO_LARGE",
message="VHA watercourse pagination exceeded the configured page limit",
details={"max_pages": settings.bathymetry_profiles_max_pages},
status_code=422,
)
url = BathymetryProfileAcquisitionService._query_url(
base,
{
"f": "json",
"where": f"\"wlasvl.vhag\" IN ({','.join(str(code) for code in chunk)})",
"outFields": "wlasvl.vhag,VHAG_TABEL.naam,VHAG_TABEL.namen",
"returnGeometry": "false",
"orderByFields": "wlasvl.vhag",
"resultOffset": str(offset),
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
},
)
payload, response_sha = BathymetryProfileAcquisitionService._fetch_json(url, settings, opener)
urls.append(url)
hashes.append(response_sha)
page_features = payload.get("features")
if not isinstance(page_features, list):
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA watercourse response does not contain a feature list",
status_code=502,
)
if response_sha in seen_page_hashes:
# The names themselves deduplicate by code, so a stuck
# provider produced no visible change while the loop, which
# ended only on exceededTransferLimit, kept requesting.
raise AppError(
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
message="VHA watercourse pagination returned the same page again",
status_code=502,
)
seen_page_hashes.add(response_sha)
for feature in page_features:
attributes = feature.get("attributes") if isinstance(feature, dict) else None
if not isinstance(attributes, dict):
continue
raw_code = attributes.get("wlasvl.vhag")
if raw_code is None:
continue
code = int(raw_code)
if code not in names:
names[code] = {
"name": attributes.get("VHAG_TABEL.naam"),
"alternative_names": attributes.get("VHAG_TABEL.namen"),
}
if not payload.get("exceededTransferLimit") or not page_features:
break
offset += len(page_features)
return names, urls, hashes
@staticmethod
def _date_from_arcgis(value: Any) -> str | None:
if not isinstance(value, (int, float)) or not math.isfinite(float(value)):
return None
try:
return datetime.fromtimestamp(float(value) / 1000.0, tz=UTC).date().isoformat()
except (OverflowError, OSError, ValueError):
return None
@staticmethod
def _numeric_or_none(value: Any) -> float | None:
if value is None:
return None
try:
normalized = float(value)
except (TypeError, ValueError):
return None
return normalized if math.isfinite(normalized) else None
@staticmethod
def _document_url(value: Any) -> str | None:
if not isinstance(value, str) or not value.strip():
return None
normalized = value.strip()
if normalized.startswith("http://vha.waterinfo.be/"):
normalized = "https://" + normalized[len("http://") :]
return normalized if normalized.startswith("https://vha.waterinfo.be/") else None
@staticmethod
def _normalize_features(
raw_features: list[dict[str, Any]],
scope_geometry,
watercourse_names: dict[int, dict[str, str | None]],
*,
partition_properties: dict[str, Any] | None = None,
) -> tuple[dict[str, Any], dict[str, Any]]:
normalized: list[dict[str, Any]] = []
dates: list[str] = []
watercourse_codes: set[int] = set()
document_count = 0
depth_count = 0
width_count = 0
for raw_feature in raw_features:
attributes = raw_feature.get("attributes")
geometry = raw_feature.get("geometry")
if not isinstance(attributes, dict) or not isinstance(geometry, dict):
continue
x = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("x"))
y = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("y"))
if x is None or y is None:
continue
point = Point(x, y)
if not scope_geometry.covers(point):
continue
raw_vhag = attributes.get("vhag")
vhag = int(raw_vhag) if isinstance(raw_vhag, (int, float)) else None
if vhag is not None:
watercourse_codes.add(vhag)
names = watercourse_names.get(vhag or -1, {})
measurement_date = BathymetryProfileAcquisitionService._date_from_arcgis(attributes.get("d_opmeti"))
if measurement_date:
dates.append(measurement_date)
document_url = BathymetryProfileAcquisitionService._document_url(attributes.get("hyperlink"))
depth = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_diepte"))
crown_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_kruinb"))
floor_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_vloerb"))
if document_url:
document_count += 1
if depth is not None:
depth_count += 1
if crown_width is not None or floor_width is not None:
width_count += 1
object_id = str(attributes.get("OBJECTID"))
normalized.append(
{
"type": "Feature",
"id": object_id,
"properties": {
"source_feature_id": object_id,
"provider_record_id": object_id,
"watercourse_vhag": vhag,
"watercourse_name": names.get("name") or (f"VHA-waterloop {vhag}" if vhag else "Onbekende waterloop"),
"watercourse_alternative_names": names.get("alternative_names"),
"profile_number": attributes.get("atlaspunt"),
"measurement_date": measurement_date,
"recorded_depth_m": depth,
"recorded_crown_width_m": crown_width,
"recorded_floor_width_m": floor_width,
"source_document_url": document_url,
"document_available": document_url is not None,
"structured_depth_available": depth is not None,
"source_code": attributes.get("bron"),
"structure_id": attributes.get("kunstwerkid"),
"provider": BathymetryProfileAcquisitionService.PROVIDER,
"measurement_semantics": "historical_cross_section_profile_point",
"vertical_reference": "document-specific",
**(partition_properties or {}),
},
"geometry": mapping(point),
}
)
return (
{
"type": "FeatureCollection",
"name": "vha_bathymetry_profiles",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": normalized,
},
{
"profile_count": len(normalized),
"document_count": document_count,
"structured_depth_count": depth_count,
"structured_width_count": width_count,
"watercourse_count": len(watercourse_codes),
"measurement_date_min": min(dates) if dates else None,
"measurement_date_max": max(dates) if dates else None,
},
)
@staticmethod
def _municipality_name(area: Area | None) -> str | None:
if area is None:
return None
normalized = str(area.name or "").strip()
prefix = "Gemeente "
if not normalized.casefold().startswith(prefix.casefold()):
return None
municipality = normalized[len(prefix) :].split(" - ", 1)[0].strip()
return municipality or None
@staticmethod
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
return (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.name == filename,
Dataset.source_name == BathymetryProfileAcquisitionService.PROVIDER,
Dataset.status == "ready",
)
.order_by(Dataset.created_at.desc())
.first()
)
@staticmethod
def _result(dataset: Dataset, *, reused: bool) -> dict[str, Any]:
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
return BathymetryProfileAcquisitionResult(
output_dataset_id=dataset.id,
reused=reused,
provider=BathymetryProfileAcquisitionService.PROVIDER,
profile_count=int(metadata.get("profile_count") or 0),
document_count=int(metadata.get("document_count") or 0),
structured_depth_count=int(metadata.get("structured_depth_count") or 0),
structured_width_count=int(metadata.get("structured_width_count") or 0),
watercourse_count=int(metadata.get("watercourse_count") or 0),
bbox_epsg4326=list(metadata.get("bbox_epsg4326") or []),
clipped_to_area_id=dataset.area_id,
measurement_date_min=metadata.get("measurement_date_min"),
measurement_date_max=metadata.get("measurement_date_max"),
attribution=BathymetryProfileAcquisitionService.ATTRIBUTION,
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
).model_dump(mode="json")
@staticmethod
def acquire(
db,
project_id: UUID,
payload: BathymetryProfileAcquireRequest,
*,
settings: Settings | None = None,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
if not resolved_settings.bathymetry_profiles_enabled:
raise AppError(
code="BATHYMETRY_NOT_CONFIGURED",
message="VHA bathymetry profile acquisition is disabled",
status_code=503,
)
bbox_values = BathymetryProfileAcquisitionService._validate_bbox(payload)
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
db, project_id, payload.area_id, bbox_values
)
area = db.get(Area, payload.area_id) if payload.area_id else None
municipality = BathymetryProfileAcquisitionService._municipality_name(area)
exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
request_identity = {
"provider": BathymetryProfileAcquisitionService.PROVIDER,
"bbox_epsg4326": list(exact_bbox),
"area_id": str(payload.area_id) if payload.area_id else None,
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
}
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
filename = f"vha_bathymetry_profiles_{request_hash[:12]}.geojson"
if not payload.force_refresh:
cached = BathymetryProfileAcquisitionService._cached_dataset(db, project_id, filename)
if cached is not None:
return BathymetryProfileAcquisitionService._result(cached, reused=True)
raw_features, fetch_provenance = BathymetryProfileAcquisitionService._fetch_profiles(
exact_bbox, resolved_settings, opener
)
vhag_codes = {
int(feature["attributes"]["vhag"])
for feature in raw_features
if isinstance(feature.get("attributes"), dict)
and isinstance(feature["attributes"].get("vhag"), (int, float))
}
names, name_urls, name_hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names(
vhag_codes, resolved_settings, opener
)
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
raw_features,
scope_geometry,
names,
partition_properties={
"partition_area_id": str(area.id),
"partition_area_name": area.name,
**({"municipality": municipality} if municipality else {}),
}
if area
else None,
)
if summary["profile_count"] == 0:
raise AppError(
code="BATHYMETRY_NO_PROFILES",
message="No VHA bathymetry profiles intersect the requested area",
status_code=404,
)
artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
acquired_at = datetime.now(UTC)
source_metadata = {
"provider": BathymetryProfileAcquisitionService.PROVIDER,
"service": "ArcGIS MapServer",
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
"theme": "bathymetry",
"layer_name": "bathymetry_profiles",
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
"partition_area_id": str(area.id) if area else None,
"partition_area_name": area.name if area else None,
"municipality": municipality,
"partitioned_source_audit": False,
"regional_partitions_complete": False,
"bbox_epsg4326": list(exact_bbox),
**summary,
"selection_aggregation": {
"metric_key": "profile_count",
"method": "feature_count",
"label": "Dwarsprofielen",
"unit": "profielen",
},
"selection_metrics": [
{
"metric_key": "recorded_depth_mean_m",
"method": "mean",
"property": "recorded_depth_m",
"label": "Gemiddelde geregistreerde diepte",
"unit": "m",
"warning": "Alleen profielen met een gestructureerde dieptewaarde; meetdata kunnen verschillen.",
},
{
"metric_key": "recorded_depth_min_m",
"method": "min",
"property": "recorded_depth_m",
"label": "Kleinste geregistreerde diepte",
"unit": "m",
},
{
"metric_key": "recorded_depth_max_m",
"method": "max",
"property": "recorded_depth_m",
"label": "Grootste geregistreerde diepte",
"unit": "m",
},
{
"metric_key": "recorded_crown_width_mean_m",
"method": "mean",
"property": "recorded_crown_width_m",
"label": "Gemiddelde geregistreerde kruinbreedte",
"unit": "m",
},
{
"metric_key": "recorded_floor_width_mean_m",
"method": "mean",
"property": "recorded_floor_width_m",
"label": "Gemiddelde geregistreerde vloerbreedte",
"unit": "m",
},
],
"attribution": BathymetryProfileAcquisitionService.ATTRIBUTION,
"license_note": BathymetryProfileAcquisitionService.LICENSE_NOTE,
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
"volume_supported": False,
}
dataset = DatasetService.import_vector_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=artifact,
source="VHA digitale atlas dwarsprofielen",
source_name=BathymetryProfileAcquisitionService.PROVIDER,
dataset_role="reference",
reference_layer_name="bathymetry_profiles",
source_version=BathymetryProfileAcquisitionService.SOURCE_VERSION,
source_metadata=source_metadata,
provenance_metadata={
"acquisition": "explicit_bounded_arcgis_feature_query",
"acquired_at": acquired_at.isoformat(),
"request_hash": request_hash,
"profile_query_urls": fetch_provenance["request_urls"],
"watercourse_name_query_urls": name_urls,
"response_sha256": [*fetch_provenance["response_sha256"], *name_hashes],
"artifact_sha256": hashlib.sha256(artifact).hexdigest(),
"candidate_count": fetch_provenance["candidate_count"],
"exact_profile_count": summary["profile_count"],
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
"scope_geometry_type": scope_geometry.geom_type,
"vertical_reference": "document-specific",
"bathymetric_surface_available": False,
"water_surface_level_available": False,
"water_volume_available": False,
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
},
)
persisted = db.get(Dataset, dataset.id)
return BathymetryProfileAcquisitionService._result(persisted, reused=False)
@staticmethod
def finalize_partitions(
db,
project_id: UUID,
payload: BathymetryPartitionFinalizeRequest,
) -> dict[str, Any]:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
expected_area_ids = set(payload.expected_area_ids)
dataset_ids = set(payload.dataset_ids)
no_profile_area_ids = set(payload.no_profile_area_ids)
areas: dict[UUID, Area] = {}
for area_id in expected_area_ids:
area = db.get(Area, area_id)
if area is None or area.project_id != project_id:
raise AppError(
code="BATHYMETRY_PARTITION_AREA_INVALID",
message="Every expected partition Area must belong to the project",
details={"area_id": str(area_id)},
status_code=400,
)
if BathymetryProfileAcquisitionService._municipality_name(area) is None:
raise AppError(
code="BATHYMETRY_PARTITION_AREA_INVALID",
message="Bathymetry partitions must use persisted municipality Areas",
details={"area_id": str(area_id), "area_name": area.name},
status_code=400,
)
areas[area_id] = area
if not no_profile_area_ids.issubset(expected_area_ids):
raise AppError(
code="BATHYMETRY_PARTITION_MANIFEST_INVALID",
message="No-profile partitions must be part of the expected Area set",
status_code=400,
)
datasets: list[Dataset] = []
data_area_ids: set[UUID] = set()
for dataset_id in payload.dataset_ids:
dataset = db.get(Dataset, dataset_id)
if (
dataset is None
or dataset.project_id != project_id
or dataset.source_name != BathymetryProfileAcquisitionService.PROVIDER
or dataset.status != "ready"
or dataset.area_id not in expected_area_ids
):
raise AppError(
code="BATHYMETRY_PARTITION_DATASET_INVALID",
message="Every partition Dataset must be a ready VHA profile Dataset scoped to an expected Area",
details={"dataset_id": str(dataset_id)},
status_code=400,
)
if dataset.area_id in data_area_ids:
raise AppError(
code="BATHYMETRY_PARTITION_DATASET_DUPLICATE",
message="A complete manifest may reference only one profile Dataset per Area",
details={"area_id": str(dataset.area_id)},
status_code=400,
)
data_area_ids.add(dataset.area_id)
datasets.append(dataset)
accounted_area_ids = data_area_ids.union(no_profile_area_ids)
if accounted_area_ids != expected_area_ids:
raise AppError(
code="BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE",
message="Every expected Area must have one ready Dataset or an explicit no-profile result",
details={
"missing_area_ids": sorted(str(value) for value in expected_area_ids - accounted_area_ids),
"unexpected_area_ids": sorted(str(value) for value in accounted_area_ids - expected_area_ids),
},
status_code=400,
)
profile_count = 0
document_count = 0
structured_depth_count = 0
dates_min: list[str] = []
dates_max: list[str] = []
shared_metadata = {
"partition_scope_key": payload.partition_scope_key,
"partition_count": len(expected_area_ids),
"data_partition_count": len(datasets),
"no_profile_partition_count": len(no_profile_area_ids),
"partition_manifest_sha256": payload.manifest_sha256,
"partition_manifest_observed_at": payload.observed_at.isoformat(),
"partitioned_source_audit": True,
"regional_partitions_complete": True,
}
shared_provenance = {
"partition_manifest_sha256": payload.manifest_sha256,
"partition_manifest_observed_at": payload.observed_at.isoformat(),
"partition_scope_key": payload.partition_scope_key,
"regional_partitions_complete": True,
"no_profile_area_ids": sorted(str(value) for value in no_profile_area_ids),
}
for dataset in datasets:
source_metadata = dict(dataset.source_metadata or {})
provenance_metadata = dict(dataset.provenance_metadata or {})
profile_count += int(source_metadata.get("profile_count") or 0)
document_count += int(source_metadata.get("document_count") or 0)
structured_depth_count += int(source_metadata.get("structured_depth_count") or 0)
if source_metadata.get("measurement_date_min"):
dates_min.append(str(source_metadata["measurement_date_min"]))
if source_metadata.get("measurement_date_max"):
dates_max.append(str(source_metadata["measurement_date_max"]))
area = areas[dataset.area_id]
source_metadata.update(
{
**shared_metadata,
"coverage_scope": payload.partition_scope_key,
"partition_area_id": str(area.id),
"partition_area_name": area.name,
"municipality": BathymetryProfileAcquisitionService._municipality_name(area),
}
)
provenance_metadata.update(shared_provenance)
dataset.source_metadata = source_metadata
dataset.provenance_metadata = provenance_metadata
if dataset_ids:
versions = (
db.query(DatasetVersion)
.filter(DatasetVersion.dataset_id.in_(dataset_ids))
.all()
)
for version in versions:
version.source_metadata = dict(
next(dataset.source_metadata for dataset in datasets if dataset.id == version.dataset_id)
)
version.provenance_metadata = dict(
next(dataset.provenance_metadata for dataset in datasets if dataset.id == version.dataset_id)
)
db.commit()
return BathymetryPartitionFinalizationResult(
partition_scope_key=payload.partition_scope_key,
regional_partitions_complete=True,
partition_count=len(expected_area_ids),
data_partition_count=len(datasets),
no_profile_partition_count=len(no_profile_area_ids),
profile_count=profile_count,
document_count=document_count,
structured_depth_count=structured_depth_count,
measurement_date_min=min(dates_min) if dates_min else None,
measurement_date_max=max(dates_max) if dates_max else None,
dataset_ids=payload.dataset_ids,
manifest_sha256=payload.manifest_sha256,
observed_at=payload.observed_at,
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
).model_dump(mode="json")
@@ -0,0 +1,366 @@
from __future__ import annotations
import io
import math
from datetime import UTC, datetime
from pathlib import Path
from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import box, mapping
from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.raster_cell_selection import select_cells
from app.models import Area, Dataset
from app.schemas.bathymetry import (
BathymetryRasterMetric,
BathymetryRasterSelectionRequest,
BathymetryRasterSelectionResponse,
BathymetryRasterSelectionSummary,
)
class BathymetryRasterAnalysisService:
SOURCE_NAME = "spw_bathymetry"
PRODUCT_KEY = "spw_bathymetry_50cm_mdng"
UNSUPPORTED_METRICS = [
"current_water_depth_m",
"water_volume_m3",
"vertical_datum_conversion",
]
LIMITATION = (
"De rasterwaarden zijn waterbodemhoogtes in mDNG uit een samengestelde SPW-opmeting "
"(2019-2022). Zonder een gelijktijdig waterpeil zijn actuele waterdiepte en watervolume "
"niet berekenbaar. mDNG wordt niet stilzwijgend naar TAW, LAT of een ander verticaal datum omgezet."
)
@staticmethod
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster" or dataset.source_name != BathymetryRasterAnalysisService.SOURCE_NAME:
raise AppError(
code="INVALID_BATHYMETRY_RASTER_DATASET",
message="Bathymetry analysis requires a governed SPW bathymetry raster dataset",
status_code=400,
)
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
raise AppError(
code="DATASET_FILE_MISSING",
message="Persisted bathymetry raster file is unavailable",
status_code=404,
)
return dataset
@staticmethod
def _metadata(dataset: Dataset) -> dict:
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
if (
metadata.get("product_key") != BathymetryRasterAnalysisService.PRODUCT_KEY
or metadata.get("theme") != "bathymetry"
or metadata.get("value_semantics") != "bed_elevation"
or metadata.get("vertical_reference") != "mDNG"
or metadata.get("source_crs") != "EPSG:3812"
):
raise AppError(
code="INVALID_BATHYMETRY_RASTER_METADATA",
message="Bathymetry raster provenance or value semantics are incomplete",
status_code=409,
)
return metadata
@staticmethod
def _selection_geometry(db, project_id: UUID, payload: BathymetryRasterSelectionRequest):
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
if payload.area_id is None:
return selection
area = db.get(Area, payload.area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(
code="INVALID_DATASET_SCOPE",
message="Area does not belong to this project",
status_code=400,
)
selection = selection.intersection(to_shape(area.geometry))
if selection.is_empty or selection.area <= 0:
raise AppError(
code="BATHYMETRY_SELECTION_OUTSIDE_AREA",
message="Selection does not overlap the selected work area",
status_code=422,
)
return selection
@staticmethod
def analyze(
db,
project_id: UUID,
dataset_id: UUID,
payload: BathymetryRasterSelectionRequest,
*,
settings: Settings | None = None,
) -> dict:
resolved_settings = settings or get_settings()
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
source_metadata = BathymetryRasterAnalysisService._metadata(dataset)
selection_4326 = BathymetryRasterAnalysisService._selection_geometry(db, project_id, payload)
try:
import numpy as np
import rasterio
from rasterio.mask import mask
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Rasterio and numpy are required for bathymetry analysis",
status_code=503,
) from exc
try:
with rasterio.open(dataset.storage_path) as source:
if source.crs is None or source.crs.to_epsg() != 3812:
raise AppError(
code="INVALID_DATASET_CRS",
message="SPW bathymetry raster CRS must be EPSG:3812",
status_code=409,
)
if source.count != 1:
raise AppError(
code="INVALID_BATHYMETRY_RASTER_BANDS",
message="SPW bathymetry requires one bed-elevation band",
status_code=409,
)
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
selection_metric = shapely_transform(transformer.transform, selection_4326)
analysis_geometry = selection_metric.intersection(box(*source.bounds))
if analysis_geometry.is_empty or analysis_geometry.area <= 0:
raise AppError(
code="BATHYMETRY_SELECTION_OUTSIDE_DATASET",
message="Selection does not overlap the persisted bathymetry raster",
status_code=422,
)
min_x, min_y, max_x, max_y = analysis_geometry.bounds
expected_cells = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil(
(max_y - min_y) / abs(source.res[1])
)
if expected_cells > resolved_settings.bathymetry_raster_max_pixels:
raise AppError(
code="BATHYMETRY_SELECTION_TOO_LARGE",
message="Bathymetry analysis exceeds the configured raster cell limit",
details={
"pixel_count": expected_cells,
"max_pixels": resolved_settings.bathymetry_raster_max_pixels,
},
status_code=422,
)
# ``all_touched`` keeps the values of cells the selection only
# clips, so a selection finer than one cell still has data to
# read. Which of those cells actually count is decided by
# ``select_cells`` below, so the normal result is unchanged.
clipped, clipped_transform = mask(
source,
[mapping(analysis_geometry)],
crop=True,
filled=False,
indexes=[1],
all_touched=True,
)
band = np.ma.asarray(clipped[0], dtype="float64")
raw = band.filled(np.nan)
cell_selection = select_cells(
analysis_geometry,
out_shape=band.shape,
transform=clipped_transform,
cell_area_m2=abs(float(source.res[0])) * abs(float(source.res[1])),
)
selected_cells = cell_selection.mask
valid_cells = selected_cells & ~np.ma.getmaskarray(band) & np.isfinite(raw)
if source.nodata is not None:
valid_cells &= ~np.isclose(raw, float(source.nodata))
values = raw[valid_cells]
if values.size == 0:
raise AppError(
code="BATHYMETRY_NO_VALID_DATA",
message="No surveyed waterbed cells occur in this selection",
status_code=422,
)
resolution_x = abs(float(source.res[0]))
resolution_y = abs(float(source.res[1]))
cell_area_m2 = resolution_x * resolution_y
except AppError:
raise
except Exception as exc:
raise AppError(
code="BATHYMETRY_ANALYSIS_FAILED",
message="The persisted bathymetry raster could not be analysed",
details={"reason": str(exc)},
status_code=500,
) from exc
def metric(key: str, label: str, value: float, unit: str, method: str) -> BathymetryRasterMetric:
return BathymetryRasterMetric(
metric_key=key,
metric_label=label,
metric_value=round(float(value), 4),
metric_unit=unit,
aggregation_method=method,
)
selected_cell_count = int(selected_cells.sum())
valid_cell_count = int(values.size)
vertical_unit = str(source_metadata["vertical_reference"])
coverage_ratio = valid_cell_count / max(1, selected_cell_count)
metrics = [
metric(
"bed_elevation_mean_m",
"Gemiddelde waterbodemhoogte",
values.mean(),
f"m {vertical_unit}",
"mean_valid_source_cells",
),
metric(
"bed_elevation_min_m",
"Laagste waterbodemhoogte",
values.min(),
f"m {vertical_unit}",
"minimum_valid_source_cells",
),
metric(
"bed_elevation_max_m",
"Hoogste waterbodemhoogte",
values.max(),
f"m {vertical_unit}",
"maximum_valid_source_cells",
),
metric(
"bed_elevation_p10_m",
"10e percentiel waterbodemhoogte",
np.percentile(values, 10),
f"m {vertical_unit}",
"percentile_10_valid_source_cells",
),
metric(
"bed_elevation_p90_m",
"90e percentiel waterbodemhoogte",
np.percentile(values, 90),
f"m {vertical_unit}",
"percentile_90_valid_source_cells",
),
metric(
"surveyed_bed_surface_ha",
"Oppervlakte met gemeten waterbodem",
valid_cell_count * cell_area_m2 / 10_000.0,
"ha",
"valid_source_cells_times_cell_area",
),
metric(
"bathymetry_coverage_pct",
"Dekking waterbodemmeting",
coverage_ratio * 100.0,
"%",
"valid_source_cells_divided_by_selected_cells",
),
]
primary = metrics[0]
response = BathymetryRasterSelectionResponse(
dataset_id=dataset.id,
product_key=BathymetryRasterAnalysisService.PRODUCT_KEY,
selection_bbox=payload.bbox,
selection_area_id=payload.area_id,
selected_cell_count=selected_cell_count,
valid_cell_count=valid_cell_count,
coverage_ratio=round(coverage_ratio, 6),
cell_selection_warning=cell_selection.warning,
resolution_m=round(max(resolution_x, resolution_y), 4),
vertical_reference=vertical_unit,
survey_period=str(source_metadata.get("survey_period") or "2019-2022"),
summary=BathymetryRasterSelectionSummary(
metric_label=primary.metric_label,
metric_value=primary.metric_value,
metric_unit=primary.metric_unit,
aggregation_method=primary.aggregation_method,
primary_metric_key=primary.metric_key,
metrics=metrics,
),
unsupported_metrics=BathymetryRasterAnalysisService.UNSUPPORTED_METRICS,
limitation_message=BathymetryRasterAnalysisService.LIMITATION,
generated_at=datetime.now(UTC).isoformat(),
)
return response.model_dump(mode="json")
@staticmethod
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
BathymetryRasterAnalysisService._metadata(dataset)
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Rasterio, numpy and Pillow are required for bathymetry rendering",
status_code=503,
) from exc
try:
with rasterio.open(dataset.storage_path) as source:
scale = min(1.0, max_dimension / max(source.width, source.height))
width = max(1, round(source.width * scale))
height = max(1, round(source.height * scale))
data = source.read(
1,
out_shape=(height, width),
masked=True,
resampling=Resampling.bilinear,
)
values = np.asarray(data.filled(np.nan), dtype="float64")
valid = np.isfinite(values) & ~np.ma.getmaskarray(data)
if source.nodata is not None:
valid &= ~np.isclose(values, float(source.nodata))
if not valid.any():
raise AppError(
code="BATHYMETRY_NO_VALID_DATA",
message="Bathymetry raster contains no renderable cells",
status_code=422,
)
low, high = np.percentile(values[valid], [2, 98])
if high <= low:
high = low + 1.0
normalized = np.clip((values - low) / (high - low), 0.0, 1.0)
normalized = np.where(valid, normalized, 0.0)
stops = np.asarray([0.0, 0.35, 0.7, 1.0])
colors = np.asarray(
[
[8, 47, 73],
[15, 118, 140],
[103, 190, 170],
[236, 224, 163],
],
dtype="float64",
)
rgba = np.zeros((height, width, 4), dtype="uint8")
for channel in range(3):
rgba[:, :, channel] = np.interp(
normalized,
stops,
colors[:, channel],
).astype("uint8")
rgba[:, :, 3] = np.where(valid, 220, 0).astype("uint8")
output = io.BytesIO()
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
return output.getvalue()
except AppError:
raise
except Exception as exc:
raise AppError(
code="BATHYMETRY_PREVIEW_FAILED",
message="The persisted bathymetry raster could not be rendered",
details={"reason": str(exc)},
status_code=500,
) from exc
@@ -0,0 +1,412 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from uuid import UUID
from geoalchemy2.shape import from_shape, to_shape
from shapely.geometry import mapping
from shapely.geometry.base import BaseGeometry
from shapely.strtree import STRtree
from shapely.validation import make_valid
from sqlalchemy import func
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Dataset, VectorFeature
from app.schemas.analysis import ChangeDetectionSummary
from app.services.vector_operations_service import VectorOperationsService
class ChangeDetectionService:
SUPPORTED_GEOMETRY_TYPES = {"Polygon", "MultiPolygon"}
@staticmethod
def compare_vector_datasets(
db: Session,
*,
project_id: UUID,
source_dataset_id: UUID,
target_dataset_id: UUID,
iou_threshold: float = 0.8,
include_unchanged: bool = True,
modified_threshold: float = 0.3,
bbox: dict[str, Any] | None = None,
area_id: UUID | None = None,
preview_limit: int = 2_000,
) -> ChangeDetectionSummary:
if source_dataset_id == target_dataset_id:
raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400)
if iou_threshold < 0 or iou_threshold > 1:
raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400)
if modified_threshold < 0 or modified_threshold > iou_threshold:
raise AppError(
code="INVALID_PARAMETERS",
message="modified_threshold must be between 0 and iou_threshold",
status_code=400,
)
source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source")
target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target")
selection_geometry = ChangeDetectionService._selection_geometry(db, project_id, bbox=bbox, area_id=area_id)
source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset, selection_geometry)
target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset, selection_geometry)
if not source_features:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422)
if not target_features:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422)
source_features = ChangeDetectionService.restrict_to_selection(source_features, selection_geometry, label="Source")
target_features = ChangeDetectionService.restrict_to_selection(target_features, selection_geometry, label="Target")
classified = ChangeDetectionService._classify_features(
source_features,
target_features,
iou_threshold=iou_threshold,
modified_threshold=modified_threshold,
)
buckets: dict[str, list[dict[str, Any]]] = {"added": [], "removed": [], "modified": [], "unchanged": []}
for item in classified:
buckets[item["change_type"]].append(
ChangeDetectionService._feature(
geometry=item["geometry"],
change_type=item["change_type"],
source_dataset_id=source_dataset_id,
target_dataset_id=target_dataset_id,
source_feature_id=item["source_feature_id"],
target_feature_id=item["target_feature_id"],
iou=item["iou"],
properties=item["properties"],
)
)
unchanged_count = len(buckets["unchanged"])
if not include_unchanged:
buckets["unchanged"] = []
geojson_features, preview_truncated = ChangeDetectionService.limit_preview(
buckets["added"] + buckets["removed"] + buckets["modified"] + buckets["unchanged"],
limit=preview_limit,
)
warnings = source_warnings + target_warnings
edge_count = sum(
1 for feature in source_features + target_features if feature.get("partially_covered")
)
if edge_count:
warnings.append(
f"{edge_count} objecten liggen deels buiten de selectie. Ze zijn volledig vergeleken, zodat de "
"selectierand zelf geen wijziging veroorzaakt."
)
if preview_truncated:
warnings.append(
f"De tellingen gelden voor de volledige selectie; de kaart toont maximaal {preview_limit} objecten, "
"wijzigingen eerst."
)
return ChangeDetectionSummary(
source_dataset_id=source_dataset_id,
target_dataset_id=target_dataset_id,
source_feature_count=len(source_features),
target_feature_count=len(target_features),
added_count=len(buckets["added"]),
removed_count=len(buckets["removed"]),
modified_count=len(buckets["modified"]),
unchanged_count=unchanged_count,
iou_threshold=iou_threshold,
modified_iou_threshold=modified_threshold,
selection_area_id=area_id,
preview_limit=preview_limit,
preview_truncated=preview_truncated,
warnings=warnings,
generated_at=datetime.now(timezone.utc),
geojson={"type": "FeatureCollection", "features": geojson_features},
)
@staticmethod
def _selection_geometry(
db: Session,
project_id: UUID,
*,
bbox: dict[str, Any] | None,
area_id: UUID | None,
) -> BaseGeometry | None:
"""Resolve the drawn rectangle against the named work area, if any."""
from app.models import Area
from shapely.geometry import box as shapely_box
selection = None
if bbox:
selection = shapely_box(
float(bbox["min_x"]), float(bbox["min_y"]), float(bbox["max_x"]), float(bbox["max_y"])
)
if area_id is None:
return selection
area = db.get(Area, area_id)
if area is None or area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
area_geometry = to_shape(area.geometry)
if selection is None:
return area_geometry
intersection = selection.intersection(area_geometry)
if intersection.is_empty or intersection.area <= 0:
raise AppError(
code="CHANGE_DETECTION_SELECTION_OUTSIDE_AREA",
message="Selection does not overlap the selected work area",
status_code=422,
)
return intersection
# Order the preview spends its budget in. An operator asking what changed
# is not helped by a cap filled with unchanged footprints.
PREVIEW_PRIORITY = {"modified": 0, "added": 1, "removed": 2, "unchanged": 3}
@staticmethod
def restrict_to_selection(
features: list[dict[str, Any]],
selection_geometry: BaseGeometry | None,
*,
label: str = "Dataset",
) -> list[dict[str, Any]]:
"""Keep the features a drawn selection reaches, and say which it cuts.
Geometry is deliberately *not* clipped. A change class describes a whole
object: comparing a clipped 2020 footprint against an unclipped 2024 one
would manufacture "modified" along the selection edge. Clipping is right
for an area metric and wrong for an identity comparison.
"""
if selection_geometry is None:
return features
kept: list[dict[str, Any]] = []
for feature in features:
geometry = feature["geometry"]
if not geometry.intersects(selection_geometry):
continue
kept.append({**feature, "partially_covered": not selection_geometry.covers(geometry)})
if not kept:
raise AppError(
code="CHANGE_DETECTION_SELECTION_EMPTY",
message=f"{label} dataset has no features inside this selection",
status_code=422,
)
return kept
@staticmethod
def limit_preview(
features: list[dict[str, Any]],
*,
limit: int,
) -> tuple[list[dict[str, Any]], bool]:
"""Cap the returned geometry without capping the counts.
``include_unchanged`` defaulted to true and nothing bounded the result,
so a regional comparison returned a FeatureCollection holding both
datasets in full. The counts describe the whole selection; the preview
describes what a map can usefully draw.
"""
if limit <= 0 or len(features) <= limit:
return features, False
ordered = sorted(
features,
key=lambda item: ChangeDetectionService.PREVIEW_PRIORITY.get(item["change_type"], 9),
)
return ordered[:limit], True
@staticmethod
def _classify_features(
source_features: list[dict[str, Any]],
target_features: list[dict[str, Any]],
*,
iou_threshold: float,
modified_threshold: float,
) -> list[dict[str, Any]]:
"""Pair source with target footprints and label how each one changed.
Matching is indexed rather than a full cross product: comparing two
municipal building layers is otherwise hundreds of millions of geometry
intersections. Sources are considered largest first so a big footprint
is not left over after a small neighbour claimed its counterpart.
"""
target_geometries = [feature["geometry"] for feature in target_features]
tree = STRtree(target_geometries) if target_geometries else None
claimed: set[int] = set()
classified: list[dict[str, Any]] = []
order = sorted(
range(len(source_features)),
key=lambda index: (-source_features[index]["geometry"].area, str(source_features[index]["feature_id"])),
)
for source_index in order:
source_feature = source_features[source_index]
geometry = source_feature["geometry"]
best_iou = 0.0
best_index: int | None = None
candidates = [] if tree is None else sorted(int(value) for value in tree.query(geometry))
for target_index in candidates:
if target_index in claimed:
continue
candidate_iou = ChangeDetectionService._iou(geometry, target_geometries[target_index])
if candidate_iou > best_iou:
best_iou = candidate_iou
best_index = target_index
if best_index is not None and best_iou >= iou_threshold:
claimed.add(best_index)
change_type = "unchanged"
elif best_index is not None and best_iou >= modified_threshold:
# The same object, redrawn: an annexe, a demolition of one wing,
# or a resurvey. Reporting it as removed + added would hide it.
claimed.add(best_index)
change_type = "modified"
else:
change_type = "removed"
classified.append(
{
"change_type": change_type,
"geometry": geometry if change_type != "modified" else target_geometries[best_index],
"source_feature_id": source_feature["feature_id"],
"target_feature_id": target_features[best_index]["feature_id"] if change_type != "removed" else None,
"iou": best_iou if best_iou > 0 else None,
"properties": source_feature["properties"],
}
)
classified.extend(
{
"change_type": "added",
"geometry": target_feature["geometry"],
"source_feature_id": None,
"target_feature_id": target_feature["feature_id"],
"iou": None,
"properties": target_feature["properties"],
}
for target_index, target_feature in enumerate(target_features)
if target_index not in claimed
)
return classified
@staticmethod
def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset:
dataset = db.get(Dataset, dataset_id)
if not dataset:
raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
if dataset.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message=f"{label} dataset does not belong to this project", status_code=400)
VectorOperationsService._require_vector_dataset(dataset)
return dataset
@staticmethod
def _load_features(
db: Session,
dataset: Dataset,
selection_geometry: BaseGeometry | None = None,
) -> tuple[list[dict[str, Any]], list[str]]:
query = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id)
if selection_geometry is not None and hasattr(query, "filter"):
# Bound the load in the database. Pulling a regional building layer
# into Python to then discard most of it costs memory and time for
# nothing, and the fallback below has no such option.
try:
query = query.filter(
func.ST_Intersects(VectorFeature.geometry, from_shape(selection_geometry, srid=4326))
)
except Exception:
# Lightweight unit-test sessions do not implement every spatial
# predicate; restrict_to_selection still bounds the population.
pass
rows = query.all()
warnings: list[str] = []
if rows:
return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings
warnings.append(f"Dataset {dataset.id} has no persisted vector_features; falling back to stored GeoJSON artifact")
_payload, raw_features = VectorOperationsService._load_dataset_payload(dataset)
extracted = VectorOperationsService._extract_geometries(raw_features)
return [
ChangeDetectionService._raw_feature_to_feature(index, raw_feature, geometry)
for index, (raw_feature, geometry) in enumerate(extracted)
], warnings
@staticmethod
def _row_to_feature(row: VectorFeature) -> dict[str, Any]:
geometry = ChangeDetectionService._valid_comparable_geometry(to_shape(row.geometry))
return {
"feature_id": str(row.source_feature_id or row.id),
"properties": dict(row.properties_json or {}),
"geometry": geometry,
}
@staticmethod
def _raw_feature_to_feature(index: int, raw_feature: dict[str, Any], geometry: BaseGeometry) -> dict[str, Any]:
properties = raw_feature.get("properties") if isinstance(raw_feature.get("properties"), dict) else {}
source_id = raw_feature.get("id") or properties.get("id") or properties.get("source_feature_id") or str(index)
return {
"feature_id": str(source_id),
"properties": dict(properties),
"geometry": ChangeDetectionService._valid_comparable_geometry(geometry),
}
@staticmethod
def _valid_comparable_geometry(geometry: BaseGeometry) -> BaseGeometry:
if geometry.is_empty:
raise AppError(code="INVALID_GEOMETRY", message="Empty geometry cannot be compared", status_code=400)
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(code="INVALID_GEOMETRY", message="Geometry cannot be repaired for comparison", status_code=400)
if geometry.geom_type not in ChangeDetectionService.SUPPORTED_GEOMETRY_TYPES:
raise AppError(
code="UNSUPPORTED_GEOMETRY",
message="Change detection supports Polygon and MultiPolygon geometries only",
details={"geometry_type": geometry.geom_type},
status_code=422,
)
return geometry
@staticmethod
def _iou(left: BaseGeometry, right: BaseGeometry) -> float:
if left.area <= 0 or right.area <= 0:
return 0.0
intersection = left.intersection(right)
if intersection.is_empty:
return 0.0
union_area = left.area + right.area - intersection.area
if union_area <= 0:
return 0.0
return float(intersection.area / union_area)
@staticmethod
def _feature(
*,
geometry: BaseGeometry,
change_type: str,
source_dataset_id: UUID,
target_dataset_id: UUID,
source_feature_id: str | None,
target_feature_id: str | None,
iou: float | None,
properties: dict[str, Any],
) -> dict[str, Any]:
return {
"type": "Feature",
"geometry": mapping(geometry),
"properties": {
**properties,
"change_type": change_type,
"source_dataset_id": str(source_dataset_id),
"target_dataset_id": str(target_dataset_id),
"source_feature_id": source_feature_id,
"target_feature_id": target_feature_id,
"iou": iou,
},
}
@@ -0,0 +1,645 @@
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any
from uuid import UUID
from geoalchemy2.shape import to_shape
from shapely.geometry import box
from shapely.ops import unary_union
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.schemas.coverage import (
CoverageBBox,
CoverageCatalogResponse,
CoverageResolutionItem,
CoverageResolveResponse,
CoverageSourceContract,
)
THEMES = (
"admin",
"buildings",
"roads",
"surface_water",
"land_cover_use",
"nature",
"population",
"parcels",
"soil",
"elevation",
"orthophoto",
"flood_climate",
"maritime_planning",
"marine_environment",
"bathymetry",
)
ZONES = (
"belgium",
"flanders",
"wallonia",
"brussels",
"belgian_north_sea",
"territorial_sea",
"exclusive_economic_zone",
"continental_shelf",
)
STATUS_ORDER = ("unsupported", "not_configured", "partial", "operational")
STATUS_RANK = {status: index for index, status in enumerate(STATUS_ORDER)}
SCOPE_AREA_NAMES = {
"belgium": "Belgium land",
"flanders": "Flanders",
"wallonia": "Wallonia",
"brussels": "Brussels-Capital Region",
"belgian_north_sea": "Belgian part of the North Sea",
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
"continental_shelf": "Belgian continental shelf beyond territorial sea",
}
DETAIL_ZONES = (
"flanders",
"wallonia",
"brussels",
"territorial_sea",
"exclusive_economic_zone",
"continental_shelf",
)
@dataclass(frozen=True)
class _SourceDefinition:
contract: CoverageSourceContract
materialized_layer_names: tuple[str, ...] = ()
materialized_source_names: tuple[str, ...] = ()
operational_themes: tuple[str, ...] = ()
def _contract(
*,
source_name: str,
display_name: str,
authority_level: str,
coverage_zones: tuple[str, ...],
themes: tuple[str, ...],
native_layers: tuple[str, ...],
geometry_types: tuple[str, ...],
acquisition_mode: str,
integration_status: str,
source_url: str,
attribution: str,
license_note: str,
limitation_message: str,
materialized_layer_names: tuple[str, ...] = (),
materialized_source_names: tuple[str, ...] = (),
operational_themes: tuple[str, ...] = (),
) -> _SourceDefinition:
return _SourceDefinition(
contract=CoverageSourceContract(
source_name=source_name,
display_name=display_name,
authority_level=authority_level,
coverage_zones=list(coverage_zones),
themes=list(themes),
native_layers=list(native_layers),
supported_geometry_types=list(geometry_types),
acquisition_mode=acquisition_mode,
integration_status=integration_status,
source_url=source_url,
attribution=attribution,
license_note=license_note,
limitation_message=limitation_message,
),
materialized_layer_names=materialized_layer_names,
materialized_source_names=materialized_source_names or (source_name,),
operational_themes=operational_themes,
)
SOURCE_DEFINITIONS = (
_contract(
source_name="ngi_adminvector",
display_name="NGI AdminVector",
authority_level="authoritative",
coverage_zones=("belgium", "flanders", "wallonia", "brussels", "belgian_north_sea"),
themes=("admin",),
native_layers=(
"belgianterritory",
"belgianmaritimezone",
"region",
"province",
"municipality",
),
geometry_types=("Polygon", "MultiPolygon"),
acquisition_mode="operator_archive",
integration_status="operational",
source_url="https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9",
attribution="National Geographic Institute (NGI), AdminVector",
license_note="CC BY 4.0",
limitation_message="Administrative reference geometry; it does not provide thematic land content.",
materialized_layer_names=(
"belgium_land_boundary",
"belgium_regions",
"belgium_provinces",
"belgium_municipalities",
),
),
_contract(
source_name="statbel",
display_name="Statbel statistical sectors and population",
authority_level="authoritative",
coverage_zones=("belgium", "flanders", "wallonia", "brussels"),
themes=("admin", "population"),
native_layers=("statistical_sectors", "population_statistics"),
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
acquisition_mode="operator_archive",
integration_status="operational",
source_url="https://statbel.fgov.be/en/open-data",
attribution="Statbel",
license_note="Consult the license of the selected Statbel release.",
limitation_message=(
"National editions require the governed plan-stage-review-apply operator; "
"population in partially selected sectors is area-weighted."
),
materialized_layer_names=("population",),
operational_themes=("population",),
),
_contract(
source_name="digitaal_vlaanderen",
display_name="Flemish authoritative services",
authority_level="authoritative",
coverage_zones=("flanders",),
themes=(
"buildings",
"roads",
"surface_water",
"land_cover_use",
"nature",
"parcels",
"soil",
"elevation",
"orthophoto",
"flood_climate",
),
native_layers=("GRB", "BWK", "DHMV", "OMWRGBMRVL", "OGRK", "Mercator"),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
acquisition_mode="bounded_api",
integration_status="operational",
source_url="https://www.vlaanderen.be/datavindplaats",
attribution="Digitaal Vlaanderen and the authoritative Flemish source owners",
license_note="Consult the license and attribution stored with each acquired dataset.",
limitation_message="Operational only for bounded products implemented by GeoIntel and materialized in the project.",
materialized_source_names=(
"grb",
"digitaal_vlaanderen_buildings_addresses_register",
"digitaal_vlaanderen_dhmv",
"digitaal_vlaanderen_orthophoto",
"vmm_flood_hazard",
"department_omgeving_thematic_raster",
"inbo_bwk_natura2000",
"dov_soil_map",
"agentschap_landbouw_zeevisserij_agricultural_parcels",
),
),
_contract(
source_name="vmm_vha_bathymetry_profiles",
display_name="VHA historische dwarsprofielen",
authority_level="authoritative",
coverage_zones=("flanders",),
themes=("bathymetry",),
native_layers=("digitale_atlas_profile_points",),
geometry_types=("Point",),
acquisition_mode="bounded_api",
integration_status="operational",
source_url="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
attribution="Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas",
license_note="Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata.",
limitation_message=(
"Historische puntmetingen met bronafhankelijke meetdatum en verticale referentie; "
"geen continue actuele bodemkaart en zonder gelijktijdig waterpeil geen watervolume."
),
materialized_source_names=("vmm_vha_bathymetry_profiles",),
),
_contract(
source_name="spw_geoportail",
display_name="SPW Geoportail Wallonie",
authority_level="authoritative",
coverage_zones=("wallonia",),
themes=(
"buildings",
"roads",
"surface_water",
"land_cover_use",
"nature",
"soil",
"elevation",
"orthophoto",
"flood_climate",
"bathymetry",
),
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
acquisition_mode="bounded_api",
integration_status="operational",
source_url="https://geoportail.wallonie.be/catalogue",
attribution="Service public de Wallonie",
license_note="Consult the license of each Geoportail Wallonie product.",
limitation_message=(
"Bounded PICC buildings, road axes and hydrography, the legally current flood-hazard polygons, "
"operator-imported SPW bathymetry and bounded SPW MNT terrain are operational; other Walloon themes remain separately governed."
),
materialized_source_names=("spw_picc", "spw_flood_hazard", "spw_walous_land_cover", "spw_bathymetry", "spw_terrain"),
operational_themes=("buildings", "roads", "surface_water", "land_cover_use", "elevation", "flood_climate", "bathymetry"),
),
_contract(
source_name="urbis",
display_name="UrbIS Brussels",
authority_level="authoritative",
coverage_zones=("brussels",),
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
acquisition_mode="bounded_api",
integration_status="operational",
source_url="https://datastore.brussels",
attribution="Brussels UrbIS",
license_note="Consult the license of the selected UrbIS dataset.",
limitation_message=(
"Bounded UrbIS buildings, cadastral parcels, street axes and Land Cover blocks are operational. "
"Permanent water uses the official WB block class; no separate hydrography network is inferred."
),
materialized_source_names=("urbis",),
operational_themes=("buildings", "parcels", "roads", "surface_water", "land_cover_use"),
),
_contract(
source_name="rbins_marine_reporting_units",
display_name="RBINS marine reporting units",
authority_level="authoritative",
coverage_zones=(
"belgian_north_sea",
"territorial_sea",
"exclusive_economic_zone",
"continental_shelf",
),
themes=("admin", "marine_environment"),
native_layers=("marine_reporting_units_2024",),
geometry_types=("Polygon", "MultiPolygon"),
acquisition_mode="operator_wfs",
integration_status="operational",
source_url=(
"https://metadata.naturalsciences.be/geonetwork/srv/api/records/"
"29f40b0d-2a3e-49a8-870a-e9b4acd4d1e3"
),
attribution="Royal Belgian Institute of Natural Sciences (RBINS), BMDC",
license_note="Reuse conditions are retained from the source metadata with every persisted artifact.",
limitation_message="The EEZ and continental shelf can share geometry while retaining different legal semantics.",
materialized_layer_names=("marine_legal_scopes",),
),
_contract(
source_name="rbins_msp_2026",
display_name="Belgian Marine Spatial Plan 2026-2034",
authority_level="authoritative",
coverage_zones=(
"belgian_north_sea",
"territorial_sea",
"exclusive_economic_zone",
"continental_shelf",
),
themes=("maritime_planning", "marine_environment"),
native_layers=("imsp26",),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon"),
acquisition_mode="operator_wfs",
integration_status="operational",
source_url="https://www.health.belgium.be/en/themes/environment/marine-environment/marine-spatial-plan",
attribution="Belgian federal Marine Environment service and RBINS",
license_note="Official source metadata and attribution are retained with the imported snapshot.",
limitation_message="The dataset represents the legally current 2026-2034 plan, not live maritime activity.",
materialized_layer_names=("marine_spatial_plan_2026",),
),
_contract(
source_name="mdk_bathymetry",
display_name="MDK Belgian North Sea depth model",
authority_level="authoritative",
coverage_zones=(
"belgian_north_sea",
"territorial_sea",
"exclusive_economic_zone",
"continental_shelf",
),
themes=("bathymetry",),
native_layers=("depth_model_20m_lat",),
geometry_types=("Raster",),
acquisition_mode="catalog_only",
integration_status="not_configured",
source_url="https://www.vlaanderen.be/datavindplaats",
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
license_note="Consult the official product license before acquisition.",
limitation_message=(
"Bounded strict-TLS WCS acquisition is implemented but stays disabled until the operator enables it "
"with a live-validated coverage id; no depths are synthesized."
),
),
)
FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
"buildings": {
"grb": ("buildings",),
"digitaal_vlaanderen_buildings_addresses_register": (),
},
"roads": {"grb": ("roads",)},
"surface_water": {"grb": ("water",)},
"land_cover_use": {
"department_omgeving_thematic_raster": (),
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
},
"nature": {"inbo_bwk_natura2000": ()},
"parcels": {
"grb": ("parcels",),
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
},
"soil": {"dov_soil_map": ()},
"elevation": {"digitaal_vlaanderen_dhmv": ()},
"orthophoto": {"digitaal_vlaanderen_orthophoto": ()},
"flood_climate": {"vmm_flood_hazard": ()},
}
REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = {
"spw_geoportail": {
"buildings": {"spw_picc": ("buildings",)},
"roads": {"spw_picc": ("roads",)},
"surface_water": {"spw_picc": ("water",)},
"land_cover_use": {"spw_walous_land_cover": ()},
"elevation": {"spw_terrain": ()},
"flood_climate": {"spw_flood_hazard": ("flood_hazard",)},
"bathymetry": {"spw_bathymetry": ()},
"orthophoto": {"spw_orthophoto": ()},
},
"urbis": {
"buildings": {"urbis": ("buildings",)},
"parcels": {"urbis": ("parcels",)},
"roads": {"urbis": ("roads",)},
"surface_water": {"urbis": ("water",)},
"land_cover_use": {"urbis": ("space_occupation", "forest")},
"orthophoto": {"urbis_orthophoto": ()},
},
}
class CoverageRegistryService:
@staticmethod
def catalog() -> CoverageCatalogResponse:
return CoverageCatalogResponse(
themes=list(THEMES),
zones=list(ZONES),
statuses=list(STATUS_ORDER),
sources=[definition.contract for definition in SOURCE_DEFINITIONS],
)
@staticmethod
def normalize_themes(themes: Iterable[str]) -> list[str]:
requested = list(dict.fromkeys(str(theme).strip().lower() for theme in themes if str(theme).strip()))
invalid = sorted(set(requested) - set(THEMES))
if invalid:
raise AppError(
code="COVERAGE_THEME_UNSUPPORTED",
message="One or more coverage themes are unsupported",
status_code=422,
details={"unsupported_themes": invalid, "supported_themes": list(THEMES)},
)
return requested or list(THEMES)
@staticmethod
def _geometry(value: Any):
if value is None:
return None
return value if hasattr(value, "__geo_interface__") else to_shape(value)
@staticmethod
def _intersected_zones(areas: list[Area], selection) -> tuple[list[str], bool]:
geometries: dict[str, Any] = {}
by_name = {area.name: area for area in areas}
for zone, area_name in SCOPE_AREA_NAMES.items():
area = by_name.get(area_name)
geometry = CoverageRegistryService._geometry(area.geometry) if area else None
if geometry is not None and not geometry.is_empty:
geometries[zone] = geometry
detail_intersections = [
zone for zone in DETAIL_ZONES if zone in geometries and geometries[zone].intersects(selection)
]
zones = detail_intersections
if not any(zone in zones for zone in ("flanders", "wallonia", "brussels")):
if "belgium" in geometries and geometries["belgium"].intersects(selection):
zones = ["belgium", *zones]
if not any(zone in zones for zone in ("territorial_sea", "exclusive_economic_zone", "continental_shelf")):
if "belgian_north_sea" in geometries and geometries["belgian_north_sea"].intersects(selection):
zones = [*zones, "belgian_north_sea"]
intersected_geometries = [geometries[zone].intersection(selection) for zone in zones if zone in geometries]
covered = unary_union(intersected_geometries) if intersected_geometries else None
outside = covered is None or covered.is_empty or not covered.covers(selection)
return zones, outside
@staticmethod
def _matching_datasets(
datasets: list[Dataset],
definition: _SourceDefinition,
theme: str,
zone: str,
selection: Any,
) -> tuple[list[Dataset], bool]:
if definition.operational_themes and theme not in definition.operational_themes:
return [], False
matches: list[Dataset] = []
bounded_scopes: list[Any] = []
zone_scoped_materialization = False
for dataset in datasets:
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
continue
# A source-name claim alone must not cause an unsafe artifact to
# appear as operational authoritative coverage.
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
continue
layer_names = definition.materialized_layer_names
if definition.contract.source_name == "digitaal_vlaanderen":
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
if dataset.source_name not in theme_sources:
continue
layer_names = theme_sources[dataset.source_name]
elif definition.contract.source_name in REGIONAL_THEME_DATASETS:
theme_sources = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme, {})
if dataset.source_name not in theme_sources:
continue
layer_names = theme_sources[dataset.source_name]
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
if isinstance(coverage_zones, str):
coverage_zones = [coverage_zones]
acquired_bbox = metadata.get("bbox_epsg4326")
if (
definition.contract.acquisition_mode == "bounded_api"
and isinstance(acquired_bbox, list)
and len(acquired_bbox) == 4
):
try:
acquired_scope = box(*(float(value) for value in acquired_bbox))
except (TypeError, ValueError):
continue
if not acquired_scope.is_valid or not acquired_scope.intersects(selection):
continue
bounded_scopes.append(acquired_scope)
elif definition.contract.acquisition_mode == "bounded_api" and coverage_zones:
zone_scoped_materialization = zone in coverage_zones or "belgium" in coverage_zones
layer_matches = not layer_names or dataset.reference_layer_name in layer_names
zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones
if layer_matches and zone_matches:
matches.append(dataset)
if not matches:
return [], False
fully_covered = True
if definition.contract.acquisition_mode == "bounded_api":
fully_covered = zone_scoped_materialization or (bool(bounded_scopes) and unary_union(bounded_scopes).covers(selection))
return matches, fully_covered
@staticmethod
def _resolve_item(
*,
zone: str,
theme: str,
datasets: list[Dataset],
selection: Any,
) -> CoverageResolutionItem:
definitions = [
definition
for definition in SOURCE_DEFINITIONS
if zone in definition.contract.coverage_zones and theme in definition.contract.themes
]
if not definitions:
return CoverageResolutionItem(
zone=zone,
theme=theme,
status="unsupported",
source_names=[],
materialized_dataset_ids=[],
limitation_message="No audited source contract supports this theme in the selected zone.",
)
materialized: list[Dataset] = []
evidence: list[dict[str, Any]] = []
source_statuses: list[str] = []
limitations: list[str] = []
for definition in definitions:
matches, fully_covered = CoverageRegistryService._matching_datasets(
datasets,
definition,
theme,
zone,
selection,
)
materialized.extend(matches)
for dataset in matches:
metadata = dataset.source_metadata if isinstance(getattr(dataset, "source_metadata", None), dict) else {}
observed_at = getattr(dataset, "observed_at", None)
published_at = metadata.get("published_at") or metadata.get("publication_date") or metadata.get("published_on")
evidence.append({
"dataset_id": dataset.id,
"source_name": str(dataset.source_name or definition.contract.source_name),
"authority_level": definition.contract.authority_level,
"source_version": getattr(dataset, "source_version", None),
"observed_at": observed_at.isoformat() if hasattr(observed_at, "isoformat") else (str(observed_at) if observed_at else None),
"published_at": str(published_at) if published_at else None,
"crs": getattr(dataset, "crs", None) or metadata.get("source_crs"),
"resolution": getattr(dataset, "resolution_json", None),
"coverage_bbox_epsg4326": metadata.get("bbox_epsg4326"),
"attribution": metadata.get("attribution") or definition.contract.attribution,
"license_note": metadata.get("license_note") or definition.contract.license_note,
"checksum_sha256": getattr(dataset, "checksum_sha256", None),
})
if matches and fully_covered:
source_statuses.append("operational")
elif matches:
source_statuses.append("partial")
elif (
definition.contract.integration_status == "operational"
and (not definition.operational_themes or theme in definition.operational_themes)
):
source_statuses.append("partial")
else:
source_statuses.append(
"not_configured"
if definition.contract.integration_status == "operational"
else definition.contract.integration_status
)
limitations.append(definition.contract.limitation_message)
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
return CoverageResolutionItem(
zone=zone,
theme=theme,
status=best_status,
source_names=[definition.contract.source_name for definition in definitions],
materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)),
evidence=list({str(item["dataset_id"]): item for item in evidence}.values()),
limitation_message=" ".join(dict.fromkeys(limitations)),
)
@staticmethod
def resolve(
db: Session,
project_id: UUID,
bbox: CoverageBBox,
themes: Iterable[str],
) -> CoverageResolveResponse:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
requested_themes = CoverageRegistryService.normalize_themes(themes)
selection = box(bbox.minx, bbox.miny, bbox.maxx, bbox.maxy)
areas = db.query(Area).filter(Area.project_id == project_id).all()
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
zones, outside_supported_scope = CoverageRegistryService._intersected_zones(areas, selection)
if not zones:
return CoverageResolveResponse(
project_id=project_id,
bbox=bbox,
requested_themes=requested_themes,
intersected_zones=[],
outside_supported_scope=True,
items=[],
warnings=["The selection does not intersect a persisted Belgium or Belgian North Sea scope."],
)
items = [
CoverageRegistryService._resolve_item(
zone=zone,
theme=theme,
datasets=datasets,
selection=selection,
)
for zone in zones
for theme in requested_themes
]
warnings = []
if outside_supported_scope:
warnings.append("Part of the selection lies outside the persisted Belgium and Belgian North Sea scopes.")
if len(zones) > 1:
warnings.append(
"The selection crosses coverage zones; results remain split and only semantically compatible metrics may be merged."
)
return CoverageResolveResponse(
project_id=project_id,
bbox=bbox,
requested_themes=requested_themes,
intersected_zones=zones,
outside_supported_scope=outside_supported_scope,
items=items,
warnings=warnings,
)
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More