JensandClaude Opus 5
08188005bd
Tile handling produced results that were wrong before any model quality
question arose:
- orthophoto tiles reached the model through PIL convert("RGB"), which
truncates the high byte of a 16-bit product and treats a 4-band RGB+NIR
tile's infrared channel as colour. Tiles are now read with rasterio, the
visible bands are chosen explicitly, and values are percentile-stretched
across all three bands together so hue is preserved;
- an object wider than the tile overlap was truncated by both tiles into two
boxes that barely intersect, so IoU suppression kept both: two false
positives and one missed footprint per seam building. Suppression now also
compares overlap against the smaller box, and boxes cut by an interior tile
edge are dropped in favour of the neighbouring tile's complete view;
- georeferencing fell back to an assumed EPSG:4326 when a manifest carried no
CRS, producing geometry that renders plausibly in the wrong place. QA
already refused such a tile; inference now fails closed too.
Segmentation QA scored candidates against every reference feature in the
dataset, so every building outside the inferred tiles counted as a false
negative. It now applies the same persisted tile coverage that detection QA
has always used, including the indexed ST_Intersects prefilter.
Duplicate suppression uses an STRtree instead of the O(n^2) scan, tiles are
predicted in batches of YOLO_BATCH_SIZE (a setting that existed but was never
read), and detection/segmentation runs can be queued through /run-async for a
polling background worker rather than holding an HTTP worker thread for
minutes of GPU work.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
412 lines
18 KiB
Python
412 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import asyncio
|
|
import re
|
|
import time
|
|
import uuid
|
|
from contextlib import asynccontextmanager
|
|
|
|
from fastapi import FastAPI, HTTPException, Request
|
|
from fastapi.exceptions import RequestValidationError
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import JSONResponse
|
|
|
|
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
|
|
from app.core.config import get_settings
|
|
from app.core.errors import AppError
|
|
from app.core.logging import configure_logging
|
|
from app.core.request_context import reset_request_id, set_request_id
|
|
from app.db.session import SessionLocal
|
|
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
|
from app.services.auth_service import AuthService
|
|
from app.services.analysis_job_worker import AnalysisJobWorker
|
|
from app.services.aoi_operation_worker import AoiOperationWorker
|
|
|
|
|
|
logger = logging.getLogger("geointel")
|
|
SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
UNSAFE_HOST = re.compile(r"[/\\@\s\x00-\x1f\x7f]")
|
|
|
|
|
|
def _to_error_payload(
|
|
code: str,
|
|
message: str,
|
|
details: dict | list | None = None,
|
|
request_id: str | None = None,
|
|
) -> dict:
|
|
return {
|
|
"error": code,
|
|
"message": message,
|
|
"details": details or {},
|
|
"request_id": request_id,
|
|
}
|
|
|
|
|
|
def create_app() -> FastAPI:
|
|
settings = get_settings()
|
|
configure_logging(settings.log_level, settings.sql_log_level)
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(_: FastAPI):
|
|
worker_stop = asyncio.Event()
|
|
worker_task = None
|
|
analysis_worker_task = None
|
|
if settings.reconcile_interrupted_runs_on_startup:
|
|
db = SessionLocal()
|
|
try:
|
|
result = RuntimeReconciliationService.reconcile(db)
|
|
logger.info(
|
|
"Runtime reconciliation completed: jobs=%s analysis_runs=%s resumed_aoi_partitions=%s exhausted_aoi_partitions=%s",
|
|
result.interrupted_jobs,
|
|
result.interrupted_analysis_runs,
|
|
result.resumed_aoi_partitions,
|
|
result.exhausted_aoi_partitions,
|
|
)
|
|
except Exception:
|
|
db.rollback()
|
|
logger.exception("Runtime reconciliation failed")
|
|
raise
|
|
finally:
|
|
db.close()
|
|
if settings.aoi_worker_enabled:
|
|
worker_task = asyncio.create_task(AoiOperationWorker.run(worker_stop, settings.aoi_worker_poll_seconds))
|
|
if settings.analysis_worker_enabled:
|
|
analysis_worker_task = asyncio.create_task(
|
|
AnalysisJobWorker.run(worker_stop, settings.analysis_worker_poll_seconds)
|
|
)
|
|
try:
|
|
yield
|
|
finally:
|
|
worker_stop.set()
|
|
for task in (worker_task, analysis_worker_task):
|
|
if task is not None:
|
|
await task
|
|
|
|
app = FastAPI(
|
|
title="GeoIntel",
|
|
version=settings.app_version,
|
|
docs_url="/docs",
|
|
redoc_url="/redoc",
|
|
lifespan=lifespan,
|
|
)
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=settings.cors_origins,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
allow_credentials=True,
|
|
)
|
|
|
|
app.include_router(health.router)
|
|
app.include_router(auth.router, prefix=settings.api_prefix)
|
|
app.include_router(analysis.router, prefix=settings.api_prefix)
|
|
app.include_router(aoi_operations.router, prefix=settings.api_prefix)
|
|
app.include_router(projects.router, prefix=settings.api_prefix)
|
|
app.include_router(areas.router, prefix=settings.api_prefix)
|
|
app.include_router(datasets.router, prefix=settings.api_prefix)
|
|
app.include_router(jobs.router, prefix=settings.api_prefix)
|
|
app.include_router(quality_checks.router, prefix=settings.api_prefix)
|
|
app.include_router(exports.router, prefix=settings.api_prefix)
|
|
app.include_router(external.router, prefix=settings.api_prefix)
|
|
app.include_router(source_registry.router, prefix=settings.api_prefix)
|
|
app.include_router(demo.router, prefix=settings.api_prefix)
|
|
app.include_router(qa.router, prefix=settings.api_prefix)
|
|
app.include_router(detection.router, prefix=settings.api_prefix)
|
|
app.include_router(segmentation.router, prefix=settings.api_prefix)
|
|
app.include_router(selection_partitions.router, prefix=settings.api_prefix)
|
|
app.include_router(temporal.router, prefix=settings.api_prefix)
|
|
app.include_router(assistant.router, prefix=settings.api_prefix)
|
|
|
|
@app.middleware("http")
|
|
async def request_identity(request: Request, call_next):
|
|
supplied_request_id = request.headers.get("x-request-id", "")
|
|
request_id = supplied_request_id if SAFE_REQUEST_ID.fullmatch(supplied_request_id) else str(uuid.uuid4())
|
|
request.state.request_id = request_id
|
|
token = set_request_id(request_id)
|
|
started_at = time.perf_counter()
|
|
raw_path = str(request.scope.get("path") or "")
|
|
try:
|
|
host = request.headers.get("host", "")
|
|
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
|
if not raw_path.startswith("/") or not host or UNSAFE_HOST.search(host):
|
|
response = JSONResponse(
|
|
status_code=400,
|
|
content=_to_error_payload(
|
|
"INVALID_REQUEST_TARGET",
|
|
"The request target or Host header is invalid",
|
|
request_id=request_id,
|
|
),
|
|
)
|
|
response.headers["x-request-id"] = request_id
|
|
return response
|
|
if content_type == "application/x-www-form-urlencoded":
|
|
response = JSONResponse(
|
|
status_code=415,
|
|
content=_to_error_payload(
|
|
"UNSUPPORTED_CONTENT_TYPE",
|
|
"URL-encoded form bodies are not supported",
|
|
request_id=request_id,
|
|
),
|
|
)
|
|
response.headers["x-request-id"] = request_id
|
|
return response
|
|
public_auth_paths = {
|
|
f"{settings.api_prefix}/auth/session",
|
|
f"{settings.api_prefix}/auth/login",
|
|
f"{settings.api_prefix}/auth/guest",
|
|
f"{settings.api_prefix}/auth/logout",
|
|
}
|
|
direct_loopback_request = (
|
|
request.client is not None
|
|
and request.client.host in {"127.0.0.1", "::1"}
|
|
and not request.headers.get("x-real-ip")
|
|
and not request.headers.get("x-forwarded-for")
|
|
)
|
|
if (
|
|
settings.auth_enabled
|
|
and raw_path.startswith(f"{settings.api_prefix}/")
|
|
and raw_path not in public_auth_paths
|
|
and not direct_loopback_request
|
|
):
|
|
principal = AuthService.verify_session_token(
|
|
request.cookies.get(auth.COOKIE_NAME),
|
|
settings,
|
|
)
|
|
if principal is None:
|
|
response = JSONResponse(
|
|
status_code=401,
|
|
content=_to_error_payload(
|
|
"AUTHENTICATION_REQUIRED",
|
|
"Meld u aan om de GeoIntel API te gebruiken.",
|
|
request_id=request_id,
|
|
),
|
|
)
|
|
response.headers["x-request-id"] = request_id
|
|
return response
|
|
request.state.auth_principal = principal
|
|
if principal.role == "guest":
|
|
project_path_prefix = f"{settings.api_prefix}/projects/"
|
|
guest_project_root = f"{project_path_prefix}{principal.project_id}"
|
|
if raw_path.startswith(project_path_prefix):
|
|
scoped_path = raw_path[len(project_path_prefix):]
|
|
requested_project_id = scoped_path.split("/", 1)[0]
|
|
if str(principal.project_id) != requested_project_id:
|
|
response = JSONResponse(
|
|
status_code=403,
|
|
content=_to_error_payload(
|
|
"GUEST_PROJECT_SCOPE_REQUIRED",
|
|
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
|
request_id=request_id,
|
|
),
|
|
)
|
|
response.headers["x-request-id"] = request_id
|
|
return response
|
|
query_project_id = request.query_params.get("project_id")
|
|
if query_project_id and query_project_id != str(principal.project_id):
|
|
response = JSONResponse(
|
|
status_code=403,
|
|
content=_to_error_payload(
|
|
"GUEST_PROJECT_SCOPE_REQUIRED",
|
|
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
|
request_id=request_id,
|
|
),
|
|
)
|
|
response.headers["x-request-id"] = request_id
|
|
return response
|
|
guest_safe_read_paths = {
|
|
f"{settings.api_prefix}/projects",
|
|
f"{settings.api_prefix}/external/providers",
|
|
f"{settings.api_prefix}/assistant/status",
|
|
f"{settings.api_prefix}/assistant/models",
|
|
f"{settings.api_prefix}/detection/models",
|
|
f"{settings.api_prefix}/detection/model-assets",
|
|
f"{settings.api_prefix}/detection/yolo/preflight",
|
|
f"{settings.api_prefix}/segmentation/models",
|
|
}
|
|
normalized_path = raw_path.rstrip("/") or "/"
|
|
guest_project_read = (
|
|
normalized_path == guest_project_root
|
|
or normalized_path.startswith(f"{guest_project_root}/")
|
|
)
|
|
is_read_request = request.method in {"GET", "HEAD", "OPTIONS"}
|
|
if is_read_request:
|
|
guest_scoped_analysis_read = (
|
|
query_project_id == str(principal.project_id)
|
|
and normalized_path.startswith(
|
|
(
|
|
f"{settings.api_prefix}/detection/",
|
|
f"{settings.api_prefix}/segmentation/",
|
|
f"{settings.api_prefix}/exports/",
|
|
)
|
|
)
|
|
)
|
|
if (
|
|
normalized_path not in guest_safe_read_paths
|
|
and not guest_project_read
|
|
and not guest_scoped_analysis_read
|
|
):
|
|
response = JSONResponse(
|
|
status_code=403,
|
|
content=_to_error_payload(
|
|
"GUEST_ROUTE_NOT_AVAILABLE",
|
|
"Deze API-route maakt geen deel uit van de afgeschermde GeoIntel-demo.",
|
|
request_id=request_id,
|
|
),
|
|
)
|
|
response.headers["x-request-id"] = request_id
|
|
return response
|
|
else:
|
|
guest_safe_post_paths = {
|
|
f"{settings.api_prefix}/demo/workflow",
|
|
f"{settings.api_prefix}/external/coverage/resolve",
|
|
}
|
|
guest_scoped_analysis_post_paths = {
|
|
f"{settings.api_prefix}/detection/run",
|
|
f"{settings.api_prefix}/segmentation/run",
|
|
f"{settings.api_prefix}/qa/detections-vs-reference",
|
|
f"{settings.api_prefix}/exports/geojson",
|
|
f"{settings.api_prefix}/exports/metadata",
|
|
f"{settings.api_prefix}/exports/report",
|
|
f"{settings.api_prefix}/exports/map-result",
|
|
}
|
|
guest_safe_post_suffixes = (
|
|
"/vector/select",
|
|
"/raster/bathymetry/select",
|
|
"/raster/terrain/select",
|
|
"/raster/flood-hazard/select",
|
|
"/raster/thematic/select",
|
|
"/raster/walous/select",
|
|
"/temporal/compare",
|
|
"/datasets/vector/partitions/select",
|
|
"/datasets/bathymetry/profiles/partitions/select",
|
|
)
|
|
is_guest_safe_post = request.method == "POST" and (
|
|
raw_path in guest_safe_post_paths
|
|
or (
|
|
raw_path in guest_scoped_analysis_post_paths
|
|
and query_project_id == str(principal.project_id)
|
|
)
|
|
or (
|
|
query_project_id == str(principal.project_id)
|
|
and raw_path.startswith(
|
|
(
|
|
f"{settings.api_prefix}/detection/runs/",
|
|
f"{settings.api_prefix}/segmentation/runs/",
|
|
)
|
|
)
|
|
and raw_path.endswith("/qa/reference")
|
|
)
|
|
or (
|
|
raw_path.startswith(project_path_prefix)
|
|
and (
|
|
raw_path.endswith(guest_safe_post_suffixes)
|
|
or raw_path.endswith("/assistant/query")
|
|
)
|
|
)
|
|
)
|
|
if not is_guest_safe_post:
|
|
response = JSONResponse(
|
|
status_code=403,
|
|
content=_to_error_payload(
|
|
"GUEST_READ_ONLY",
|
|
"Gasttoegang is een tijdelijke, alleen-lezen demo. Meld u aan als operator om gegevens te wijzigen of taken te starten.",
|
|
request_id=request_id,
|
|
),
|
|
)
|
|
response.headers["x-request-id"] = request_id
|
|
return response
|
|
response = await call_next(request)
|
|
response.headers["x-request-id"] = request_id
|
|
logger.info(
|
|
"request_complete request_id=%s method=%s path=%s status=%s duration_ms=%.1f",
|
|
request_id,
|
|
request.method,
|
|
raw_path,
|
|
response.status_code,
|
|
(time.perf_counter() - started_at) * 1000,
|
|
)
|
|
return response
|
|
finally:
|
|
reset_request_id(token)
|
|
|
|
@app.exception_handler(AppError)
|
|
async def app_error(request: Request, exc: AppError): # noqa: ARG001
|
|
return JSONResponse(
|
|
status_code=exc.status_code,
|
|
content=_to_error_payload(
|
|
exc.code,
|
|
exc.message,
|
|
exc.details,
|
|
request_id=request.state.request_id,
|
|
),
|
|
)
|
|
|
|
@app.exception_handler(HTTPException)
|
|
async def http_error(request: Request, exc: HTTPException): # noqa: ARG001
|
|
code = "HTTP_ERROR"
|
|
message = str(exc.detail)
|
|
details = {}
|
|
if isinstance(exc.detail, dict):
|
|
code = str(exc.detail.get("error") or exc.detail.get("code") or code)
|
|
message = str(exc.detail.get("message") or message)
|
|
raw_details = exc.detail.get("details")
|
|
details = raw_details if isinstance(raw_details, (dict, list)) else {}
|
|
return JSONResponse(
|
|
status_code=exc.status_code,
|
|
content=_to_error_payload(
|
|
code,
|
|
message,
|
|
details,
|
|
request_id=request.state.request_id,
|
|
),
|
|
)
|
|
|
|
@app.exception_handler(RequestValidationError)
|
|
async def validation_error(request: Request, exc: RequestValidationError): # noqa: ARG001
|
|
return JSONResponse(
|
|
status_code=422,
|
|
content=_to_error_payload(
|
|
"VALIDATION_ERROR",
|
|
"Validation failed",
|
|
exc.errors(),
|
|
request_id=request.state.request_id,
|
|
),
|
|
)
|
|
|
|
@app.exception_handler(Exception)
|
|
async def unexpected_error(request: Request, exc: Exception):
|
|
logger.exception(
|
|
"Unhandled request error request_id=%s method=%s path=%s",
|
|
request.state.request_id,
|
|
request.method,
|
|
str(request.scope.get("path") or ""),
|
|
)
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=_to_error_payload(
|
|
"INTERNAL_ERROR",
|
|
"Unexpected server error",
|
|
{"type": exc.__class__.__name__},
|
|
request_id=request.state.request_id,
|
|
),
|
|
)
|
|
|
|
return app
|
|
|
|
|
|
app = create_app()
|
|
|
|
|
|
def main() -> None:
|
|
import uvicorn
|
|
|
|
settings = get_settings()
|
|
uvicorn.run(
|
|
"app.main:app",
|
|
host="0.0.0.0",
|
|
port=8000,
|
|
reload=settings.app_env == "development",
|
|
)
|