Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
293fb07890 | ||
|
|
faeb58ef6d |
@@ -1,19 +1,51 @@
|
|||||||
|
# Build context exclusions for deploy/unraid/Dockerfile.all-in-one.
|
||||||
|
# Everything listed here is NOT sent to the Docker daemon.
|
||||||
|
# Keep this aggressive: the all-in-one image only needs
|
||||||
|
# backend/, frontend/, fixtures/, scripts/, deploy/ and VERSION.
|
||||||
|
|
||||||
.git
|
.git
|
||||||
|
.gitea
|
||||||
|
.github
|
||||||
.venv
|
.venv
|
||||||
venv
|
venv
|
||||||
__pycache__
|
__pycache__
|
||||||
*.pyc
|
*.pyc
|
||||||
.pytest_cache
|
.pytest_cache
|
||||||
|
.mypy_cache
|
||||||
|
.ruff_cache
|
||||||
|
|
||||||
|
# Node dependencies at every level (root test harness + frontend).
|
||||||
|
node_modules
|
||||||
|
**/node_modules
|
||||||
frontend/node_modules
|
frontend/node_modules
|
||||||
frontend/dist
|
frontend/dist
|
||||||
frontend/*.tsbuildinfo
|
frontend/*.tsbuildinfo
|
||||||
|
**/*.tsbuildinfo
|
||||||
|
frontend-src.tar.gz
|
||||||
|
|
||||||
backend/.pytest_cache
|
backend/.pytest_cache
|
||||||
backend/**/*.pyc
|
backend/**/*.pyc
|
||||||
backend/**/__pycache__
|
backend/**/__pycache__
|
||||||
|
|
||||||
|
# Accidental self-copy of the repository into itself.
|
||||||
|
# Without this the build context is duplicated and the build stalls.
|
||||||
|
/geointel
|
||||||
|
|
||||||
|
# Local work products, never needed inside the image.
|
||||||
|
.codex-input
|
||||||
|
test-results
|
||||||
|
playwright-report
|
||||||
|
artifacts
|
||||||
|
RELEASE_NOTES
|
||||||
|
adr
|
||||||
|
docs
|
||||||
|
checklists
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# Runtime data and secrets: mounted at runtime, never baked in.
|
||||||
storage
|
storage
|
||||||
postgres-data
|
postgres-data
|
||||||
|
backups
|
||||||
datasets/raw
|
datasets/raw
|
||||||
datasets/processed
|
datasets/processed
|
||||||
datasets/cache
|
datasets/cache
|
||||||
@@ -21,3 +53,5 @@ exports
|
|||||||
models
|
models
|
||||||
|
|
||||||
.env
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
|
|||||||
+91
-1
@@ -4,9 +4,30 @@ GEOINTEL_API_PREFIX=/api/v1
|
|||||||
DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1
|
DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1
|
||||||
STORAGE_ROOT=./storage
|
STORAGE_ROOT=./storage
|
||||||
MAX_UPLOAD_MB=500
|
MAX_UPLOAD_MB=500
|
||||||
|
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64
|
||||||
CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202
|
CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202
|
||||||
|
|
||||||
|
# Optional single-operator access gate. Store only a PBKDF2-SHA256 hash and
|
||||||
|
# a unique 32+ character signing secret. Guest access is opt-in and should be
|
||||||
|
# enabled only on a dedicated demo installation with bounded public data.
|
||||||
|
GEOINTEL_AUTH_ENABLED=false
|
||||||
|
GEOINTEL_AUTH_REQUIRE_HTTPS=false
|
||||||
|
GEOINTEL_AUTH_USERNAME=
|
||||||
|
GEOINTEL_AUTH_PASSWORD_HASH=
|
||||||
|
GEOINTEL_AUTH_SESSION_SECRET=
|
||||||
|
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
|
||||||
|
GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202
|
||||||
|
GEOINTEL_AUTHENTIK_ISSUER=
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_ID=
|
||||||
|
GEOINTEL_AUTHENTIK_CLIENT_SECRET=
|
||||||
|
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL=
|
||||||
|
GEOINTEL_GUEST_ACCESS_ENABLED=false
|
||||||
|
GEOINTEL_GUEST_DISPLAY_NAME=Gast
|
||||||
|
GEOINTEL_GUEST_SESSION_TTL_SECONDS=7200
|
||||||
ORTHOPHOTO_ENABLED=true
|
ORTHOPHOTO_ENABLED=true
|
||||||
ORTHOPHOTO_WMS_URL=https://geo.api.vlaanderen.be/OMWRGBMRVL/wms
|
ORTHOPHOTO_WMS_URL=https://geo.api.vlaanderen.be/OMWRGBMRVL/wms
|
||||||
|
SPW_ORTHOPHOTO_WMS_URL=https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_LAST/MapServer/WMSServer
|
||||||
|
BRUSSELS_ORTHOPHOTO_WMS_URL=https://geoservices-grid.irisnet.be/geoserver/urbisgrid/ows
|
||||||
ORTHOPHOTO_WMS_LAYER=Ortho
|
ORTHOPHOTO_WMS_LAYER=Ortho
|
||||||
ORTHOPHOTO_RESOLUTION_M=1.0
|
ORTHOPHOTO_RESOLUTION_M=1.0
|
||||||
ORTHOPHOTO_MIN_SIDE_M=128
|
ORTHOPHOTO_MIN_SIDE_M=128
|
||||||
@@ -28,6 +49,12 @@ GRB_CACHE_TTL_HOURS=24
|
|||||||
OFFICIAL_VECTOR_ENABLED=true
|
OFFICIAL_VECTOR_ENABLED=true
|
||||||
BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs
|
BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs
|
||||||
DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs
|
DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs
|
||||||
|
SPW_PICC_ENABLED=true
|
||||||
|
SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer
|
||||||
|
SPW_FLOOD_HAZARD_ENABLED=true
|
||||||
|
SPW_FLOOD_HAZARD_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer
|
||||||
|
URBIS_ENABLED=true
|
||||||
|
URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows
|
||||||
OFFICIAL_VECTOR_MIN_SIDE_M=10
|
OFFICIAL_VECTOR_MIN_SIDE_M=10
|
||||||
OFFICIAL_VECTOR_MAX_SIDE_M=20000
|
OFFICIAL_VECTOR_MAX_SIDE_M=20000
|
||||||
OFFICIAL_VECTOR_PAGE_SIZE=1000
|
OFFICIAL_VECTOR_PAGE_SIZE=1000
|
||||||
@@ -63,6 +90,7 @@ BATHYMETRY_PROFILES_ENABLED=true
|
|||||||
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
|
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
|
||||||
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
|
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
|
||||||
BATHYMETRY_PROFILES_PAGE_SIZE=1000
|
BATHYMETRY_PROFILES_PAGE_SIZE=1000
|
||||||
|
BATHYMETRY_PROFILES_MAX_PAGES=200
|
||||||
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
||||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
||||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
||||||
@@ -70,6 +98,14 @@ MDK_BATHYMETRY_PROBE_ENABLED=true
|
|||||||
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
|
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
|
||||||
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
|
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
|
||||||
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
|
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
|
||||||
|
# Bounded MDK acquisition stays fail-closed until the readiness probe reports
|
||||||
|
# "reachable" and an advertised coverage id is configured explicitly.
|
||||||
|
MDK_BATHYMETRY_ACQUISITION_ENABLED=false
|
||||||
|
MDK_BATHYMETRY_COVERAGE_ID=
|
||||||
|
MDK_BATHYMETRY_REQUEST_CRS=EPSG:4326
|
||||||
|
MDK_BATHYMETRY_MAX_BBOX_DEG2=0.25
|
||||||
|
MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS=120
|
||||||
|
MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB=160
|
||||||
THEMATIC_RASTER_ENABLED=true
|
THEMATIC_RASTER_ENABLED=true
|
||||||
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
|
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
|
||||||
THEMATIC_RASTER_MIN_SIDE_M=100
|
THEMATIC_RASTER_MIN_SIDE_M=100
|
||||||
@@ -77,19 +113,62 @@ THEMATIC_RASTER_MAX_SIDE_M=60000
|
|||||||
THEMATIC_RASTER_MAX_PIXELS=30000000
|
THEMATIC_RASTER_MAX_PIXELS=30000000
|
||||||
THEMATIC_RASTER_TIMEOUT_SECONDS=300
|
THEMATIC_RASTER_TIMEOUT_SECONDS=300
|
||||||
THEMATIC_RASTER_MAX_RESPONSE_MB=160
|
THEMATIC_RASTER_MAX_RESPONSE_MB=160
|
||||||
|
WALOUS_ENABLED=true
|
||||||
|
WALOUS_SOURCE_DIR=/app/storage/source-cache/walous
|
||||||
|
WALOUS_ANALYSIS_RESOLUTION_M=10
|
||||||
|
WALOUS_MAX_SIDE_M=60000
|
||||||
|
WALOUS_MAX_PIXELS=36000000
|
||||||
YOLO_ENABLED=false
|
YOLO_ENABLED=false
|
||||||
YOLO_MODELS_DIR=/app/models
|
YOLO_MODELS_DIR=/app/models
|
||||||
YOLO_MODEL_PATH=
|
YOLO_MODEL_PATH=
|
||||||
YOLO_MODEL_ID=yolo-configured
|
YOLO_MODEL_ID=yolo-configured
|
||||||
YOLO_MODEL_DISPLAY_NAME=Configured YOLO detector
|
YOLO_MODEL_DISPLAY_NAME=Configured YOLO detector
|
||||||
YOLO_MODEL_VERSION=
|
YOLO_MODEL_VERSION=
|
||||||
|
YOLO_MODEL_CLASSES=building
|
||||||
|
YOLO_ENFORCE_VALIDATION_SCOPE=false
|
||||||
|
# Required when scope enforcement is enabled. The manifest is bound to exact
|
||||||
|
# model bytes and contains the allowed EPSG:4326 validation geometry.
|
||||||
|
YOLO_VALIDATION_SCOPE_MANIFEST_PATH=
|
||||||
|
YOLO_VALIDATION_SCOPE_MANIFEST_SHA256=
|
||||||
|
# Deprecated display metadata; never used as an inference authorization gate.
|
||||||
|
YOLO_VALIDATED_AREA_NAMES=Mol,Kempen
|
||||||
YOLO_CONFIG_DIR=./storage/ultralytics
|
YOLO_CONFIG_DIR=./storage/ultralytics
|
||||||
YOLO_DEVICE=cpu
|
YOLO_DEVICE=cpu
|
||||||
|
YOLO_REQUIRE_CUDA=false
|
||||||
YOLO_IMAGE_SIZE=640
|
YOLO_IMAGE_SIZE=640
|
||||||
YOLO_MAX_TILES=100
|
YOLO_MAX_TILES=100
|
||||||
YOLO_MAX_DETECTIONS=1000
|
YOLO_MAX_DETECTIONS=1000
|
||||||
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
|
YOLO_DUPLICATE_IOU_THRESHOLD=0.5
|
||||||
|
# Tiles per model.predict call. 1 leaves most of a modern GPU idle on a run of
|
||||||
|
# a hundred tiles; raise it until VRAM is the limit.
|
||||||
YOLO_BATCH_SIZE=1
|
YOLO_BATCH_SIZE=1
|
||||||
|
# Drop boxes truncated by an interior tile edge. Because tiles overlap, the
|
||||||
|
# neighbouring tile saw the same object whole, so the truncated half is a
|
||||||
|
# duplicate and a shape error at once. Boxes on the outer raster edge are kept.
|
||||||
|
YOLO_SUPPRESS_TILE_EDGE_DETECTIONS=true
|
||||||
|
# Intersection over the smaller box. scripts/evaluate_belgium_building_candidate.py
|
||||||
|
# freezes this during calibration (--containment-nms) before the protected test.
|
||||||
|
# Serving a promoted model at a different value means the runtime suppresses
|
||||||
|
# detections its gate counted, so set this to the value the candidate was gated at.
|
||||||
|
YOLO_CONTAINMENT_NMS_THRESHOLD=0.85
|
||||||
|
# Segmentation carries its own value: masks and boxes overlap differently,
|
||||||
|
# so one threshold need not fit both.
|
||||||
|
SEGMENTATION_CONTAINMENT_NMS_THRESHOLD=0.85
|
||||||
|
|
||||||
|
# Local segmentation models. GeoIntel never downloads model weights
|
||||||
|
# automatically; point these to existing local files to enable inference.
|
||||||
|
YOLO_SEG_ENABLED=false
|
||||||
|
YOLO_SEG_MODEL_PATH=
|
||||||
|
YOLO_SEG_MODEL_ID=yolo-seg-configured
|
||||||
|
YOLO_SEG_MODEL_DISPLAY_NAME=Configured YOLO segmentation
|
||||||
|
YOLO_SEG_MODEL_VERSION=
|
||||||
|
SAM_ENABLED=false
|
||||||
|
SAM_MODEL_PATH=
|
||||||
|
SAM_MODEL_ID=sam-configured
|
||||||
|
SAM_MODEL_DISPLAY_NAME=Configured SAM segmentation
|
||||||
|
SAM_MODEL_VERSION=
|
||||||
|
SEGMENTATION_MAX_MASKS_PER_TILE=300
|
||||||
|
SEGMENTATION_DUPLICATE_IOU_THRESHOLD=0.5
|
||||||
ENABLE_GRB_WFS=false
|
ENABLE_GRB_WFS=false
|
||||||
GRB_WFS_URL=
|
GRB_WFS_URL=
|
||||||
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
|
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
|
||||||
@@ -115,5 +194,16 @@ GEOINTEL_POSTGIS_DATA_PATH=./postgres-data
|
|||||||
GEOINTEL_POSTGRES_DB=geointel
|
GEOINTEL_POSTGRES_DB=geointel
|
||||||
GEOINTEL_POSTGRES_USER=geointel
|
GEOINTEL_POSTGRES_USER=geointel
|
||||||
GEOINTEL_POSTGRES_PASSWORD=geointel
|
GEOINTEL_POSTGRES_PASSWORD=geointel
|
||||||
GEOINTEL_CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202
|
GEOINTEL_CORS_ORIGINS=https://geointel.example.com,http://localhost:1202,http://127.0.0.1:1202
|
||||||
GEOINTEL_MAX_UPLOAD_MB=500
|
GEOINTEL_MAX_UPLOAD_MB=500
|
||||||
|
GEOINTEL_AOI_WORKER_ENABLED=false
|
||||||
|
GEOINTEL_AOI_WORKER_POLL_SECONDS=2
|
||||||
|
# Executes queued detection.run / segmentation.run jobs from POST
|
||||||
|
# /detection/run-async, so tiled GPU inference never blocks an HTTP request.
|
||||||
|
GEOINTEL_ANALYSIS_WORKER_ENABLED=false
|
||||||
|
GEOINTEL_ANALYSIS_WORKER_POLL_SECONDS=2
|
||||||
|
# Analysis consumes only artifacts under STORAGE_ROOT: a tile manifest path
|
||||||
|
# arrives in the request and a manifest entry may name an absolute tile
|
||||||
|
# path, so without this an API field is an unbounded filesystem reference.
|
||||||
|
# Provisioning workflows that stage tiles elsewhere before ingest can opt out.
|
||||||
|
GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS=false
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
*.sh text eol=lf
|
*.sh text eol=lf
|
||||||
|
deploy/unraid/gosu-setpriv text eol=lf
|
||||||
*.py text eol=lf
|
*.py text eol=lf
|
||||||
*.yml text eol=lf
|
*.yml text eol=lf
|
||||||
*.yaml text eol=lf
|
*.yaml text eol=lf
|
||||||
@@ -10,3 +11,7 @@ Dockerfile text eol=lf
|
|||||||
*.ts text eol=lf
|
*.ts text eol=lf
|
||||||
*.css text eol=lf
|
*.css text eol=lf
|
||||||
*.json text eol=lf
|
*.json text eol=lf
|
||||||
|
|
||||||
|
# Retained audit exports preserve source bytes, including line endings and
|
||||||
|
# tool-emitted whitespace.
|
||||||
|
artifacts/evidence/accuracy/** -text -whitespace
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
name: Managed validation
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
profile:
|
||||||
|
description: Allowlisted validation profile
|
||||||
|
required: true
|
||||||
|
default: full
|
||||||
|
type: choice
|
||||||
|
options: [test, lint, typecheck, build, security, full]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: managed-validation-${{ gitea.repository }}-${{ gitea.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
full:
|
||||||
|
# Gitea Actions does not consistently evaluate the GitHub-style `||`
|
||||||
|
# expression for pull-request runs without workflow inputs.
|
||||||
|
name: Managed repository validation
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 60
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
|
with:
|
||||||
|
python-version: "3.11"
|
||||||
|
cache: pip
|
||||||
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
|
with:
|
||||||
|
node-version: "20"
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: frontend/package-lock.json
|
||||||
|
- name: Validate the requested profile against the real projects
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
REQUESTED_PROFILE: ${{ inputs.profile }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
profile="${REQUESTED_PROFILE:-full}"
|
||||||
|
case "${profile}" in
|
||||||
|
test|lint|typecheck|build|security|full) ;;
|
||||||
|
*) echo "Profile is not allowlisted" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
git diff --check
|
||||||
|
if git grep -nE '^(<<<<<<< |=======$|>>>>>>> )' -- . ':!*.lock' ':!*.patch'; then
|
||||||
|
echo "Unresolved merge markers detected" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# MANAGED_FAST_PATH: documentation and this baseline workflow cannot
|
||||||
|
# affect the shipped runtime. Keep the required status check, but do
|
||||||
|
# not install toolchains or execute the full product suite.
|
||||||
|
if [[ -n "${GITHUB_BASE_REF:-}" ]]; then
|
||||||
|
git fetch --no-tags --depth=1 origin "${GITHUB_BASE_REF}"
|
||||||
|
managed_base="origin/${GITHUB_BASE_REF}"
|
||||||
|
git diff --check "${managed_base}..HEAD"
|
||||||
|
mapfile -t managed_changed_files < <(
|
||||||
|
git diff --name-only --diff-filter=ACMR "${managed_base}..HEAD"
|
||||||
|
)
|
||||||
|
managed_runtime_change=0
|
||||||
|
for managed_path in "${managed_changed_files[@]}"; do
|
||||||
|
case "${managed_path}" in
|
||||||
|
*.md|*.mdx|docs/*|.github/ISSUE_TEMPLATE/*|.gitea/ISSUE_TEMPLATE/*|.gitea/runner-scope.sh|.gitea/workflows/managed-validation.yml)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
managed_runtime_change=1
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
if [[ "${#managed_changed_files[@]}" -gt 0 && "${managed_runtime_change}" -eq 0 ]]; then
|
||||||
|
printf 'Managed validation fast path: %s non-runtime file(s); full product suite skipped.\n' \
|
||||||
|
"${#managed_changed_files[@]}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi python scripts/verify_repository_layout.py
|
||||||
|
|
||||||
|
python -m pip install --disable-pip-version-check --require-hashes -r backend/requirements-ci.lock
|
||||||
|
python -m pip install --disable-pip-version-check --no-deps -e backend
|
||||||
|
(cd frontend && npm ci)
|
||||||
|
|
||||||
|
case "${profile}" in
|
||||||
|
test)
|
||||||
|
(cd backend && python -m pytest -W error::DeprecationWarning)
|
||||||
|
(cd frontend && npm run test:unit)
|
||||||
|
;;
|
||||||
|
lint)
|
||||||
|
python -m ruff check backend scripts tests
|
||||||
|
(cd frontend && npm run lint --if-present)
|
||||||
|
;;
|
||||||
|
typecheck)
|
||||||
|
(cd frontend && npm run typecheck)
|
||||||
|
;;
|
||||||
|
build)
|
||||||
|
python -m compileall backend/app
|
||||||
|
(cd frontend && npm run build)
|
||||||
|
;;
|
||||||
|
security)
|
||||||
|
python -m pip install --disable-pip-version-check pip-audit==2.10.1
|
||||||
|
bash scripts/audit_python_dependencies.sh
|
||||||
|
(cd frontend && npm audit --audit-level=high)
|
||||||
|
;;
|
||||||
|
full)
|
||||||
|
PYTHON_BIN=python bash scripts/run_readiness_check.sh
|
||||||
|
;;
|
||||||
|
esac
|
||||||
@@ -1,9 +1,8 @@
|
|||||||
name: GeoIntel release gates
|
name: GeoIntel release gates
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
|
||||||
branches: [main, develop, "codex/**", "build/**"]
|
|
||||||
pull_request:
|
pull_request:
|
||||||
|
push:
|
||||||
branches: [main, develop]
|
branches: [main, develop]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
@@ -12,21 +11,30 @@ permissions:
|
|||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: geointel-release-${{ gitea.ref }}
|
group: geointel-release-${{ gitea.ref }}
|
||||||
cancel-in-progress: true
|
# A cancelled HTTP caller does not terminate the allowlisted controller
|
||||||
|
# process that already owns the production lock. Queue a newer revision
|
||||||
|
# instead of orphaning an in-flight backup or deploy.
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
quality:
|
quality:
|
||||||
name: Compile, test, contracts and builds
|
name: Compile, test, contracts and builds
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 45
|
timeout-minutes: 60
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- uses: actions/setup-python@v5
|
- name: Secret scan
|
||||||
|
run: >-
|
||||||
|
docker run --rm
|
||||||
|
--volume "$PWD:/repo:ro"
|
||||||
|
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
|
||||||
|
filesystem /repo --only-verified --no-update
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -53,7 +61,9 @@ jobs:
|
|||||||
docker compose config > artifacts/docker-compose.resolved.yml
|
docker compose config > artifacts/docker-compose.resolved.yml
|
||||||
- name: Publish quality evidence
|
- name: Publish quality evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
# Gitea Actions currently exposes the GHES-compatible artifact API;
|
||||||
|
# upload-artifact v4 deliberately refuses that API.
|
||||||
|
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
|
||||||
with:
|
with:
|
||||||
name: quality-evidence
|
name: quality-evidence
|
||||||
path: |
|
path: |
|
||||||
@@ -67,13 +77,13 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -90,7 +100,7 @@ jobs:
|
|||||||
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
||||||
- name: Publish dependency evidence
|
- name: Publish dependency evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
|
||||||
with:
|
with:
|
||||||
name: dependency-audits
|
name: dependency-audits
|
||||||
path: |
|
path: |
|
||||||
@@ -101,41 +111,81 @@ jobs:
|
|||||||
retention-days: 30
|
retention-days: 30
|
||||||
|
|
||||||
container:
|
container:
|
||||||
name: GIS image, SBOM and container scan
|
name: Production AI image, SBOM and container scan
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 60
|
timeout-minutes: 120
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- name: Build non-AI release image
|
- name: Build production AI release image
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ gitea.sha }}
|
RELEASE_SHA: ${{ gitea.sha }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p artifacts
|
mkdir -p artifacts
|
||||||
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
|
||||||
docker build \
|
docker build \
|
||||||
-f deploy/unraid/Dockerfile.all-in-one \
|
-f deploy/unraid/Dockerfile.all-in-one \
|
||||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
--build-arg GEOINTEL_INSTALL_AI=true \
|
||||||
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
||||||
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
||||||
-t "geointel-ci:$RELEASE_SHA-gis" \
|
--build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
|
||||||
|
-t "geointel-ci:$RELEASE_SHA-ai" \
|
||||||
.
|
.
|
||||||
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json
|
IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
|
||||||
|
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
|
||||||
|
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
|
||||||
- name: Generate SPDX SBOM
|
- name: Generate SPDX SBOM
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ gitea.sha }}
|
RELEASE_SHA: ${{ gitea.sha }}
|
||||||
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis"
|
GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
|
||||||
|
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
|
||||||
|
SYFT_PARALLELISM: "1"
|
||||||
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
|
||||||
- name: Enforce container vulnerability policy
|
- name: Enforce container vulnerability policy
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ gitea.sha }}
|
RELEASE_SHA: ${{ gitea.sha }}
|
||||||
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis"
|
GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
|
||||||
|
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
|
||||||
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/scan_container_image.sh "$IMAGE_ID"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
- name: Remove temporary image archive
|
||||||
|
if: always()
|
||||||
|
run: >-
|
||||||
|
rm -f -- artifacts/geointel-image.tar
|
||||||
|
artifacts/geointel-image.tar.image-id
|
||||||
|
artifacts/geointel-image.tar.partial.*
|
||||||
- name: Publish container evidence
|
- name: Publish container evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
|
||||||
with:
|
with:
|
||||||
name: container-evidence
|
name: container-evidence
|
||||||
path: |
|
path: |
|
||||||
artifacts/image-inspect.json
|
artifacts/image-inspect.json
|
||||||
|
artifacts/image-id.txt
|
||||||
artifacts/geointel-sbom.spdx.json
|
artifacts/geointel-sbom.spdx.json
|
||||||
artifacts/geointel-container-vulnerabilities.json
|
artifacts/geointel-container-vulnerabilities.json
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
retention-days: 30
|
retention-days: 30
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
name: Deploy exact gated revision to Unraid
|
||||||
|
needs: [quality, dependency-audit, container]
|
||||||
|
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
|
||||||
|
runs-on: unraid-deploy
|
||||||
|
# The first byte-complete storage snapshot can exceed 100 GiB. Keep the
|
||||||
|
# gated caller attached for the full conservative backup/build window;
|
||||||
|
# the controller and deploy script still serialize every mutation.
|
||||||
|
timeout-minutes: 720
|
||||||
|
steps:
|
||||||
|
- name: Deploy only after every release gate is green
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
docker exec gitea-deploy-control \
|
||||||
|
/opt/gitea-deploy/deploy.py deploy \
|
||||||
|
"${{ gitea.repository }}" "${{ gitea.sha }}"
|
||||||
|
|||||||
@@ -20,13 +20,19 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 45
|
timeout-minutes: 45
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- uses: actions/setup-python@v5
|
- name: Secret scan
|
||||||
|
run: >-
|
||||||
|
docker run --rm
|
||||||
|
--volume "$PWD:/repo:ro"
|
||||||
|
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
|
||||||
|
filesystem /repo --only-verified --no-update
|
||||||
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -53,7 +59,7 @@ jobs:
|
|||||||
docker compose config > artifacts/docker-compose.resolved.yml
|
docker compose config > artifacts/docker-compose.resolved.yml
|
||||||
- name: Publish quality evidence
|
- name: Publish quality evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: quality-evidence
|
name: quality-evidence
|
||||||
path: |
|
path: |
|
||||||
@@ -67,13 +73,13 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
cache: pip
|
cache: pip
|
||||||
cache-dependency-path: backend/requirements-ci.lock
|
cache-dependency-path: backend/requirements-ci.lock
|
||||||
- uses: actions/setup-node@v4
|
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||||
with:
|
with:
|
||||||
node-version: "20"
|
node-version: "20"
|
||||||
cache: npm
|
cache: npm
|
||||||
@@ -90,7 +96,7 @@ jobs:
|
|||||||
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
npm audit --audit-level=high --json > ../artifacts/npm-audit.json
|
||||||
- name: Publish dependency evidence
|
- name: Publish dependency evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: dependency-audits
|
name: dependency-audits
|
||||||
path: |
|
path: |
|
||||||
@@ -101,40 +107,63 @@ jobs:
|
|||||||
retention-days: 30
|
retention-days: 30
|
||||||
|
|
||||||
container:
|
container:
|
||||||
name: GIS image, SBOM and container scan
|
name: Production AI image, SBOM and container scan
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 60
|
timeout-minutes: 120
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
- name: Build non-AI release image
|
- name: Build production AI release image
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ github.sha }}
|
RELEASE_SHA: ${{ github.sha }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p artifacts
|
mkdir -p artifacts
|
||||||
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
|
||||||
docker build \
|
docker build \
|
||||||
-f deploy/unraid/Dockerfile.all-in-one \
|
-f deploy/unraid/Dockerfile.all-in-one \
|
||||||
--build-arg GEOINTEL_INSTALL_AI=false \
|
--build-arg GEOINTEL_INSTALL_AI=true \
|
||||||
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
|
||||||
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
|
||||||
-t "geointel-ci:$RELEASE_SHA-gis" \
|
--build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
|
||||||
|
-t "geointel-ci:$RELEASE_SHA-ai" \
|
||||||
.
|
.
|
||||||
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json
|
IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
|
||||||
|
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
|
||||||
|
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
|
||||||
- name: Generate SPDX SBOM
|
- name: Generate SPDX SBOM
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ github.sha }}
|
RELEASE_SHA: ${{ github.sha }}
|
||||||
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis"
|
GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
|
||||||
|
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
|
||||||
|
SYFT_PARALLELISM: "1"
|
||||||
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
|
||||||
- name: Enforce container vulnerability policy
|
- name: Enforce container vulnerability policy
|
||||||
env:
|
env:
|
||||||
RELEASE_SHA: ${{ github.sha }}
|
RELEASE_SHA: ${{ github.sha }}
|
||||||
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis"
|
GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
|
||||||
|
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
|
||||||
|
run: |
|
||||||
|
IMAGE_ID="$(cat artifacts/image-id.txt)"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
bash scripts/scan_container_image.sh "$IMAGE_ID"
|
||||||
|
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
|
||||||
|
- name: Remove temporary image archive
|
||||||
|
if: always()
|
||||||
|
run: >-
|
||||||
|
rm -f -- artifacts/geointel-image.tar
|
||||||
|
artifacts/geointel-image.tar.image-id
|
||||||
|
artifacts/geointel-image.tar.partial.*
|
||||||
- name: Publish container evidence
|
- name: Publish container evidence
|
||||||
if: always()
|
if: always()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: container-evidence
|
name: container-evidence
|
||||||
path: |
|
path: |
|
||||||
artifacts/image-inspect.json
|
artifacts/image-inspect.json
|
||||||
|
artifacts/image-id.txt
|
||||||
artifacts/geointel-sbom.spdx.json
|
artifacts/geointel-sbom.spdx.json
|
||||||
artifacts/geointel-container-vulnerabilities.json
|
artifacts/geointel-container-vulnerabilities.json
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
|
|||||||
+13
-1
@@ -4,6 +4,8 @@ __pycache__/
|
|||||||
.venv/
|
.venv/
|
||||||
venv/
|
venv/
|
||||||
.env
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
*.egg-info/
|
*.egg-info/
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
.ruff_cache/
|
.ruff_cache/
|
||||||
@@ -14,7 +16,10 @@ dist/
|
|||||||
build/
|
build/
|
||||||
*.tsbuildinfo
|
*.tsbuildinfo
|
||||||
|
|
||||||
# Large local data
|
# Local source-transfer archives; preserve them outside version control.
|
||||||
|
/frontend-src.tar.gz
|
||||||
|
|
||||||
|
# Generated evidence, model output and operational snapshots stay outside Git.
|
||||||
/artifacts/
|
/artifacts/
|
||||||
/.cache/
|
/.cache/
|
||||||
/datasets/raw/*
|
/datasets/raw/*
|
||||||
@@ -54,3 +59,10 @@ build/
|
|||||||
.DS_Store
|
.DS_Store
|
||||||
.vscode/
|
.vscode/
|
||||||
.idea/
|
.idea/
|
||||||
|
|
||||||
|
# Local investigation scratch is never repository input.
|
||||||
|
/.codex-input/
|
||||||
|
/.codex-artifacts/
|
||||||
|
/.playwright-mcp/
|
||||||
|
/.mcp.json
|
||||||
|
/data/
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[extend]
|
||||||
|
useDefault = true
|
||||||
|
|
||||||
|
[[allowlists]]
|
||||||
|
description = "Public registry identifiers that resemble generic API keys"
|
||||||
|
regexTarget = "match"
|
||||||
|
regexes = [
|
||||||
|
'''key="population_density_2019"''',
|
||||||
|
'''source_key="inbo_bwk_natura2000"''',
|
||||||
|
'''product_key": "population_density_2019"''',
|
||||||
|
'''key="spw_flood_hazard_2021"''',
|
||||||
|
'''key="phase2-mutation"''',
|
||||||
|
]
|
||||||
+22
-2897
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
|||||||
|
# Contributing
|
||||||
|
|
||||||
|
Keep GeoIntel changes reproducible, privacy-preserving and free from generated workspace state.
|
||||||
|
|
||||||
|
- Branch from the protected default branch.
|
||||||
|
- Use synthetic or explicitly redistributable imagery, coordinates and datasets in tests and documentation.
|
||||||
|
- Do not commit `.codex-input` contents, SQLite/WAL/SHM files, archives, contact sheets, model outputs, private maps, credentials or local deployment data.
|
||||||
|
- Document the origin, license and intended use of every external dataset, model and media asset.
|
||||||
|
- Add tests for behavioural changes and run the documented backend, frontend, geospatial and managed validation gates.
|
||||||
|
- Explain data-model, coordinate-system, model, privacy and deployment impact in the pull request.
|
||||||
|
|
||||||
|
Report security issues according to `SECURITY.md`.
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright 2026 Jens / ITWorx.tech
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
@@ -1,292 +1,282 @@
|
|||||||
# GeoIntel Belgium and the Belgian North Sea
|
# GeoIntel
|
||||||
|
|
||||||
GeoIntel is a map-first GeoAI Workbench for Belgium and the Belgian North Sea.
|
<p align="center">
|
||||||
It combines governed official-source coverage, raster/vector processing,
|
<img src="frontend/public/geointel-icon-180.png" width="92" alt="GeoIntel logo">
|
||||||
historical comparison, computer vision, QA/QC and geospatial exports.
|
</p>
|
||||||
|
|
||||||
Mol and the Kempen remain deep regression and model-validation references. The
|
<p align="center"><strong>Evidence-first GeoAI voor België en de Belgische Noordzee.</strong></p>
|
||||||
release scope is all of Belgium plus legally labelled Belgian maritime zones;
|
|
||||||
source coverage remains explicit per theme and jurisdiction.
|
|
||||||
|
|
||||||
GeoIntel is not a generic dashboard or chatbot. The core product is:
|
GeoIntel is een kaartgerichte GeoAI-workbench waarin officiële databronnen,
|
||||||
|
ruimtelijke analyse, PyTorch-computervisie, QA/QC en export samenkomen. Het
|
||||||
|
platform bewaart niet alleen een resultaat, maar ook de bron, ruimtelijke
|
||||||
|
context, modelversie en het bewijs waarmee dat resultaat gecontroleerd kan
|
||||||
|
worden.
|
||||||
|
|
||||||
> data → processing → geospatial output → QA/QC → export
|
De release ondersteunt heel België en de juridisch onderscheiden Belgische
|
||||||
|
Noordzeezones. Regionale bronnen blijven herkenbaar: GRB, PICC, UrbIS en
|
||||||
|
maritieme datasets worden niet voorgesteld alsof ze dezelfde semantiek,
|
||||||
|
resolutie of actualiteit hebben.
|
||||||
|
|
||||||
## Current milestone
|
> **Data → ruimtelijke verwerking → AI-resultaat → kwaliteitsbewijs → export**
|
||||||
|
|
||||||
**v1.0.0-rc.1 - Belgium/North Sea release candidate**
|

|
||||||
|
|
||||||
The canonical release controls are:
|
## Waarom GeoIntel?
|
||||||
|
|
||||||
- `docs/00-start/START_HERE.md`
|
Veel geoportalen tonen lagen. GeoIntel ondersteunt een volledige,
|
||||||
- `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md`
|
controleerbare analyseketen:
|
||||||
- `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`
|
|
||||||
- `docs/RELEASE_RUNBOOK.md`
|
|
||||||
- `docs/KNOWN_LIMITATIONS.md`
|
|
||||||
- `docs/DEFINITION_OF_DONE.md`
|
|
||||||
|
|
||||||
Older milestone and sprint handoff files remain historical evidence. They do
|
- selecteer een officiële grens of teken een eigen Area of Interest;
|
||||||
not override the active national/maritime scope freeze or RC roadmap.
|
- combineer vector-, raster- en historische bronnen met expliciete dekking;
|
||||||
|
- voer GIS- en CUDA-versnelde beeldanalyse uit;
|
||||||
|
- vergelijk kandidaatresultaten met referentiedata;
|
||||||
|
- inspecteer false positives, false negatives, overlap en provenance;
|
||||||
|
- exporteer alleen wat als reproduceerbaar resultaat is vastgelegd.
|
||||||
|
|
||||||
## Core V1 vertical slice
|
Mol en de Kempen blijven de gouden regressiegebieden. De productgrens is heel
|
||||||
|
België plus de juridisch benoemde Belgische maritieme zones.
|
||||||
|
|
||||||
The first implementation target is:
|
## Product in beeld
|
||||||
|
|
||||||
1. Project + Area creation.
|
De onderstaande screenshots tonen de huidige applicatie met publieke
|
||||||
2. Dataset registration/upload and metadata extraction.
|
demodata. De gastmodus toont een projectgebonden demowerkruimte
|
||||||
3. Reference building layer loading.
|
met dezelfde kaart-, bron-, model-, analyse-, QA- en exportflow als een
|
||||||
4. Predicted detection layer loading/import.
|
operator. Alleen beheer, instellingen, uploads, bronconfiguratie,
|
||||||
5. QA/QC matching against reference polygons.
|
projectbeheer en evidence-review blijven afgeschermd.
|
||||||
6. Metrics and false positive/false negative outputs.
|
|
||||||
7. GeoJSON export.
|
|
||||||
8. Minimal map/workbench UI.
|
|
||||||
|
|
||||||
## Primary stack
|
### Interactieve projectketen
|
||||||
|
|
||||||
- Frontend: React, TypeScript, MapLibre GL, Deck.gl, Tailwind.
|
De landingspagina vertaalt de technische keten naar vier interactieve
|
||||||
- Backend: FastAPI, Python.
|
schakels. De illustratie reageert op selectie, bronnen, analyse en bewijs en
|
||||||
- Database: PostgreSQL + PostGIS.
|
respecteert automatisch `prefers-reduced-motion`.
|
||||||
- GIS processing: GeoPandas, Shapely, Rasterio, PyProj, GDAL.
|
|
||||||
- AI: PyTorch, Ultralytics YOLO, SAM-compatible architecture.
|
|
||||||
- Jobs: Redis + RQ.
|
|
||||||
- Storage: local filesystem first, MinIO-compatible later.
|
|
||||||
|
|
||||||
## Codex instructions
|

|
||||||
|
|
||||||
Codex must start with:
|

|
||||||
|
|
||||||
1. `docs/00-start/START_HERE.md`
|
### Kaart als werkruimte
|
||||||
2. `prompts/codex/M11_ARCHITECT_MASTER_PROMPT.md`
|
|
||||||
|
|
||||||
Then follow the build order in:
|
De kaart blijft het primaire werkblad. Thema, broncontext, selectie en
|
||||||
|
resultaten zijn vanuit dezelfde ruimtelijke context bereikbaar.
|
||||||
|
|
||||||
- `docs/build/BUILD_ORDER_DEPENDENCY_GRAPH.md`
|

|
||||||
- `docs/build/CODEX_OPERATING_SYSTEM.md`
|
|
||||||
|
|
||||||
Before every implementation pass, run available preflight/smoke scripts where applicable.
|
Op een breed scherm krijgt de kaart extra ruimte terwijl de themakolom en de
|
||||||
|
controleerbare analysestappen zichtbaar blijven.
|
||||||
|
|
||||||
## Repo principle
|

|
||||||
|
|
||||||
This is a documentation-driven engineering repo. The documentation is not decorative; it is the control system for autonomous implementation.
|
### Kwaliteit vóór resultaat
|
||||||
|
|
||||||
## Fastest Day 1 command path
|
QA/QC is een eerste-klas workflow. Bewaarde controles koppelen scores aan
|
||||||
|
objectbewijs, kandidaat- en referentielagen en technische provenance.
|
||||||
|
|
||||||
```bash
|

|
||||||
make readiness
|
|
||||||
|
### Mobiele werkruimte
|
||||||
|
|
||||||
|
Dezelfde kaartgerichte workflow blijft bruikbaar op een smal scherm. Thema's,
|
||||||
|
selectieacties en de kaart worden gestapeld zonder de actieve werkcontext te
|
||||||
|
verbergen.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
## Kernmogelijkheden
|
||||||
|
|
||||||
|
| Domein | Mogelijkheden |
|
||||||
|
| --- | --- |
|
||||||
|
| Werkgebieden | Officiële grenzen, vrije AOI-selectie, clipping en buffering |
|
||||||
|
| Bronnen | GeoJSON, vector, raster, orthofoto, historische en regionale catalogi |
|
||||||
|
| GIS | CRS-normalisatie, spatial joins, rasterstatistiek, tile- en selectieanalyse |
|
||||||
|
| GeoAI | PyTorch, Ultralytics YOLO en SAM-compatibele abstrahering |
|
||||||
|
| Tijd | Immutable DatasetVersions en vergelijking van ruimtelijke snapshots |
|
||||||
|
| QA/QC | Precision, recall, F1, IoU, false-positive/-negative evidence en reviews |
|
||||||
|
| Export | Reproduceerbare GeoJSON- en artefactexports met provenance |
|
||||||
|
| Runtime | DockerMan-native Unraid-container met PostGIS, backend en frontend |
|
||||||
|
|
||||||
|
## Architectuur
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
UI["React + MapLibre workbench"] --> API["FastAPI contracts"]
|
||||||
|
API --> GIS["GeoPandas / Shapely / Rasterio"]
|
||||||
|
API --> JOBS["Job-queue in PostgreSQL"]
|
||||||
|
JOBS --> AI["PyTorch / YOLO / SAM"]
|
||||||
|
GIS --> DB[("PostgreSQL + PostGIS")]
|
||||||
|
AI --> DB
|
||||||
|
DB --> QA["QA/QC + provenance"]
|
||||||
|
QA --> UI
|
||||||
|
QA --> EXPORT["GeoJSON / artifacts"]
|
||||||
```
|
```
|
||||||
|
|
||||||
## Unraid / Tower deployment
|
De publieke endpoints volgen [docs/API_CONTRACTS.md](docs/API_CONTRACTS.md).
|
||||||
|
PostGIS en de persistentieregels volgen
|
||||||
|
[docs/DATABASE_IMPLEMENTATION_PLAN.md](docs/DATABASE_IMPLEMENTATION_PLAN.md).
|
||||||
|
|
||||||
GeoIntel runs on Unraid as an all-in-one DockerMan-native container. The container embeds PostGIS, runs the FastAPI backend internally, and serves the frontend through nginx on one editable web port.
|
## Stack
|
||||||
|
|
||||||
Unraid template assets live in:
|
- **Frontend:** React 18, TypeScript, MapLibre GL, Lucide en Vite
|
||||||
|
- **Backend:** FastAPI en Python
|
||||||
|
- **Spatial:** GeoPandas, Shapely, Rasterio, PyProj en GDAL
|
||||||
|
- **AI:** PyTorch, Ultralytics YOLO en SAM-compatibele segmentatie
|
||||||
|
- **Data:** PostgreSQL/PostGIS, lokale artefactopslag en immutable versions
|
||||||
|
- **Jobs:** job-tabel in PostgreSQL met achtergrondworkers in het API-proces
|
||||||
|
- **Deployment:** één DockerMan-native Unraid-container
|
||||||
|
|
||||||
- `deploy/unraid/geointel.env.example`
|
## Lokaal starten
|
||||||
- `deploy/unraid/geointel-unraid-template.xml`
|
|
||||||
- `deploy/unraid/geointel-icon.svg`
|
|
||||||
- `deploy/unraid/geointel-icon.png`
|
|
||||||
- `docker-compose.unraid.yml`
|
|
||||||
|
|
||||||
Copy the Unraid env template to `.env` in the checkout and edit ports/paths there:
|
Vereisten: Python 3.11+, Node 20.19+ of 22.12+ en PostgreSQL/PostGIS.
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /mnt/user/appdata/geointel
|
|
||||||
cp deploy/unraid/geointel.env.example .env
|
|
||||||
nano .env
|
|
||||||
docker build -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest .
|
|
||||||
bash deploy/unraid/run-dockerman-container.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
Common editable values:
|
|
||||||
|
|
||||||
```env
|
|
||||||
GEOINTEL_FRONTEND_PORT=1202
|
|
||||||
GEOINTEL_STORAGE_PATH=/mnt/user/appdata/geointel/storage
|
|
||||||
GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data
|
|
||||||
```
|
|
||||||
|
|
||||||
The backend and PostGIS ports are intentionally not exposed to the LAN in the all-in-one runtime. See `deploy/unraid/README.md` for full setup, port-change and cleanup notes.
|
|
||||||
|
|
||||||
On Tower/Unraid, `scripts/deploy_tower.ps1` and `scripts/deploy_tower.sh` validate the Compose reference but build with plain `docker build`, then automatically install the editable DockerMan template as `/boot/config/plugins/dockerMan/templates-user/my-geointel.xml`, install the PNG icon as `/boot/config/plugins/dockerMan/images/geointel-icon.png`, remove any old Compose-owned `geointel` container and start the final container with DockerMan labels.
|
|
||||||
|
|
||||||
## Sprint 2 quick start
|
|
||||||
|
|
||||||
- Update dependencies:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python -m pip install -e backend/.[dev]
|
python -m pip install -e backend/.[dev]
|
||||||
cd frontend && npm install
|
cd frontend
|
||||||
|
npm install
|
||||||
|
npm run start
|
||||||
```
|
```
|
||||||
|
|
||||||
- Run full readiness checks (with no scope expansion):
|
Voor de volledige lokale stack:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
docker compose up --build
|
||||||
|
```
|
||||||
|
|
||||||
|
Open daarna `http://localhost:1202`.
|
||||||
|
|
||||||
|
### Gastdemo
|
||||||
|
|
||||||
|
Met `GEOINTEL_AUTH_ENABLED=true` en de expliciete opt-in
|
||||||
|
`GEOINTEL_GUEST_ACCESS_ENABLED=true` biedt de toegangspagina een
|
||||||
|
kortlevende, projectgebonden demosessie. De gast heeft binnen het ingestelde
|
||||||
|
demoproject dezelfde kaart-, bronselectie-, modelselectie-, analyse-, QA- en
|
||||||
|
exportmogelijkheden als een operator. Alleen beheerfuncties zoals instellingen,
|
||||||
|
uploads, bronconfiguratie, projectbeheer en evidence-review blijven geblokkeerd;
|
||||||
|
dit is geen multi-user- of tenantisolatie. Gasttoegang staat standaard uit en
|
||||||
|
hoort alleen op een aparte demo-installatie met publieke data.
|
||||||
|
|
||||||
|
Een optionele Authentik OIDC-login kan naast de lokale operatorlogin worden
|
||||||
|
ingeschakeld. GeoIntel gebruikt daarbij PKCE, state, nonce, issuer-/audience-
|
||||||
|
controle en één expliciet toegelaten, geverifieerd e-mailadres. De lokale
|
||||||
|
operatorlogin blijft altijd het herstelpad.
|
||||||
|
|
||||||
|
## NVIDIA/CUDA
|
||||||
|
|
||||||
|
Productie-AI gebruikt de NVIDIA GPU van de server. De runtime faalt gesloten
|
||||||
|
wanneer CUDA vereist maar niet beschikbaar is; ontbrekende modelconfiguratie
|
||||||
|
wordt als `not_configured` getoond en nooit als geslaagde inferentie.
|
||||||
|
|
||||||
|
Corpus-, kalibratie-, test- en pure-background-gates blijven gescheiden: een
|
||||||
|
experimentele label- of modelvariant wordt pas actief nadat alle toepasselijke
|
||||||
|
kwaliteitscontroles slagen. Exacte runtime- en hardwaregegevens horen bij de
|
||||||
|
lokale deployment, niet bij de publieke broncode.
|
||||||
|
|
||||||
|
Controleer GPU-zichtbaarheid in de container met:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel nvidia-smi
|
||||||
|
```
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
## Training en labelkwaliteit
|
||||||
|
|
||||||
|
GeoIntel behandelt officiële gebouwgrondvlakken niet automatisch als perfecte
|
||||||
|
daklabels. Voor trainingsdata worden temporele geldigheid, ruimtelijke leakage,
|
||||||
|
bronklasse, zichtbaarheid en pure-background-gedrag afzonderlijk gecontroleerd.
|
||||||
|
Een directe polygon-overlay maakt bovendien zichtbaar of de officiële geometrie
|
||||||
|
op het bijbehorende luchtbeeld aansluit vóór omzetting naar YOLO-boxen. Voor
|
||||||
|
productresultaten blijft de taakgeschikte officiële bron doorslaggevend; een
|
||||||
|
AI-detectie is controleerbaar voorstelbewijs zolang een taakgebonden releasegate
|
||||||
|
niet aantoonbaar anders beslist.
|
||||||
|
|
||||||
|
Modelactivatie vereist naast het oudere diagnostische promotierapport ook een
|
||||||
|
geslaagd, governed Phase-4/5 release-gaterapport dat exact dezelfde candidate
|
||||||
|
key, model-SHA-256 en benchmarkmanifest-SHA-256 bindt. Zie
|
||||||
|
[Accuracy, modelgrenzen en bewijs](docs/ACCURACY.md). Er wordt geen
|
||||||
|
100%-accuratieclaim gemaakt; modeloutput blijft controleerbaar voorstelbewijs.
|
||||||
|
|
||||||
|
## Portfolio case study
|
||||||
|
|
||||||
|
De presentatieklare case study combineert productvraag, interface, PyTorch- en
|
||||||
|
NVIDIA-keten, architectuur en resultaten in één visueel document:
|
||||||
|
|
||||||
|
- [GeoIntel case study PDF](output/pdf/geointel-case-study.pdf)
|
||||||
|
- [Donkere cover in hoge resolutie](frontend/public/portfolio/geointel-dark-case-study-cover.png)
|
||||||
|
- [Architectuurvisual](docs/assets/portfolio/geointel-architecture.png)
|
||||||
|
|
||||||
|
De PDF kan reproduceerbaar opnieuw worden opgebouwd met:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python scripts/create_portfolio_case_study.py
|
||||||
|
```
|
||||||
|
|
||||||
|
## Kwaliteitscontrole
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Backend
|
||||||
python -m compileall backend/app
|
python -m compileall backend/app
|
||||||
cd backend && python -m pytest
|
cd backend && python -m pytest
|
||||||
cd ../frontend && npm run typecheck && npm run build
|
|
||||||
bash scripts/run_readiness_check.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
- Raster workflow validation command (backend only):
|
# Frontend
|
||||||
|
cd frontend
|
||||||
```bash
|
|
||||||
bash scripts/smoke_backend_import.sh
|
|
||||||
cd backend && python -c "from app.main import app; print(app.title)"
|
|
||||||
```
|
|
||||||
|
|
||||||
If `rasterio` is not installed, raster metadata endpoints return `RASTER_PROCESSING_UNAVAILABLE` and the frontend displays the
|
|
||||||
state as failed until the dependency is added.
|
|
||||||
|
|
||||||
## Sprint 4 raster foundation
|
|
||||||
|
|
||||||
- Raster operations now support:
|
|
||||||
- raster metadata extraction,
|
|
||||||
- raster preview generation,
|
|
||||||
- raster clip by area (with provenance on derived datasets),
|
|
||||||
- raster tile generation with manifest output.
|
|
||||||
- Raster services are dependency-aware:
|
|
||||||
- if `rasterio` is unavailable, endpoints return `RASTER_PROCESSING_UNAVAILABLE`.
|
|
||||||
- if preview dependencies (`numpy`, `pillow`) are unavailable, preview generation is unavailable with a clear error.
|
|
||||||
- Enable raster stack explicitly when needed:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd backend && python -m pip install -e .[dev,raster]
|
|
||||||
```
|
|
||||||
|
|
||||||
## Sprint 5 raster analytics hardening
|
|
||||||
|
|
||||||
- Added raster band statistics (min/max/mean/std, nodata ratio/count, valid pixel count, dtype, optional histograms).
|
|
||||||
- Added raster reproject workflow with CRS validation and provenance persistence.
|
|
||||||
- Extended tile manifest expectations (`tile_set_id`, `tile_size`, `overlap`, `bounds`, `source_raster_id`, `tile_paths`, `tile_server`).
|
|
||||||
- Clarified raster operation availability in frontend/backend docs (`RASTER_PROCESSING_UNAVAILABLE` and invalid-CRS cases).
|
|
||||||
|
|
||||||
- Raster workflow command set (where available):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd backend
|
|
||||||
python -m pip install -e .[dev,raster]
|
|
||||||
python -m pytest
|
|
||||||
cd ../frontend
|
|
||||||
npm run typecheck
|
npm run typecheck
|
||||||
|
npm run test:unit
|
||||||
npm run build
|
npm run build
|
||||||
```
|
```
|
||||||
|
|
||||||
Then give Codex the prompt in:
|
Een wijziging is pas afgerond wanneer de toepasselijke criteria uit
|
||||||
|
[docs/DEFINITION_OF_DONE.md](docs/DEFINITION_OF_DONE.md) aantoonbaar zijn
|
||||||
|
afgevinkt.
|
||||||
|
|
||||||
- `prompts/codex/final/DAY_1_MASTER_PROMPT.md`
|
## Unraid-deployment
|
||||||
|
|
||||||
|
De ondersteunde productieopstelling bundelt PostGIS, backend en frontend in één
|
||||||
|
Unraid-container. Kopieer de voorbeeldconfiguratie, vul de verplichte
|
||||||
|
authenticatie- en opslaginstellingen in en gebruik de releaseworkflow met
|
||||||
|
backup-, healthcheck- en rollbackcontrole.
|
||||||
|
|
||||||
## M13 Codex optimization
|
Zie [deploy/unraid/README.md](deploy/unraid/README.md) en
|
||||||
|
[docs/RELEASE_RUNBOOK.md](docs/RELEASE_RUNBOOK.md) voor configuratie,
|
||||||
|
healthchecks en rollback.
|
||||||
|
|
||||||
For the first serious Codex build run, use:
|
## Documentatiekaart
|
||||||
|
|
||||||
- `prompts/codex/m13/DAY_1_OPTIMIZED_MASTER_PROMPT.md`
|
- [Start hier](docs/00-start/START_HERE.md)
|
||||||
|
- [Belgium/North Sea scope freeze](docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md)
|
||||||
|
- [Actieve release-roadmap](docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md)
|
||||||
|
- [API-contracten](docs/API_CONTRACTS.md)
|
||||||
|
- [Database-implementatieplan](docs/DATABASE_IMPLEMENTATION_PLAN.md)
|
||||||
|
- [Bekende beperkingen](docs/KNOWN_LIMITATIONS.md)
|
||||||
|
- [Release-runbook](docs/RELEASE_RUNBOOK.md)
|
||||||
|
|
||||||
Codex should also use the relevant reusable skill under `skills/` for each implementation pass. Validate the optimization assets with:
|
## Portfolio-assets
|
||||||
|
|
||||||
```bash
|
De map [`docs/assets/portfolio`](docs/assets/portfolio) bevat presentatieklare
|
||||||
make m13
|
screenshots, de geanimeerde projectketen, architectuur- en AI-visuals. De drie
|
||||||
```
|
originele campagnebeelden staan in
|
||||||
|
[`frontend/public/portfolio`](frontend/public/portfolio). Geoptimaliseerde
|
||||||
|
WebP-varianten worden door de applicatie gebruikt; de PNG-bronnen blijven
|
||||||
|
beschikbaar voor drukwerk en portfolio-opmaak.
|
||||||
|
|
||||||
The full readiness path remains:
|
Actuele rasterassets:
|
||||||
|
|
||||||
```bash
|
| Asset | Gebruik |
|
||||||
make readiness
|
| --- | --- |
|
||||||
```
|
| `geointel-landing-hero.png` | Desktop hero en projectintroductie |
|
||||||
|
| `geointel-interactive-story.png` | Vierstappenworkflow en bewijsvoering |
|
||||||
|
| `geointel-workbench-map.png` | Kaartgerichte gastwerkruimte |
|
||||||
|
| `geointel-workbench-wide.png` | Brede kaartwerkruimte voor desktopportfolio's |
|
||||||
|
| `geointel-workbench-quality.png` | QA/QC, metrics en objectbewijs |
|
||||||
|
| `geointel-landing-mobile.png` | Mobiele landing |
|
||||||
|
| `geointel-workbench-mobile.png` | Mobiele kaartworkflow |
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
## M14 Build Launch
|
## Status
|
||||||
|
|
||||||
For the first serious implementation run, use:
|
Actieve mijlpaal: **v1.0.0 — Belgium and Belgian North Sea**.
|
||||||
|
|
||||||
- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md`
|
GeoIntel is een project van Jens / ITWorx.tech.
|
||||||
- `docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md`
|
|
||||||
- `docs/40-build-launch/CODEX_STOP_RULES.md`
|
|
||||||
- `prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md`
|
|
||||||
|
|
||||||
Validate launch assets with:
|
## Licentie
|
||||||
|
|
||||||
```bash
|
GeoIntel is beschikbaar onder de [Apache License 2.0](LICENSE).
|
||||||
make m14
|
|
||||||
```
|
|
||||||
|
|
||||||
Full readiness remains:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make readiness
|
|
||||||
```
|
|
||||||
|
|
||||||
## Sprint 1 execution (Sprint 1 only)
|
|
||||||
|
|
||||||
From a clean machine:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd backend && python -m pip install -e .[dev]
|
|
||||||
cd ..
|
|
||||||
make backend-install
|
|
||||||
make frontend-install
|
|
||||||
make readiness
|
|
||||||
```
|
|
||||||
|
|
||||||
Copy `.env.example` to `.env` only when you want local overrides. Docker Compose has safe defaults for the local PostGIS/backend/frontend stack and does not require a root `.env` file to exist.
|
|
||||||
|
|
||||||
With Docker Compose, open the workbench at `http://localhost:1202`.
|
|
||||||
|
|
||||||
The Docker frontend is served by nginx and proxies `/api` and `/health` to the backend container, so browser clients should use the frontend URL only, for example `http://192.168.10.150:1202` on a LAN host.
|
|
||||||
|
|
||||||
Runtime containers include healthchecks for PostGIS, backend and frontend. After
|
|
||||||
startup, inspect them with:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose ps
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify the browser-facing API proxy after rebuilding Docker images:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash scripts/verify_browser_runtime.sh http://localhost:1202 http://localhost:8000/health
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify the Docker GIS runtime after rebuilding the backend image:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash scripts/verify_gis_runtime.sh http://localhost:1202
|
|
||||||
```
|
|
||||||
|
|
||||||
On the LAN host use the published browser URL, for example:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash scripts/verify_gis_runtime.sh http://192.168.10.150:1202
|
|
||||||
```
|
|
||||||
|
|
||||||
Load the explicit offline demo workflow:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -X POST http://192.168.10.150:1202/api/v1/demo/workflow
|
|
||||||
```
|
|
||||||
|
|
||||||
If `/api/v1/projects` returns frontend HTML instead of a JSON envelope, rebuild
|
|
||||||
and restart the frontend container.
|
|
||||||
|
|
||||||
Useful direct verification commands:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m compileall backend/app
|
|
||||||
cd backend && python -c "from app.main import app; print(app.title)"
|
|
||||||
python -m pytest
|
|
||||||
cd ../frontend && npm run typecheck
|
|
||||||
cd ../frontend && npm run build
|
|
||||||
docker compose config
|
|
||||||
bash scripts/run_readiness_check.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
If `make` or `docker` are unavailable in your shell, run the equivalent script entrypoints directly:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash scripts/backend_install.sh
|
|
||||||
bash scripts/backend_test.sh
|
|
||||||
bash scripts/frontend_install.sh
|
|
||||||
bash scripts/frontend_typecheck.sh
|
|
||||||
bash scripts/frontend_build.sh
|
|
||||||
bash scripts/run_readiness_check.sh
|
|
||||||
```
|
|
||||||
|
|||||||
+38
@@ -0,0 +1,38 @@
|
|||||||
|
# Security Policy
|
||||||
|
|
||||||
|
## Supported code
|
||||||
|
|
||||||
|
Security fixes target the current `main` release line. Historical research,
|
||||||
|
calibration and development branches are not supported release channels unless
|
||||||
|
explicitly documented otherwise.
|
||||||
|
|
||||||
|
## Reporting vulnerabilities
|
||||||
|
|
||||||
|
Report suspected security issues privately to the repository owner. Do not put
|
||||||
|
credentials, access tokens, private infrastructure, precise sensitive
|
||||||
|
locations, proprietary imagery or datasets, model artifacts, production
|
||||||
|
database contents, personal data or exploit-sensitive evidence in a public
|
||||||
|
issue.
|
||||||
|
|
||||||
|
Include the affected commit and component, minimal reproduction conditions
|
||||||
|
using synthetic or public data where possible, expected and observed
|
||||||
|
behaviour, impact, and whether the issue affects ingestion, geospatial
|
||||||
|
processing, authentication or authorization, uploads, archive or path handling,
|
||||||
|
model inference, reports and exports, deployment, data retention or secrets.
|
||||||
|
|
||||||
|
## Repository and data boundary
|
||||||
|
|
||||||
|
Geospatial inputs and derived outputs can reveal people, assets and sensitive
|
||||||
|
locations. Treat screenshots, contact sheets, coordinates, source archives,
|
||||||
|
SQLite/WAL state, model output and exports as potentially sensitive. These
|
||||||
|
belong in controlled external storage, not the source repository.
|
||||||
|
|
||||||
|
Local Codex inputs, calibration databases, runtime data, generated reports and
|
||||||
|
cross-project scratch are not release source. Never commit live `.env` files,
|
||||||
|
private keys, production backups or databases, customer or operator data,
|
||||||
|
unpublished imagery, credentials or private datasets. Public examples and tests
|
||||||
|
must use synthetic or explicitly redistributable material.
|
||||||
|
|
||||||
|
## Disclosure
|
||||||
|
|
||||||
|
Coordinate remediation and disclosure with the repository owner before publishing details that would materially increase exploitation risk.
|
||||||
@@ -8,3 +8,5 @@ storage
|
|||||||
dist
|
dist
|
||||||
node_modules
|
node_modules
|
||||||
.env
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
|
|||||||
+109
-19
@@ -142,6 +142,7 @@ bash scripts/live_migration_smoke.sh
|
|||||||
- `GET /api/v1/detection/models`
|
- `GET /api/v1/detection/models`
|
||||||
- `GET /api/v1/detection/model-assets`
|
- `GET /api/v1/detection/model-assets`
|
||||||
- `POST /api/v1/detection/run`
|
- `POST /api/v1/detection/run`
|
||||||
|
- `POST /api/v1/detection/run-async` (production browser path)
|
||||||
- `GET /api/v1/detection/runs/{analysis_run_id}`
|
- `GET /api/v1/detection/runs/{analysis_run_id}`
|
||||||
- `GET /api/v1/detection/runs/{analysis_run_id}/detections`
|
- `GET /api/v1/detection/runs/{analysis_run_id}/detections`
|
||||||
- YOLO/PyTorch real inference is not enabled in Sprint 8.
|
- YOLO/PyTorch real inference is not enabled in Sprint 8.
|
||||||
@@ -184,6 +185,7 @@ bash scripts/live_migration_smoke.sh
|
|||||||
- Added segmentation endpoints:
|
- Added segmentation endpoints:
|
||||||
- `GET /api/v1/segmentation/models`
|
- `GET /api/v1/segmentation/models`
|
||||||
- `POST /api/v1/segmentation/run`
|
- `POST /api/v1/segmentation/run`
|
||||||
|
- `POST /api/v1/segmentation/run-async` (production browser path)
|
||||||
- `GET /api/v1/segmentation/runs`
|
- `GET /api/v1/segmentation/runs`
|
||||||
- `GET /api/v1/segmentation/runs/{analysis_run_id}`
|
- `GET /api/v1/segmentation/runs/{analysis_run_id}`
|
||||||
- `GET /api/v1/segmentation/runs/{analysis_run_id}/segmentations`
|
- `GET /api/v1/segmentation/runs/{analysis_run_id}/segmentations`
|
||||||
@@ -191,6 +193,11 @@ bash scripts/live_migration_smoke.sh
|
|||||||
- `POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference`
|
- `POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference`
|
||||||
- Real SAM and YOLO-seg inference are not enabled in Sprint 9.
|
- Real SAM and YOLO-seg inference are not enabled in Sprint 9.
|
||||||
- Mask paths are provenance/debug artifacts; persisted PostGIS geometry is authoritative for QA, map display and GeoJSON.
|
- Mask paths are provenance/debug artifacts; persisted PostGIS geometry is authoritative for QA, map display and GeoJSON.
|
||||||
|
- Current configured detection and segmentation run through the async analysis
|
||||||
|
worker (`GEOINTEL_ANALYSIS_WORKER_ENABLED`) and are followed through
|
||||||
|
`GET /api/v1/projects/{project_id}/jobs/{job_id}`. The Unraid profile sets
|
||||||
|
`YOLO_REQUIRE_CUDA=true`, so both pipelines fail closed instead of silently
|
||||||
|
falling back from NVIDIA CUDA to CPU.
|
||||||
|
|
||||||
## Sprint 17 additions
|
## Sprint 17 additions
|
||||||
- Added export foundation backed by the existing `exports` table.
|
- Added export foundation backed by the existing `exports` table.
|
||||||
@@ -453,7 +460,7 @@ To validate the full configured-YOLO runtime path against Docker/Tower after a
|
|||||||
model is mounted and selected, run:
|
model is mounted and selected, run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash scripts/verify_model_asset_detection_workflow.sh http://192.168.10.150:1202
|
bash scripts/verify_model_asset_detection_workflow.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
The smoke uses the existing demo raster to generate a tile manifest, selects a
|
The smoke uses the existing demo raster to generate a tile manifest, selects a
|
||||||
@@ -470,7 +477,7 @@ the runtime host, then run:
|
|||||||
```bash
|
```bash
|
||||||
REAL_RASTER_PATH=/mnt/user/appdata/geointel/data/orthophoto.tif \
|
REAL_RASTER_PATH=/mnt/user/appdata/geointel/data/orthophoto.tif \
|
||||||
REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/data/reference-buildings.geojson \
|
REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/data/reference-buildings.geojson \
|
||||||
bash scripts/verify_real_data_detection_qa_workflow.sh http://192.168.10.150:1202
|
bash scripts/verify_real_data_detection_qa_workflow.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
This smoke refuses missing/unsupported inputs, uploads the raster and reference
|
This smoke refuses missing/unsupported inputs, uploads the raster and reference
|
||||||
@@ -549,7 +556,7 @@ For model-quality calibration, run the confidence sweep wrapper:
|
|||||||
REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \
|
REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \
|
||||||
REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \
|
REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \
|
||||||
CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \
|
CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \
|
||||||
bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202
|
bash scripts/run_detection_calibration_sweep.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
The sweep creates one real persisted workflow run per threshold, fetches the
|
The sweep creates one real persisted workflow run per threshold, fetches the
|
||||||
@@ -569,7 +576,7 @@ QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
|
|||||||
QUALITY_TILE_SIZES="512 640" \
|
QUALITY_TILE_SIZES="512 640" \
|
||||||
QUALITY_TILE_OVERLAPS="64" \
|
QUALITY_TILE_OVERLAPS="64" \
|
||||||
QUALITY_THRESHOLDS="0.50 0.15" \
|
QUALITY_THRESHOLDS="0.50 0.15" \
|
||||||
bash scripts/run_detection_quality_matrix.sh http://192.168.10.150:1202
|
bash scripts/run_detection_quality_matrix.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
The matrix creates one real persisted workflow run per combination and writes
|
The matrix creates one real persisted workflow run per combination and writes
|
||||||
@@ -587,7 +594,7 @@ QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
|
|||||||
QUALITY_TILE_SIZES="512 640" \
|
QUALITY_TILE_SIZES="512 640" \
|
||||||
QUALITY_TILE_OVERLAPS="64" \
|
QUALITY_TILE_OVERLAPS="64" \
|
||||||
QUALITY_THRESHOLDS="0.50 0.15" \
|
QUALITY_THRESHOLDS="0.50 0.15" \
|
||||||
bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202
|
bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
The combined `multi_sample_quality_summary.json` reports per-sample and overall
|
The combined `multi_sample_quality_summary.json` reports per-sample and overall
|
||||||
@@ -604,7 +611,7 @@ QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-bu
|
|||||||
QUALITY_TILE_SIZES="640" \
|
QUALITY_TILE_SIZES="640" \
|
||||||
QUALITY_TILE_OVERLAPS="64" \
|
QUALITY_TILE_OVERLAPS="64" \
|
||||||
QUALITY_THRESHOLDS="0.25 0.15 0.05" \
|
QUALITY_THRESHOLDS="0.25 0.15 0.05" \
|
||||||
bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.168.10.150:1202
|
bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
This path uploads only background rasters, runs configured-YOLO detection and
|
This path uploads only background rasters, runs configured-YOLO detection and
|
||||||
@@ -617,7 +624,7 @@ evidence bundle:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
CALIBRATION_SUMMARY_PATH=/mnt/user/appdata/geointel/artifacts/detection-calibration/20260707T002103Z/calibration_summary.json \
|
CALIBRATION_SUMMARY_PATH=/mnt/user/appdata/geointel/artifacts/detection-calibration/20260707T002103Z/calibration_summary.json \
|
||||||
bash scripts/export_detection_calibration_evidence.sh http://192.168.10.150:1202
|
bash scripts/export_detection_calibration_evidence.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
The bundle writes combined QA evidence GeoJSON plus a standalone HTML/SVG review
|
The bundle writes combined QA evidence GeoJSON plus a standalone HTML/SVG review
|
||||||
@@ -731,7 +738,7 @@ python scripts/seed_demo_workflow.py --json
|
|||||||
In Docker Compose on a LAN host:
|
In Docker Compose on a LAN host:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -X POST http://192.168.10.150:1202/api/v1/demo/workflow
|
curl -X POST http://192.0.2.10:1202/api/v1/demo/workflow
|
||||||
```
|
```
|
||||||
|
|
||||||
### QA/QC result listing
|
### QA/QC result listing
|
||||||
@@ -816,7 +823,7 @@ After rebuilding a Docker/LAN deployment, verify the end-to-end demo and export
|
|||||||
flow through the browser-facing frontend proxy:
|
flow through the browser-facing frontend proxy:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash scripts/verify_demo_export_workflow.sh http://192.168.10.150:1202
|
bash scripts/verify_demo_export_workflow.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
The script seeds the explicit demo workflow, verifies persisted QA/QC results,
|
The script seeds the explicit demo workflow, verifies persisted QA/QC results,
|
||||||
@@ -879,7 +886,7 @@ bash scripts/verify_gis_runtime.sh http://localhost:1202
|
|||||||
On a NAS or server host, use the published LAN URL:
|
On a NAS or server host, use the published LAN URL:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash scripts/verify_gis_runtime.sh http://192.168.10.150:1202
|
bash scripts/verify_gis_runtime.sh http://192.0.2.10:1202
|
||||||
```
|
```
|
||||||
|
|
||||||
The script calls `/api/v1/system/capabilities` through the frontend proxy and
|
The script calls `/api/v1/system/capabilities` through the frontend proxy and
|
||||||
@@ -1336,7 +1343,10 @@ not copied into queryable properties.
|
|||||||
`GET /api/v1/projects/{project_id}/datasets/orthophoto/products` lists the
|
`GET /api/v1/projects/{project_id}/datasets/orthophoto/products` lists the
|
||||||
governed product allowlist. `POST .../datasets/orthophoto/acquire` accepts an
|
governed product allowlist. `POST .../datasets/orthophoto/acquire` accepts an
|
||||||
explicit EPSG:4326 map rectangle plus `product_key` and stores the official
|
explicit EPSG:4326 map rectangle plus `product_key` and stores the official
|
||||||
Digitaal Vlaanderen WMS response as a canonical EPSG:31370 raster Dataset. The
|
regional WMS response as a canonical EPSG:31370 raster Dataset. Digitaal
|
||||||
|
Vlaanderen, SPW (`wallonia_latest`) and Paradigm UrbIS (`brussels_latest`) are
|
||||||
|
allowlisted. The two regional products are bound to persisted Wallonia and
|
||||||
|
Brussels-Capital Region Areas. The
|
||||||
default safety envelope is 128-1,024 m per side, 1 m/pixel, 32 MiB and a
|
default safety envelope is 128-1,024 m per side, 1 m/pixel, 32 MiB and a
|
||||||
24-hour exact-request cache. It runs synchronously behind the existing Job
|
24-hour exact-request cache. It runs synchronously behind the existing Job
|
||||||
abstraction and never during startup.
|
abstraction and never during startup.
|
||||||
@@ -1348,11 +1358,16 @@ from configured-YOLO/current-GRB QA. `GET .../datasets/{dataset_id}/raster/image
|
|||||||
is the constrained binary PNG endpoint used by the MapLibre image overlay.
|
is the constrained binary PNG endpoint used by the MapLibre image overlay.
|
||||||
|
|
||||||
Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`,
|
Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`,
|
||||||
|
`SPW_ORTHOPHOTO_WMS_URL`, `BRUSSELS_ORTHOPHOTO_WMS_URL`,
|
||||||
`ORTHOPHOTO_WMS_LAYER`, `ORTHOPHOTO_RESOLUTION_M`,
|
`ORTHOPHOTO_WMS_LAYER`, `ORTHOPHOTO_RESOLUTION_M`,
|
||||||
`ORTHOPHOTO_MIN_SIDE_M`, `ORTHOPHOTO_MAX_SIDE_M`,
|
`ORTHOPHOTO_MIN_SIDE_M`, `ORTHOPHOTO_MAX_SIDE_M`,
|
||||||
`ORTHOPHOTO_TIMEOUT_SECONDS`, `ORTHOPHOTO_MAX_RESPONSE_MB` and
|
`ORTHOPHOTO_TIMEOUT_SECONDS`, `ORTHOPHOTO_MAX_RESPONSE_MB` and
|
||||||
`ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile
|
`ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile
|
||||||
unless a separately verified deployment/model profile requires a change.
|
unless a separately verified deployment/model profile requires a change.
|
||||||
|
An explicit bounded request may provide `resolution_m` down to the governed
|
||||||
|
product's native resolution. This is intended for reviewed training corpora;
|
||||||
|
the service rejects source oversampling and records rolling-latest observation
|
||||||
|
time as unknown per pixel rather than equating it with download time.
|
||||||
|
|
||||||
Before a future `most_recent` source release is allowed into a governed pixel
|
Before a future `most_recent` source release is allowed into a governed pixel
|
||||||
stage, run the metadata-only preflight for the exact intended rectangle:
|
stage, run the metadata-only preflight for the exact intended rectangle:
|
||||||
@@ -1599,6 +1614,53 @@ Datasets. A full-Flanders raster request remains blocked by the same 60 km and
|
|||||||
`Gemeente ...` as `coverage_scope=municipality`; regional Area clipping is
|
`Gemeente ...` as `coverage_scope=municipality`; regional Area clipping is
|
||||||
stored as `bounded_selection`.
|
stored as `bounded_selection`.
|
||||||
|
|
||||||
|
## Walloon WALOUS land cover and flood hazard
|
||||||
|
|
||||||
|
The Wallonia map flow uses bounded PICC vector products, the queryable legal
|
||||||
|
SPW flood-hazard polygon layer and provisioned official WALOUS land-cover
|
||||||
|
rasters. Provision the 2018, 2020 and 2023 source editions once in the persistent
|
||||||
|
storage mount:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/provision_walous_sources.py \
|
||||||
|
--years 2018 2020 2023 \
|
||||||
|
--destination /app/storage/source-cache/walous
|
||||||
|
```
|
||||||
|
|
||||||
|
The provisioner verifies advertised archive sizes, safe ZIP structure,
|
||||||
|
EPSG:3812, one band, 1 m cells, the official non-contiguous class codes
|
||||||
|
`1,2,3,4,5,6,7,8,9,80,90` and SHA-256 checksums. It does not run at
|
||||||
|
application startup. `GET .../datasets/walous/products` therefore reports
|
||||||
|
`source_not_provisioned` for each edition whose source file is absent.
|
||||||
|
|
||||||
|
For a bounded Walloon selection the browser persists the latest edition and
|
||||||
|
all other configured comparable editions. `POST .../raster/walous/select`
|
||||||
|
returns cell-area hectares; the temporal API compares the same semantic metric
|
||||||
|
keys for 2018, 2020 and 2023. The 2018 stacked classes use the official visible-
|
||||||
|
class crosswalk and retain the earlier-method limitation. WALOUS is land cover,
|
||||||
|
not legal land use, ownership,
|
||||||
|
tree count, timber volume or water volume.
|
||||||
|
|
||||||
|
The class semantics follow the official raster codes, not display-list
|
||||||
|
positions: 1 artificial ground, 2 above-ground construction, 3 railway, 4 bare
|
||||||
|
soil, 5 surface water, 6 rotating herbaceous cover, 7 continuous herbaceous
|
||||||
|
cover, 8/9 trees above 3 m and 80/90 woody cover up to 3 m. Observation ranges
|
||||||
|
are retained from the SPW metadata rather than replaced by arbitrary year-end
|
||||||
|
dates.
|
||||||
|
|
||||||
|
Settings: `WALOUS_ENABLED`, `WALOUS_SOURCE_DIR`,
|
||||||
|
`WALOUS_ANALYSIS_RESOLUTION_M`, `WALOUS_MAX_SIDE_M` and
|
||||||
|
`WALOUS_MAX_PIXELS`. The SPW flood polygon adapter uses
|
||||||
|
`SPW_FLOOD_HAZARD_ENABLED` and `SPW_FLOOD_HAZARD_MAPSERVER_URL`.
|
||||||
|
|
||||||
|
The official Walloon 2021-2022 1 m MNT is an explicit operator asset. Provision
|
||||||
|
it once with `scripts/provision_spw_terrain_source.py`; the runtime then reads
|
||||||
|
only bounded windows and persists 5 m analysis derivatives. The full 0.5 m
|
||||||
|
artifact remains intentionally excluded because it adds no V1 metric and is
|
||||||
|
about 213 GB. Settings: `SPW_TERRAIN_ENABLED`, `SPW_TERRAIN_SOURCE_DIR`,
|
||||||
|
`SPW_TERRAIN_ANALYSIS_RESOLUTION_M`, `SPW_TERRAIN_MAX_SIDE_M` and
|
||||||
|
`SPW_TERRAIN_MAX_PIXELS`.
|
||||||
|
|
||||||
Provision the official DOV soil polygons for Mol through the existing vector
|
Provision the official DOV soil polygons for Mol through the existing vector
|
||||||
upload path:
|
upload path:
|
||||||
|
|
||||||
@@ -1806,7 +1868,8 @@ matched name with `--show-names`.
|
|||||||
performs a bounded official VHA ArcGIS query, exact persisted-Area clipping,
|
performs a bounded official VHA ArcGIS query, exact persisted-Area clipping,
|
||||||
watercourse-name normalization and ordinary Dataset/VectorFeature persistence.
|
watercourse-name normalization and ordinary Dataset/VectorFeature persistence.
|
||||||
`GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as
|
`GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as
|
||||||
operational and the audited MDK/SPW candidates as unavailable for acquisition.
|
operational, MDK as probe-only and the pinned SPW raster operator as
|
||||||
|
operational.
|
||||||
|
|
||||||
Runtime controls are `BATHYMETRY_PROFILES_ENABLED`,
|
Runtime controls are `BATHYMETRY_PROFILES_ENABLED`,
|
||||||
`BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`,
|
`BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`,
|
||||||
@@ -1854,21 +1917,48 @@ readiness state such as TLS or endpoint failure. Runtime controls are
|
|||||||
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
|
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
|
||||||
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
|
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
|
||||||
|
|
||||||
## Governed forest, agriculture, nature and soil acquisition
|
### SPW waterbed raster
|
||||||
|
|
||||||
|
The official 2023-05-23 SPW bathymetry ZIP is integrated only through the
|
||||||
|
bounded operator. Stage the immutable ZIP under persistent storage and run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/import_spw_bathymetry.py \
|
||||||
|
--base-url http://127.0.0.1:8000 \
|
||||||
|
--project-name "Belgium and North Sea Workbench" \
|
||||||
|
--area "RC Golden - Wallonia urban-rural" \
|
||||||
|
--bbox 4.85,50.45,4.87,50.47 \
|
||||||
|
--raw-zip /app/storage/operator-evidence/spw-bathymetry/2023-05-23/raw/BATHY_50CM_ALTITUDE_DNG_GEOTIFF_3812.zip \
|
||||||
|
--output-dir /app/storage/operator-evidence/spw-bathymetry/2023-05-23/derived
|
||||||
|
```
|
||||||
|
|
||||||
|
The script validates the pinned official checksum, safe archive members,
|
||||||
|
EPSG:3812, one Float32 band, approximately 0.5 m cells and nodata `-9999`.
|
||||||
|
It then creates a bounded COG and uploads it through `/datasets/upload`.
|
||||||
|
`POST .../raster/bathymetry/select` returns waterbed elevation in mDNG,
|
||||||
|
surveyed surface and coverage. Current depth, volume and datum conversion stay
|
||||||
|
unavailable without a compatible water-surface source. Selection analysis is
|
||||||
|
bounded by `BATHYMETRY_RASTER_MAX_PIXELS` (30 million by default).
|
||||||
|
|
||||||
|
## Governed regional official-vector acquisition
|
||||||
|
|
||||||
The thematic raster registry includes forest and agricultural land-use masks
|
The thematic raster registry includes forest and agricultural land-use masks
|
||||||
derived from Landgebruik Vlaanderen 2025 classes 12 and 13/14. They use the
|
derived from Landgebruik Vlaanderen 2025 classes 12 and 13/14. They use the
|
||||||
existing thematic acquisition and selection routes.
|
existing thematic acquisition and selection routes.
|
||||||
|
|
||||||
Two polygon products are exposed through
|
Eight fixed products are exposed through
|
||||||
`/datasets/official-vector/products` and
|
`/datasets/official-vector/products` and
|
||||||
`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and the DOV
|
`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and DOV soil
|
||||||
digital soil map. Both require an EPSG:4326 rectangle, optionally intersect it
|
for Flanders; PICC buildings, roads, hydrographic axes and surfaces for
|
||||||
with a persisted Area, clip in EPSG:31370 and persist through
|
Wallonia; and UrbIS buildings and cadastral parcels for Brussels. All require
|
||||||
`DatasetService.import_vector_bytes`.
|
an EPSG:4326 rectangle, clip in a provider-appropriate metric CRS and persist
|
||||||
|
through `DatasetService.import_vector_bytes`. SPW/PICC and UrbIS additionally
|
||||||
|
require a persisted exact regional coverage Area and never write directly to
|
||||||
|
`vector_features`.
|
||||||
|
|
||||||
Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`,
|
Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`,
|
||||||
`DOV_SOIL_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`,
|
`DOV_SOIL_WFS_URL`, `SPW_PICC_ENABLED`, `SPW_PICC_MAPSERVER_URL`,
|
||||||
|
`URBIS_ENABLED`, `URBIS_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`,
|
||||||
`OFFICIAL_VECTOR_MAX_SIDE_M`, `OFFICIAL_VECTOR_PAGE_SIZE`,
|
`OFFICIAL_VECTOR_MAX_SIDE_M`, `OFFICIAL_VECTOR_PAGE_SIZE`,
|
||||||
`OFFICIAL_VECTOR_MAX_PAGES`, `OFFICIAL_VECTOR_MAX_FEATURES`,
|
`OFFICIAL_VECTOR_MAX_PAGES`, `OFFICIAL_VECTOR_MAX_FEATURES`,
|
||||||
`OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`,
|
`OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`,
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
"""Add resumable AOI parent and partition operations."""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from geoalchemy2 import Geometry
|
||||||
|
|
||||||
|
|
||||||
|
revision = "202607260001"
|
||||||
|
down_revision = "202607160001"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"aoi_operations",
|
||||||
|
sa.Column("id", sa.UUID(), primary_key=True),
|
||||||
|
sa.Column("project_id", sa.UUID(), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False),
|
||||||
|
sa.Column("area_id", sa.UUID(), sa.ForeignKey("areas.id", ondelete="SET NULL")),
|
||||||
|
sa.Column("parent_job_id", sa.UUID(), sa.ForeignKey("jobs.id", ondelete="SET NULL")),
|
||||||
|
sa.Column("operation_type", sa.String(128), nullable=False),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("geometry", Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False),
|
||||||
|
sa.Column("request_json", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("plan_json", sa.JSON(), nullable=False),
|
||||||
|
sa.Column("result_json", sa.JSON()),
|
||||||
|
sa.Column("error_message", sa.Text()),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
|
sa.CheckConstraint("status IN ('queued', 'running', 'partial', 'success', 'failed', 'cancelled')", name="ck_aoi_operations_status"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_aoi_operations_project_status", "aoi_operations", ["project_id", "status"])
|
||||||
|
op.create_index("ix_aoi_operations_geometry", "aoi_operations", ["geometry"], postgresql_using="gist")
|
||||||
|
op.create_table(
|
||||||
|
"aoi_operation_partitions",
|
||||||
|
sa.Column("id", sa.UUID(), primary_key=True),
|
||||||
|
sa.Column("operation_id", sa.UUID(), sa.ForeignKey("aoi_operations.id", ondelete="CASCADE"), nullable=False),
|
||||||
|
sa.Column("child_job_id", sa.UUID(), sa.ForeignKey("jobs.id", ondelete="SET NULL")),
|
||||||
|
sa.Column("partition_key", sa.String(255), nullable=False),
|
||||||
|
sa.Column("provider_key", sa.String(120), nullable=False),
|
||||||
|
sa.Column("product_key", sa.String(120), nullable=False),
|
||||||
|
sa.Column("ordinal", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("status", sa.String(32), nullable=False),
|
||||||
|
sa.Column("geometry", Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False),
|
||||||
|
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
|
||||||
|
sa.Column("checkpoint_json", sa.JSON()),
|
||||||
|
sa.Column("result_json", sa.JSON()),
|
||||||
|
sa.Column("error_message", sa.Text()),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
|
sa.CheckConstraint("status IN ('queued', 'running', 'success', 'failed', 'skipped')", name="ck_aoi_operation_partitions_status"),
|
||||||
|
sa.UniqueConstraint("operation_id", "partition_key", name="uq_aoi_operation_partition_key"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_aoi_operation_partitions_operation_status", "aoi_operation_partitions", ["operation_id", "status"])
|
||||||
|
op.create_index("ix_aoi_operation_partitions_geometry", "aoi_operation_partitions", ["geometry"], postgresql_using="gist")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("aoi_operation_partitions")
|
||||||
|
op.drop_table("aoi_operations")
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,43 @@
|
|||||||
|
"""Configure the immutable model source registry for governed snapshots.
|
||||||
|
|
||||||
|
The phase-2 seed intentionally registered model artifacts as unknown. Runtime
|
||||||
|
model provenance now records exact immutable snapshots, so the server-owned
|
||||||
|
registry must advertise that configured capability. The write guard is only
|
||||||
|
disabled for this narrowly-scoped, versioned migration and is restored in the
|
||||||
|
same transaction.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
|
||||||
|
revision = "202608230001"
|
||||||
|
down_revision = "202608010001"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def _set_status(*, freshness_status: str, ingest_status: str) -> None:
|
||||||
|
op.execute("ALTER TABLE source_registry DISABLE TRIGGER trg_source_registry_write_guard")
|
||||||
|
op.execute(
|
||||||
|
f"""
|
||||||
|
UPDATE source_registry
|
||||||
|
SET freshness_status = '{freshness_status}',
|
||||||
|
ingest_status = '{ingest_status}',
|
||||||
|
registry_metadata_json = (
|
||||||
|
registry_metadata_json::jsonb ||
|
||||||
|
'{{"runtime_model_contract": {{"key": "geointel.model.pytorch", "version": "1.0.0"}}}}'::jsonb
|
||||||
|
)::json,
|
||||||
|
updated_at = now()
|
||||||
|
WHERE source_key = 'model'
|
||||||
|
AND registry_metadata_json ->> 'registry_owner' = 'server'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.execute("ALTER TABLE source_registry ENABLE TRIGGER trg_source_registry_write_guard")
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
_set_status(freshness_status="current", ingest_status="configured")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
_set_status(freshness_status="unknown", ingest_status="registered")
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import Request
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
|
||||||
|
|
||||||
|
def guest_project_scope(request: Request) -> UUID | None:
|
||||||
|
principal = getattr(request.state, "auth_principal", None)
|
||||||
|
if getattr(principal, "role", None) != "guest":
|
||||||
|
return None
|
||||||
|
project_id = getattr(principal, "project_id", None)
|
||||||
|
if isinstance(project_id, UUID):
|
||||||
|
return project_id
|
||||||
|
raise AppError(
|
||||||
|
code="GUEST_PROJECT_SCOPE_REQUIRED",
|
||||||
|
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||||
|
status_code=403,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def assert_guest_project_scope(request: Request, project_id: UUID) -> None:
|
||||||
|
guest_project_id = guest_project_scope(request)
|
||||||
|
if guest_project_id is not None and project_id != guest_project_id:
|
||||||
|
raise AppError(
|
||||||
|
code="GUEST_PROJECT_SCOPE_REQUIRED",
|
||||||
|
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||||
|
status_code=403,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def guest_scoped_project_filter(
|
||||||
|
request: Request,
|
||||||
|
requested_project_id: UUID | None,
|
||||||
|
) -> UUID | None:
|
||||||
|
guest_project_id = guest_project_scope(request)
|
||||||
|
if guest_project_id is None:
|
||||||
|
return requested_project_id
|
||||||
|
if requested_project_id is not None:
|
||||||
|
assert_guest_project_scope(request, requested_project_id)
|
||||||
|
return guest_project_id
|
||||||
@@ -1 +1,15 @@
|
|||||||
__all__ = ["analysis", "areas", "assistant", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
|
__all__ = [
|
||||||
|
"analysis",
|
||||||
|
"areas",
|
||||||
|
"assistant",
|
||||||
|
"auth",
|
||||||
|
"datasets",
|
||||||
|
"exports",
|
||||||
|
"external",
|
||||||
|
"health",
|
||||||
|
"jobs",
|
||||||
|
"projects",
|
||||||
|
"qa",
|
||||||
|
"source_registry",
|
||||||
|
"temporal",
|
||||||
|
]
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends, Request
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.guest_scope import assert_guest_project_scope
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.models import Dataset
|
from app.models import Dataset
|
||||||
@@ -18,12 +19,15 @@ router = APIRouter(prefix="/analysis", tags=["analysis"])
|
|||||||
@router.post("/change-detection", response_model=Envelope[JobRead])
|
@router.post("/change-detection", response_model=Envelope[JobRead])
|
||||||
def run_change_detection(
|
def run_change_detection(
|
||||||
payload: ChangeDetectionRequest,
|
payload: ChangeDetectionRequest,
|
||||||
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
source_dataset = db.get(Dataset, payload.source_dataset_id)
|
source_dataset = db.get(Dataset, payload.source_dataset_id)
|
||||||
if not source_dataset:
|
if not source_dataset:
|
||||||
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
|
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
|
||||||
|
assert_guest_project_scope(request, source_dataset.project_id)
|
||||||
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
|
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
|
||||||
|
ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target")
|
||||||
job = JobService.run_sync_job(
|
job = JobService.run_sync_job(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=source_dataset.project_id,
|
project_id=source_dataset.project_id,
|
||||||
@@ -36,7 +40,11 @@ def run_change_detection(
|
|||||||
source_dataset_id=payload.source_dataset_id,
|
source_dataset_id=payload.source_dataset_id,
|
||||||
target_dataset_id=payload.target_dataset_id,
|
target_dataset_id=payload.target_dataset_id,
|
||||||
iou_threshold=payload.iou_threshold,
|
iou_threshold=payload.iou_threshold,
|
||||||
|
modified_threshold=payload.modified_threshold,
|
||||||
include_unchanged=payload.include_unchanged,
|
include_unchanged=payload.include_unchanged,
|
||||||
|
bbox=payload.bbox.model_dump() if payload.bbox is not None else None,
|
||||||
|
area_id=payload.area_id,
|
||||||
|
preview_limit=payload.preview_limit,
|
||||||
).model_dump(mode="json"),
|
).model_dump(mode="json"),
|
||||||
)
|
)
|
||||||
return envelope(job)
|
return envelope(job)
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.db.session import get_db
|
||||||
|
from app.schemas.aoi_operation import AoiOperationCreate, AoiOperationList, AoiOperationRead, AoiPartitionCheckpoint, AoiPartitionComplete, AoiPartitionFail, AoiPartitionRead
|
||||||
|
from app.schemas.common import Envelope
|
||||||
|
from app.services.aoi_operation_service import AoiOperationService
|
||||||
|
from app.services.aoi_operation_executor import AoiOperationExecutor
|
||||||
|
from app.utils.response import envelope
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/projects/{project_id}/aoi-operations", tags=["aoi-operations"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", status_code=201, response_model=Envelope[AoiOperationRead])
|
||||||
|
def create_operation(project_id: UUID, payload: AoiOperationCreate, db: Session = Depends(get_db)):
|
||||||
|
return envelope(AoiOperationService.create(db, project_id, payload))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=Envelope[AoiOperationList])
|
||||||
|
def list_operations(project_id: UUID, limit: int = Query(default=50, ge=1, le=200), db: Session = Depends(get_db)):
|
||||||
|
return envelope(AoiOperationService.list(db, project_id, limit))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{operation_id}", response_model=Envelope[AoiOperationRead])
|
||||||
|
def read_operation(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
|
||||||
|
return envelope(AoiOperationService.read(db, project_id, operation_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{operation_id}/partitions/claim", response_model=Envelope[AoiPartitionRead | None])
|
||||||
|
def claim_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
|
||||||
|
partition = AoiOperationService.claim_next(db, project_id, operation_id)
|
||||||
|
return envelope(AoiPartitionRead.model_validate(partition).model_dump() if partition else None)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{operation_id}/execute-next", response_model=Envelope[AoiOperationRead])
|
||||||
|
def execute_next_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
|
||||||
|
return envelope(AoiOperationExecutor.execute_next(db, project_id, operation_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{operation_id}/partitions/{partition_id}/checkpoint", response_model=Envelope[AoiPartitionRead])
|
||||||
|
def checkpoint_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionCheckpoint, db: Session = Depends(get_db)):
|
||||||
|
partition = AoiOperationService.checkpoint(db, project_id, operation_id, partition_id, payload.checkpoint_json)
|
||||||
|
return envelope(AoiPartitionRead.model_validate(partition).model_dump())
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{operation_id}/partitions/{partition_id}/complete", response_model=Envelope[AoiOperationRead])
|
||||||
|
def complete_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionComplete, db: Session = Depends(get_db)):
|
||||||
|
return envelope(AoiOperationService.complete(db, project_id, operation_id, partition_id, payload.result_json, payload.skipped))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{operation_id}/partitions/{partition_id}/fail", response_model=Envelope[AoiOperationRead])
|
||||||
|
def fail_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionFail, db: Session = Depends(get_db)):
|
||||||
|
return envelope(AoiOperationService.fail(db, project_id, operation_id, partition_id, payload.error_message, payload.retryable, payload.details))
|
||||||
@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
|
|||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.models import Area
|
from app.models import Area
|
||||||
from app.schemas import Envelope
|
from app.schemas import Envelope
|
||||||
from app.schemas.area import AreaCreate, AreaList, AreaRead, AreaUpdate
|
from app.schemas.area import AreaCreate, AreaList, AreaRead, AreaUpdate, MunicipalitySearchList
|
||||||
from app.services.area_service import AreaService
|
from app.services.area_service import AreaService
|
||||||
from app.utils.response import envelope
|
from app.utils.response import envelope
|
||||||
|
|
||||||
@@ -33,6 +33,23 @@ def create_area(project_id: UUID, payload: AreaCreate, db: Session = Depends(get
|
|||||||
return envelope(AreaService.serialize_area(area))
|
return envelope(AreaService.serialize_area(area))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/municipalities", response_model=Envelope[MunicipalitySearchList])
|
||||||
|
def search_municipalities(
|
||||||
|
project_id: UUID,
|
||||||
|
query: str = Query(default="", max_length=120),
|
||||||
|
limit: int = Query(default=20, ge=1, le=50),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
items, total = AreaService.search_municipalities(db, project_id, query, limit)
|
||||||
|
return envelope({"items": items, "total": total})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/municipalities/{niscode}/activate", response_model=Envelope[AreaRead])
|
||||||
|
def activate_municipality(project_id: UUID, niscode: str, db: Session = Depends(get_db)):
|
||||||
|
area = AreaService.activate_municipality(db, project_id, niscode)
|
||||||
|
return envelope(AreaService.serialize_area(area))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{area_id}", response_model=Envelope[AreaRead])
|
@router.get("/{area_id}", response_model=Envelope[AreaRead])
|
||||||
def get_area(
|
def get_area(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
|
|||||||
@@ -0,0 +1,326 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from ipaddress import ip_address, ip_network
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Request, Response, status
|
||||||
|
from fastapi.responses import RedirectResponse
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.db.session import get_db
|
||||||
|
from app.schemas.auth import AuthLoginRequest, AuthSession, AuthSessionEnvelope
|
||||||
|
from app.services.auth_service import AuthPrincipal, AuthService
|
||||||
|
from app.services.authentik_oidc_service import AuthentikOidcService
|
||||||
|
from app.services.demo_workflow_service import DemoWorkflowService
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||||
|
COOKIE_NAME = "geointel_session"
|
||||||
|
OIDC_FLOW_COOKIE_NAME = "geointel_oidc_flow"
|
||||||
|
logger = logging.getLogger("geointel.auth")
|
||||||
|
_TRUSTED_PROXY_NETWORKS = (
|
||||||
|
ip_network("127.0.0.0/8"),
|
||||||
|
ip_network("::1/128"),
|
||||||
|
ip_network("172.16.0.0/12"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _peer_is_trusted_proxy(request: Request) -> bool:
|
||||||
|
if request.client is None:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
peer_address = ip_address(request.client.host)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
return any(peer_address in network for network in _TRUSTED_PROXY_NETWORKS)
|
||||||
|
|
||||||
|
|
||||||
|
def _request_is_https(request: Request) -> bool:
|
||||||
|
if request.url.scheme == "https":
|
||||||
|
return True
|
||||||
|
if not _peer_is_trusted_proxy(request):
|
||||||
|
return False
|
||||||
|
forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",", 1)[0].strip().lower()
|
||||||
|
return forwarded_proto == "https"
|
||||||
|
|
||||||
|
|
||||||
|
def _client_host(request: Request) -> str:
|
||||||
|
peer = request.client.host if request.client else "unknown"
|
||||||
|
if not _peer_is_trusted_proxy(request):
|
||||||
|
return peer
|
||||||
|
forwarded = request.headers.get("x-real-ip", "").strip()
|
||||||
|
if not forwarded:
|
||||||
|
return peer
|
||||||
|
try:
|
||||||
|
return str(ip_address(forwarded))
|
||||||
|
except ValueError:
|
||||||
|
return peer
|
||||||
|
|
||||||
|
|
||||||
|
def _session_from_principal(
|
||||||
|
principal: AuthPrincipal,
|
||||||
|
*,
|
||||||
|
guest_access_enabled: bool,
|
||||||
|
authentik_enabled: bool,
|
||||||
|
) -> AuthSession:
|
||||||
|
return AuthSession(
|
||||||
|
authentication_required=True,
|
||||||
|
authenticated=True,
|
||||||
|
username=principal.username,
|
||||||
|
expires_at=datetime.fromtimestamp(principal.expires_at, tz=UTC),
|
||||||
|
role=principal.role,
|
||||||
|
guest_access_enabled=guest_access_enabled,
|
||||||
|
authentik_enabled=authentik_enabled,
|
||||||
|
guest_project_id=principal.project_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _session_payload(request: Request) -> AuthSession:
|
||||||
|
settings = get_settings()
|
||||||
|
guest_access_enabled = settings.auth_enabled and settings.guest_access_enabled
|
||||||
|
authentik_enabled = AuthentikOidcService(settings).enabled
|
||||||
|
if not settings.auth_enabled:
|
||||||
|
return AuthSession(
|
||||||
|
authentication_required=False,
|
||||||
|
authenticated=True,
|
||||||
|
guest_access_enabled=False,
|
||||||
|
authentik_enabled=False,
|
||||||
|
)
|
||||||
|
principal = AuthService.verify_session_token(request.cookies.get(COOKIE_NAME), settings)
|
||||||
|
if principal is None:
|
||||||
|
return AuthSession(
|
||||||
|
authentication_required=True,
|
||||||
|
authenticated=False,
|
||||||
|
guest_access_enabled=guest_access_enabled,
|
||||||
|
authentik_enabled=authentik_enabled,
|
||||||
|
)
|
||||||
|
return _session_from_principal(
|
||||||
|
principal,
|
||||||
|
guest_access_enabled=guest_access_enabled,
|
||||||
|
authentik_enabled=authentik_enabled,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _set_session_cookie(
|
||||||
|
*,
|
||||||
|
request: Request,
|
||||||
|
response: Response,
|
||||||
|
token: str,
|
||||||
|
max_age: int,
|
||||||
|
) -> None:
|
||||||
|
response.set_cookie(
|
||||||
|
key=COOKIE_NAME,
|
||||||
|
value=token,
|
||||||
|
max_age=max_age,
|
||||||
|
httponly=True,
|
||||||
|
secure=_request_is_https(request),
|
||||||
|
samesite="strict",
|
||||||
|
path="/",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/session", response_model=AuthSessionEnvelope)
|
||||||
|
def session(request: Request) -> AuthSessionEnvelope:
|
||||||
|
return AuthSessionEnvelope(data=_session_payload(request))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/login", response_model=AuthSessionEnvelope)
|
||||||
|
def login(payload: AuthLoginRequest, request: Request, response: Response) -> AuthSessionEnvelope:
|
||||||
|
settings = get_settings()
|
||||||
|
if not settings.auth_enabled:
|
||||||
|
raise AppError(
|
||||||
|
code="AUTHENTICATION_DISABLED",
|
||||||
|
message="Operator authentication is not enabled on this runtime",
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
)
|
||||||
|
if settings.auth_require_https and not _request_is_https(request):
|
||||||
|
raise AppError(
|
||||||
|
code="AUTH_HTTPS_REQUIRED",
|
||||||
|
message="Operator authentication requires HTTPS on this runtime",
|
||||||
|
status_code=status.HTTP_426_UPGRADE_REQUIRED,
|
||||||
|
)
|
||||||
|
client_host = _client_host(request)
|
||||||
|
throttle_key = f"{client_host}:{payload.username.casefold()}"
|
||||||
|
retry_after = AuthService.retry_after_seconds(throttle_key)
|
||||||
|
if retry_after:
|
||||||
|
raise AppError(
|
||||||
|
code="LOGIN_RATE_LIMITED",
|
||||||
|
message="Te veel mislukte aanmeldpogingen. Probeer later opnieuw.",
|
||||||
|
details={"retry_after_seconds": retry_after},
|
||||||
|
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||||
|
)
|
||||||
|
if not AuthService.credentials_match(payload.username, payload.password, settings):
|
||||||
|
AuthService.record_failure(throttle_key)
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_CREDENTIALS",
|
||||||
|
message="Gebruikersnaam of wachtwoord is onjuist.",
|
||||||
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||||
|
)
|
||||||
|
AuthService.clear_failures(throttle_key)
|
||||||
|
token = AuthService.create_session_token(payload.username, settings)
|
||||||
|
principal = AuthService.verify_session_token(token, settings)
|
||||||
|
if principal is None: # pragma: no cover - defensive invariant
|
||||||
|
raise AppError(
|
||||||
|
code="SESSION_CREATION_FAILED",
|
||||||
|
message="De beveiligde sessie kon niet worden aangemaakt.",
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
)
|
||||||
|
_set_session_cookie(
|
||||||
|
request=request,
|
||||||
|
response=response,
|
||||||
|
token=token,
|
||||||
|
max_age=settings.auth_session_ttl_seconds,
|
||||||
|
)
|
||||||
|
return AuthSessionEnvelope(
|
||||||
|
data=_session_from_principal(
|
||||||
|
principal,
|
||||||
|
guest_access_enabled=settings.guest_access_enabled,
|
||||||
|
authentik_enabled=AuthentikOidcService(settings).enabled,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/authentik/start")
|
||||||
|
def authentik_start(request: Request) -> RedirectResponse:
|
||||||
|
settings = get_settings()
|
||||||
|
service = AuthentikOidcService(settings)
|
||||||
|
try:
|
||||||
|
location, flow = service.start()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Authentik authorization start failed: %s", type(exc).__name__)
|
||||||
|
raise AppError(
|
||||||
|
code="AUTHENTIK_UNAVAILABLE",
|
||||||
|
message="Authentik is momenteel niet beschikbaar.",
|
||||||
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||||
|
) from exc
|
||||||
|
response = RedirectResponse(location, status_code=status.HTTP_302_FOUND)
|
||||||
|
response.set_cookie(
|
||||||
|
OIDC_FLOW_COOKIE_NAME,
|
||||||
|
flow,
|
||||||
|
max_age=600,
|
||||||
|
httponly=True,
|
||||||
|
secure=True,
|
||||||
|
samesite="lax",
|
||||||
|
path=f"{settings.api_prefix}/auth/authentik",
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/authentik/callback")
|
||||||
|
def authentik_callback(
|
||||||
|
request: Request,
|
||||||
|
code: str = "",
|
||||||
|
state: str = "",
|
||||||
|
) -> RedirectResponse:
|
||||||
|
settings = get_settings()
|
||||||
|
service = AuthentikOidcService(settings)
|
||||||
|
base_url = settings.public_base_url.rstrip("/")
|
||||||
|
try:
|
||||||
|
service.finish(
|
||||||
|
code=code,
|
||||||
|
state=state,
|
||||||
|
flow_cookie=request.cookies.get(OIDC_FLOW_COOKIE_NAME, ""),
|
||||||
|
)
|
||||||
|
token = AuthService.create_session_token(
|
||||||
|
settings.auth_username or "operator",
|
||||||
|
settings,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Authentik callback rejected: %s", type(exc).__name__)
|
||||||
|
response = RedirectResponse(
|
||||||
|
f"{base_url}/?authentik=error",
|
||||||
|
status_code=status.HTTP_302_FOUND,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
response = RedirectResponse(
|
||||||
|
f"{base_url}/",
|
||||||
|
status_code=status.HTTP_302_FOUND,
|
||||||
|
)
|
||||||
|
_set_session_cookie(
|
||||||
|
request=request,
|
||||||
|
response=response,
|
||||||
|
token=token,
|
||||||
|
max_age=settings.auth_session_ttl_seconds,
|
||||||
|
)
|
||||||
|
response.delete_cookie(
|
||||||
|
OIDC_FLOW_COOKIE_NAME,
|
||||||
|
path=f"{settings.api_prefix}/auth/authentik",
|
||||||
|
secure=True,
|
||||||
|
httponly=True,
|
||||||
|
samesite="lax",
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/guest", response_model=AuthSessionEnvelope)
|
||||||
|
def guest_login(
|
||||||
|
request: Request,
|
||||||
|
response: Response,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> AuthSessionEnvelope:
|
||||||
|
settings = get_settings()
|
||||||
|
if not settings.auth_enabled or not settings.guest_access_enabled:
|
||||||
|
raise AppError(
|
||||||
|
code="GUEST_ACCESS_DISABLED",
|
||||||
|
message="Gasttoegang is niet ingeschakeld op deze GeoIntel-installatie.",
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
)
|
||||||
|
|
||||||
|
client_host = _client_host(request)
|
||||||
|
retry_after = AuthService.consume_guest_request(
|
||||||
|
f"guest-login:{client_host}",
|
||||||
|
max_requests=settings.guest_login_requests_per_minute,
|
||||||
|
)
|
||||||
|
if retry_after:
|
||||||
|
raise AppError(
|
||||||
|
code="GUEST_LOGIN_RATE_LIMITED",
|
||||||
|
message="Too many guest sessions were requested. Try again later.",
|
||||||
|
details={"retry_after_seconds": retry_after},
|
||||||
|
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||||
|
)
|
||||||
|
|
||||||
|
demo = DemoWorkflowService.seed(db)
|
||||||
|
token = AuthService.create_session_token(
|
||||||
|
settings.guest_display_name,
|
||||||
|
settings,
|
||||||
|
role="guest",
|
||||||
|
project_id=demo.project_id,
|
||||||
|
ttl_seconds=settings.guest_session_ttl_seconds,
|
||||||
|
)
|
||||||
|
principal = AuthService.verify_session_token(token, settings)
|
||||||
|
if principal is None: # pragma: no cover - defensive invariant
|
||||||
|
raise AppError(
|
||||||
|
code="SESSION_CREATION_FAILED",
|
||||||
|
message="De tijdelijke gastensessie kon niet worden aangemaakt.",
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
)
|
||||||
|
_set_session_cookie(
|
||||||
|
request=request,
|
||||||
|
response=response,
|
||||||
|
token=token,
|
||||||
|
max_age=settings.guest_session_ttl_seconds,
|
||||||
|
)
|
||||||
|
return AuthSessionEnvelope(
|
||||||
|
data=_session_from_principal(
|
||||||
|
principal,
|
||||||
|
guest_access_enabled=True,
|
||||||
|
authentik_enabled=AuthentikOidcService(settings).enabled,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/logout", response_model=AuthSessionEnvelope)
|
||||||
|
def logout(response: Response) -> AuthSessionEnvelope:
|
||||||
|
settings = get_settings()
|
||||||
|
response.delete_cookie(key=COOKIE_NAME, path="/", httponly=True, samesite="strict")
|
||||||
|
return AuthSessionEnvelope(
|
||||||
|
data=AuthSession(
|
||||||
|
authentication_required=settings.auth_enabled,
|
||||||
|
authenticated=not settings.auth_enabled,
|
||||||
|
guest_access_enabled=settings.auth_enabled and settings.guest_access_enabled,
|
||||||
|
authentik_enabled=AuthentikOidcService(settings).enabled,
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -5,19 +5,22 @@ from datetime import datetime
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response
|
||||||
from fastapi import UploadFile
|
from fastapi import UploadFile
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from app.models import Area, Project
|
from app.core.config import get_settings
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.core.public_demo import is_public_demo_project
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
|
from app.models import Area, Project
|
||||||
from app.schemas import (
|
from app.schemas import (
|
||||||
BathymetryPartitionFinalizationResult,
|
BathymetryPartitionFinalizationResult,
|
||||||
BathymetrySourceProbeRead,
|
BathymetrySourceProbeRead,
|
||||||
BathymetrySourceRead,
|
BathymetrySourceRead,
|
||||||
DatasetList,
|
DatasetList,
|
||||||
DhmvProductRead,
|
DhmvProductRead,
|
||||||
|
SpwTerrainAcquireRequest,
|
||||||
|
SpwTerrainProductRead,
|
||||||
Envelope,
|
Envelope,
|
||||||
FloodHazardProductRead,
|
FloodHazardProductRead,
|
||||||
FloodHazardSelectionResponse,
|
FloodHazardSelectionResponse,
|
||||||
@@ -48,6 +51,9 @@ from app.schemas import (
|
|||||||
FloodHazardSelectionRequest,
|
FloodHazardSelectionRequest,
|
||||||
BathymetryPartitionFinalizeRequest,
|
BathymetryPartitionFinalizeRequest,
|
||||||
BathymetryProfileAcquireRequest,
|
BathymetryProfileAcquireRequest,
|
||||||
|
BathymetryRasterSelectionRequest,
|
||||||
|
BathymetryRasterSelectionResponse,
|
||||||
|
MdkBathymetryAcquireRequest,
|
||||||
ThematicRasterAcquireRequest,
|
ThematicRasterAcquireRequest,
|
||||||
ThematicRasterProductRead,
|
ThematicRasterProductRead,
|
||||||
ThematicRasterSelectionResponse,
|
ThematicRasterSelectionResponse,
|
||||||
@@ -85,13 +91,17 @@ from app.services.grb_acquisition_service import GrbAcquisitionService
|
|||||||
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
|
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
|
||||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||||
|
from app.services.spw_terrain_service import SpwTerrainService
|
||||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||||
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
||||||
|
from app.services.bathymetry_raster_analysis_service import BathymetryRasterAnalysisService
|
||||||
|
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
|
||||||
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
||||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||||
|
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||||
from app.utils.response import envelope
|
from app.utils.response import envelope
|
||||||
|
|
||||||
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
|
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
|
||||||
@@ -154,6 +164,12 @@ async def upload_dataset(
|
|||||||
source_version: str | None = Form(None),
|
source_version: str | None = Form(None),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
|
if is_public_demo_project(project_id):
|
||||||
|
raise AppError(
|
||||||
|
code="PUBLIC_DEMO_UPLOAD_FORBIDDEN",
|
||||||
|
message="Operator uploads are not accepted in the public demo project.",
|
||||||
|
status_code=403,
|
||||||
|
)
|
||||||
if area_id is not None:
|
if area_id is not None:
|
||||||
area = db.get(Area, area_id)
|
area = db.get(Area, area_id)
|
||||||
if not area:
|
if not area:
|
||||||
@@ -237,6 +253,33 @@ def list_dhmv_products(project_id: UUID, db: Session = Depends(get_db)):
|
|||||||
return envelope({"items": items, "total": len(items)})
|
return envelope({"items": items, "total": len(items)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/datasets/spw-terrain/acquire", response_model=Envelope[JobRead])
|
||||||
|
def acquire_bounded_spw_terrain(
|
||||||
|
project_id: UUID,
|
||||||
|
payload: SpwTerrainAcquireRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
job = JobService.run_sync_job(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
job_type="raster.spw-terrain.acquire",
|
||||||
|
parameters=payload.model_dump(mode="json"),
|
||||||
|
operation=lambda: SpwTerrainService.acquire(db, project_id, payload),
|
||||||
|
)
|
||||||
|
return envelope(job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/datasets/spw-terrain/products",
|
||||||
|
response_model=Envelope[ItemList[SpwTerrainProductRead]],
|
||||||
|
)
|
||||||
|
def list_spw_terrain_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||||
|
if not db.get(Project, project_id):
|
||||||
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
items = SpwTerrainService.list_products()
|
||||||
|
return envelope({"items": items, "total": len(items)})
|
||||||
|
|
||||||
|
|
||||||
@router.post("/datasets/grb/acquire", response_model=Envelope[JobRead])
|
@router.post("/datasets/grb/acquire", response_model=Envelope[JobRead])
|
||||||
def acquire_bounded_grb(
|
def acquire_bounded_grb(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
@@ -339,6 +382,25 @@ def probe_mdk_bathymetry_readiness(project_id: UUID, db: Session = Depends(get_d
|
|||||||
return envelope(MdkBathymetryProbeService.probe())
|
return envelope(MdkBathymetryProbeService.probe())
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/datasets/bathymetry/mdk/acquire",
|
||||||
|
response_model=Envelope[JobRead],
|
||||||
|
)
|
||||||
|
def acquire_bounded_mdk_bathymetry(
|
||||||
|
project_id: UUID,
|
||||||
|
payload: MdkBathymetryAcquireRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
job = JobService.run_sync_job(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
job_type="raster.mdk_bathymetry.acquire",
|
||||||
|
parameters=payload.model_dump(mode="json"),
|
||||||
|
operation=lambda: MdkBathymetryAcquisitionService.acquire(db, project_id, payload),
|
||||||
|
)
|
||||||
|
return envelope(job)
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/datasets/bathymetry/profiles/acquire",
|
"/datasets/bathymetry/profiles/acquire",
|
||||||
response_model=Envelope[JobRead],
|
response_model=Envelope[JobRead],
|
||||||
@@ -435,6 +497,33 @@ def list_thematic_raster_products(project_id: UUID, db: Session = Depends(get_db
|
|||||||
return envelope({"items": items, "total": len(items)})
|
return envelope({"items": items, "total": len(items)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/datasets/walous/acquire", response_model=Envelope[JobRead])
|
||||||
|
def acquire_bounded_walous_land_cover(
|
||||||
|
project_id: UUID,
|
||||||
|
payload: ThematicRasterAcquireRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
job = JobService.run_sync_job(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
job_type="raster.walous.acquire",
|
||||||
|
parameters=payload.model_dump(mode="json"),
|
||||||
|
operation=lambda: WalousLandCoverService.acquire(db, project_id, payload),
|
||||||
|
)
|
||||||
|
return envelope(job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/datasets/walous/products",
|
||||||
|
response_model=Envelope[ItemList[ThematicRasterProductRead]],
|
||||||
|
)
|
||||||
|
def list_walous_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||||
|
if not db.get(Project, project_id):
|
||||||
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
items = WalousLandCoverService.list_products()
|
||||||
|
return envelope({"items": items, "total": len(items)})
|
||||||
|
|
||||||
|
|
||||||
@router.get("/datasets", response_model=Envelope[DatasetList])
|
@router.get("/datasets", response_model=Envelope[DatasetList])
|
||||||
def list_datasets(
|
def list_datasets(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
@@ -630,16 +719,17 @@ def select_vector_features(
|
|||||||
payload.bbox.model_dump(),
|
payload.bbox.model_dump(),
|
||||||
selection_area.geometry,
|
selection_area.geometry,
|
||||||
)
|
)
|
||||||
full_dataset_area = covers_full_area and VectorFeatureService.can_use_full_area_fast_path(
|
dataset_is_preclipped_to_area = VectorFeatureService.can_use_full_area_fast_path(
|
||||||
dataset,
|
dataset,
|
||||||
selection_area.id,
|
selection_area.id,
|
||||||
)
|
)
|
||||||
|
full_dataset_area = covers_full_area and dataset_is_preclipped_to_area
|
||||||
preclipped_partition_filter = VectorFeatureService.preclipped_partition_filter(
|
preclipped_partition_filter = VectorFeatureService.preclipped_partition_filter(
|
||||||
dataset,
|
dataset,
|
||||||
getattr(selection_area, "name", None),
|
getattr(selection_area, "name", None),
|
||||||
)
|
)
|
||||||
selection_kwargs.update(
|
selection_kwargs.update(
|
||||||
selection_geometry=selection_geometry,
|
selection_geometry=None if dataset_is_preclipped_to_area else selection_geometry,
|
||||||
selection_area_id=selection_area.id,
|
selection_area_id=selection_area.id,
|
||||||
full_dataset_area=full_dataset_area,
|
full_dataset_area=full_dataset_area,
|
||||||
preclipped_partition_filter=preclipped_partition_filter,
|
preclipped_partition_filter=preclipped_partition_filter,
|
||||||
@@ -652,7 +742,7 @@ def select_vector_features(
|
|||||||
"total_feature_count": result.get("total_feature_count"),
|
"total_feature_count": result.get("total_feature_count"),
|
||||||
}
|
}
|
||||||
if selection_area is not None:
|
if selection_area is not None:
|
||||||
summary_kwargs["selection_geometry"] = selection_geometry
|
summary_kwargs["selection_geometry"] = None if dataset_is_preclipped_to_area else selection_geometry
|
||||||
summary_kwargs["full_dataset_area"] = full_dataset_area
|
summary_kwargs["full_dataset_area"] = full_dataset_area
|
||||||
summary_kwargs["preclipped_partition_filter"] = preclipped_partition_filter
|
summary_kwargs["preclipped_partition_filter"] = preclipped_partition_filter
|
||||||
result["summary"] = VectorFeatureService.summarize_features_by_bbox(db, **summary_kwargs)
|
result["summary"] = VectorFeatureService.summarize_features_by_bbox(db, **summary_kwargs)
|
||||||
@@ -888,6 +978,33 @@ def raster_terrain_image(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/datasets/{dataset_id}/raster/bathymetry/select",
|
||||||
|
response_model=Envelope[BathymetryRasterSelectionResponse],
|
||||||
|
)
|
||||||
|
def raster_bathymetry_selection(
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
payload: BathymetryRasterSelectionRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
return envelope(BathymetryRasterAnalysisService.analyze(db, project_id, dataset_id, payload))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/datasets/{dataset_id}/raster/bathymetry/image")
|
||||||
|
def raster_bathymetry_image(
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
content = BathymetryRasterAnalysisService.render_png(db, project_id, dataset_id)
|
||||||
|
return Response(
|
||||||
|
content=content,
|
||||||
|
media_type="image/png",
|
||||||
|
headers={"Cache-Control": "private, max-age=86400"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/datasets/{dataset_id}/raster/flood-hazard/select",
|
"/datasets/{dataset_id}/raster/flood-hazard/select",
|
||||||
response_model=Envelope[FloodHazardSelectionResponse],
|
response_model=Envelope[FloodHazardSelectionResponse],
|
||||||
@@ -954,6 +1071,33 @@ def raster_thematic_image(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/datasets/{dataset_id}/raster/walous/select",
|
||||||
|
response_model=Envelope[ThematicRasterSelectionResponse],
|
||||||
|
)
|
||||||
|
def raster_walous_selection(
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
payload: ThematicRasterSelectionRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
return envelope(WalousLandCoverService.analyze(db, project_id, dataset_id, payload))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/datasets/{dataset_id}/raster/walous/image")
|
||||||
|
def raster_walous_image(
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
content = WalousLandCoverService.render_png(db, project_id, dataset_id)
|
||||||
|
return Response(
|
||||||
|
content=content,
|
||||||
|
media_type="image/png",
|
||||||
|
headers={"Cache-Control": "private, max-age=86400"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
"/datasets/{dataset_id}/raster/stats",
|
"/datasets/{dataset_id}/raster/stats",
|
||||||
response_model=Envelope[RasterStatsResponse],
|
response_model=Envelope[RasterStatsResponse],
|
||||||
@@ -1035,11 +1179,14 @@ def raster_tile_dataset(
|
|||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
dataset_id: UUID,
|
dataset_id: UUID,
|
||||||
payload: RasterTileRequest,
|
payload: RasterTileRequest,
|
||||||
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||||
if dataset.project_id != project_id:
|
if dataset.project_id != project_id:
|
||||||
raise HTTPException(status_code=404, detail="Dataset not found")
|
raise HTTPException(status_code=404, detail="Dataset not found")
|
||||||
|
principal = getattr(request.state, "auth_principal", None)
|
||||||
|
guest_max_tiles = get_settings().yolo_max_tiles if getattr(principal, "role", None) == "guest" else None
|
||||||
job = _run_job_sync(
|
job = _run_job_sync(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
@@ -1052,6 +1199,7 @@ def raster_tile_dataset(
|
|||||||
tile_size=payload.tile_size,
|
tile_size=payload.tile_size,
|
||||||
overlap=payload.overlap,
|
overlap=payload.overlap,
|
||||||
output_name=payload.output_name,
|
output_name=payload.output_name,
|
||||||
|
max_tiles=guest_max_tiles,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
return envelope(job)
|
return envelope(job)
|
||||||
|
|||||||
@@ -2,14 +2,21 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends, Query, Request
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.guest_scope import (
|
||||||
|
assert_guest_project_scope,
|
||||||
|
guest_project_scope,
|
||||||
|
guest_scoped_project_filter,
|
||||||
|
)
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.schemas import (
|
from app.schemas import (
|
||||||
AnalysisQaResponse,
|
AnalysisQaResponse,
|
||||||
DetectionListResponse,
|
DetectionListResponse,
|
||||||
DetectionModelsResponse,
|
DetectionModelsResponse,
|
||||||
|
DetectionComparisonRequest,
|
||||||
|
DetectionComparisonResponse,
|
||||||
DetectionQaRequest,
|
DetectionQaRequest,
|
||||||
DetectionRead,
|
DetectionRead,
|
||||||
DetectionRunListResponse,
|
DetectionRunListResponse,
|
||||||
@@ -18,9 +25,12 @@ from app.schemas import (
|
|||||||
DetectionRunResponse,
|
DetectionRunResponse,
|
||||||
Envelope,
|
Envelope,
|
||||||
GeoJsonFeatureCollection,
|
GeoJsonFeatureCollection,
|
||||||
|
JobRead,
|
||||||
ModelAssetListResponse,
|
ModelAssetListResponse,
|
||||||
YoloPreflightResponse,
|
YoloPreflightResponse,
|
||||||
)
|
)
|
||||||
|
from app.services.detection_comparison_service import DetectionComparisonService
|
||||||
|
from app.services.dataset_service import DatasetService
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
@@ -45,18 +55,25 @@ def get_yolo_preflight(
|
|||||||
tile_manifest_path: str | None = None,
|
tile_manifest_path: str | None = None,
|
||||||
check_model_load: bool = False,
|
check_model_load: bool = False,
|
||||||
model_asset_id: str | None = None,
|
model_asset_id: str | None = None,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
return envelope(
|
return envelope(
|
||||||
YoloPreflightService.run(
|
YoloPreflightService.run(
|
||||||
tile_manifest_path=tile_manifest_path,
|
tile_manifest_path=tile_manifest_path,
|
||||||
check_model_load=check_model_load,
|
check_model_load=check_model_load,
|
||||||
model_asset_id=model_asset_id,
|
model_asset_id=model_asset_id,
|
||||||
|
db=db,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/run", response_model=Envelope[DetectionRunResponse])
|
@router.post("/run", response_model=Envelope[DetectionRunResponse])
|
||||||
def run_detection(payload: DetectionRunRequest, db: Session = Depends(get_db)) -> dict:
|
def run_detection(
|
||||||
|
payload: DetectionRunRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
assert_guest_project_scope(request, payload.project_id)
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=payload.project_id,
|
project_id=payload.project_id,
|
||||||
@@ -71,18 +88,60 @@ def run_detection(payload: DetectionRunRequest, db: Session = Depends(get_db)) -
|
|||||||
return envelope(result.model_dump())
|
return envelope(result.model_dump())
|
||||||
|
|
||||||
|
|
||||||
@router.get("/runs", response_model=Envelope[DetectionRunListResponse])
|
@router.post("/run-async", response_model=Envelope[JobRead])
|
||||||
def list_detection_runs(
|
def queue_detection(
|
||||||
project_id: UUID | None = None,
|
payload: DetectionRunRequest,
|
||||||
dataset_id: UUID | None = None,
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
return envelope(DetectionService.list_runs(db, project_id=project_id, dataset_id=dataset_id).model_dump())
|
"""Queue a detection run for the background worker.
|
||||||
|
|
||||||
|
Tiled GPU inference takes minutes; ``POST /detection/run`` performs it
|
||||||
|
inside the request and is only appropriate for a handful of tiles. Poll
|
||||||
|
``GET /jobs/{id}`` for the queued run instead.
|
||||||
|
"""
|
||||||
|
|
||||||
|
assert_guest_project_scope(request, payload.project_id)
|
||||||
|
job = DetectionService.enqueue_detection(
|
||||||
|
db=db,
|
||||||
|
project_id=payload.project_id,
|
||||||
|
dataset_id=payload.dataset_id,
|
||||||
|
model_id=payload.model_id,
|
||||||
|
model_asset_id=payload.model_asset_id,
|
||||||
|
confidence_threshold=payload.confidence_threshold,
|
||||||
|
class_filter=payload.class_filter,
|
||||||
|
tile_manifest_path=payload.tile_manifest_path,
|
||||||
|
parameters_json=payload.parameters_json,
|
||||||
|
)
|
||||||
|
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/runs", response_model=Envelope[DetectionRunListResponse])
|
||||||
|
def list_detection_runs(
|
||||||
|
request: Request,
|
||||||
|
project_id: UUID | None = None,
|
||||||
|
dataset_id: UUID | None = None,
|
||||||
|
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
project_id = guest_scoped_project_filter(request, project_id)
|
||||||
|
return envelope(
|
||||||
|
DetectionService.list_runs(
|
||||||
|
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
|
||||||
|
).model_dump()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/runs/{analysis_run_id}", response_model=Envelope[DetectionRunRead])
|
@router.get("/runs/{analysis_run_id}", response_model=Envelope[DetectionRunRead])
|
||||||
def get_detection_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> dict:
|
def get_detection_run(
|
||||||
return envelope(DetectionService.get_run(db, analysis_run_id).model_dump())
|
analysis_run_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
run = DetectionService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
|
return envelope(run.model_dump())
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
@@ -91,11 +150,22 @@ def get_detection_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> d
|
|||||||
)
|
)
|
||||||
def list_detection_run_detections(
|
def list_detection_run_detections(
|
||||||
analysis_run_id: UUID,
|
analysis_run_id: UUID,
|
||||||
|
request: Request,
|
||||||
dataset_id: UUID | None = None,
|
dataset_id: UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
run = DetectionService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
DetectionService.list_detections(
|
DetectionService.list_detections(
|
||||||
db,
|
db,
|
||||||
@@ -103,6 +173,8 @@ def list_detection_run_detections(
|
|||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
min_confidence=min_confidence,
|
min_confidence=min_confidence,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
).model_dump()
|
).model_dump()
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -113,11 +185,22 @@ def list_detection_run_detections(
|
|||||||
)
|
)
|
||||||
def list_dataset_detections(
|
def list_dataset_detections(
|
||||||
dataset_id: UUID,
|
dataset_id: UUID,
|
||||||
|
request: Request,
|
||||||
analysis_run_id: UUID | None = None,
|
analysis_run_id: UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||||
|
assert_guest_project_scope(request, dataset.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
DetectionService.list_detections(
|
DetectionService.list_detections(
|
||||||
db,
|
db,
|
||||||
@@ -125,13 +208,21 @@ def list_dataset_detections(
|
|||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
min_confidence=min_confidence,
|
min_confidence=min_confidence,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
).model_dump()
|
).model_dump()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/detections/{detection_id}", response_model=Envelope[DetectionRead])
|
@router.get("/detections/{detection_id}", response_model=Envelope[DetectionRead])
|
||||||
def get_detection(detection_id: UUID, db: Session = Depends(get_db)) -> dict:
|
def get_detection(
|
||||||
return envelope(DetectionService.get_detection(db, detection_id).model_dump())
|
detection_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
detection = DetectionService.get_detection(db, detection_id)
|
||||||
|
assert_guest_project_scope(request, detection.project_id)
|
||||||
|
return envelope(detection.model_dump())
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
@@ -140,13 +231,24 @@ def get_detection(detection_id: UUID, db: Session = Depends(get_db)) -> dict:
|
|||||||
)
|
)
|
||||||
def get_detection_run_geojson(
|
def get_detection_run_geojson(
|
||||||
analysis_run_id: UUID,
|
analysis_run_id: UUID,
|
||||||
|
request: Request,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
run = DetectionService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
DetectionService.detections_to_geojson(
|
DetectionService.detections_to_geojson(
|
||||||
db,
|
db,
|
||||||
|
limit=limit,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
min_confidence=min_confidence,
|
min_confidence=min_confidence,
|
||||||
@@ -160,14 +262,25 @@ def get_detection_run_geojson(
|
|||||||
)
|
)
|
||||||
def get_dataset_detection_geojson(
|
def get_dataset_detection_geojson(
|
||||||
dataset_id: UUID,
|
dataset_id: UUID,
|
||||||
|
request: Request,
|
||||||
analysis_run_id: UUID | None = None,
|
analysis_run_id: UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||||
|
assert_guest_project_scope(request, dataset.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
DetectionService.detections_to_geojson(
|
DetectionService.detections_to_geojson(
|
||||||
db,
|
db,
|
||||||
|
limit=limit,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
@@ -176,6 +289,27 @@ def get_dataset_detection_geojson(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/runs/compare", response_model=Envelope[DetectionComparisonResponse])
|
||||||
|
def compare_detection_runs(payload: DetectionComparisonRequest, db: Session = Depends(get_db)) -> dict:
|
||||||
|
"""Rank several runs against one reference on average precision.
|
||||||
|
|
||||||
|
The workbench ranks model variants by a stored F1 measured at each
|
||||||
|
variant's own confidence threshold, which orders the thresholds as much as
|
||||||
|
the models. Average precision describes the whole ranking a model produced.
|
||||||
|
Comparability is reported first: runs over different rasters, different
|
||||||
|
references or different inference coverage are not alternatives.
|
||||||
|
"""
|
||||||
|
|
||||||
|
return envelope(
|
||||||
|
DetectionComparisonService.compare_runs(
|
||||||
|
db,
|
||||||
|
analysis_run_ids=payload.analysis_run_ids,
|
||||||
|
reference_dataset_id=payload.reference_dataset_id,
|
||||||
|
iou_threshold=payload.iou_threshold,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/runs/{analysis_run_id}/qa/reference",
|
"/runs/{analysis_run_id}/qa/reference",
|
||||||
response_model=Envelope[AnalysisQaResponse],
|
response_model=Envelope[AnalysisQaResponse],
|
||||||
@@ -183,8 +317,12 @@ def get_dataset_detection_geojson(
|
|||||||
def compare_detection_run_with_reference(
|
def compare_detection_run_with_reference(
|
||||||
analysis_run_id: UUID,
|
analysis_run_id: UUID,
|
||||||
payload: DetectionQaRequest,
|
payload: DetectionQaRequest,
|
||||||
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
run = DetectionService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
DetectionService.compare_detections_with_reference(
|
DetectionService.compare_detections_with_reference(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -193,5 +331,6 @@ def compare_detection_run_with_reference(
|
|||||||
iou_threshold=payload.iou_threshold,
|
iou_threshold=payload.iou_threshold,
|
||||||
class_name=payload.class_name,
|
class_name=payload.class_name,
|
||||||
min_confidence=payload.min_confidence,
|
min_confidence=payload.min_confidence,
|
||||||
|
calibration_thresholds=payload.calibration_thresholds,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -2,10 +2,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
from fastapi import APIRouter, Depends, Query, Request
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.guest_scope import assert_guest_project_scope, guest_project_scope
|
||||||
|
from app.core.errors import AppError
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.schemas import Envelope
|
from app.schemas import Envelope
|
||||||
from app.schemas.export import (
|
from app.schemas.export import (
|
||||||
@@ -19,13 +21,30 @@ from app.schemas.export import (
|
|||||||
ReportExportRequest,
|
ReportExportRequest,
|
||||||
)
|
)
|
||||||
from app.services.export_service import ExportService
|
from app.services.export_service import ExportService
|
||||||
|
from app.services.dataset_service import DatasetService
|
||||||
|
from app.services.detection_service import DetectionService
|
||||||
|
from app.services.segmentation_service import SegmentationService
|
||||||
from app.utils.response import envelope
|
from app.utils.response import envelope
|
||||||
|
|
||||||
router = APIRouter(prefix="/exports", tags=["exports"])
|
router = APIRouter(prefix="/exports", tags=["exports"])
|
||||||
|
|
||||||
|
|
||||||
@router.post("/geojson", response_model=Envelope[ExportCreateResponse])
|
@router.post("/geojson", response_model=Envelope[ExportCreateResponse])
|
||||||
def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db)):
|
def export_geojson(
|
||||||
|
payload: GeoJsonExportRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
if payload.export_kind in {"dataset", "vector_selection"} and payload.dataset_id is not None:
|
||||||
|
dataset = DatasetService.get_dataset(db, payload.dataset_id)
|
||||||
|
assert_guest_project_scope(request, dataset.project_id)
|
||||||
|
elif payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
|
||||||
|
run = DetectionService.get_run(db, payload.analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
|
elif payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
|
||||||
|
run = SegmentationService.get_run(db, payload.analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
if payload.export_kind == "vector_selection" and payload.dataset_id is not None and payload.bbox is not None:
|
if payload.export_kind == "vector_selection" and payload.dataset_id is not None and payload.bbox is not None:
|
||||||
return envelope(
|
return envelope(
|
||||||
ExportService.export_vector_selection_geojson(
|
ExportService.export_vector_selection_geojson(
|
||||||
@@ -39,7 +58,12 @@ def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db))
|
|||||||
)
|
)
|
||||||
if payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
|
if payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
|
||||||
return envelope(
|
return envelope(
|
||||||
ExportService.export_detection_run_geojson(db, payload.analysis_run_id, payload.name).model_dump(mode="json")
|
ExportService.export_detection_run_geojson(
|
||||||
|
db,
|
||||||
|
payload.analysis_run_id,
|
||||||
|
payload.name,
|
||||||
|
intended_use=payload.intended_use,
|
||||||
|
).model_dump(mode="json")
|
||||||
)
|
)
|
||||||
if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
|
if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
|
||||||
return envelope(
|
return envelope(
|
||||||
@@ -47,21 +71,40 @@ def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db))
|
|||||||
)
|
)
|
||||||
if payload.dataset_id is not None:
|
if payload.dataset_id is not None:
|
||||||
return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json"))
|
return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json"))
|
||||||
return envelope({})
|
raise AppError(
|
||||||
|
code="INVALID_EXPORT_REQUEST",
|
||||||
|
message="GeoJSON export request does not match any supported export target",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/metadata", response_model=Envelope[ExportCreateResponse])
|
@router.post("/metadata", response_model=Envelope[ExportCreateResponse])
|
||||||
def export_project_metadata(payload: MetadataExportRequest, db: Session = Depends(get_db)):
|
def export_project_metadata(
|
||||||
|
payload: MetadataExportRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
assert_guest_project_scope(request, payload.project_id)
|
||||||
return envelope(ExportService.export_project_metadata(db, payload.project_id, payload.name).model_dump(mode="json"))
|
return envelope(ExportService.export_project_metadata(db, payload.project_id, payload.name).model_dump(mode="json"))
|
||||||
|
|
||||||
|
|
||||||
@router.post("/report", response_model=Envelope[ExportCreateResponse])
|
@router.post("/report", response_model=Envelope[ExportCreateResponse])
|
||||||
def export_project_report(payload: ReportExportRequest, db: Session = Depends(get_db)):
|
def export_project_report(
|
||||||
|
payload: ReportExportRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
assert_guest_project_scope(request, payload.project_id)
|
||||||
return envelope(ExportService.export_project_report(db, payload.project_id, payload.name).model_dump(mode="json"))
|
return envelope(ExportService.export_project_report(db, payload.project_id, payload.name).model_dump(mode="json"))
|
||||||
|
|
||||||
|
|
||||||
@router.post("/map-result", response_model=Envelope[ExportCreateResponse])
|
@router.post("/map-result", response_model=Envelope[ExportCreateResponse])
|
||||||
def export_map_result(payload: MapResultExportRequest, db: Session = Depends(get_db)):
|
def export_map_result(
|
||||||
|
payload: MapResultExportRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
assert_guest_project_scope(request, payload.project_id)
|
||||||
return envelope(ExportService.export_map_result(db, payload).model_dump(mode="json"))
|
return envelope(ExportService.export_map_result(db, payload).model_dump(mode="json"))
|
||||||
|
|
||||||
|
|
||||||
@@ -71,25 +114,35 @@ def export_map_result(payload: MapResultExportRequest, db: Session = Depends(get
|
|||||||
)
|
)
|
||||||
def list_project_exports(
|
def list_project_exports(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
|
request: Request,
|
||||||
limit: int = Query(default=50, ge=1, le=100),
|
limit: int = Query(default=50, ge=1, le=100),
|
||||||
offset: int = Query(default=0, ge=0),
|
offset: int = Query(default=0, ge=0),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
|
assert_guest_project_scope(request, project_id)
|
||||||
return envelope(ExportService.list_project_exports(db, project_id, limit=limit, offset=offset).model_dump(mode="json"))
|
return envelope(ExportService.list_project_exports(db, project_id, limit=limit, offset=offset).model_dump(mode="json"))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{export_id}", response_model=Envelope[ExportRead])
|
@router.get("/{export_id}", response_model=Envelope[ExportRead])
|
||||||
def get_export(export_id: UUID, db: Session = Depends(get_db)):
|
def get_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
|
||||||
return envelope(ExportService.get_export(db, export_id).model_dump(mode="json"))
|
export = ExportService.get_export(db, export_id)
|
||||||
|
assert_guest_project_scope(request, export.project_id)
|
||||||
|
return envelope(export.model_dump(mode="json"))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{export_id}/download")
|
@router.get("/{export_id}/download")
|
||||||
def download_export(export_id: UUID, db: Session = Depends(get_db)):
|
def download_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
export = ExportService.get_export(db, export_id)
|
||||||
|
assert_guest_project_scope(request, export.project_id)
|
||||||
path = ExportService.get_export_download_path(db, export_id)
|
path = ExportService.get_export_download_path(db, export_id)
|
||||||
media_type = "text/html" if path.suffix.lower() in {".html", ".htm"} else "application/json"
|
media_type = "text/html" if path.suffix.lower() in {".html", ".htm"} else "application/json"
|
||||||
return FileResponse(path, filename=path.name, media_type=media_type)
|
return FileResponse(path, filename=path.name, media_type=media_type)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{export_id}/content", response_model=Envelope[ExportContentResponse])
|
@router.get("/{export_id}/content", response_model=Envelope[ExportContentResponse])
|
||||||
def get_export_content(export_id: UUID, db: Session = Depends(get_db)):
|
def get_export_content(export_id: UUID, request: Request, db: Session = Depends(get_db)):
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
export = ExportService.get_export(db, export_id)
|
||||||
|
assert_guest_project_scope(request, export.project_id)
|
||||||
return envelope(ExportService.get_export_content(db, export_id).model_dump(mode="json"))
|
return envelope(ExportService.get_export_content(db, export_id).model_dump(mode="json"))
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends, Request
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
@@ -43,6 +43,19 @@ def _assert_project_exists(db: Session, project_id):
|
|||||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_guest_project_scope(request: Request, project_id) -> None:
|
||||||
|
principal = getattr(request.state, "auth_principal", None)
|
||||||
|
if (
|
||||||
|
getattr(principal, "role", None) == "guest"
|
||||||
|
and getattr(principal, "project_id", None) != project_id
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="GUEST_PROJECT_SCOPE_REQUIRED",
|
||||||
|
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||||
|
status_code=403,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _normalize_layer_input(layers: list[str] | None) -> list[str]:
|
def _normalize_layer_input(layers: list[str] | None) -> list[str]:
|
||||||
return [layer.strip() for layer in (layers or []) if isinstance(layer, str) and layer.strip()]
|
return [layer.strip() for layer in (layers or []) if isinstance(layer, str) and layer.strip()]
|
||||||
|
|
||||||
@@ -66,7 +79,12 @@ def get_coverage_catalog() -> dict:
|
|||||||
|
|
||||||
|
|
||||||
@router.post("/coverage/resolve", response_model=Envelope[CoverageResolveResponse])
|
@router.post("/coverage/resolve", response_model=Envelope[CoverageResolveResponse])
|
||||||
def resolve_project_coverage(payload: CoverageResolveRequest, db: Session = Depends(get_db)) -> dict:
|
def resolve_project_coverage(
|
||||||
|
payload: CoverageResolveRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
_assert_guest_project_scope(request, payload.project_id)
|
||||||
result = CoverageRegistryService.resolve(
|
result = CoverageRegistryService.resolve(
|
||||||
db,
|
db,
|
||||||
project_id=payload.project_id,
|
project_id=payload.project_id,
|
||||||
|
|||||||
@@ -47,21 +47,18 @@ def _database_checks() -> dict[str, str]:
|
|||||||
with get_engine().connect() as connection:
|
with get_engine().connect() as connection:
|
||||||
connection.execute(text("SELECT 1"))
|
connection.execute(text("SELECT 1"))
|
||||||
checks["database"] = "ok"
|
checks["database"] = "ok"
|
||||||
postgis_version = connection.execute(
|
connection.execute(
|
||||||
text("SELECT PostGIS_Version()")
|
text("SELECT PostGIS_Version()")
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
checks["postgis"] = f"ok:{postgis_version}"
|
checks["postgis"] = "ok"
|
||||||
database_head = connection.execute(
|
database_head = connection.execute(
|
||||||
text("SELECT version_num FROM alembic_version")
|
text("SELECT version_num FROM alembic_version")
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
expected_heads = _expected_migration_heads()
|
expected_heads = _expected_migration_heads()
|
||||||
if len(expected_heads) == 1 and database_head == expected_heads[0]:
|
if len(expected_heads) == 1 and database_head == expected_heads[0]:
|
||||||
checks["migration"] = f"ok:{database_head}"
|
checks["migration"] = "ok"
|
||||||
else:
|
else:
|
||||||
checks["migration"] = (
|
checks["migration"] = "degraded"
|
||||||
f"degraded:database={database_head};"
|
|
||||||
f"expected={','.join(expected_heads) or 'none'}"
|
|
||||||
)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
return checks
|
return checks
|
||||||
return checks
|
return checks
|
||||||
@@ -94,9 +91,7 @@ def _readiness_payload() -> HealthResponse:
|
|||||||
return HealthResponse(
|
return HealthResponse(
|
||||||
status="ok" if ready else "degraded",
|
status="ok" if ready else "degraded",
|
||||||
service="geointel-backend",
|
service="geointel-backend",
|
||||||
version=settings.app_version,
|
version="public",
|
||||||
build_sha=settings.build_sha,
|
|
||||||
build_time=settings.build_time,
|
|
||||||
database=checks["database"],
|
database=checks["database"],
|
||||||
postgis=checks["postgis"],
|
postgis=checks["postgis"],
|
||||||
migration=checks["migration"],
|
migration=checks["migration"],
|
||||||
@@ -107,13 +102,10 @@ def _readiness_payload() -> HealthResponse:
|
|||||||
|
|
||||||
@router.get("/health/live", response_model=HealthResponse)
|
@router.get("/health/live", response_model=HealthResponse)
|
||||||
def liveness() -> HealthResponse:
|
def liveness() -> HealthResponse:
|
||||||
settings = get_settings()
|
|
||||||
return HealthResponse(
|
return HealthResponse(
|
||||||
status="ok",
|
status="ok",
|
||||||
service="geointel-backend",
|
service="geointel-backend",
|
||||||
version=settings.app_version,
|
version="public",
|
||||||
build_sha=settings.build_sha,
|
|
||||||
build_time=settings.build_time,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -147,6 +139,11 @@ def capabilities() -> SystemCapabilitiesEnvelope:
|
|||||||
)
|
)
|
||||||
yolo_configured = bool(configured_yolo and configured_yolo.configured)
|
yolo_configured = bool(configured_yolo and configured_yolo.configured)
|
||||||
yolo_status = configured_yolo.status if configured_yolo else "not_configured"
|
yolo_status = configured_yolo.status if configured_yolo else "not_configured"
|
||||||
|
configured_sam = ModelRegistryService.get_model_capability(
|
||||||
|
settings.sam_model_id,
|
||||||
|
settings=settings,
|
||||||
|
task_type="segmentation",
|
||||||
|
)
|
||||||
postgis_ready = _database_checks()["postgis"].startswith("ok:")
|
postgis_ready = _database_checks()["postgis"].startswith("ok:")
|
||||||
return SystemCapabilitiesEnvelope(
|
return SystemCapabilitiesEnvelope(
|
||||||
data=SystemCapabilities(
|
data=SystemCapabilities(
|
||||||
@@ -155,7 +152,7 @@ def capabilities() -> SystemCapabilitiesEnvelope:
|
|||||||
geopandas=_dependency_enabled("geopandas"),
|
geopandas=_dependency_enabled("geopandas"),
|
||||||
yolo=yolo_configured,
|
yolo=yolo_configured,
|
||||||
yolo_status=yolo_status,
|
yolo_status=yolo_status,
|
||||||
sam=False,
|
sam=bool(configured_sam and configured_sam.configured),
|
||||||
grb="bounded",
|
grb="bounded",
|
||||||
sentinel="planned",
|
sentinel="planned",
|
||||||
version=settings.app_version,
|
version=settings.app_version,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
from typing import Literal
|
from typing import Literal
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
@@ -17,12 +17,34 @@ router = APIRouter(prefix="/projects", tags=["projects"])
|
|||||||
|
|
||||||
@router.get("", response_model=Envelope[ProjectList])
|
@router.get("", response_model=Envelope[ProjectList])
|
||||||
def list_projects(
|
def list_projects(
|
||||||
|
request: Request,
|
||||||
limit: int = Query(default=50, ge=1, le=200),
|
limit: int = Query(default=50, ge=1, le=200),
|
||||||
offset: int = Query(default=0, ge=0),
|
offset: int = Query(default=0, ge=0),
|
||||||
name: str | None = Query(default=None, min_length=1, max_length=255),
|
name: str | None = Query(default=None, min_length=1, max_length=255),
|
||||||
project_status: Literal["active", "archived", "all"] = Query(default="active", alias="status"),
|
project_status: Literal["active", "archived", "all"] = Query(default="active", alias="status"),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
|
principal = getattr(request.state, "auth_principal", None)
|
||||||
|
if principal is not None and principal.role == "guest":
|
||||||
|
project = ProjectService.get_project(db, principal.project_id)
|
||||||
|
status_matches = bool(
|
||||||
|
project is not None
|
||||||
|
and (project_status == "all" or project.status == project_status)
|
||||||
|
)
|
||||||
|
name_matches = bool(
|
||||||
|
project is not None
|
||||||
|
and (name is None or name.casefold() in project.name.casefold())
|
||||||
|
)
|
||||||
|
matches = project is not None and status_matches and name_matches
|
||||||
|
visible = [project] if matches and offset == 0 else []
|
||||||
|
return envelope(
|
||||||
|
{
|
||||||
|
"items": [ProjectRead.model_validate(item).model_dump() for item in visible[:limit]],
|
||||||
|
"total": 1 if matches else 0,
|
||||||
|
"limit": limit,
|
||||||
|
"offset": offset,
|
||||||
|
}
|
||||||
|
)
|
||||||
projects, total = ProjectService.list_projects(
|
projects, total = ProjectService.list_projects(
|
||||||
db,
|
db,
|
||||||
limit=limit,
|
limit=limit,
|
||||||
|
|||||||
@@ -40,9 +40,22 @@ def list_quality_checks(
|
|||||||
def get_quality_check_evidence_geojson(
|
def get_quality_check_evidence_geojson(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
quality_check_id: UUID,
|
quality_check_id: UUID,
|
||||||
|
limit: int = Query(
|
||||||
|
default=QualityEvidenceService.DEFAULT_EVIDENCE_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=100_000,
|
||||||
|
description="Maximum evidence features to draw; 0 returns everything. Misses and false positives first.",
|
||||||
|
),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id))
|
return envelope(
|
||||||
|
QualityEvidenceService.evidence_geojson(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
quality_check_id=quality_check_id,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
|
|||||||
@@ -2,14 +2,20 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends, Query, Request
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.guest_scope import (
|
||||||
|
assert_guest_project_scope,
|
||||||
|
guest_project_scope,
|
||||||
|
guest_scoped_project_filter,
|
||||||
|
)
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.schemas import (
|
from app.schemas import (
|
||||||
AnalysisQaResponse,
|
AnalysisQaResponse,
|
||||||
Envelope,
|
Envelope,
|
||||||
GeoJsonFeatureCollection,
|
GeoJsonFeatureCollection,
|
||||||
|
JobRead,
|
||||||
SegmentationListResponse,
|
SegmentationListResponse,
|
||||||
SegmentationModelsResponse,
|
SegmentationModelsResponse,
|
||||||
SegmentationQaRequest,
|
SegmentationQaRequest,
|
||||||
@@ -20,6 +26,8 @@ from app.schemas import (
|
|||||||
SegmentationRunResponse,
|
SegmentationRunResponse,
|
||||||
)
|
)
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
|
from app.services.dataset_service import DatasetService
|
||||||
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.segmentation_service import SegmentationService
|
from app.services.segmentation_service import SegmentationService
|
||||||
from app.utils.response import envelope
|
from app.utils.response import envelope
|
||||||
|
|
||||||
@@ -32,7 +40,12 @@ def list_segmentation_models() -> dict:
|
|||||||
|
|
||||||
|
|
||||||
@router.post("/run", response_model=Envelope[SegmentationRunResponse])
|
@router.post("/run", response_model=Envelope[SegmentationRunResponse])
|
||||||
def run_segmentation(payload: SegmentationRunRequest, db: Session = Depends(get_db)) -> dict:
|
def run_segmentation(
|
||||||
|
payload: SegmentationRunRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
assert_guest_project_scope(request, payload.project_id)
|
||||||
result = SegmentationService.run_segmentation(
|
result = SegmentationService.run_segmentation(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=payload.project_id,
|
project_id=payload.project_id,
|
||||||
@@ -46,18 +59,58 @@ def run_segmentation(payload: SegmentationRunRequest, db: Session = Depends(get_
|
|||||||
return envelope(result.model_dump())
|
return envelope(result.model_dump())
|
||||||
|
|
||||||
|
|
||||||
@router.get("/runs", response_model=Envelope[SegmentationRunListResponse])
|
@router.post("/run-async", response_model=Envelope[JobRead])
|
||||||
def list_segmentation_runs(
|
def queue_segmentation(
|
||||||
project_id: UUID | None = None,
|
payload: SegmentationRunRequest,
|
||||||
dataset_id: UUID | None = None,
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
return envelope(SegmentationService.list_runs(db, project_id=project_id, dataset_id=dataset_id).model_dump())
|
"""Queue a segmentation run for the background worker.
|
||||||
|
|
||||||
|
Configured segmentation walks the same tile manifest as detection and is
|
||||||
|
just as unsuited to running inside the request. Poll ``GET /jobs/{id}``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
assert_guest_project_scope(request, payload.project_id)
|
||||||
|
job = SegmentationService.enqueue_segmentation(
|
||||||
|
db=db,
|
||||||
|
project_id=payload.project_id,
|
||||||
|
dataset_id=payload.dataset_id,
|
||||||
|
model_id=payload.model_id,
|
||||||
|
confidence_threshold=payload.confidence_threshold,
|
||||||
|
class_filter=payload.class_filter,
|
||||||
|
tile_manifest_path=payload.tile_manifest_path,
|
||||||
|
parameters_json=payload.parameters_json,
|
||||||
|
)
|
||||||
|
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/runs", response_model=Envelope[SegmentationRunListResponse])
|
||||||
|
def list_segmentation_runs(
|
||||||
|
request: Request,
|
||||||
|
project_id: UUID | None = None,
|
||||||
|
dataset_id: UUID | None = None,
|
||||||
|
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
project_id = guest_scoped_project_filter(request, project_id)
|
||||||
|
return envelope(
|
||||||
|
SegmentationService.list_runs(
|
||||||
|
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
|
||||||
|
).model_dump()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/runs/{analysis_run_id}", response_model=Envelope[SegmentationRunRead])
|
@router.get("/runs/{analysis_run_id}", response_model=Envelope[SegmentationRunRead])
|
||||||
def get_segmentation_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> dict:
|
def get_segmentation_run(
|
||||||
return envelope(SegmentationService.get_run(db, analysis_run_id).model_dump())
|
analysis_run_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
run = SegmentationService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
|
return envelope(run.model_dump())
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
@@ -66,14 +119,27 @@ def get_segmentation_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -
|
|||||||
)
|
)
|
||||||
def list_segmentation_run_outputs(
|
def list_segmentation_run_outputs(
|
||||||
analysis_run_id: UUID,
|
analysis_run_id: UUID,
|
||||||
|
request: Request,
|
||||||
dataset_id: UUID | None = None,
|
dataset_id: UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
run = SegmentationService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
SegmentationService.list_segmentations(
|
SegmentationService.list_segmentations(
|
||||||
db,
|
db,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
@@ -88,14 +154,27 @@ def list_segmentation_run_outputs(
|
|||||||
)
|
)
|
||||||
def list_dataset_segmentations(
|
def list_dataset_segmentations(
|
||||||
dataset_id: UUID,
|
dataset_id: UUID,
|
||||||
|
request: Request,
|
||||||
analysis_run_id: UUID | None = None,
|
analysis_run_id: UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
|
offset: int = Query(default=0, ge=0),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||||
|
assert_guest_project_scope(request, dataset.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
SegmentationService.list_segmentations(
|
SegmentationService.list_segmentations(
|
||||||
db,
|
db,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
@@ -105,8 +184,14 @@ def list_dataset_segmentations(
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/segmentations/{segmentation_id}", response_model=Envelope[SegmentationRead])
|
@router.get("/segmentations/{segmentation_id}", response_model=Envelope[SegmentationRead])
|
||||||
def get_segmentation(segmentation_id: UUID, db: Session = Depends(get_db)) -> dict:
|
def get_segmentation(
|
||||||
return envelope(SegmentationService.get_segmentation(db, segmentation_id).model_dump())
|
segmentation_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
segmentation = SegmentationService.get_segmentation(db, segmentation_id)
|
||||||
|
assert_guest_project_scope(request, segmentation.project_id)
|
||||||
|
return envelope(segmentation.model_dump())
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
@@ -115,13 +200,24 @@ def get_segmentation(segmentation_id: UUID, db: Session = Depends(get_db)) -> di
|
|||||||
)
|
)
|
||||||
def get_segmentation_run_geojson(
|
def get_segmentation_run_geojson(
|
||||||
analysis_run_id: UUID,
|
analysis_run_id: UUID,
|
||||||
|
request: Request,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
run = SegmentationService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
SegmentationService.segmentations_to_geojson(
|
SegmentationService.segmentations_to_geojson(
|
||||||
db,
|
db,
|
||||||
|
limit=limit,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
min_confidence=min_confidence,
|
min_confidence=min_confidence,
|
||||||
@@ -135,14 +231,25 @@ def get_segmentation_run_geojson(
|
|||||||
)
|
)
|
||||||
def get_dataset_segmentation_geojson(
|
def get_dataset_segmentation_geojson(
|
||||||
dataset_id: UUID,
|
dataset_id: UUID,
|
||||||
|
request: Request,
|
||||||
analysis_run_id: UUID | None = None,
|
analysis_run_id: UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int = Query(
|
||||||
|
default=DetectionService.DEFAULT_RESULT_LIMIT,
|
||||||
|
ge=0,
|
||||||
|
le=50_000,
|
||||||
|
description="Maximum results to return; 0 returns everything. Highest confidence first.",
|
||||||
|
),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||||
|
assert_guest_project_scope(request, dataset.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
SegmentationService.segmentations_to_geojson(
|
SegmentationService.segmentations_to_geojson(
|
||||||
db,
|
db,
|
||||||
|
limit=limit,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
@@ -158,8 +265,12 @@ def get_dataset_segmentation_geojson(
|
|||||||
def compare_segmentation_run_with_reference(
|
def compare_segmentation_run_with_reference(
|
||||||
analysis_run_id: UUID,
|
analysis_run_id: UUID,
|
||||||
payload: SegmentationQaRequest,
|
payload: SegmentationQaRequest,
|
||||||
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
if guest_project_scope(request) is not None:
|
||||||
|
run = SegmentationService.get_run(db, analysis_run_id)
|
||||||
|
assert_guest_project_scope(request, run.project_id)
|
||||||
return envelope(
|
return envelope(
|
||||||
SegmentationService.compare_segmentations_with_reference(
|
SegmentationService.compare_segmentations_with_reference(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -168,5 +279,6 @@ def compare_segmentation_run_with_reference(
|
|||||||
iou_threshold=payload.iou_threshold,
|
iou_threshold=payload.iou_threshold,
|
||||||
class_name=payload.class_name,
|
class_name=payload.class_name,
|
||||||
min_confidence=payload.min_confidence,
|
min_confidence=payload.min_confidence,
|
||||||
|
calibration_thresholds=payload.calibration_thresholds,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.db.session import get_db
|
||||||
|
from app.models import Area, Dataset
|
||||||
|
from app.schemas.common import Envelope
|
||||||
|
from app.schemas.operations import VectorSelectionResponse
|
||||||
|
from app.schemas.selection_partitions import VectorPartitionSelectionRequest
|
||||||
|
from app.services.vector_feature_service import VectorFeatureService
|
||||||
|
from app.utils.response import envelope
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/projects/{project_id}", tags=["selection-partitions"])
|
||||||
|
|
||||||
|
|
||||||
|
def _product_identity(dataset: Dataset) -> str:
|
||||||
|
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||||
|
return str(metadata.get("product_key") or dataset.reference_layer_name or "")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/datasets/vector/partitions/select",
|
||||||
|
response_model=Envelope[VectorSelectionResponse],
|
||||||
|
)
|
||||||
|
def select_vector_partitions(
|
||||||
|
project_id: UUID,
|
||||||
|
payload: VectorPartitionSelectionRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
datasets = db.query(Dataset).filter(Dataset.id.in_(payload.dataset_ids)).all()
|
||||||
|
by_id = {dataset.id: dataset for dataset in datasets}
|
||||||
|
ordered = [by_id.get(dataset_id) for dataset_id in payload.dataset_ids]
|
||||||
|
if any(dataset is None or dataset.project_id != project_id for dataset in ordered):
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="One or more selection partitions were not found", status_code=404)
|
||||||
|
typed_datasets = [dataset for dataset in ordered if dataset is not None]
|
||||||
|
if any(dataset.dataset_type not in {"vector", "geojson"} or dataset.status != "ready" for dataset in typed_datasets):
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_VECTOR_PARTITIONS",
|
||||||
|
message="Every selection partition must be a ready vector dataset",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
source_names = {dataset.source_name for dataset in typed_datasets}
|
||||||
|
product_keys = {_product_identity(dataset) for dataset in typed_datasets}
|
||||||
|
if len(source_names) != 1 or len(product_keys) != 1:
|
||||||
|
raise AppError(
|
||||||
|
code="VECTOR_PARTITION_SOURCE_MISMATCH",
|
||||||
|
message="Selection partitions must belong to one governed source product",
|
||||||
|
details={"source_names": sorted(str(value) for value in source_names), "product_keys": sorted(product_keys)},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
|
||||||
|
selection_geometry = None
|
||||||
|
selection_area_id = None
|
||||||
|
if payload.area_id is not None:
|
||||||
|
selection_area = db.get(Area, payload.area_id)
|
||||||
|
if selection_area is None or selection_area.project_id != project_id:
|
||||||
|
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||||
|
selection_geometry, _covers_full_area = VectorFeatureService.constrain_bbox_to_area(
|
||||||
|
payload.bbox.model_dump(),
|
||||||
|
selection_area.geometry,
|
||||||
|
)
|
||||||
|
selection_area_id = selection_area.id
|
||||||
|
|
||||||
|
representative = typed_datasets[0]
|
||||||
|
dataset_ids = [dataset.id for dataset in typed_datasets]
|
||||||
|
result = VectorFeatureService.select_features_by_bbox(
|
||||||
|
db,
|
||||||
|
dataset_id=representative.id,
|
||||||
|
dataset_ids=dataset_ids,
|
||||||
|
bbox=payload.bbox.model_dump(),
|
||||||
|
limit=payload.limit,
|
||||||
|
dataset=representative,
|
||||||
|
selection_geometry=selection_geometry,
|
||||||
|
selection_area_id=selection_area_id,
|
||||||
|
deduplicate_source_features=True,
|
||||||
|
)
|
||||||
|
result.update(
|
||||||
|
partition_count=len(dataset_ids),
|
||||||
|
source_name=representative.source_name,
|
||||||
|
dataset_ids=dataset_ids,
|
||||||
|
)
|
||||||
|
return envelope(VectorSelectionResponse(**result).model_dump(exclude_none=True))
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy import func, or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.db.session import get_db
|
||||||
|
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
|
||||||
|
from app.schemas import (
|
||||||
|
DatasetLineageEdgeRead,
|
||||||
|
DatasetProvenanceRead,
|
||||||
|
DatasetQuarantineRead,
|
||||||
|
Envelope,
|
||||||
|
ItemList,
|
||||||
|
SourceRegistryDetailRead,
|
||||||
|
SourceRegistryRead,
|
||||||
|
SourceSnapshotRead,
|
||||||
|
)
|
||||||
|
from app.utils.response import envelope
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(tags=["source-registry"])
|
||||||
|
|
||||||
|
|
||||||
|
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
|
||||||
|
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
|
||||||
|
def list_source_registry(
|
||||||
|
classification: str | None = None,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
query = (
|
||||||
|
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
|
||||||
|
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
|
||||||
|
)
|
||||||
|
if classification:
|
||||||
|
query = query.filter(SourceRegistry.classification == classification.strip().lower())
|
||||||
|
rows = (
|
||||||
|
query.group_by(SourceRegistry.id)
|
||||||
|
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
items = [_source_read(source, snapshot_count=count) for source, count in rows]
|
||||||
|
return envelope({"items": items, "total": len(items)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
|
||||||
|
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
|
||||||
|
normalized_key = source_key.strip().lower()
|
||||||
|
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
|
||||||
|
if source is None:
|
||||||
|
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
|
||||||
|
snapshots = (
|
||||||
|
db.query(SourceSnapshot)
|
||||||
|
.filter(SourceSnapshot.source_registry_id == source.id)
|
||||||
|
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
detail = SourceRegistryDetailRead(
|
||||||
|
source=_source_read(source, snapshot_count=len(snapshots)),
|
||||||
|
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
|
||||||
|
)
|
||||||
|
return envelope(detail)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/projects/{project_id}/datasets/{dataset_id}/provenance",
|
||||||
|
response_model=Envelope[DatasetProvenanceRead],
|
||||||
|
)
|
||||||
|
def get_dataset_provenance(
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
if db.get(Project, project_id) is None:
|
||||||
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if dataset is None or dataset.project_id != project_id:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
|
||||||
|
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
|
||||||
|
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
|
||||||
|
lineage = (
|
||||||
|
db.query(DatasetLineageEdge)
|
||||||
|
.filter(
|
||||||
|
or_(
|
||||||
|
DatasetLineageEdge.parent_dataset_id == dataset.id,
|
||||||
|
DatasetLineageEdge.child_dataset_id == dataset.id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
quarantines = (
|
||||||
|
db.query(DatasetQuarantine)
|
||||||
|
.filter(DatasetQuarantine.dataset_id == dataset.id)
|
||||||
|
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
result = DatasetProvenanceRead(
|
||||||
|
dataset_id=dataset.id,
|
||||||
|
source=_source_read(source) if source else None,
|
||||||
|
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
|
||||||
|
data_contract_key=dataset.data_contract_key,
|
||||||
|
data_contract_version=dataset.data_contract_version,
|
||||||
|
validation_status=dataset.validation_status,
|
||||||
|
validation_report_json=dataset.validation_report_json,
|
||||||
|
provenance_status=dataset.provenance_status,
|
||||||
|
lineage_status=dataset.lineage_status,
|
||||||
|
quarantine_status=dataset.quarantine_status,
|
||||||
|
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
|
||||||
|
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
|
||||||
|
)
|
||||||
|
return envelope(result)
|
||||||
+298
-3
@@ -1,4 +1,6 @@
|
|||||||
from pydantic import Field, field_validator
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
from pydantic import AliasChoices, Field, field_validator, model_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
|
|
||||||
@@ -12,24 +14,119 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
|
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
|
||||||
app_version: str = Field(
|
app_version: str = Field(
|
||||||
default="1.0.0-rc.1",
|
default="1.0.0",
|
||||||
validation_alias="GEOINTEL_APP_VERSION",
|
validation_alias="GEOINTEL_APP_VERSION",
|
||||||
)
|
)
|
||||||
build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
|
build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
|
||||||
build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
|
build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
|
||||||
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
|
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
|
||||||
|
auth_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_ENABLED")
|
||||||
|
auth_require_https: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_REQUIRE_HTTPS")
|
||||||
|
auth_username: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_USERNAME")
|
||||||
|
auth_password_hash: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_PASSWORD_HASH")
|
||||||
|
auth_session_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_SESSION_SECRET")
|
||||||
|
authentik_issuer: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ISSUER")
|
||||||
|
authentik_client_id: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_ID")
|
||||||
|
authentik_client_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_SECRET")
|
||||||
|
authentik_allowed_email: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ALLOWED_EMAIL")
|
||||||
|
public_base_url: str = Field(
|
||||||
|
default="http://localhost:1202",
|
||||||
|
validation_alias="GEOINTEL_PUBLIC_BASE_URL",
|
||||||
|
)
|
||||||
|
auth_session_ttl_seconds: int = Field(
|
||||||
|
default=43_200,
|
||||||
|
ge=900,
|
||||||
|
le=604_800,
|
||||||
|
validation_alias="GEOINTEL_AUTH_SESSION_TTL_SECONDS",
|
||||||
|
)
|
||||||
|
guest_access_enabled: bool = Field(
|
||||||
|
default=False,
|
||||||
|
validation_alias="GEOINTEL_GUEST_ACCESS_ENABLED",
|
||||||
|
)
|
||||||
|
guest_display_name: str = Field(
|
||||||
|
default="Gast",
|
||||||
|
min_length=1,
|
||||||
|
max_length=64,
|
||||||
|
validation_alias="GEOINTEL_GUEST_DISPLAY_NAME",
|
||||||
|
)
|
||||||
|
guest_session_ttl_seconds: int = Field(
|
||||||
|
default=7_200,
|
||||||
|
ge=900,
|
||||||
|
le=86_400,
|
||||||
|
validation_alias="GEOINTEL_GUEST_SESSION_TTL_SECONDS",
|
||||||
|
)
|
||||||
|
guest_login_requests_per_minute: int = Field(
|
||||||
|
default=3,
|
||||||
|
ge=1,
|
||||||
|
le=60,
|
||||||
|
validation_alias="GEOINTEL_GUEST_LOGIN_REQUESTS_PER_MINUTE",
|
||||||
|
)
|
||||||
|
guest_compute_requests_per_minute: int = Field(
|
||||||
|
default=4,
|
||||||
|
ge=1,
|
||||||
|
le=120,
|
||||||
|
validation_alias="GEOINTEL_GUEST_COMPUTE_REQUESTS_PER_MINUTE",
|
||||||
|
)
|
||||||
|
guest_compute_max_concurrency: int = Field(
|
||||||
|
default=2,
|
||||||
|
ge=1,
|
||||||
|
le=16,
|
||||||
|
validation_alias="GEOINTEL_GUEST_COMPUTE_MAX_CONCURRENCY",
|
||||||
|
)
|
||||||
database_url: str = Field(
|
database_url: str = Field(
|
||||||
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
|
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
|
||||||
validation_alias="DATABASE_URL",
|
validation_alias="DATABASE_URL",
|
||||||
)
|
)
|
||||||
storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT")
|
storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT")
|
||||||
max_upload_mb: int = Field(default=500, validation_alias="MAX_UPLOAD_MB")
|
# Analysis consumes only artifacts under storage_root. Provisioning
|
||||||
|
# workflows that stage tiles elsewhere before ingest can opt out.
|
||||||
|
allow_external_artifact_paths: bool = Field(
|
||||||
|
default=False, validation_alias="GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS"
|
||||||
|
)
|
||||||
|
max_upload_mb: int = Field(
|
||||||
|
default=500,
|
||||||
|
ge=1,
|
||||||
|
le=2_048,
|
||||||
|
validation_alias=AliasChoices("GEOINTEL_MAX_UPLOAD_MB", "MAX_UPLOAD_MB"),
|
||||||
|
)
|
||||||
|
max_in_memory_vector_mb: int = Field(
|
||||||
|
default=64,
|
||||||
|
ge=1,
|
||||||
|
le=256,
|
||||||
|
validation_alias="GEOINTEL_MAX_IN_MEMORY_VECTOR_MB",
|
||||||
|
)
|
||||||
|
max_raster_pixels: int = Field(
|
||||||
|
default=40_000_000,
|
||||||
|
ge=1,
|
||||||
|
le=500_000_000,
|
||||||
|
validation_alias="GEOINTEL_MAX_RASTER_PIXELS",
|
||||||
|
)
|
||||||
|
max_raster_bands: int = Field(
|
||||||
|
default=16,
|
||||||
|
ge=1,
|
||||||
|
le=256,
|
||||||
|
validation_alias="GEOINTEL_MAX_RASTER_BANDS",
|
||||||
|
)
|
||||||
|
max_decoded_raster_mb: int = Field(
|
||||||
|
default=1024,
|
||||||
|
ge=16,
|
||||||
|
le=8192,
|
||||||
|
validation_alias="GEOINTEL_MAX_DECODED_RASTER_MB",
|
||||||
|
)
|
||||||
orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED")
|
orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED")
|
||||||
orthophoto_wms_url: str = Field(
|
orthophoto_wms_url: str = Field(
|
||||||
default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
||||||
validation_alias="ORTHOPHOTO_WMS_URL",
|
validation_alias="ORTHOPHOTO_WMS_URL",
|
||||||
)
|
)
|
||||||
orthophoto_wms_layer: str = Field(default="Ortho", validation_alias="ORTHOPHOTO_WMS_LAYER")
|
orthophoto_wms_layer: str = Field(default="Ortho", validation_alias="ORTHOPHOTO_WMS_LAYER")
|
||||||
|
spw_orthophoto_wms_url: str = Field(
|
||||||
|
default="https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_LAST/MapServer/WMSServer",
|
||||||
|
validation_alias="SPW_ORTHOPHOTO_WMS_URL",
|
||||||
|
)
|
||||||
|
brussels_orthophoto_wms_url: str = Field(
|
||||||
|
default="https://geoservices-grid.irisnet.be/geoserver/urbisgrid/ows",
|
||||||
|
validation_alias="BRUSSELS_ORTHOPHOTO_WMS_URL",
|
||||||
|
)
|
||||||
orthophoto_resolution_m: float = Field(default=1.0, gt=0, validation_alias="ORTHOPHOTO_RESOLUTION_M")
|
orthophoto_resolution_m: float = Field(default=1.0, gt=0, validation_alias="ORTHOPHOTO_RESOLUTION_M")
|
||||||
orthophoto_min_side_m: float = Field(default=128.0, gt=0, validation_alias="ORTHOPHOTO_MIN_SIDE_M")
|
orthophoto_min_side_m: float = Field(default=128.0, gt=0, validation_alias="ORTHOPHOTO_MIN_SIDE_M")
|
||||||
orthophoto_max_side_m: float = Field(default=1024.0, gt=0, validation_alias="ORTHOPHOTO_MAX_SIDE_M")
|
orthophoto_max_side_m: float = Field(default=1024.0, gt=0, validation_alias="ORTHOPHOTO_MAX_SIDE_M")
|
||||||
@@ -152,6 +249,27 @@ class Settings(BaseSettings):
|
|||||||
le=8760,
|
le=8760,
|
||||||
validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS",
|
validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS",
|
||||||
)
|
)
|
||||||
|
spw_picc_enabled: bool = Field(default=True, validation_alias="SPW_PICC_ENABLED")
|
||||||
|
spw_picc_mapserver_url: str = Field(
|
||||||
|
default=(
|
||||||
|
"https://geoservices.wallonie.be/arcgis/rest/services/"
|
||||||
|
"TOPOGRAPHIE/PICC_VDIFF/MapServer"
|
||||||
|
),
|
||||||
|
validation_alias="SPW_PICC_MAPSERVER_URL",
|
||||||
|
)
|
||||||
|
spw_flood_hazard_enabled: bool = Field(default=True, validation_alias="SPW_FLOOD_HAZARD_ENABLED")
|
||||||
|
spw_flood_hazard_mapserver_url: str = Field(
|
||||||
|
default=(
|
||||||
|
"https://geoservices.wallonie.be/arcgis/rest/services/"
|
||||||
|
"EAU/ALEA_INOND/MapServer"
|
||||||
|
),
|
||||||
|
validation_alias="SPW_FLOOD_HAZARD_MAPSERVER_URL",
|
||||||
|
)
|
||||||
|
urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED")
|
||||||
|
urbis_wfs_url: str = Field(
|
||||||
|
default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows",
|
||||||
|
validation_alias="URBIS_WFS_URL",
|
||||||
|
)
|
||||||
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
|
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
|
||||||
dhmv_wcs_url: str = Field(
|
dhmv_wcs_url: str = Field(
|
||||||
default="https://geo.api.vlaanderen.be/DHMV/wcs",
|
default="https://geo.api.vlaanderen.be/DHMV/wcs",
|
||||||
@@ -195,6 +313,12 @@ class Settings(BaseSettings):
|
|||||||
le=250_000,
|
le=250_000,
|
||||||
validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES",
|
validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES",
|
||||||
)
|
)
|
||||||
|
bathymetry_profiles_max_pages: int = Field(
|
||||||
|
default=200,
|
||||||
|
ge=1,
|
||||||
|
le=5_000,
|
||||||
|
validation_alias="BATHYMETRY_PROFILES_MAX_PAGES",
|
||||||
|
)
|
||||||
bathymetry_profiles_timeout_seconds: int = Field(
|
bathymetry_profiles_timeout_seconds: int = Field(
|
||||||
default=120,
|
default=120,
|
||||||
ge=1,
|
ge=1,
|
||||||
@@ -207,6 +331,11 @@ class Settings(BaseSettings):
|
|||||||
le=256,
|
le=256,
|
||||||
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
|
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
|
||||||
)
|
)
|
||||||
|
bathymetry_raster_max_pixels: int = Field(
|
||||||
|
default=30_000_000,
|
||||||
|
ge=1,
|
||||||
|
validation_alias="BATHYMETRY_RASTER_MAX_PIXELS",
|
||||||
|
)
|
||||||
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
|
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
|
||||||
mdk_bathymetry_wcs_url: str = Field(
|
mdk_bathymetry_wcs_url: str = Field(
|
||||||
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
||||||
@@ -229,11 +358,58 @@ class Settings(BaseSettings):
|
|||||||
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
||||||
validation_alias="THEMATIC_RASTER_WCS_URL",
|
validation_alias="THEMATIC_RASTER_WCS_URL",
|
||||||
)
|
)
|
||||||
|
mdk_bathymetry_acquisition_enabled: bool = Field(
|
||||||
|
default=False,
|
||||||
|
validation_alias="MDK_BATHYMETRY_ACQUISITION_ENABLED",
|
||||||
|
)
|
||||||
|
mdk_bathymetry_coverage_id: str | None = Field(default=None, validation_alias="MDK_BATHYMETRY_COVERAGE_ID")
|
||||||
|
mdk_bathymetry_request_crs: str = Field(default="EPSG:4326", validation_alias="MDK_BATHYMETRY_REQUEST_CRS")
|
||||||
|
mdk_bathymetry_max_bbox_deg2: float = Field(
|
||||||
|
default=0.25,
|
||||||
|
gt=0,
|
||||||
|
validation_alias="MDK_BATHYMETRY_MAX_BBOX_DEG2",
|
||||||
|
)
|
||||||
|
mdk_bathymetry_acquisition_timeout_seconds: int = Field(
|
||||||
|
default=120,
|
||||||
|
ge=1,
|
||||||
|
validation_alias="MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS",
|
||||||
|
)
|
||||||
|
mdk_bathymetry_acquisition_max_response_mb: int = Field(
|
||||||
|
default=160,
|
||||||
|
ge=1,
|
||||||
|
validation_alias="MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB",
|
||||||
|
)
|
||||||
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
|
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
|
||||||
thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
|
thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
|
||||||
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
|
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
|
||||||
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
|
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
|
||||||
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
||||||
|
walous_enabled: bool = Field(default=True, validation_alias="WALOUS_ENABLED")
|
||||||
|
walous_source_dir: str = Field(
|
||||||
|
default="/app/storage/source-cache/walous",
|
||||||
|
validation_alias="WALOUS_SOURCE_DIR",
|
||||||
|
)
|
||||||
|
walous_analysis_resolution_m: float = Field(
|
||||||
|
default=10.0,
|
||||||
|
ge=1.0,
|
||||||
|
le=100.0,
|
||||||
|
validation_alias="WALOUS_ANALYSIS_RESOLUTION_M",
|
||||||
|
)
|
||||||
|
walous_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="WALOUS_MAX_SIDE_M")
|
||||||
|
walous_max_pixels: int = Field(default=36_000_000, ge=1, validation_alias="WALOUS_MAX_PIXELS")
|
||||||
|
spw_terrain_enabled: bool = Field(default=True, validation_alias="SPW_TERRAIN_ENABLED")
|
||||||
|
spw_terrain_source_dir: str = Field(
|
||||||
|
default="/app/storage/source-cache/spw-terrain",
|
||||||
|
validation_alias="SPW_TERRAIN_SOURCE_DIR",
|
||||||
|
)
|
||||||
|
spw_terrain_analysis_resolution_m: float = Field(
|
||||||
|
default=5.0,
|
||||||
|
ge=1.0,
|
||||||
|
le=10.0,
|
||||||
|
validation_alias="SPW_TERRAIN_ANALYSIS_RESOLUTION_M",
|
||||||
|
)
|
||||||
|
spw_terrain_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="SPW_TERRAIN_MAX_SIDE_M")
|
||||||
|
spw_terrain_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="SPW_TERRAIN_MAX_PIXELS")
|
||||||
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
||||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||||
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
|
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
|
||||||
@@ -241,6 +417,14 @@ class Settings(BaseSettings):
|
|||||||
default=False,
|
default=False,
|
||||||
validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
|
validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
|
||||||
)
|
)
|
||||||
|
aoi_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AOI_WORKER_ENABLED")
|
||||||
|
aoi_worker_poll_seconds: float = Field(default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_AOI_WORKER_POLL_SECONDS")
|
||||||
|
# Executes queued detection.run / segmentation.run jobs so tiled GPU
|
||||||
|
# inference never blocks an HTTP request.
|
||||||
|
analysis_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_ANALYSIS_WORKER_ENABLED")
|
||||||
|
analysis_worker_poll_seconds: float = Field(
|
||||||
|
default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_ANALYSIS_WORKER_POLL_SECONDS"
|
||||||
|
)
|
||||||
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
||||||
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
|
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
|
||||||
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
|
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
|
||||||
@@ -248,12 +432,67 @@ class Settings(BaseSettings):
|
|||||||
yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID")
|
yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID")
|
||||||
yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME")
|
yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME")
|
||||||
yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION")
|
yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION")
|
||||||
|
yolo_model_classes: str = Field(default="building", validation_alias="YOLO_MODEL_CLASSES")
|
||||||
|
yolo_enforce_validation_scope: bool = Field(default=False, validation_alias="YOLO_ENFORCE_VALIDATION_SCOPE")
|
||||||
|
yolo_validation_scope_manifest_path: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_PATH",
|
||||||
|
)
|
||||||
|
yolo_validation_scope_manifest_sha256: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_SHA256",
|
||||||
|
)
|
||||||
|
# Deprecated compatibility field. Mutable Area names are never an
|
||||||
|
# inference authorization boundary; deployments must use the immutable
|
||||||
|
# checksum-bound scope manifest above.
|
||||||
|
yolo_validated_area_names: str = Field(default="Mol,Kempen", validation_alias="YOLO_VALIDATED_AREA_NAMES")
|
||||||
yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE")
|
yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE")
|
||||||
|
yolo_require_cuda: bool = Field(default=False, validation_alias="YOLO_REQUIRE_CUDA")
|
||||||
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
|
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
|
||||||
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
|
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
|
||||||
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
|
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
|
||||||
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
|
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
|
||||||
|
yolo_suppress_tile_edge_detections: bool = Field(
|
||||||
|
default=True, validation_alias="YOLO_SUPPRESS_TILE_EDGE_DETECTIONS"
|
||||||
|
)
|
||||||
|
# Intersection over the smaller box. The candidate evaluation freezes this
|
||||||
|
# during calibration; serving a promoted model at a different value means
|
||||||
|
# the runtime suppresses detections the gate counted.
|
||||||
|
yolo_containment_nms_threshold: float = Field(
|
||||||
|
default=0.85, ge=0.0, le=1.0, validation_alias="YOLO_CONTAINMENT_NMS_THRESHOLD"
|
||||||
|
)
|
||||||
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
|
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
|
||||||
|
yolo_seg_enabled: bool = Field(default=False, validation_alias="YOLO_SEG_ENABLED")
|
||||||
|
yolo_seg_model_path: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_PATH")
|
||||||
|
yolo_seg_model_id: str = Field(default="yolo-seg-configured", validation_alias="YOLO_SEG_MODEL_ID")
|
||||||
|
yolo_seg_model_display_name: str = Field(
|
||||||
|
default="Configured YOLO segmentation",
|
||||||
|
validation_alias="YOLO_SEG_MODEL_DISPLAY_NAME",
|
||||||
|
)
|
||||||
|
yolo_seg_model_version: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_VERSION")
|
||||||
|
sam_enabled: bool = Field(default=False, validation_alias="SAM_ENABLED")
|
||||||
|
sam_model_path: str | None = Field(default=None, validation_alias="SAM_MODEL_PATH")
|
||||||
|
sam_model_id: str = Field(default="sam-configured", validation_alias="SAM_MODEL_ID")
|
||||||
|
sam_model_display_name: str = Field(
|
||||||
|
default="Configured SAM segmentation",
|
||||||
|
validation_alias="SAM_MODEL_DISPLAY_NAME",
|
||||||
|
)
|
||||||
|
sam_model_version: str | None = Field(default=None, validation_alias="SAM_MODEL_VERSION")
|
||||||
|
segmentation_max_masks_per_tile: int = Field(default=300, ge=1, validation_alias="SEGMENTATION_MAX_MASKS_PER_TILE")
|
||||||
|
# Masks and boxes overlap differently, so segmentation carries its own
|
||||||
|
# containment value rather than borrowing the detector's.
|
||||||
|
segmentation_containment_nms_threshold: float = Field(
|
||||||
|
default=0.85,
|
||||||
|
ge=0.0,
|
||||||
|
le=1.0,
|
||||||
|
validation_alias="SEGMENTATION_CONTAINMENT_NMS_THRESHOLD",
|
||||||
|
)
|
||||||
|
segmentation_duplicate_iou_threshold: float = Field(
|
||||||
|
default=0.5,
|
||||||
|
ge=0.0,
|
||||||
|
le=1.0,
|
||||||
|
validation_alias="SEGMENTATION_DUPLICATE_IOU_THRESHOLD",
|
||||||
|
)
|
||||||
ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED")
|
ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED")
|
||||||
ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL")
|
ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL")
|
||||||
ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL")
|
ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL")
|
||||||
@@ -284,6 +523,62 @@ class Settings(BaseSettings):
|
|||||||
raise ValueError("OLLAMA_BASE_URL must use http or https")
|
raise ValueError("OLLAMA_BASE_URL must use http or https")
|
||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_operator_auth(self) -> "Settings":
|
||||||
|
self.guest_display_name = self.guest_display_name.strip()
|
||||||
|
if not self.guest_display_name:
|
||||||
|
raise ValueError("GEOINTEL_GUEST_DISPLAY_NAME must not be blank")
|
||||||
|
for field_name in (
|
||||||
|
"authentik_issuer",
|
||||||
|
"authentik_client_id",
|
||||||
|
"authentik_client_secret",
|
||||||
|
"authentik_allowed_email",
|
||||||
|
):
|
||||||
|
value = getattr(self, field_name)
|
||||||
|
setattr(self, field_name, value.strip() if value else None)
|
||||||
|
self.public_base_url = self.public_base_url.strip().rstrip("/")
|
||||||
|
authentik_values = (
|
||||||
|
self.authentik_issuer,
|
||||||
|
self.authentik_client_id,
|
||||||
|
self.authentik_client_secret,
|
||||||
|
self.authentik_allowed_email,
|
||||||
|
)
|
||||||
|
if any(authentik_values) and not all(authentik_values):
|
||||||
|
raise ValueError("All GEOINTEL_AUTHENTIK_* values must be configured together")
|
||||||
|
if all(authentik_values):
|
||||||
|
if not self.auth_enabled:
|
||||||
|
raise ValueError("GEOINTEL_AUTH_ENABLED must be true when Authentik is configured")
|
||||||
|
for label, value in (
|
||||||
|
("GEOINTEL_AUTHENTIK_ISSUER", self.authentik_issuer),
|
||||||
|
("GEOINTEL_PUBLIC_BASE_URL", self.public_base_url),
|
||||||
|
):
|
||||||
|
parsed = urlsplit(str(value))
|
||||||
|
if (
|
||||||
|
parsed.scheme != "https"
|
||||||
|
or not parsed.hostname
|
||||||
|
or parsed.username
|
||||||
|
or parsed.password
|
||||||
|
or parsed.query
|
||||||
|
or parsed.fragment
|
||||||
|
):
|
||||||
|
raise ValueError(f"{label} must be an absolute HTTPS URL without credentials, query or fragment")
|
||||||
|
public_url = urlsplit(self.public_base_url)
|
||||||
|
if public_url.path not in ("", "/"):
|
||||||
|
raise ValueError("GEOINTEL_PUBLIC_BASE_URL must not contain a path")
|
||||||
|
if "@" not in str(self.authentik_allowed_email) or any(
|
||||||
|
character.isspace() for character in str(self.authentik_allowed_email)
|
||||||
|
):
|
||||||
|
raise ValueError("GEOINTEL_AUTHENTIK_ALLOWED_EMAIL must be one valid e-mail address")
|
||||||
|
if not self.auth_enabled:
|
||||||
|
return self
|
||||||
|
if not (self.auth_username or "").strip():
|
||||||
|
raise ValueError("GEOINTEL_AUTH_USERNAME is required when authentication is enabled")
|
||||||
|
if not (self.auth_password_hash or "").startswith("pbkdf2_sha256$"):
|
||||||
|
raise ValueError("GEOINTEL_AUTH_PASSWORD_HASH must be a PBKDF2-SHA256 hash")
|
||||||
|
if len(self.auth_session_secret or "") < 32:
|
||||||
|
raise ValueError("GEOINTEL_AUTH_SESSION_SECRET must contain at least 32 characters")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
def get_settings() -> Settings:
|
def get_settings() -> Settings:
|
||||||
return Settings()
|
return Settings()
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
# Stable server-owned identity: a public session must never attach itself to an
|
||||||
|
# operator project merely because the display names happen to match.
|
||||||
|
PUBLIC_DEMO_PROJECT_ID = UUID("6f7e6f12-9b62-4a3f-a5a0-4b3bb6b2c901")
|
||||||
|
PUBLIC_DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
|
||||||
|
PUBLIC_DEMO_PROJECT_MARKER = "geointel:public-demo:v1"
|
||||||
|
|
||||||
|
|
||||||
|
def is_public_demo_project(project_id: UUID) -> bool:
|
||||||
|
return project_id == PUBLIC_DEMO_PROJECT_ID
|
||||||
+249
-3
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import asyncio
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
@@ -11,18 +12,21 @@ from fastapi.exceptions import RequestValidationError
|
|||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from app.api.routes import analysis, areas, assistant, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, temporal
|
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.core.logging import configure_logging
|
from app.core.logging import configure_logging
|
||||||
from app.core.request_context import reset_request_id, set_request_id
|
from app.core.request_context import reset_request_id, set_request_id
|
||||||
from app.db.session import SessionLocal
|
from app.db.session import SessionLocal
|
||||||
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||||
|
from app.services.auth_service import AuthService
|
||||||
|
from app.services.analysis_job_worker import AnalysisJobWorker
|
||||||
|
from app.services.aoi_operation_worker import AoiOperationWorker
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger("geointel")
|
logger = logging.getLogger("geointel")
|
||||||
SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||||
UNSAFE_HOST = re.compile(r"[/\\@\s\x00-\x1f\x7f]")
|
UNSAFE_HOST = re.compile(r"[/\\@?#\s\x00-\x1f\x7f]")
|
||||||
|
|
||||||
|
|
||||||
def _to_error_payload(
|
def _to_error_payload(
|
||||||
@@ -45,14 +49,19 @@ def create_app() -> FastAPI:
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(_: FastAPI):
|
async def lifespan(_: FastAPI):
|
||||||
|
worker_stop = asyncio.Event()
|
||||||
|
worker_task = None
|
||||||
|
analysis_worker_task = None
|
||||||
if settings.reconcile_interrupted_runs_on_startup:
|
if settings.reconcile_interrupted_runs_on_startup:
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
result = RuntimeReconciliationService.reconcile(db)
|
result = RuntimeReconciliationService.reconcile(db)
|
||||||
logger.info(
|
logger.info(
|
||||||
"Runtime reconciliation completed: jobs=%s analysis_runs=%s",
|
"Runtime reconciliation completed: jobs=%s analysis_runs=%s resumed_aoi_partitions=%s exhausted_aoi_partitions=%s",
|
||||||
result.interrupted_jobs,
|
result.interrupted_jobs,
|
||||||
result.interrupted_analysis_runs,
|
result.interrupted_analysis_runs,
|
||||||
|
result.resumed_aoi_partitions,
|
||||||
|
result.exhausted_aoi_partitions,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
@@ -60,7 +69,19 @@ def create_app() -> FastAPI:
|
|||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
if settings.aoi_worker_enabled:
|
||||||
|
worker_task = asyncio.create_task(AoiOperationWorker.run(worker_stop, settings.aoi_worker_poll_seconds))
|
||||||
|
if settings.analysis_worker_enabled:
|
||||||
|
analysis_worker_task = asyncio.create_task(
|
||||||
|
AnalysisJobWorker.run(worker_stop, settings.analysis_worker_poll_seconds)
|
||||||
|
)
|
||||||
|
try:
|
||||||
yield
|
yield
|
||||||
|
finally:
|
||||||
|
worker_stop.set()
|
||||||
|
for task in (worker_task, analysis_worker_task):
|
||||||
|
if task is not None:
|
||||||
|
await task
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="GeoIntel",
|
title="GeoIntel",
|
||||||
@@ -79,7 +100,9 @@ def create_app() -> FastAPI:
|
|||||||
)
|
)
|
||||||
|
|
||||||
app.include_router(health.router)
|
app.include_router(health.router)
|
||||||
|
app.include_router(auth.router, prefix=settings.api_prefix)
|
||||||
app.include_router(analysis.router, prefix=settings.api_prefix)
|
app.include_router(analysis.router, prefix=settings.api_prefix)
|
||||||
|
app.include_router(aoi_operations.router, prefix=settings.api_prefix)
|
||||||
app.include_router(projects.router, prefix=settings.api_prefix)
|
app.include_router(projects.router, prefix=settings.api_prefix)
|
||||||
app.include_router(areas.router, prefix=settings.api_prefix)
|
app.include_router(areas.router, prefix=settings.api_prefix)
|
||||||
app.include_router(datasets.router, prefix=settings.api_prefix)
|
app.include_router(datasets.router, prefix=settings.api_prefix)
|
||||||
@@ -87,10 +110,12 @@ def create_app() -> FastAPI:
|
|||||||
app.include_router(quality_checks.router, prefix=settings.api_prefix)
|
app.include_router(quality_checks.router, prefix=settings.api_prefix)
|
||||||
app.include_router(exports.router, prefix=settings.api_prefix)
|
app.include_router(exports.router, prefix=settings.api_prefix)
|
||||||
app.include_router(external.router, prefix=settings.api_prefix)
|
app.include_router(external.router, prefix=settings.api_prefix)
|
||||||
|
app.include_router(source_registry.router, prefix=settings.api_prefix)
|
||||||
app.include_router(demo.router, prefix=settings.api_prefix)
|
app.include_router(demo.router, prefix=settings.api_prefix)
|
||||||
app.include_router(qa.router, prefix=settings.api_prefix)
|
app.include_router(qa.router, prefix=settings.api_prefix)
|
||||||
app.include_router(detection.router, prefix=settings.api_prefix)
|
app.include_router(detection.router, prefix=settings.api_prefix)
|
||||||
app.include_router(segmentation.router, prefix=settings.api_prefix)
|
app.include_router(segmentation.router, prefix=settings.api_prefix)
|
||||||
|
app.include_router(selection_partitions.router, prefix=settings.api_prefix)
|
||||||
app.include_router(temporal.router, prefix=settings.api_prefix)
|
app.include_router(temporal.router, prefix=settings.api_prefix)
|
||||||
app.include_router(assistant.router, prefix=settings.api_prefix)
|
app.include_router(assistant.router, prefix=settings.api_prefix)
|
||||||
|
|
||||||
@@ -102,6 +127,7 @@ def create_app() -> FastAPI:
|
|||||||
token = set_request_id(request_id)
|
token = set_request_id(request_id)
|
||||||
started_at = time.perf_counter()
|
started_at = time.perf_counter()
|
||||||
raw_path = str(request.scope.get("path") or "")
|
raw_path = str(request.scope.get("path") or "")
|
||||||
|
guest_compute_acquired = False
|
||||||
try:
|
try:
|
||||||
host = request.headers.get("host", "")
|
host = request.headers.get("host", "")
|
||||||
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||||
@@ -127,6 +153,224 @@ def create_app() -> FastAPI:
|
|||||||
)
|
)
|
||||||
response.headers["x-request-id"] = request_id
|
response.headers["x-request-id"] = request_id
|
||||||
return response
|
return response
|
||||||
|
public_auth_paths = {
|
||||||
|
f"{settings.api_prefix}/auth/session",
|
||||||
|
f"{settings.api_prefix}/auth/login",
|
||||||
|
f"{settings.api_prefix}/auth/guest",
|
||||||
|
f"{settings.api_prefix}/auth/logout",
|
||||||
|
f"{settings.api_prefix}/auth/authentik/start",
|
||||||
|
f"{settings.api_prefix}/auth/authentik/callback",
|
||||||
|
}
|
||||||
|
direct_loopback_request = (
|
||||||
|
request.client is not None
|
||||||
|
and request.client.host in {"127.0.0.1", "::1"}
|
||||||
|
and not request.headers.get("x-real-ip")
|
||||||
|
and not request.headers.get("x-forwarded-for")
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
settings.auth_enabled
|
||||||
|
and raw_path.startswith(f"{settings.api_prefix}/")
|
||||||
|
and raw_path not in public_auth_paths
|
||||||
|
and not direct_loopback_request
|
||||||
|
):
|
||||||
|
principal = AuthService.verify_session_token(
|
||||||
|
request.cookies.get(auth.COOKIE_NAME),
|
||||||
|
settings,
|
||||||
|
)
|
||||||
|
if principal is None:
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=401,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"AUTHENTICATION_REQUIRED",
|
||||||
|
"Meld u aan om de GeoIntel API te gebruiken.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
request.state.auth_principal = principal
|
||||||
|
if principal.role == "guest":
|
||||||
|
project_path_prefix = f"{settings.api_prefix}/projects/"
|
||||||
|
guest_project_root = f"{project_path_prefix}{principal.project_id}"
|
||||||
|
if raw_path.startswith(project_path_prefix):
|
||||||
|
scoped_path = raw_path[len(project_path_prefix):]
|
||||||
|
requested_project_id = scoped_path.split("/", 1)[0]
|
||||||
|
if str(principal.project_id) != requested_project_id:
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=403,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"GUEST_PROJECT_SCOPE_REQUIRED",
|
||||||
|
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
query_project_id = request.query_params.get("project_id")
|
||||||
|
if query_project_id and query_project_id != str(principal.project_id):
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=403,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"GUEST_PROJECT_SCOPE_REQUIRED",
|
||||||
|
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
guest_safe_read_paths = {
|
||||||
|
f"{settings.api_prefix}/projects",
|
||||||
|
f"{settings.api_prefix}/external/providers",
|
||||||
|
f"{settings.api_prefix}/assistant/status",
|
||||||
|
f"{settings.api_prefix}/assistant/models",
|
||||||
|
f"{settings.api_prefix}/detection/models",
|
||||||
|
f"{settings.api_prefix}/detection/model-assets",
|
||||||
|
f"{settings.api_prefix}/detection/yolo/preflight",
|
||||||
|
f"{settings.api_prefix}/segmentation/models",
|
||||||
|
}
|
||||||
|
normalized_path = raw_path.rstrip("/") or "/"
|
||||||
|
guest_project_read = (
|
||||||
|
normalized_path == guest_project_root
|
||||||
|
or normalized_path.startswith(f"{guest_project_root}/")
|
||||||
|
)
|
||||||
|
is_read_request = request.method in {"GET", "HEAD", "OPTIONS"}
|
||||||
|
if is_read_request:
|
||||||
|
if (
|
||||||
|
normalized_path == f"{settings.api_prefix}/detection/yolo/preflight"
|
||||||
|
and request.query_params.get("check_model_load", "").lower() in {"1", "true", "yes", "on"}
|
||||||
|
):
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=403,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"GUEST_MODEL_LOAD_FORBIDDEN",
|
||||||
|
"Model loading is available to authenticated operators only.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
guest_scoped_analysis_read = (
|
||||||
|
query_project_id == str(principal.project_id)
|
||||||
|
and normalized_path.startswith(
|
||||||
|
(
|
||||||
|
f"{settings.api_prefix}/detection/",
|
||||||
|
f"{settings.api_prefix}/segmentation/",
|
||||||
|
f"{settings.api_prefix}/exports/",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
normalized_path not in guest_safe_read_paths
|
||||||
|
and not guest_project_read
|
||||||
|
and not guest_scoped_analysis_read
|
||||||
|
):
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=403,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"GUEST_ROUTE_NOT_AVAILABLE",
|
||||||
|
"Deze API-route maakt geen deel uit van de afgeschermde GeoIntel-demo.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
else:
|
||||||
|
guest_safe_post_paths = {
|
||||||
|
f"{settings.api_prefix}/demo/workflow",
|
||||||
|
f"{settings.api_prefix}/external/coverage/resolve",
|
||||||
|
f"{settings.api_prefix}/analysis/change-detection",
|
||||||
|
}
|
||||||
|
guest_scoped_analysis_post_paths = {
|
||||||
|
f"{settings.api_prefix}/detection/run",
|
||||||
|
f"{settings.api_prefix}/detection/run-async",
|
||||||
|
f"{settings.api_prefix}/segmentation/run",
|
||||||
|
f"{settings.api_prefix}/segmentation/run-async",
|
||||||
|
f"{settings.api_prefix}/qa/detections-vs-reference",
|
||||||
|
f"{settings.api_prefix}/exports/geojson",
|
||||||
|
f"{settings.api_prefix}/exports/metadata",
|
||||||
|
f"{settings.api_prefix}/exports/report",
|
||||||
|
f"{settings.api_prefix}/exports/map-result",
|
||||||
|
}
|
||||||
|
guest_safe_post_suffixes = (
|
||||||
|
"/acquire",
|
||||||
|
"/vector/select",
|
||||||
|
"/vector/select/derive",
|
||||||
|
"/raster/tile",
|
||||||
|
"/raster/bathymetry/select",
|
||||||
|
"/raster/terrain/select",
|
||||||
|
"/raster/flood-hazard/select",
|
||||||
|
"/raster/thematic/select",
|
||||||
|
"/raster/walous/select",
|
||||||
|
"/temporal/compare",
|
||||||
|
"/datasets/vector/partitions/select",
|
||||||
|
"/datasets/bathymetry/profiles/partitions/select",
|
||||||
|
)
|
||||||
|
is_guest_safe_post = request.method == "POST" and (
|
||||||
|
raw_path in guest_safe_post_paths
|
||||||
|
or (
|
||||||
|
raw_path in guest_scoped_analysis_post_paths
|
||||||
|
and query_project_id == str(principal.project_id)
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
query_project_id == str(principal.project_id)
|
||||||
|
and raw_path.startswith(
|
||||||
|
(
|
||||||
|
f"{settings.api_prefix}/detection/runs/",
|
||||||
|
f"{settings.api_prefix}/segmentation/runs/",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
and raw_path.endswith("/qa/reference")
|
||||||
|
)
|
||||||
|
or (
|
||||||
|
raw_path.startswith(project_path_prefix)
|
||||||
|
and (
|
||||||
|
raw_path.endswith(guest_safe_post_suffixes)
|
||||||
|
or raw_path.endswith("/assistant/query")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not is_guest_safe_post:
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=403,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"GUEST_READ_ONLY",
|
||||||
|
"Gasttoegang laat alleen projectgebonden demo-analyses toe. Meld u aan als operator voor beheerwijzigingen.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
retry_after = AuthService.consume_guest_request(
|
||||||
|
f"guest-compute:{principal.session_id}",
|
||||||
|
max_requests=settings.guest_compute_requests_per_minute,
|
||||||
|
)
|
||||||
|
if retry_after:
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=429,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"GUEST_COMPUTE_RATE_LIMITED",
|
||||||
|
"The public demo compute budget is temporarily exhausted.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["retry-after"] = str(retry_after)
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
|
guest_compute_acquired = AuthService.try_acquire_guest_compute(
|
||||||
|
max_concurrency=settings.guest_compute_max_concurrency,
|
||||||
|
)
|
||||||
|
if not guest_compute_acquired:
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=429,
|
||||||
|
content=_to_error_payload(
|
||||||
|
"GUEST_COMPUTE_BUSY",
|
||||||
|
"The public demo is already processing its maximum number of jobs.",
|
||||||
|
request_id=request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
response.headers["retry-after"] = "10"
|
||||||
|
response.headers["x-request-id"] = request_id
|
||||||
|
return response
|
||||||
response = await call_next(request)
|
response = await call_next(request)
|
||||||
response.headers["x-request-id"] = request_id
|
response.headers["x-request-id"] = request_id
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -139,6 +383,8 @@ def create_app() -> FastAPI:
|
|||||||
)
|
)
|
||||||
return response
|
return response
|
||||||
finally:
|
finally:
|
||||||
|
if guest_compute_acquired:
|
||||||
|
AuthService.release_guest_compute()
|
||||||
reset_request_id(token)
|
reset_request_id(token)
|
||||||
|
|
||||||
@app.exception_handler(AppError)
|
@app.exception_handler(AppError)
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
from app.models import *
|
from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API
|
||||||
|
|||||||
@@ -1,9 +1,33 @@
|
|||||||
from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
from .entities import (
|
||||||
|
AoiOperation,
|
||||||
|
AoiOperationPartition,
|
||||||
|
AnalysisRun,
|
||||||
|
Area,
|
||||||
|
Dataset,
|
||||||
|
DatasetLineageEdge,
|
||||||
|
DatasetQuarantine,
|
||||||
|
DatasetVersion,
|
||||||
|
Detection,
|
||||||
|
DetectionReview,
|
||||||
|
Export,
|
||||||
|
Job,
|
||||||
|
Metric,
|
||||||
|
Project,
|
||||||
|
QualityCheck,
|
||||||
|
Segmentation,
|
||||||
|
SourceRegistry,
|
||||||
|
SourceSnapshot,
|
||||||
|
VectorFeature,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"AnalysisRun",
|
"AnalysisRun",
|
||||||
|
"AoiOperation",
|
||||||
|
"AoiOperationPartition",
|
||||||
"Area",
|
"Area",
|
||||||
"Dataset",
|
"Dataset",
|
||||||
|
"DatasetLineageEdge",
|
||||||
|
"DatasetQuarantine",
|
||||||
"DatasetVersion",
|
"DatasetVersion",
|
||||||
"Detection",
|
"Detection",
|
||||||
"DetectionReview",
|
"DetectionReview",
|
||||||
@@ -13,5 +37,7 @@ __all__ = [
|
|||||||
"Project",
|
"Project",
|
||||||
"QualityCheck",
|
"QualityCheck",
|
||||||
"Segmentation",
|
"Segmentation",
|
||||||
|
"SourceRegistry",
|
||||||
|
"SourceSnapshot",
|
||||||
"VectorFeature",
|
"VectorFeature",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -12,13 +12,44 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|||||||
from app.db.base import Base
|
from app.db.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
SOURCE_CLASSIFICATIONS = (
|
||||||
|
"authoritative",
|
||||||
|
"corroborative",
|
||||||
|
"contextual",
|
||||||
|
"derived",
|
||||||
|
"experimental",
|
||||||
|
)
|
||||||
|
SOURCE_FRESHNESS_STATUSES = (
|
||||||
|
"unknown",
|
||||||
|
"current",
|
||||||
|
"due",
|
||||||
|
"stale",
|
||||||
|
"not_applicable",
|
||||||
|
"review_required",
|
||||||
|
)
|
||||||
|
SOURCE_INGEST_STATUSES = (
|
||||||
|
"registered",
|
||||||
|
"configured",
|
||||||
|
"not_configured",
|
||||||
|
"available",
|
||||||
|
"ingested",
|
||||||
|
"failed",
|
||||||
|
"quarantined",
|
||||||
|
"legacy_unverified",
|
||||||
|
)
|
||||||
|
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||||
|
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||||
|
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
|
||||||
|
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
|
||||||
|
|
||||||
|
|
||||||
class Project(Base):
|
class Project(Base):
|
||||||
__tablename__ = "projects"
|
__tablename__ = "projects"
|
||||||
|
|
||||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
region: Mapped[str] = mapped_column(String(120), default="Kempen")
|
region: Mapped[str] = mapped_column(String(120), default="Belgium and Belgian North Sea")
|
||||||
status: Mapped[str] = mapped_column(String(32), default="active")
|
status: Mapped[str] = mapped_column(String(32), default="active")
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
@@ -42,6 +73,123 @@ class Area(Base):
|
|||||||
project: Mapped[Project] = relationship("Project", back_populates="areas")
|
project: Mapped[Project] = relationship("Project", back_populates="areas")
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRegistry(Base):
|
||||||
|
"""Server-owned source identity and authority contract.
|
||||||
|
|
||||||
|
Dataset metadata remains descriptive until a governed importer binds a
|
||||||
|
dataset to both this registry entry and an immutable SourceSnapshot.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "source_registry"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
|
||||||
|
CheckConstraint(
|
||||||
|
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
|
||||||
|
name="ck_source_registry_classification",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||||
|
name="ck_source_registry_freshness_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||||
|
"'failed', 'quarantined', 'legacy_unverified')",
|
||||||
|
name="ck_source_registry_ingest_status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
classification: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
|
||||||
|
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||||
|
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
freshness_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||||
|
)
|
||||||
|
ingest_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="registered", server_default="registered"
|
||||||
|
)
|
||||||
|
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||||
|
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
|
||||||
|
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
|
||||||
|
)
|
||||||
|
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
|
||||||
|
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
|
||||||
|
|
||||||
|
|
||||||
|
class SourceSnapshot(Base):
|
||||||
|
"""Immutable source-version evidence recorded by governed ingestion."""
|
||||||
|
|
||||||
|
__tablename__ = "source_snapshots"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
|
||||||
|
CheckConstraint(
|
||||||
|
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||||
|
name="ck_source_snapshots_freshness_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||||
|
"'failed', 'quarantined', 'legacy_unverified')",
|
||||||
|
name="ck_source_snapshots_ingest_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
|
||||||
|
name="ck_source_snapshots_checksum_sha256",
|
||||||
|
),
|
||||||
|
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
|
||||||
|
Index("ix_source_snapshots_checksum", "checksum_sha256"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
source_registry_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||||
|
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
freshness_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||||
|
)
|
||||||
|
ingest_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="registered", server_default="registered"
|
||||||
|
)
|
||||||
|
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||||
|
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
|
||||||
|
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
|
||||||
|
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
|
||||||
|
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
|
||||||
|
|
||||||
|
|
||||||
class Dataset(Base):
|
class Dataset(Base):
|
||||||
__tablename__ = "datasets"
|
__tablename__ = "datasets"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
@@ -49,6 +197,27 @@ class Dataset(Base):
|
|||||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||||
name="ck_datasets_temporal_valid_range",
|
name="ck_datasets_temporal_valid_range",
|
||||||
),
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||||
|
name="ck_datasets_validation_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_datasets_provenance_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_datasets_lineage_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"quarantine_status IN ('not_quarantined', 'quarantined')",
|
||||||
|
name="ck_datasets_quarantine_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||||
|
name="ck_datasets_ingest_key_not_blank",
|
||||||
|
),
|
||||||
|
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
|
||||||
Index(
|
Index(
|
||||||
"ix_datasets_project_temporal_series_observed",
|
"ix_datasets_project_temporal_series_observed",
|
||||||
"project_id",
|
"project_id",
|
||||||
@@ -69,6 +238,7 @@ class Dataset(Base):
|
|||||||
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
UUID(as_uuid=True),
|
UUID(as_uuid=True),
|
||||||
ForeignKey("datasets.id", ondelete="SET NULL"),
|
ForeignKey("datasets.id", ondelete="SET NULL"),
|
||||||
@@ -84,6 +254,43 @@ class Dataset(Base):
|
|||||||
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
validation_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="not_validated",
|
||||||
|
server_default="not_validated",
|
||||||
|
comment="not_validated | passed | failed",
|
||||||
|
)
|
||||||
|
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
provenance_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
|
lineage_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
|
quarantine_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="not_quarantined",
|
||||||
|
server_default="not_quarantined",
|
||||||
|
comment="not_quarantined | quarantined",
|
||||||
|
)
|
||||||
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
@@ -106,6 +313,23 @@ class Dataset(Base):
|
|||||||
back_populates="dataset",
|
back_populates="dataset",
|
||||||
cascade="all, delete-orphan",
|
cascade="all, delete-orphan",
|
||||||
)
|
)
|
||||||
|
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
|
||||||
|
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
|
||||||
|
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||||
|
"DatasetLineageEdge",
|
||||||
|
foreign_keys="DatasetLineageEdge.parent_dataset_id",
|
||||||
|
back_populates="parent_dataset",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
)
|
||||||
|
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||||
|
"DatasetLineageEdge",
|
||||||
|
foreign_keys="DatasetLineageEdge.child_dataset_id",
|
||||||
|
back_populates="child_dataset",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
)
|
||||||
|
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
|
||||||
|
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DatasetVersion(Base):
|
class DatasetVersion(Base):
|
||||||
@@ -115,7 +339,24 @@ class DatasetVersion(Base):
|
|||||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||||
name="ck_dataset_versions_temporal_valid_range",
|
name="ck_dataset_versions_temporal_valid_range",
|
||||||
),
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||||
|
name="ck_dataset_versions_validation_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_dataset_versions_provenance_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_dataset_versions_lineage_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||||
|
name="ck_dataset_versions_ingest_key_not_blank",
|
||||||
|
),
|
||||||
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
||||||
|
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
@@ -127,11 +368,135 @@ class DatasetVersion(Base):
|
|||||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
validation_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="not_validated",
|
||||||
|
server_default="not_validated",
|
||||||
|
comment="not_validated | passed | failed",
|
||||||
|
)
|
||||||
|
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
provenance_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
|
lineage_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
||||||
|
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
|
||||||
|
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
|
||||||
|
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetLineageEdge(Base):
|
||||||
|
"""Immutable relationship between input/output datasets and transforms."""
|
||||||
|
|
||||||
|
__tablename__ = "dataset_lineage_edges"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
|
||||||
|
UniqueConstraint(
|
||||||
|
"parent_dataset_id",
|
||||||
|
"child_dataset_id",
|
||||||
|
"relation_type",
|
||||||
|
"transformation_name",
|
||||||
|
name="uq_dataset_lineage_edges_relation",
|
||||||
|
),
|
||||||
|
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
|
||||||
|
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
parent_dataset: Mapped[Dataset] = relationship(
|
||||||
|
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
|
||||||
|
)
|
||||||
|
child_dataset: Mapped[Dataset] = relationship(
|
||||||
|
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetQuarantine(Base):
|
||||||
|
"""Durable fail-closed record for rejected or doubtful source artifacts."""
|
||||||
|
|
||||||
|
__tablename__ = "dataset_quarantines"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
|
||||||
|
name="ck_dataset_quarantines_target_present",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('quarantined', 'released', 'rejected')",
|
||||||
|
name="ck_dataset_quarantines_status",
|
||||||
|
),
|
||||||
|
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
|
||||||
|
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
stage: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="quarantined", server_default="quarantined"
|
||||||
|
)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
|
||||||
|
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
|
||||||
|
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
|
||||||
|
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
|
||||||
|
|
||||||
|
|
||||||
class VectorFeature(Base):
|
class VectorFeature(Base):
|
||||||
@@ -361,3 +726,63 @@ class Job(Base):
|
|||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperation(Base):
|
||||||
|
__tablename__ = "aoi_operations"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('queued', 'running', 'partial', 'success', 'failed', 'cancelled')",
|
||||||
|
name="ck_aoi_operations_status",
|
||||||
|
),
|
||||||
|
Index("ix_aoi_operations_project_status", "project_id", "status"),
|
||||||
|
Index("ix_aoi_operations_geometry", "geometry", postgresql_using="gist"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
|
||||||
|
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
|
||||||
|
parent_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||||
|
operation_type: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||||
|
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
|
||||||
|
request_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
plan_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperationPartition(Base):
|
||||||
|
__tablename__ = "aoi_operation_partitions"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('queued', 'running', 'success', 'failed', 'skipped')",
|
||||||
|
name="ck_aoi_operation_partitions_status",
|
||||||
|
),
|
||||||
|
UniqueConstraint("operation_id", "partition_key", name="uq_aoi_operation_partition_key"),
|
||||||
|
Index("ix_aoi_operation_partitions_operation_status", "operation_id", "status"),
|
||||||
|
Index("ix_aoi_operation_partitions_geometry", "geometry", postgresql_using="gist"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
operation_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("aoi_operations.id", ondelete="CASCADE"), nullable=False)
|
||||||
|
child_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
|
||||||
|
partition_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
provider_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
product_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
ordinal: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||||
|
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
|
||||||
|
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)
|
||||||
|
checkpoint_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|||||||
@@ -32,6 +32,14 @@ from .source_catalog import (
|
|||||||
SourceCatalogProbeReport,
|
SourceCatalogProbeReport,
|
||||||
SourceCatalogProbeSummary,
|
SourceCatalogProbeSummary,
|
||||||
)
|
)
|
||||||
|
from .source_registry import (
|
||||||
|
DatasetProvenanceRead,
|
||||||
|
DatasetLineageEdgeRead,
|
||||||
|
DatasetQuarantineRead,
|
||||||
|
SourceRegistryDetailRead,
|
||||||
|
SourceRegistryRead,
|
||||||
|
SourceSnapshotRead,
|
||||||
|
)
|
||||||
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
||||||
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
||||||
from .official_vector import (
|
from .official_vector import (
|
||||||
@@ -48,6 +56,8 @@ from .detection import (
|
|||||||
DetectionRunListResponse,
|
DetectionRunListResponse,
|
||||||
DetectionRunRead,
|
DetectionRunRead,
|
||||||
DetectionRunRequest,
|
DetectionRunRequest,
|
||||||
|
DetectionComparisonRequest,
|
||||||
|
DetectionComparisonResponse,
|
||||||
DetectionRunResponse,
|
DetectionRunResponse,
|
||||||
ModelAssetListResponse,
|
ModelAssetListResponse,
|
||||||
ModelAssetRead,
|
ModelAssetRead,
|
||||||
@@ -78,6 +88,11 @@ from .dhmv import (
|
|||||||
TerrainSelectionResponse,
|
TerrainSelectionResponse,
|
||||||
TerrainSelectionSummary,
|
TerrainSelectionSummary,
|
||||||
)
|
)
|
||||||
|
from .spw_terrain import (
|
||||||
|
SpwTerrainAcquireRequest,
|
||||||
|
SpwTerrainAcquisitionResult,
|
||||||
|
SpwTerrainProductRead,
|
||||||
|
)
|
||||||
from .flood_hazard import (
|
from .flood_hazard import (
|
||||||
FloodHazardAcquireRequest,
|
FloodHazardAcquireRequest,
|
||||||
FloodHazardAcquisitionResult,
|
FloodHazardAcquisitionResult,
|
||||||
@@ -93,8 +108,14 @@ from .bathymetry import (
|
|||||||
BathymetryPartitionFinalizationResult,
|
BathymetryPartitionFinalizationResult,
|
||||||
BathymetryProfileAcquireRequest,
|
BathymetryProfileAcquireRequest,
|
||||||
BathymetryProfileAcquisitionResult,
|
BathymetryProfileAcquisitionResult,
|
||||||
|
BathymetryRasterMetric,
|
||||||
|
BathymetryRasterSelectionRequest,
|
||||||
|
BathymetryRasterSelectionResponse,
|
||||||
|
BathymetryRasterSelectionSummary,
|
||||||
BathymetrySourceProbeRead,
|
BathymetrySourceProbeRead,
|
||||||
BathymetrySourceRead,
|
BathymetrySourceRead,
|
||||||
|
MdkBathymetryAcquireRequest,
|
||||||
|
MdkBathymetryAcquisitionResult,
|
||||||
)
|
)
|
||||||
from .thematic_raster import (
|
from .thematic_raster import (
|
||||||
ThematicRasterAcquireRequest,
|
ThematicRasterAcquireRequest,
|
||||||
@@ -196,6 +217,12 @@ __all__ = [
|
|||||||
"SourceCatalogProbeItem",
|
"SourceCatalogProbeItem",
|
||||||
"SourceCatalogProbeReport",
|
"SourceCatalogProbeReport",
|
||||||
"SourceCatalogProbeSummary",
|
"SourceCatalogProbeSummary",
|
||||||
|
"SourceRegistryRead",
|
||||||
|
"SourceRegistryDetailRead",
|
||||||
|
"SourceSnapshotRead",
|
||||||
|
"DatasetLineageEdgeRead",
|
||||||
|
"DatasetQuarantineRead",
|
||||||
|
"DatasetProvenanceRead",
|
||||||
"GrbRefreshLayerPlan",
|
"GrbRefreshLayerPlan",
|
||||||
"GrbRefreshPlan",
|
"GrbRefreshPlan",
|
||||||
"GrbRefreshPlanSummary",
|
"GrbRefreshPlanSummary",
|
||||||
@@ -213,6 +240,8 @@ __all__ = [
|
|||||||
"DetectionRunListResponse",
|
"DetectionRunListResponse",
|
||||||
"DetectionRunRead",
|
"DetectionRunRead",
|
||||||
"DetectionRunRequest",
|
"DetectionRunRequest",
|
||||||
|
"DetectionComparisonRequest",
|
||||||
|
"DetectionComparisonResponse",
|
||||||
"DetectionRunResponse",
|
"DetectionRunResponse",
|
||||||
"ModelAssetListResponse",
|
"ModelAssetListResponse",
|
||||||
"ModelAssetRead",
|
"ModelAssetRead",
|
||||||
@@ -242,6 +271,9 @@ __all__ = [
|
|||||||
"DhmvAcquireRequest",
|
"DhmvAcquireRequest",
|
||||||
"DhmvAcquisitionResult",
|
"DhmvAcquisitionResult",
|
||||||
"DhmvProductRead",
|
"DhmvProductRead",
|
||||||
|
"SpwTerrainAcquireRequest",
|
||||||
|
"SpwTerrainAcquisitionResult",
|
||||||
|
"SpwTerrainProductRead",
|
||||||
"TerrainMetric",
|
"TerrainMetric",
|
||||||
"TerrainPartitionSelectionRequest",
|
"TerrainPartitionSelectionRequest",
|
||||||
"TerrainSelectionRequest",
|
"TerrainSelectionRequest",
|
||||||
@@ -257,10 +289,16 @@ __all__ = [
|
|||||||
"FloodHazardSelectionSummary",
|
"FloodHazardSelectionSummary",
|
||||||
"BathymetryProfileAcquireRequest",
|
"BathymetryProfileAcquireRequest",
|
||||||
"BathymetryProfileAcquisitionResult",
|
"BathymetryProfileAcquisitionResult",
|
||||||
|
"BathymetryRasterMetric",
|
||||||
|
"BathymetryRasterSelectionRequest",
|
||||||
|
"BathymetryRasterSelectionResponse",
|
||||||
|
"BathymetryRasterSelectionSummary",
|
||||||
"BathymetryPartitionFinalizeRequest",
|
"BathymetryPartitionFinalizeRequest",
|
||||||
"BathymetryPartitionFinalizationResult",
|
"BathymetryPartitionFinalizationResult",
|
||||||
"BathymetrySourceProbeRead",
|
"BathymetrySourceProbeRead",
|
||||||
"BathymetrySourceRead",
|
"BathymetrySourceRead",
|
||||||
|
"MdkBathymetryAcquireRequest",
|
||||||
|
"MdkBathymetryAcquisitionResult",
|
||||||
"ThematicRasterAcquireRequest",
|
"ThematicRasterAcquireRequest",
|
||||||
"ThematicRasterAcquisitionResult",
|
"ThematicRasterAcquisitionResult",
|
||||||
"ThematicRasterMetric",
|
"ThematicRasterMetric",
|
||||||
|
|||||||
@@ -5,12 +5,22 @@ from uuid import UUID
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from app.schemas.operations import VectorSelectionBBox
|
||||||
|
|
||||||
|
|
||||||
class ChangeDetectionRequest(BaseModel):
|
class ChangeDetectionRequest(BaseModel):
|
||||||
source_dataset_id: UUID
|
source_dataset_id: UUID
|
||||||
target_dataset_id: UUID
|
target_dataset_id: UUID
|
||||||
iou_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
|
iou_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
|
||||||
|
# Below this the two footprints are separate objects rather than one that
|
||||||
|
# was redrawn; between the two thresholds the change class is "modified".
|
||||||
|
modified_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
|
||||||
include_unchanged: bool = True
|
include_unchanged: bool = True
|
||||||
|
# Without a selection the comparison covers both datasets in full, which is
|
||||||
|
# rarely the question and never a response a map can draw.
|
||||||
|
bbox: VectorSelectionBBox | None = None
|
||||||
|
area_id: UUID | None = None
|
||||||
|
preview_limit: int = Field(default=2_000, ge=1, le=20_000)
|
||||||
|
|
||||||
|
|
||||||
class ChangeDetectionSummary(BaseModel):
|
class ChangeDetectionSummary(BaseModel):
|
||||||
@@ -20,8 +30,17 @@ class ChangeDetectionSummary(BaseModel):
|
|||||||
target_feature_count: int
|
target_feature_count: int
|
||||||
added_count: int
|
added_count: int
|
||||||
removed_count: int
|
removed_count: int
|
||||||
|
# A footprint that was redrawn rather than demolished and rebuilt. Without
|
||||||
|
# this class it appeared as one removal plus one addition.
|
||||||
|
modified_count: int = 0
|
||||||
unchanged_count: int
|
unchanged_count: int
|
||||||
iou_threshold: float
|
iou_threshold: float
|
||||||
|
modified_iou_threshold: float | None = None
|
||||||
|
selection_area_id: UUID | None = None
|
||||||
|
# Counts describe the whole selection; the GeoJSON is capped so a regional
|
||||||
|
# comparison does not return both datasets in one response.
|
||||||
|
preview_limit: int | None = None
|
||||||
|
preview_truncated: bool = False
|
||||||
warnings: list[str] = Field(default_factory=list)
|
warnings: list[str] = Field(default_factory=list)
|
||||||
generated_at: datetime
|
generated_at: datetime
|
||||||
geojson: dict
|
geojson: dict
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from app.schemas.operations import VectorSelectionBBox
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperationCreate(BaseModel):
|
||||||
|
area_id: UUID | None = None
|
||||||
|
bbox: VectorSelectionBBox | None = None
|
||||||
|
operation_type: str = Field(min_length=1, max_length=128)
|
||||||
|
provider_key: str = Field(min_length=1, max_length=120)
|
||||||
|
product_key: str = Field(min_length=1, max_length=120)
|
||||||
|
coverage_zone: str | None = Field(default=None, max_length=64)
|
||||||
|
max_partition_side_m: float | None = Field(default=None, gt=0, le=60_000)
|
||||||
|
max_attempts: int = Field(default=3, ge=1, le=10)
|
||||||
|
parameters_json: dict = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class AoiPartitionRead(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
partition_key: str
|
||||||
|
provider_key: str
|
||||||
|
product_key: str
|
||||||
|
ordinal: int
|
||||||
|
status: str
|
||||||
|
attempt_count: int
|
||||||
|
max_attempts: int
|
||||||
|
checkpoint_json: dict | None = None
|
||||||
|
result_json: dict | None = None
|
||||||
|
error_message: str | None = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperationRead(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
project_id: UUID
|
||||||
|
area_id: UUID | None = None
|
||||||
|
parent_job_id: UUID | None = None
|
||||||
|
operation_type: str
|
||||||
|
status: str
|
||||||
|
request_json: dict
|
||||||
|
plan_json: dict
|
||||||
|
result_json: dict | None = None
|
||||||
|
error_message: str | None = None
|
||||||
|
progress: float
|
||||||
|
partition_counts: dict[str, int]
|
||||||
|
partitions: list[AoiPartitionRead] = Field(default_factory=list)
|
||||||
|
created_at: datetime | None = None
|
||||||
|
started_at: datetime | None = None
|
||||||
|
finished_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperationList(BaseModel):
|
||||||
|
items: list[AoiOperationRead]
|
||||||
|
total: int
|
||||||
|
|
||||||
|
|
||||||
|
class AoiPartitionCheckpoint(BaseModel):
|
||||||
|
checkpoint_json: dict = Field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
class AoiPartitionComplete(BaseModel):
|
||||||
|
result_json: dict = Field(default_factory=dict)
|
||||||
|
skipped: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class AoiPartitionFail(BaseModel):
|
||||||
|
error_message: str = Field(min_length=1, max_length=4000)
|
||||||
|
retryable: bool = True
|
||||||
|
details: dict = Field(default_factory=dict)
|
||||||
@@ -14,6 +14,7 @@ class AreaCreate(BaseModel):
|
|||||||
|
|
||||||
class AreaUpdate(BaseModel):
|
class AreaUpdate(BaseModel):
|
||||||
name: str | None = None
|
name: str | None = None
|
||||||
|
geometry: dict | None = None
|
||||||
crs: str | None = None
|
crs: str | None = None
|
||||||
|
|
||||||
|
|
||||||
@@ -39,3 +40,16 @@ class AreaList(BaseModel):
|
|||||||
total: int
|
total: int
|
||||||
limit: int
|
limit: int
|
||||||
offset: int
|
offset: int
|
||||||
|
|
||||||
|
|
||||||
|
class MunicipalitySearchItem(BaseModel):
|
||||||
|
niscode: str
|
||||||
|
name: str
|
||||||
|
name_nl: str | None = None
|
||||||
|
name_fr: str | None = None
|
||||||
|
name_de: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class MunicipalitySearchList(BaseModel):
|
||||||
|
items: list[MunicipalitySearchItem]
|
||||||
|
total: int
|
||||||
|
|||||||
@@ -66,12 +66,28 @@ class AssistantTemporalSeries(BaseModel):
|
|||||||
observation_count: int
|
observation_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class AssistantEstimateDisclosure(BaseModel):
|
||||||
|
"""A value in the answer that the source itself calls an estimate.
|
||||||
|
|
||||||
|
Derived from metric metadata rather than from the generated sentences, so
|
||||||
|
the disclosure is present whatever wording the model chose.
|
||||||
|
"""
|
||||||
|
|
||||||
|
theme: str
|
||||||
|
label: str
|
||||||
|
unit: str
|
||||||
|
source: str
|
||||||
|
dataset_id: UUID
|
||||||
|
reason: str
|
||||||
|
|
||||||
|
|
||||||
class AssistantQueryResponse(BaseModel):
|
class AssistantQueryResponse(BaseModel):
|
||||||
answer: str
|
answer: str
|
||||||
model: str
|
model: str
|
||||||
scope_label: str
|
scope_label: str
|
||||||
context_metrics: list[AssistantContextMetric]
|
context_metrics: list[AssistantContextMetric]
|
||||||
temporal_series: list[AssistantTemporalSeries]
|
temporal_series: list[AssistantTemporalSeries]
|
||||||
|
estimate_disclosures: list[AssistantEstimateDisclosure] = Field(default_factory=list)
|
||||||
source_dataset_ids: list[UUID]
|
source_dataset_ids: list[UUID]
|
||||||
warnings: list[str]
|
warnings: list[str]
|
||||||
generated_at: datetime
|
generated_at: datetime
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Literal
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from app.schemas.common import Envelope
|
||||||
|
|
||||||
|
|
||||||
|
class AuthLoginRequest(BaseModel):
|
||||||
|
username: str = Field(min_length=1, max_length=128)
|
||||||
|
password: str = Field(min_length=1, max_length=1024)
|
||||||
|
|
||||||
|
|
||||||
|
class AuthSession(BaseModel):
|
||||||
|
authentication_required: bool
|
||||||
|
authenticated: bool
|
||||||
|
username: str | None = None
|
||||||
|
expires_at: datetime | None = None
|
||||||
|
role: Literal["operator", "guest"] | None = None
|
||||||
|
guest_access_enabled: bool = False
|
||||||
|
authentik_enabled: bool = False
|
||||||
|
guest_project_id: UUID | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class AuthSessionEnvelope(Envelope[AuthSession]):
|
||||||
|
pass
|
||||||
@@ -109,3 +109,65 @@ class BathymetrySourceProbeRead(BaseModel):
|
|||||||
checked_at: datetime
|
checked_at: datetime
|
||||||
message: str
|
message: str
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
|
|
||||||
|
|
||||||
|
class MdkBathymetryAcquireRequest(BaseModel):
|
||||||
|
bbox: VectorSelectionBBox
|
||||||
|
area_id: UUID | None = None
|
||||||
|
force_refresh: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class MdkBathymetryAcquisitionResult(BaseModel):
|
||||||
|
output_dataset_id: UUID
|
||||||
|
reused: bool
|
||||||
|
provider: str
|
||||||
|
coverage_id: str
|
||||||
|
bbox_epsg4326: list[float]
|
||||||
|
vertical_reference: str
|
||||||
|
resolution_m: float = Field(gt=0)
|
||||||
|
attribution: str
|
||||||
|
limitation_message: str
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetryRasterSelectionRequest(BaseModel):
|
||||||
|
bbox: VectorSelectionBBox
|
||||||
|
area_id: UUID | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetryRasterMetric(BaseModel):
|
||||||
|
metric_key: str
|
||||||
|
metric_label: str
|
||||||
|
metric_value: float
|
||||||
|
metric_unit: str
|
||||||
|
aggregation_method: str
|
||||||
|
is_estimate: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetryRasterSelectionSummary(BaseModel):
|
||||||
|
metric_label: str
|
||||||
|
metric_value: float
|
||||||
|
metric_unit: str
|
||||||
|
aggregation_method: str
|
||||||
|
primary_metric_key: str
|
||||||
|
metrics: list[BathymetryRasterMetric]
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetryRasterSelectionResponse(BaseModel):
|
||||||
|
dataset_id: UUID
|
||||||
|
product_key: str
|
||||||
|
selection_bbox: VectorSelectionBBox
|
||||||
|
selection_area_id: UUID | None = None
|
||||||
|
selected_cell_count: int = Field(ge=1)
|
||||||
|
valid_cell_count: int = Field(ge=1)
|
||||||
|
coverage_ratio: float = Field(ge=0, le=1)
|
||||||
|
# Set when the drawn selection is smaller than one source cell and the
|
||||||
|
# analysis was widened to the cells it touches, so the value covers more
|
||||||
|
# ground than was requested.
|
||||||
|
cell_selection_warning: str | None = None
|
||||||
|
resolution_m: float = Field(gt=0)
|
||||||
|
vertical_reference: str
|
||||||
|
survey_period: str
|
||||||
|
summary: BathymetryRasterSelectionSummary
|
||||||
|
unsupported_metrics: list[str]
|
||||||
|
limitation_message: str
|
||||||
|
generated_at: str
|
||||||
|
|||||||
@@ -65,9 +65,25 @@ class CoverageResolutionItem(BaseModel):
|
|||||||
status: CoverageStatus
|
status: CoverageStatus
|
||||||
source_names: list[str]
|
source_names: list[str]
|
||||||
materialized_dataset_ids: list[UUID]
|
materialized_dataset_ids: list[UUID]
|
||||||
|
evidence: list["CoverageEvidenceItem"] = Field(default_factory=list)
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
|
|
||||||
|
|
||||||
|
class CoverageEvidenceItem(BaseModel):
|
||||||
|
dataset_id: UUID
|
||||||
|
source_name: str
|
||||||
|
authority_level: CoverageAuthority
|
||||||
|
source_version: str | None = None
|
||||||
|
observed_at: str | None = None
|
||||||
|
published_at: str | None = None
|
||||||
|
crs: str | None = None
|
||||||
|
resolution: dict | None = None
|
||||||
|
coverage_bbox_epsg4326: list[float] | None = None
|
||||||
|
attribution: str | None = None
|
||||||
|
license_note: str | None = None
|
||||||
|
checksum_sha256: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class CoverageResolveResponse(BaseModel):
|
class CoverageResolveResponse(BaseModel):
|
||||||
project_id: UUID
|
project_id: UUID
|
||||||
bbox: CoverageBBox
|
bbox: CoverageBBox
|
||||||
|
|||||||
@@ -35,6 +35,16 @@ class DatasetCreateResponse(BaseModel):
|
|||||||
reference_layer_name: str | None = None
|
reference_layer_name: str | None = None
|
||||||
source_metadata: dict | None = None
|
source_metadata: dict | None = None
|
||||||
provenance_metadata: dict | None = None
|
provenance_metadata: dict | None = None
|
||||||
|
ingest_key: str | None = None
|
||||||
|
source_registry_id: UUID | None = None
|
||||||
|
source_snapshot_id: UUID | None = None
|
||||||
|
data_contract_key: str | None = None
|
||||||
|
data_contract_version: str | None = None
|
||||||
|
validation_status: str | None = None
|
||||||
|
validation_report_json: dict | None = None
|
||||||
|
provenance_status: str | None = None
|
||||||
|
lineage_status: str | None = None
|
||||||
|
quarantine_status: str | None = None
|
||||||
imported_at: datetime | None = None
|
imported_at: datetime | None = None
|
||||||
temporal_series_key: str | None = None
|
temporal_series_key: str | None = None
|
||||||
observed_at: datetime | None = None
|
observed_at: datetime | None = None
|
||||||
@@ -97,6 +107,15 @@ class DatasetVersionRead(BaseModel):
|
|||||||
checksum_sha256: str | None = None
|
checksum_sha256: str | None = None
|
||||||
source_metadata: dict | None = None
|
source_metadata: dict | None = None
|
||||||
provenance_metadata: dict | None = None
|
provenance_metadata: dict | None = None
|
||||||
|
ingest_key: str | None = None
|
||||||
|
source_registry_id: UUID | None = None
|
||||||
|
source_snapshot_id: UUID | None = None
|
||||||
|
data_contract_key: str | None = None
|
||||||
|
data_contract_version: str | None = None
|
||||||
|
validation_status: str | None = None
|
||||||
|
validation_report_json: dict | None = None
|
||||||
|
provenance_status: str | None = None
|
||||||
|
lineage_status: str | None = None
|
||||||
created_at: datetime | None = None
|
created_at: datetime | None = None
|
||||||
|
|
||||||
model_config = {"from_attributes": True}
|
model_config = {"from_attributes": True}
|
||||||
|
|||||||
@@ -18,6 +18,11 @@ class DetectionModelCapability(BaseModel):
|
|||||||
status: str
|
status: str
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
version: str | None = None
|
version: str | None = None
|
||||||
|
training_scope: str | None = None
|
||||||
|
validation_scope: str | None = None
|
||||||
|
validated_regions: list[str] = Field(default_factory=list)
|
||||||
|
nationally_validated: bool = False
|
||||||
|
operator_review_required: bool = True
|
||||||
|
|
||||||
|
|
||||||
class DetectionModelsResponse(BaseModel):
|
class DetectionModelsResponse(BaseModel):
|
||||||
@@ -37,6 +42,10 @@ class ModelAssetRead(BaseModel):
|
|||||||
size_bytes: int
|
size_bytes: int
|
||||||
sha256: str
|
sha256: str
|
||||||
active: bool
|
active: bool
|
||||||
|
runtime_available: bool
|
||||||
|
runtime_status: str
|
||||||
|
governed_validation_status: str
|
||||||
|
promotion_status: str
|
||||||
status: str
|
status: str
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
will_download_models: bool = False
|
will_download_models: bool = False
|
||||||
@@ -68,6 +77,28 @@ class DetectionQaRequest(BaseModel):
|
|||||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||||
class_name: str | None = None
|
class_name: str | None = None
|
||||||
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||||
|
# Confidence cuts to report alongside the run's own operating point. They
|
||||||
|
# are read off the one matching pass, so a sweep costs no extra inference.
|
||||||
|
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionComparisonRequest(BaseModel):
|
||||||
|
"""Place several runs side by side against one reference."""
|
||||||
|
|
||||||
|
analysis_run_ids: list[UUID] = Field(min_length=2, max_length=12)
|
||||||
|
reference_dataset_id: UUID
|
||||||
|
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionComparisonResponse(BaseModel):
|
||||||
|
reference_dataset_id: UUID
|
||||||
|
iou_threshold: float
|
||||||
|
# Whether these runs answer the same question at all, and why not if they
|
||||||
|
# do not. Numbers from incomparable runs are reported but never ranked as
|
||||||
|
# if they were alternatives.
|
||||||
|
comparability: dict
|
||||||
|
ranking_metric: str
|
||||||
|
rows: list[dict]
|
||||||
|
|
||||||
|
|
||||||
class DetectionRunResponse(BaseModel):
|
class DetectionRunResponse(BaseModel):
|
||||||
@@ -105,7 +136,11 @@ class DetectionRunRead(BaseModel):
|
|||||||
|
|
||||||
class DetectionRunListResponse(BaseModel):
|
class DetectionRunListResponse(BaseModel):
|
||||||
items: list[DetectionRunRead]
|
items: list[DetectionRunRead]
|
||||||
|
# ``total`` counts every run; ``items`` is the most recent page of them.
|
||||||
total: int
|
total: int
|
||||||
|
limit: int | None = None
|
||||||
|
offset: int = 0
|
||||||
|
truncated: bool = False
|
||||||
|
|
||||||
|
|
||||||
class DetectionRead(BaseModel):
|
class DetectionRead(BaseModel):
|
||||||
@@ -128,7 +163,11 @@ class DetectionRead(BaseModel):
|
|||||||
|
|
||||||
class DetectionListResponse(BaseModel):
|
class DetectionListResponse(BaseModel):
|
||||||
items: list[DetectionRead]
|
items: list[DetectionRead]
|
||||||
|
# ``total`` is the complete population; ``items`` is one page of it.
|
||||||
total: int
|
total: int
|
||||||
|
limit: int | None = None
|
||||||
|
offset: int = 0
|
||||||
|
truncated: bool = False
|
||||||
|
|
||||||
|
|
||||||
class YoloPreflightChecks(BaseModel):
|
class YoloPreflightChecks(BaseModel):
|
||||||
@@ -136,8 +175,11 @@ class YoloPreflightChecks(BaseModel):
|
|||||||
|
|
||||||
enabled: bool
|
enabled: bool
|
||||||
dependencies_available: bool | None = None
|
dependencies_available: bool | None = None
|
||||||
|
accelerator_ready: bool | None = None
|
||||||
model_path_set: bool | None = None
|
model_path_set: bool | None = None
|
||||||
model_file_exists: bool | None = None
|
model_file_exists: bool | None = None
|
||||||
|
model_provenance_manifest_path: str | None = None
|
||||||
|
model_provenance_valid: bool | None = None
|
||||||
model_load_requested: bool
|
model_load_requested: bool
|
||||||
model_load_ok: bool | None = None
|
model_load_ok: bool | None = None
|
||||||
manifest_path_set: bool | None = None
|
manifest_path_set: bool | None = None
|
||||||
@@ -155,6 +197,8 @@ class YoloRuntimeDetails(BaseModel):
|
|||||||
torch_version: str | None = None
|
torch_version: str | None = None
|
||||||
ultralytics_version: str | None = None
|
ultralytics_version: str | None = None
|
||||||
cuda_available: bool | None = None
|
cuda_available: bool | None = None
|
||||||
|
configured_device: str
|
||||||
|
cuda_required: bool
|
||||||
|
|
||||||
|
|
||||||
class YoloPreflightResponse(BaseModel):
|
class YoloPreflightResponse(BaseModel):
|
||||||
|
|||||||
@@ -53,6 +53,10 @@ class DetectionReviewSummary(BaseModel):
|
|||||||
false_positive_total: int
|
false_positive_total: int
|
||||||
false_negative_total: int
|
false_negative_total: int
|
||||||
decision_counts: dict[str, int]
|
decision_counts: dict[str, int]
|
||||||
|
# The score with the operator's verdicts applied, next to the raw one. A
|
||||||
|
# finding adjudicated as a reference gap is not the model's error, and an
|
||||||
|
# interval covers what the unreviewed remainder could still turn out to be.
|
||||||
|
reviewed_metrics: dict | None = None
|
||||||
|
|
||||||
|
|
||||||
class DetectionReviewList(BaseModel):
|
class DetectionReviewList(BaseModel):
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ class TerrainSelectionRequest(BaseModel):
|
|||||||
|
|
||||||
class TerrainPartitionSelectionRequest(TerrainSelectionRequest):
|
class TerrainPartitionSelectionRequest(TerrainSelectionRequest):
|
||||||
product_key: str = "dtm_1m"
|
product_key: str = "dtm_1m"
|
||||||
|
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
|
||||||
|
|
||||||
|
|
||||||
class TerrainMetric(BaseModel):
|
class TerrainMetric(BaseModel):
|
||||||
@@ -89,6 +90,10 @@ class TerrainSelectionResponse(BaseModel):
|
|||||||
sample_count: int
|
sample_count: int
|
||||||
slope_sample_count: int
|
slope_sample_count: int
|
||||||
coverage_ratio: float
|
coverage_ratio: float
|
||||||
|
# Set when the drawn selection is smaller than one source cell and the
|
||||||
|
# analysis was widened to the cells it touches, so the value covers more
|
||||||
|
# ground than was requested.
|
||||||
|
cell_selection_warning: str | None = None
|
||||||
resolution_m: float
|
resolution_m: float
|
||||||
vertical_reference: str
|
vertical_reference: str
|
||||||
summary: TerrainSelectionSummary
|
summary: TerrainSelectionSummary
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from app.schemas.operations import VectorSelectionBBox
|
|||||||
|
|
||||||
|
|
||||||
ExportKind = Literal["dataset", "detection_run", "segmentation_run", "vector_selection"]
|
ExportKind = Literal["dataset", "detection_run", "segmentation_run", "vector_selection"]
|
||||||
|
DetectionExportIntendedUse = Literal["review", "operational"]
|
||||||
MapResultMode = Literal["current", "evolution"]
|
MapResultMode = Literal["current", "evolution"]
|
||||||
|
|
||||||
|
|
||||||
@@ -21,6 +22,7 @@ class GeoJsonExportRequest(BaseModel):
|
|||||||
name: str | None = None
|
name: str | None = None
|
||||||
bbox: VectorSelectionBBox | None = None
|
bbox: VectorSelectionBBox | None = None
|
||||||
limit: int = 250
|
limit: int = 250
|
||||||
|
intended_use: DetectionExportIntendedUse = "review"
|
||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def validate_target(self) -> "GeoJsonExportRequest":
|
def validate_target(self) -> "GeoJsonExportRequest":
|
||||||
@@ -33,6 +35,8 @@ class GeoJsonExportRequest(BaseModel):
|
|||||||
raise ValueError("bbox is required for vector selection GeoJSON exports")
|
raise ValueError("bbox is required for vector selection GeoJSON exports")
|
||||||
if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None:
|
if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None:
|
||||||
raise ValueError("analysis_run_id is required for run GeoJSON exports")
|
raise ValueError("analysis_run_id is required for run GeoJSON exports")
|
||||||
|
if self.intended_use == "operational" and self.export_kind != "detection_run":
|
||||||
|
raise ValueError("operational intended_use is supported only for detection run exports")
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ class FloodHazardSelectionRequest(BaseModel):
|
|||||||
|
|
||||||
class FloodHazardPartitionSelectionRequest(FloodHazardSelectionRequest):
|
class FloodHazardPartitionSelectionRequest(FloodHazardSelectionRequest):
|
||||||
product_key: str = "pluviaal_current_t100"
|
product_key: str = "pluviaal_current_t100"
|
||||||
|
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
|
||||||
|
|
||||||
|
|
||||||
class FloodHazardMetric(BaseModel):
|
class FloodHazardMetric(BaseModel):
|
||||||
@@ -92,9 +93,17 @@ class FloodHazardSelectionResponse(BaseModel):
|
|||||||
return_period_years: int
|
return_period_years: int
|
||||||
selection_bbox: VectorSelectionBBox
|
selection_bbox: VectorSelectionBBox
|
||||||
selection_area_id: UUID | None = None
|
selection_area_id: UUID | None = None
|
||||||
|
# Three populations kept apart: cells drawn, cells the model covers, and
|
||||||
|
# cells with a positive modelled depth. ``inundated_fraction`` is a share
|
||||||
|
# of the modelled cells, and is null when nothing was modelled — absence
|
||||||
|
# of a model is not evidence of zero risk.
|
||||||
selected_cell_count: int
|
selected_cell_count: int
|
||||||
|
valid_cell_count: int = 0
|
||||||
|
no_data_cell_count: int = 0
|
||||||
|
data_coverage_ratio: float = 1.0
|
||||||
inundated_cell_count: int
|
inundated_cell_count: int
|
||||||
inundated_fraction: float
|
inundated_fraction: float | None = None
|
||||||
|
coverage_warning: str | None = None
|
||||||
resolution_m: float
|
resolution_m: float
|
||||||
summary: FloodHazardSelectionSummary
|
summary: FloodHazardSelectionSummary
|
||||||
unsupported_metrics: list[str]
|
unsupported_metrics: list[str]
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ class OfficialVectorProductRead(BaseModel):
|
|||||||
attribution: str
|
attribution: str
|
||||||
license_note: str
|
license_note: str
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
|
coverage_zones: list[str]
|
||||||
|
|
||||||
|
|
||||||
class OfficialVectorAcquisitionResult(BaseModel):
|
class OfficialVectorAcquisitionResult(BaseModel):
|
||||||
|
|||||||
@@ -236,7 +236,13 @@ class VectorSelectionSummary(BaseModel):
|
|||||||
metric_unit: str
|
metric_unit: str
|
||||||
aggregation_method: str
|
aggregation_method: str
|
||||||
primary_metric_key: str | None = None
|
primary_metric_key: str | None = None
|
||||||
|
# ``feature_count`` counts whole features that touch the selection, while
|
||||||
|
# area and length metrics clip to it. These fields say how far the two
|
||||||
|
# populations diverge, so the numbers on one panel can be read together.
|
||||||
feature_count: int
|
feature_count: int
|
||||||
|
fully_covered_feature_count: int | None = None
|
||||||
|
partially_covered_feature_count: int | None = None
|
||||||
|
selection_edge_warning: str | None = None
|
||||||
is_estimate: bool = False
|
is_estimate: bool = False
|
||||||
warning: str | None = None
|
warning: str | None = None
|
||||||
metrics: list[VectorSelectionMetric] = Field(default_factory=list)
|
metrics: list[VectorSelectionMetric] = Field(default_factory=list)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from .operations import VectorSelectionBBox
|
from .operations import VectorSelectionBBox
|
||||||
|
|
||||||
@@ -12,6 +12,7 @@ class OrthophotoAcquireRequest(BaseModel):
|
|||||||
area_id: UUID | None = None
|
area_id: UUID | None = None
|
||||||
product_key: str = "most_recent"
|
product_key: str = "most_recent"
|
||||||
force_refresh: bool = False
|
force_refresh: bool = False
|
||||||
|
resolution_m: float | None = Field(default=None, ge=0.1, le=2.0)
|
||||||
|
|
||||||
|
|
||||||
class OrthophotoProductRead(BaseModel):
|
class OrthophotoProductRead(BaseModel):
|
||||||
@@ -24,6 +25,10 @@ class OrthophotoProductRead(BaseModel):
|
|||||||
color_mode: str
|
color_mode: str
|
||||||
catalog_url: str
|
catalog_url: str
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
|
provider: str
|
||||||
|
coverage_zone: str
|
||||||
|
attribution: str
|
||||||
|
license_note: str
|
||||||
|
|
||||||
|
|
||||||
class OrthophotoAcquisitionResult(BaseModel):
|
class OrthophotoAcquisitionResult(BaseModel):
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from pydantic import BaseModel
|
|||||||
class ProjectCreate(BaseModel):
|
class ProjectCreate(BaseModel):
|
||||||
name: str
|
name: str
|
||||||
description: str | None = None
|
description: str | None = None
|
||||||
region: str | None = "Kempen"
|
region: str | None = "Belgium and Belgian North Sea"
|
||||||
|
|
||||||
|
|
||||||
class ProjectUpdate(BaseModel):
|
class ProjectUpdate(BaseModel):
|
||||||
|
|||||||
@@ -19,8 +19,14 @@ class QaProviderComparisonRequest(BaseModel):
|
|||||||
class QaProviderComparisonResult(BaseModel):
|
class QaProviderComparisonResult(BaseModel):
|
||||||
status: str
|
status: str
|
||||||
warnings: list[str] = Field(default_factory=list)
|
warnings: list[str] = Field(default_factory=list)
|
||||||
|
# Counts of the population that was actually matched, so that
|
||||||
|
# ``matches + false_positives == candidate_feature_count`` holds even when
|
||||||
|
# an area filter or an unparseable geometry removed features. The ``_raw``
|
||||||
|
# fields keep the untouched dataset totals visible next to them.
|
||||||
candidate_feature_count: int
|
candidate_feature_count: int
|
||||||
reference_feature_count: int
|
reference_feature_count: int
|
||||||
|
candidate_feature_count_raw: int | None = None
|
||||||
|
reference_feature_count_raw: int | None = None
|
||||||
matches: int
|
matches: int
|
||||||
false_positives: int
|
false_positives: int
|
||||||
false_negatives: int
|
false_negatives: int
|
||||||
@@ -83,7 +89,13 @@ class QualityEvidenceResponse(BaseModel):
|
|||||||
candidate_dataset_id: UUID | None = None
|
candidate_dataset_id: UUID | None = None
|
||||||
reference_dataset_id: UUID
|
reference_dataset_id: UUID
|
||||||
analysis_run_id: UUID | None = None
|
analysis_run_id: UUID | None = None
|
||||||
|
# The overlay is capped so a regional check stays reviewable; the counts in
|
||||||
|
# the quality check itself are always complete.
|
||||||
feature_count: int
|
feature_count: int
|
||||||
|
total_feature_count: int | None = None
|
||||||
|
role_counts: dict[str, int] = Field(default_factory=dict)
|
||||||
|
truncated: bool = False
|
||||||
|
limit: int | None = None
|
||||||
warnings: list[str] = Field(default_factory=list)
|
warnings: list[str] = Field(default_factory=list)
|
||||||
geojson: GeoJsonFeatureCollection
|
geojson: GeoJsonFeatureCollection
|
||||||
|
|
||||||
@@ -109,6 +121,8 @@ class AnalysisQaResponse(BaseModel):
|
|||||||
coverage: dict[str, Any] | None = None
|
coverage: dict[str, Any] | None = None
|
||||||
temporal_compatibility: dict[str, Any] | None = None
|
temporal_compatibility: dict[str, Any] | None = None
|
||||||
box_to_footprint_diagnostics: dict[str, Any] | None = None
|
box_to_footprint_diagnostics: dict[str, Any] | None = None
|
||||||
|
precision_recall_curve: dict[str, Any] | None = None
|
||||||
|
calibration_sweep: list[dict[str, Any]] = Field(default_factory=list)
|
||||||
match_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
match_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
||||||
false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
||||||
false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ class SegmentationQaRequest(BaseModel):
|
|||||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||||
class_name: str | None = None
|
class_name: str | None = None
|
||||||
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||||
|
# Read off the one matching pass, exactly as for detection.
|
||||||
|
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
|
||||||
|
|
||||||
|
|
||||||
class SegmentationRunResponse(BaseModel):
|
class SegmentationRunResponse(BaseModel):
|
||||||
@@ -69,7 +71,11 @@ class SegmentationRunRead(BaseModel):
|
|||||||
|
|
||||||
class SegmentationRunListResponse(BaseModel):
|
class SegmentationRunListResponse(BaseModel):
|
||||||
items: list[SegmentationRunRead]
|
items: list[SegmentationRunRead]
|
||||||
|
# ``total`` counts every run; ``items`` is the most recent page of them.
|
||||||
total: int
|
total: int
|
||||||
|
limit: int | None = None
|
||||||
|
offset: int = 0
|
||||||
|
truncated: bool = False
|
||||||
|
|
||||||
|
|
||||||
class SegmentationRead(BaseModel):
|
class SegmentationRead(BaseModel):
|
||||||
@@ -96,4 +102,9 @@ class SegmentationRead(BaseModel):
|
|||||||
|
|
||||||
class SegmentationListResponse(BaseModel):
|
class SegmentationListResponse(BaseModel):
|
||||||
items: list[SegmentationRead]
|
items: list[SegmentationRead]
|
||||||
|
# ``total`` describes the complete filtered population; ``items`` is one
|
||||||
|
# stable confidence-ranked page of it.
|
||||||
total: int
|
total: int
|
||||||
|
limit: int | None = None
|
||||||
|
offset: int = 0
|
||||||
|
truncated: bool = False
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from .operations import VectorSelectionBBox
|
||||||
|
|
||||||
|
|
||||||
|
class VectorPartitionSelectionRequest(BaseModel):
|
||||||
|
dataset_ids: list[UUID] = Field(min_length=1, max_length=4096)
|
||||||
|
bbox: VectorSelectionBBox
|
||||||
|
area_id: UUID | None = None
|
||||||
|
limit: int = Field(default=1000, ge=1, le=1000)
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRegistryRead(BaseModel):
|
||||||
|
"""Read-only, server-owned source-authority definition."""
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
source_key: str
|
||||||
|
display_name: str
|
||||||
|
classification: str
|
||||||
|
authority_name: str
|
||||||
|
authority_scope_json: dict
|
||||||
|
provider_adapter_key: str | None = None
|
||||||
|
source_url: str | None = None
|
||||||
|
license_name: str
|
||||||
|
license_url: str | None = None
|
||||||
|
usage_restrictions: str
|
||||||
|
default_crs: str
|
||||||
|
default_units: str
|
||||||
|
spatial_resolution_json: dict
|
||||||
|
temporal_coverage_json: dict
|
||||||
|
geographic_coverage_json: dict
|
||||||
|
expected_geometry_types_json: list
|
||||||
|
expected_attributes_json: dict
|
||||||
|
usage_policy_json: dict
|
||||||
|
freshness_status: str
|
||||||
|
ingest_status: str
|
||||||
|
known_limitations_json: list
|
||||||
|
registry_metadata_json: dict
|
||||||
|
created_at: datetime | None = None
|
||||||
|
updated_at: datetime | None = None
|
||||||
|
snapshot_count: int = 0
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class SourceSnapshotRead(BaseModel):
|
||||||
|
"""Immutable version/snapshot evidence attached to an imported dataset."""
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
source_registry_id: UUID
|
||||||
|
snapshot_key: str
|
||||||
|
source_version: str | None = None
|
||||||
|
snapshot_at: datetime | None = None
|
||||||
|
fetched_at: datetime | None = None
|
||||||
|
source_url: str | None = None
|
||||||
|
checksum_sha256: str | None = None
|
||||||
|
crs: str | None = None
|
||||||
|
units: str | None = None
|
||||||
|
spatial_resolution_json: dict
|
||||||
|
temporal_coverage_json: dict
|
||||||
|
geographic_coverage_json: dict
|
||||||
|
observed_schema_json: dict
|
||||||
|
freshness_status: str
|
||||||
|
ingest_status: str
|
||||||
|
known_limitations_json: list
|
||||||
|
snapshot_metadata_json: dict
|
||||||
|
created_at: datetime | None = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRegistryDetailRead(BaseModel):
|
||||||
|
source: SourceRegistryRead
|
||||||
|
snapshots: list[SourceSnapshotRead]
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetLineageEdgeRead(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
parent_dataset_id: UUID
|
||||||
|
child_dataset_id: UUID
|
||||||
|
parent_dataset_version_id: UUID | None = None
|
||||||
|
child_dataset_version_id: UUID | None = None
|
||||||
|
relation_type: str
|
||||||
|
transformation_name: str
|
||||||
|
transformation_version: str | None = None
|
||||||
|
parameters_json: dict | None = None
|
||||||
|
input_checksum_sha256: str | None = None
|
||||||
|
output_checksum_sha256: str | None = None
|
||||||
|
created_at: datetime | None = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetQuarantineRead(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
dataset_id: UUID | None = None
|
||||||
|
dataset_version_id: UUID | None = None
|
||||||
|
source_snapshot_id: UUID | None = None
|
||||||
|
stage: str
|
||||||
|
reason_code: str
|
||||||
|
details_json: dict | None = None
|
||||||
|
artifact_path: str | None = None
|
||||||
|
artifact_checksum_sha256: str | None = None
|
||||||
|
status: str
|
||||||
|
created_at: datetime | None = None
|
||||||
|
resolved_at: datetime | None = None
|
||||||
|
resolved_by: str | None = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetProvenanceRead(BaseModel):
|
||||||
|
dataset_id: UUID
|
||||||
|
source: SourceRegistryRead | None = None
|
||||||
|
snapshot: SourceSnapshotRead | None = None
|
||||||
|
data_contract_key: str | None = None
|
||||||
|
data_contract_version: str | None = None
|
||||||
|
validation_status: str | None = None
|
||||||
|
validation_report_json: dict | None = None
|
||||||
|
provenance_status: str | None = None
|
||||||
|
lineage_status: str | None = None
|
||||||
|
quarantine_status: str | None = None
|
||||||
|
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
|
||||||
|
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from .operations import VectorSelectionBBox
|
||||||
|
|
||||||
|
|
||||||
|
class SpwTerrainAcquireRequest(BaseModel):
|
||||||
|
bbox: VectorSelectionBBox
|
||||||
|
area_id: UUID | None = None
|
||||||
|
product_key: str = "spw_mnt_1m_2021_2022"
|
||||||
|
resolution_m: float | None = Field(default=None, ge=1.0, le=10.0)
|
||||||
|
force_refresh: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class SpwTerrainProductRead(BaseModel):
|
||||||
|
key: str
|
||||||
|
display_name: str
|
||||||
|
surface_model: str
|
||||||
|
source_filename: str
|
||||||
|
native_resolution_m: float
|
||||||
|
analysis_resolution_m: float
|
||||||
|
source_crs: str
|
||||||
|
vertical_reference: str
|
||||||
|
acquisition_period: str
|
||||||
|
catalog_url: str
|
||||||
|
attribution: str
|
||||||
|
license_note: str
|
||||||
|
limitation_message: str
|
||||||
|
coverage_zones: list[str]
|
||||||
|
configured: bool
|
||||||
|
status: str
|
||||||
|
|
||||||
|
|
||||||
|
class SpwTerrainAcquisitionResult(BaseModel):
|
||||||
|
output_dataset_id: UUID
|
||||||
|
reused: bool
|
||||||
|
provider: str
|
||||||
|
product_key: str
|
||||||
|
display_name: str
|
||||||
|
surface_model: str
|
||||||
|
native_resolution_m: float
|
||||||
|
resolution_m: float
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
valid_pixel_count: int
|
||||||
|
nodata_value: float
|
||||||
|
bbox_epsg4326: list[float]
|
||||||
|
bbox_epsg3812: list[float]
|
||||||
|
vertical_reference: str
|
||||||
|
acquisition_period: str
|
||||||
|
attribution: str
|
||||||
|
limitation_message: str
|
||||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from .operations import VectorSelectionBBox
|
from .operations import VectorSelectionBBox
|
||||||
|
|
||||||
@@ -32,6 +32,30 @@ class ThematicRasterProductRead(BaseModel):
|
|||||||
legend_max_label: str
|
legend_max_label: str
|
||||||
included_source_values: list[int]
|
included_source_values: list[int]
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
|
analysis_resolution_m: float | None = None
|
||||||
|
coverage_zones: list[str] = Field(default_factory=list)
|
||||||
|
configured: bool = True
|
||||||
|
status: str = "configured"
|
||||||
|
|
||||||
|
|
||||||
|
class WalousAcquisitionResult(BaseModel):
|
||||||
|
output_dataset_id: UUID
|
||||||
|
reused: bool
|
||||||
|
provider: str
|
||||||
|
product_key: str
|
||||||
|
display_name: str
|
||||||
|
theme: str
|
||||||
|
metric_kind: str
|
||||||
|
resolution_m: float
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
valid_pixel_count: int
|
||||||
|
bbox_epsg4326: list[float]
|
||||||
|
bbox_epsg3812: list[float]
|
||||||
|
observation_year: int
|
||||||
|
source_value_unit: str
|
||||||
|
attribution: str
|
||||||
|
limitation_message: str
|
||||||
|
|
||||||
|
|
||||||
class ThematicRasterAcquisitionResult(BaseModel):
|
class ThematicRasterAcquisitionResult(BaseModel):
|
||||||
@@ -89,6 +113,10 @@ class ThematicRasterSelectionResponse(BaseModel):
|
|||||||
selected_cell_count: int
|
selected_cell_count: int
|
||||||
valid_cell_count: int
|
valid_cell_count: int
|
||||||
coverage_ratio: float
|
coverage_ratio: float
|
||||||
|
# Set when the drawn selection is smaller than one source cell and the
|
||||||
|
# analysis was widened to the cells it touches, so the value covers more
|
||||||
|
# ground than was requested.
|
||||||
|
cell_selection_warning: str | None = None
|
||||||
resolution_m: float
|
resolution_m: float
|
||||||
observation_year: int
|
observation_year: int
|
||||||
summary: ThematicRasterSelectionSummary
|
summary: ThematicRasterSelectionSummary
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Background execution for queued analysis runs.
|
||||||
|
|
||||||
|
Tiled GPU inference is minutes of work. Running it inside the HTTP request
|
||||||
|
holds a worker thread for the whole duration, times the client out and leaves
|
||||||
|
the operator without progress. Queued ``detection.run`` and
|
||||||
|
``segmentation.run`` jobs are picked up here instead, mirroring the polling
|
||||||
|
worker the AOI operations already use so the runtime keeps one job model.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.db.session import SessionLocal
|
||||||
|
from app.models import Job
|
||||||
|
|
||||||
|
logger = logging.getLogger("geointel.analysis_worker")
|
||||||
|
|
||||||
|
|
||||||
|
class AnalysisJobWorker:
|
||||||
|
HANDLED_JOB_TYPES = ("detection.run", "segmentation.run")
|
||||||
|
BATCH_SIZE = 4
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _uuid(value: Any) -> UUID | None:
|
||||||
|
if isinstance(value, UUID):
|
||||||
|
return value
|
||||||
|
try:
|
||||||
|
return UUID(str(value))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _dispatch(db, job: Job) -> Any:
|
||||||
|
# Imported lazily: both services import each other's helpers, and the
|
||||||
|
# worker must not add a third edge to that cycle at module load.
|
||||||
|
from app.services.detection_service import DetectionService
|
||||||
|
from app.services.segmentation_service import SegmentationService
|
||||||
|
|
||||||
|
parameters = job.parameters_json if isinstance(job.parameters_json, dict) else {}
|
||||||
|
project_id = AnalysisJobWorker._uuid(parameters.get("project_id"))
|
||||||
|
dataset_id = AnalysisJobWorker._uuid(parameters.get("dataset_id"))
|
||||||
|
if project_id is None or dataset_id is None:
|
||||||
|
raise ValueError("Queued analysis job is missing project_id or dataset_id")
|
||||||
|
|
||||||
|
common = {
|
||||||
|
"db": db,
|
||||||
|
"project_id": project_id,
|
||||||
|
"dataset_id": dataset_id,
|
||||||
|
"model_id": parameters.get("model_id"),
|
||||||
|
"confidence_threshold": float(parameters.get("confidence_threshold") or 0.0),
|
||||||
|
"class_filter": parameters.get("class_filter") or [],
|
||||||
|
"tile_manifest_path": parameters.get("tile_manifest_path"),
|
||||||
|
"parameters_json": parameters.get("parameters_json") or {},
|
||||||
|
"existing_job": job,
|
||||||
|
}
|
||||||
|
if job.job_type == "detection.run":
|
||||||
|
return DetectionService.run_detection(
|
||||||
|
model_asset_id=parameters.get("model_asset_id"),
|
||||||
|
**common,
|
||||||
|
)
|
||||||
|
return SegmentationService.run_segmentation(**common)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def claim(db, job: Job) -> bool:
|
||||||
|
"""Take the job out of the queue, atomically. Returns whether we won.
|
||||||
|
|
||||||
|
Selecting and then updating in a second statement lets two workers —
|
||||||
|
a restarted process overlapping the previous one, or a second replica —
|
||||||
|
both start tiled GPU inference on the same row. The conditional update
|
||||||
|
makes exactly one caller see a row count of 1; the AOI worker beside
|
||||||
|
this one already claims with FOR UPDATE SKIP LOCKED for the same reason.
|
||||||
|
"""
|
||||||
|
|
||||||
|
claimed = (
|
||||||
|
db.query(Job)
|
||||||
|
.filter(Job.id == job.id, Job.status == "queued")
|
||||||
|
.update({Job.status: "running"}, synchronize_session=False)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
if not claimed:
|
||||||
|
return False
|
||||||
|
job.status = "running"
|
||||||
|
return True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _finalize(db, job: Job, result: Any) -> None:
|
||||||
|
"""Close a job the handler left open.
|
||||||
|
|
||||||
|
The analysis services normally set the terminal status themselves.
|
||||||
|
If one returns without doing so, recording the outcome here is what
|
||||||
|
keeps the job from sitting in "running" for ever.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if job.status != "running":
|
||||||
|
return
|
||||||
|
status = getattr(result, "status", None)
|
||||||
|
if status == "success":
|
||||||
|
job.status = "success"
|
||||||
|
job.result_json = {
|
||||||
|
"detection_count": getattr(result, "detection_count", None),
|
||||||
|
"segmentation_count": getattr(result, "segmentation_count", None),
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
job.status = "failed"
|
||||||
|
job.error_message = getattr(result, "message", None) or "Analysis run did not complete"
|
||||||
|
job.result_json = {"error_code": getattr(result, "error_code", None) or "ANALYSIS_JOB_INCOMPLETE"}
|
||||||
|
db.add(job)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _mark_failed(db, job: Job, *, code: str, message: str) -> None:
|
||||||
|
try:
|
||||||
|
db.rollback()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
job.status = "failed"
|
||||||
|
job.error_message = message
|
||||||
|
job.result_json = {"error_code": code, "message": message}
|
||||||
|
db.add(job)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def run_once(db=None) -> int:
|
||||||
|
"""Execute one batch of queued analysis jobs. Returns the batch size."""
|
||||||
|
|
||||||
|
owns_session = db is None
|
||||||
|
session = db if db is not None else SessionLocal()
|
||||||
|
try:
|
||||||
|
rows = [
|
||||||
|
job
|
||||||
|
for job in (
|
||||||
|
session.query(Job)
|
||||||
|
.filter(Job.status == "queued")
|
||||||
|
.filter(Job.job_type.in_(AnalysisJobWorker.HANDLED_JOB_TYPES))
|
||||||
|
.order_by(Job.created_at)
|
||||||
|
.limit(AnalysisJobWorker.BATCH_SIZE)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if job.job_type in AnalysisJobWorker.HANDLED_JOB_TYPES and job.status == "queued"
|
||||||
|
]
|
||||||
|
claimed_count = 0
|
||||||
|
for job in rows:
|
||||||
|
if not AnalysisJobWorker.claim(session, job):
|
||||||
|
# Another worker took it between the select and the claim.
|
||||||
|
continue
|
||||||
|
claimed_count += 1
|
||||||
|
try:
|
||||||
|
result = AnalysisJobWorker._dispatch(session, job)
|
||||||
|
AnalysisJobWorker._finalize(session, job, result)
|
||||||
|
except Exception as exc:
|
||||||
|
code = getattr(exc, "code", None) or "ANALYSIS_JOB_INTERNAL_ERROR"
|
||||||
|
message = getattr(exc, "message", None) or str(exc) or "Unexpected analysis job failure"
|
||||||
|
AnalysisJobWorker._mark_failed(session, job, code=str(code), message=str(message))
|
||||||
|
logger.exception("Analysis job failed job_id=%s job_type=%s", job.id, job.job_type)
|
||||||
|
return claimed_count
|
||||||
|
finally:
|
||||||
|
if owns_session:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
|
||||||
|
while not stop_event.is_set():
|
||||||
|
processed = await asyncio.to_thread(AnalysisJobWorker.run_once)
|
||||||
|
if processed == 0:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
|
||||||
|
except TimeoutError:
|
||||||
|
pass
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from geoalchemy2.shape import to_shape
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import AoiOperation, AoiOperationPartition
|
||||||
|
from app.schemas.grb import GrbAcquireRequest
|
||||||
|
from app.schemas.dhmv import DhmvAcquireRequest
|
||||||
|
from app.schemas.spw_terrain import SpwTerrainAcquireRequest
|
||||||
|
from app.schemas.official_vector import OfficialVectorAcquireRequest
|
||||||
|
from app.schemas.flood_hazard import FloodHazardAcquireRequest
|
||||||
|
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
|
||||||
|
from app.schemas.bathymetry import (
|
||||||
|
BathymetryProfileAcquireRequest,
|
||||||
|
MdkBathymetryAcquireRequest,
|
||||||
|
)
|
||||||
|
from app.schemas.job import JobCreate
|
||||||
|
from app.schemas.operations import VectorSelectionBBox
|
||||||
|
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||||
|
from app.services.aoi_operation_service import AoiOperationService
|
||||||
|
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||||
|
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||||
|
from app.services.spw_terrain_service import SpwTerrainService
|
||||||
|
from app.services.official_vector_acquisition_service import (
|
||||||
|
OfficialVectorAcquisitionService,
|
||||||
|
)
|
||||||
|
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||||
|
from app.services.thematic_raster_acquisition_service import (
|
||||||
|
ThematicRasterAcquisitionService,
|
||||||
|
)
|
||||||
|
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||||
|
from app.services.bathymetry_profile_acquisition_service import (
|
||||||
|
BathymetryProfileAcquisitionService,
|
||||||
|
)
|
||||||
|
from app.services.mdk_bathymetry_acquisition_service import (
|
||||||
|
MdkBathymetryAcquisitionService,
|
||||||
|
)
|
||||||
|
from app.services.job_service import JobService
|
||||||
|
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperationExecutor:
|
||||||
|
"""Execute one bounded partition through an existing governed provider."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def execute_next(db, project_id: UUID, operation_id: UUID) -> dict:
|
||||||
|
partition = AoiOperationService.claim_next(db, project_id, operation_id)
|
||||||
|
if partition is None:
|
||||||
|
AoiOperationService._refresh_parent(db, operation_id)
|
||||||
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
operation = db.get(AoiOperation, operation_id)
|
||||||
|
child = JobService.create_job(
|
||||||
|
db,
|
||||||
|
JobCreate(
|
||||||
|
job_type=f"aoi.{operation.operation_type}.partition",
|
||||||
|
project_id=project_id,
|
||||||
|
parameters_json={
|
||||||
|
"aoi_operation_id": str(operation_id),
|
||||||
|
"partition_id": str(partition.id),
|
||||||
|
"partition_key": partition.partition_key,
|
||||||
|
"provider_key": partition.provider_key,
|
||||||
|
"product_key": partition.product_key,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
partition = db.get(AoiOperationPartition, partition.id)
|
||||||
|
partition.child_job_id = child.id
|
||||||
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
|
JobService.mark_running(db, child.id)
|
||||||
|
try:
|
||||||
|
result = AoiOperationExecutor._dispatch(
|
||||||
|
db, project_id, operation, partition
|
||||||
|
)
|
||||||
|
output_id = (
|
||||||
|
result.get("output_dataset_id") if isinstance(result, dict) else None
|
||||||
|
)
|
||||||
|
JobService.mark_success(
|
||||||
|
db,
|
||||||
|
child.id,
|
||||||
|
result=result,
|
||||||
|
output_dataset_id=UUID(str(output_id)) if output_id else None,
|
||||||
|
)
|
||||||
|
return AoiOperationService.complete(
|
||||||
|
db, project_id, operation_id, partition.id, result
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
JobService.mark_failed(
|
||||||
|
db, child.id, exc.message, {"code": exc.code, "details": exc.details}
|
||||||
|
)
|
||||||
|
return AoiOperationService.fail(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
operation_id,
|
||||||
|
partition.id,
|
||||||
|
exc.message,
|
||||||
|
AoiOperationExecutor._retryable(exc),
|
||||||
|
{"code": exc.code, "details": exc.details},
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
try:
|
||||||
|
db.rollback()
|
||||||
|
JobService.mark_failed(
|
||||||
|
db,
|
||||||
|
child.id,
|
||||||
|
"Unexpected partition execution error",
|
||||||
|
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
AoiOperationService.fail(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
operation_id,
|
||||||
|
partition.id,
|
||||||
|
"Unexpected partition execution error",
|
||||||
|
True,
|
||||||
|
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _dispatch(
|
||||||
|
db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition
|
||||||
|
) -> dict:
|
||||||
|
geometry = to_shape(partition.geometry)
|
||||||
|
min_x, min_y, max_x, max_y = geometry.bounds
|
||||||
|
bbox = VectorSelectionBBox(
|
||||||
|
min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326"
|
||||||
|
)
|
||||||
|
force_refresh = bool(
|
||||||
|
(operation.request_json or {})
|
||||||
|
.get("parameters_json", {})
|
||||||
|
.get("force_refresh", False)
|
||||||
|
)
|
||||||
|
if partition.provider_key == "grb":
|
||||||
|
return GrbAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
GrbAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "orthophoto":
|
||||||
|
return OrthophotoAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
OrthophotoAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "dhmv":
|
||||||
|
return DhmvAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
DhmvAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "spw_terrain":
|
||||||
|
return SpwTerrainService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
SpwTerrainAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "official_vector":
|
||||||
|
return OfficialVectorAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
OfficialVectorAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "flood_hazard":
|
||||||
|
return FloodHazardAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
FloodHazardAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "thematic_raster":
|
||||||
|
return ThematicRasterAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
ThematicRasterAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "walous":
|
||||||
|
return WalousLandCoverService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
ThematicRasterAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "bathymetry_profiles":
|
||||||
|
return BathymetryProfileAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
BathymetryProfileAcquireRequest(
|
||||||
|
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if partition.provider_key == "mdk_bathymetry":
|
||||||
|
return MdkBathymetryAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
MdkBathymetryAcquireRequest(
|
||||||
|
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PROVIDER_UNSUPPORTED",
|
||||||
|
message="No governed AOI executor is registered for this provider",
|
||||||
|
details={"provider_key": partition.provider_key},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _retryable(error: AppError) -> bool:
|
||||||
|
return error.status_code >= 500 or error.code.endswith(
|
||||||
|
("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")
|
||||||
|
)
|
||||||
@@ -0,0 +1,481 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from hashlib import sha256
|
||||||
|
import math
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from geoalchemy2.shape import from_shape, to_shape
|
||||||
|
from pyproj import Transformer
|
||||||
|
from shapely.geometry import MultiPolygon, Polygon, box
|
||||||
|
from shapely.ops import transform
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.models import AoiOperation, AoiOperationPartition, Area, Project
|
||||||
|
from app.schemas.aoi_operation import AoiOperationCreate
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperationService:
|
||||||
|
MAX_PARTITIONS = 4096
|
||||||
|
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||||
|
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||||
|
SCOPE_AREA_NAMES = {
|
||||||
|
"belgium": "Belgium land",
|
||||||
|
"flanders": "Flanders",
|
||||||
|
"wallonia": "Wallonia",
|
||||||
|
"brussels": "Brussels-Capital Region",
|
||||||
|
"belgian_north_sea": "Belgian part of the North Sea",
|
||||||
|
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
||||||
|
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
||||||
|
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
|
||||||
|
if db.get(Project, project_id) is None:
|
||||||
|
raise AppError(
|
||||||
|
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
|
||||||
|
)
|
||||||
|
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
|
||||||
|
if payload.coverage_zone:
|
||||||
|
geometry = AoiOperationService._clip_to_zone(
|
||||||
|
db, project_id, geometry, payload.coverage_zone
|
||||||
|
)
|
||||||
|
geometry = AoiOperationService._as_multipolygon(geometry)
|
||||||
|
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
|
||||||
|
partition_side_m = AoiOperationService._partition_side(
|
||||||
|
payload.provider_key, payload.max_partition_side_m
|
||||||
|
)
|
||||||
|
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
|
||||||
|
operation_id = uuid4()
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
operation = AoiOperation(
|
||||||
|
id=operation_id,
|
||||||
|
project_id=project_id,
|
||||||
|
area_id=payload.area_id,
|
||||||
|
operation_type=payload.operation_type,
|
||||||
|
status="queued",
|
||||||
|
geometry=from_shape(geometry, srid=4326),
|
||||||
|
request_json=payload.model_dump(mode="json", exclude_none=True),
|
||||||
|
plan_json={
|
||||||
|
"partition_strategy": "epsg31370_square_grid_intersection_v1",
|
||||||
|
"max_partition_side_m": partition_side_m,
|
||||||
|
"budget_source": "governed_provider_registry"
|
||||||
|
if payload.max_partition_side_m is None
|
||||||
|
else "stricter_operator_override",
|
||||||
|
"partition_count": len(cells),
|
||||||
|
"provider_key": payload.provider_key,
|
||||||
|
"product_key": payload.product_key,
|
||||||
|
},
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
db.add(operation)
|
||||||
|
for ordinal, cell in enumerate(cells):
|
||||||
|
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
|
||||||
|
wgs84 = AoiOperationService._as_multipolygon(wgs84)
|
||||||
|
digest = sha256(wgs84.wkb).hexdigest()[:20]
|
||||||
|
db.add(
|
||||||
|
AoiOperationPartition(
|
||||||
|
id=uuid4(),
|
||||||
|
operation_id=operation_id,
|
||||||
|
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
|
||||||
|
provider_key=payload.provider_key,
|
||||||
|
product_key=payload.product_key,
|
||||||
|
ordinal=ordinal,
|
||||||
|
status="queued",
|
||||||
|
geometry=from_shape(wgs84, srid=4326),
|
||||||
|
attempt_count=0,
|
||||||
|
max_attempts=payload.max_attempts,
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
|
||||||
|
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
|
||||||
|
if area_name is None:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
|
||||||
|
message="Unknown governed coverage zone",
|
||||||
|
details={"coverage_zone": zone},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
scope = (
|
||||||
|
db.query(Area)
|
||||||
|
.filter(Area.project_id == project_id, Area.name == area_name)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if scope is None:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
|
||||||
|
message="The governed coverage-zone geometry is not persisted in this project",
|
||||||
|
details={"coverage_zone": zone},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
clipped = geometry.intersection(to_shape(scope.geometry))
|
||||||
|
if clipped.is_empty:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_OUTSIDE_PROVIDER_ZONE",
|
||||||
|
message="The AOI does not intersect the provider coverage zone",
|
||||||
|
details={"coverage_zone": zone},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return clipped
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _as_multipolygon(geometry) -> MultiPolygon:
|
||||||
|
if isinstance(geometry, Polygon):
|
||||||
|
return MultiPolygon([geometry])
|
||||||
|
if isinstance(geometry, MultiPolygon):
|
||||||
|
return geometry
|
||||||
|
polygons = [
|
||||||
|
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
|
||||||
|
]
|
||||||
|
if not polygons:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_GEOMETRY_EMPTY",
|
||||||
|
message="AOI contains no polygonal area after clipping",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return MultiPolygon(polygons)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _partition_side(provider_key: str, requested: float | None) -> float:
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
|
def raster_side(
|
||||||
|
max_side_m: float, max_pixels: int, resolution_m: float
|
||||||
|
) -> float:
|
||||||
|
# Keep every square grid cell within both the provider's spatial
|
||||||
|
# extent limit and its decoded-pixel budget. The small safety
|
||||||
|
# margin absorbs ceil/edge rounding in the acquisition services.
|
||||||
|
pixel_limited_side = (
|
||||||
|
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
|
||||||
|
)
|
||||||
|
return min(float(max_side_m), pixel_limited_side)
|
||||||
|
|
||||||
|
budgets = {
|
||||||
|
"orthophoto": float(settings.orthophoto_max_side_m),
|
||||||
|
"grb": float(settings.grb_max_side_m),
|
||||||
|
"dhmv": raster_side(
|
||||||
|
settings.dhmv_max_side_m,
|
||||||
|
settings.dhmv_max_pixels,
|
||||||
|
settings.dhmv_resolution_m,
|
||||||
|
),
|
||||||
|
"spw_terrain": raster_side(
|
||||||
|
settings.spw_terrain_max_side_m,
|
||||||
|
settings.spw_terrain_max_pixels,
|
||||||
|
settings.spw_terrain_analysis_resolution_m,
|
||||||
|
),
|
||||||
|
"official_vector": 20_000.0,
|
||||||
|
"flood_hazard": raster_side(
|
||||||
|
settings.flood_hazard_max_side_m,
|
||||||
|
settings.flood_hazard_max_pixels,
|
||||||
|
settings.flood_hazard_resolution_m,
|
||||||
|
),
|
||||||
|
"thematic_raster": raster_side(
|
||||||
|
settings.thematic_raster_max_side_m,
|
||||||
|
settings.thematic_raster_max_pixels,
|
||||||
|
10.0,
|
||||||
|
),
|
||||||
|
"walous": raster_side(
|
||||||
|
settings.walous_max_side_m,
|
||||||
|
settings.walous_max_pixels,
|
||||||
|
settings.walous_analysis_resolution_m,
|
||||||
|
),
|
||||||
|
"bathymetry_profiles": 20_000.0,
|
||||||
|
"mdk_bathymetry": 20_000.0,
|
||||||
|
}
|
||||||
|
if provider_key not in budgets:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PROVIDER_UNSUPPORTED",
|
||||||
|
message="No governed partition budget is registered for this provider",
|
||||||
|
details={"provider_key": provider_key},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
governed = budgets[provider_key]
|
||||||
|
return min(governed, float(requested)) if requested is not None else governed
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
|
||||||
|
if (payload.area_id is None) == (payload.bbox is None):
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_SELECTION_REQUIRED",
|
||||||
|
message="Provide exactly one area_id or bbox",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
if payload.area_id is not None:
|
||||||
|
area = db.get(Area, payload.area_id)
|
||||||
|
if area is None or area.project_id != project_id:
|
||||||
|
raise AppError(
|
||||||
|
code="AREA_NOT_FOUND", message="Area not found", status_code=404
|
||||||
|
)
|
||||||
|
return to_shape(area.geometry)
|
||||||
|
bbox = payload.bbox
|
||||||
|
if bbox is None or bbox.crs != "EPSG:4326":
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_AOI_CRS",
|
||||||
|
message="AOI bbox must use EPSG:4326",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _partition(geometry, side_m: float) -> list:
|
||||||
|
min_x, min_y, max_x, max_y = geometry.bounds
|
||||||
|
columns = max(1, math.ceil((max_x - min_x) / side_m))
|
||||||
|
rows = max(1, math.ceil((max_y - min_y) / side_m))
|
||||||
|
if columns * rows > AoiOperationService.MAX_PARTITIONS:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PARTITION_LIMIT_EXCEEDED",
|
||||||
|
message="AOI requires too many bounded partitions",
|
||||||
|
details={
|
||||||
|
"candidate_count": columns * rows,
|
||||||
|
"max_partitions": AoiOperationService.MAX_PARTITIONS,
|
||||||
|
},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
partitions = []
|
||||||
|
for row in range(rows):
|
||||||
|
for column in range(columns):
|
||||||
|
clipped = geometry.intersection(
|
||||||
|
box(
|
||||||
|
min_x + column * side_m,
|
||||||
|
min_y + row * side_m,
|
||||||
|
min(min_x + (column + 1) * side_m, max_x),
|
||||||
|
min(min_y + (row + 1) * side_m, max_y),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not clipped.is_empty and clipped.area > 0:
|
||||||
|
partitions.append(clipped)
|
||||||
|
return partitions
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def read(db, project_id: UUID, operation_id: UUID) -> dict:
|
||||||
|
operation = db.get(AoiOperation, operation_id)
|
||||||
|
if operation is None or operation.project_id != project_id:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_OPERATION_NOT_FOUND",
|
||||||
|
message="AOI operation not found",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
|
partitions = (
|
||||||
|
db.query(AoiOperationPartition)
|
||||||
|
.filter(AoiOperationPartition.operation_id == operation_id)
|
||||||
|
.order_by(AoiOperationPartition.ordinal)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
counts = Counter(partition.status for partition in partitions)
|
||||||
|
complete = counts["success"] + counts["skipped"]
|
||||||
|
return {
|
||||||
|
"id": operation.id,
|
||||||
|
"project_id": operation.project_id,
|
||||||
|
"area_id": operation.area_id,
|
||||||
|
"parent_job_id": operation.parent_job_id,
|
||||||
|
"operation_type": operation.operation_type,
|
||||||
|
"status": operation.status,
|
||||||
|
"request_json": operation.request_json,
|
||||||
|
"plan_json": operation.plan_json,
|
||||||
|
"result_json": operation.result_json,
|
||||||
|
"error_message": operation.error_message,
|
||||||
|
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
|
||||||
|
"partition_counts": dict(counts),
|
||||||
|
"partitions": partitions,
|
||||||
|
"created_at": operation.created_at,
|
||||||
|
"started_at": operation.started_at,
|
||||||
|
"finished_at": operation.finished_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def list(db, project_id: UUID, limit: int = 50) -> dict:
|
||||||
|
rows = (
|
||||||
|
db.query(AoiOperation)
|
||||||
|
.filter(AoiOperation.project_id == project_id)
|
||||||
|
.order_by(AoiOperation.created_at.desc())
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
|
||||||
|
"total": len(rows),
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def claim_next(db, project_id: UUID, operation_id: UUID):
|
||||||
|
operation = db.get(AoiOperation, operation_id)
|
||||||
|
if operation is None or operation.project_id != project_id:
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_OPERATION_NOT_FOUND",
|
||||||
|
message="AOI operation not found",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
|
partition = (
|
||||||
|
db.query(AoiOperationPartition)
|
||||||
|
.filter(
|
||||||
|
AoiOperationPartition.operation_id == operation_id,
|
||||||
|
AoiOperationPartition.status == "queued",
|
||||||
|
)
|
||||||
|
.order_by(AoiOperationPartition.ordinal)
|
||||||
|
.with_for_update(skip_locked=True)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if partition is None:
|
||||||
|
return None
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
partition.status = "running"
|
||||||
|
partition.started_at = now
|
||||||
|
partition.attempt_count += 1
|
||||||
|
partition.error_message = None
|
||||||
|
operation.status = "running"
|
||||||
|
operation.started_at = operation.started_at or now
|
||||||
|
db.add(partition)
|
||||||
|
db.add(operation)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(partition)
|
||||||
|
return partition
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def checkpoint(
|
||||||
|
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
|
||||||
|
):
|
||||||
|
partition = AoiOperationService._partition_row(
|
||||||
|
db, project_id, operation_id, partition_id
|
||||||
|
)
|
||||||
|
if partition.status != "running":
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PARTITION_NOT_RUNNING",
|
||||||
|
message="Only a running partition can be checkpointed",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
partition.checkpoint_json = checkpoint
|
||||||
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(partition)
|
||||||
|
return partition
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def complete(
|
||||||
|
db,
|
||||||
|
project_id: UUID,
|
||||||
|
operation_id: UUID,
|
||||||
|
partition_id: UUID,
|
||||||
|
result: dict,
|
||||||
|
skipped: bool = False,
|
||||||
|
):
|
||||||
|
partition = AoiOperationService._partition_row(
|
||||||
|
db, project_id, operation_id, partition_id
|
||||||
|
)
|
||||||
|
if partition.status == "success" or partition.status == "skipped":
|
||||||
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
if partition.status != "running":
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PARTITION_NOT_RUNNING",
|
||||||
|
message="Only a running partition can complete",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
partition.status = "skipped" if skipped else "success"
|
||||||
|
partition.result_json = result
|
||||||
|
partition.finished_at = datetime.now(timezone.utc)
|
||||||
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
|
AoiOperationService._refresh_parent(db, operation_id)
|
||||||
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def fail(
|
||||||
|
db,
|
||||||
|
project_id: UUID,
|
||||||
|
operation_id: UUID,
|
||||||
|
partition_id: UUID,
|
||||||
|
message: str,
|
||||||
|
retryable: bool,
|
||||||
|
details: dict,
|
||||||
|
):
|
||||||
|
partition = AoiOperationService._partition_row(
|
||||||
|
db, project_id, operation_id, partition_id
|
||||||
|
)
|
||||||
|
partition.error_message = message
|
||||||
|
partition.result_json = {"details": details}
|
||||||
|
partition.status = (
|
||||||
|
"queued"
|
||||||
|
if retryable and partition.attempt_count < partition.max_attempts
|
||||||
|
else "failed"
|
||||||
|
)
|
||||||
|
partition.finished_at = (
|
||||||
|
None if partition.status == "queued" else datetime.now(timezone.utc)
|
||||||
|
)
|
||||||
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
|
AoiOperationService._refresh_parent(db, operation_id)
|
||||||
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _partition_row(db, project_id, operation_id, partition_id):
|
||||||
|
operation = db.get(AoiOperation, operation_id)
|
||||||
|
partition = db.get(AoiOperationPartition, partition_id)
|
||||||
|
if (
|
||||||
|
operation is None
|
||||||
|
or operation.project_id != project_id
|
||||||
|
or partition is None
|
||||||
|
or partition.operation_id != operation_id
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PARTITION_NOT_FOUND",
|
||||||
|
message="AOI partition not found",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
|
return partition
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _refresh_parent(db, operation_id):
|
||||||
|
operation = db.get(AoiOperation, operation_id)
|
||||||
|
partitions = (
|
||||||
|
db.query(AoiOperationPartition)
|
||||||
|
.filter(AoiOperationPartition.operation_id == operation_id)
|
||||||
|
.order_by(AoiOperationPartition.ordinal)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
statuses = [partition.status for partition in partitions]
|
||||||
|
output_dataset_ids = []
|
||||||
|
for partition in partitions:
|
||||||
|
output_id = (
|
||||||
|
(partition.result_json or {}).get("output_dataset_id")
|
||||||
|
if isinstance(partition.result_json, dict)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
if output_id and str(output_id) not in output_dataset_ids:
|
||||||
|
output_dataset_ids.append(str(output_id))
|
||||||
|
operation.result_json = {
|
||||||
|
"partition_count": len(partitions),
|
||||||
|
"completed_partition_count": sum(
|
||||||
|
status in {"success", "skipped"} for status in statuses
|
||||||
|
),
|
||||||
|
"failed_partition_count": statuses.count("failed"),
|
||||||
|
"output_dataset_ids": output_dataset_ids,
|
||||||
|
"merge_contract": "source_aware_spatial_union",
|
||||||
|
"vector_deduplication": "source_feature_id_then_geometry",
|
||||||
|
"raster_deduplication": "governed_mosaic_grid",
|
||||||
|
"complete_coverage": bool(statuses)
|
||||||
|
and all(status in {"success", "skipped"} for status in statuses),
|
||||||
|
}
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
if statuses and all(status in {"success", "skipped"} for status in statuses):
|
||||||
|
operation.status = "success"
|
||||||
|
operation.finished_at = now
|
||||||
|
operation.error_message = None
|
||||||
|
elif "failed" in statuses and not any(
|
||||||
|
status in {"queued", "running"} for status in statuses
|
||||||
|
):
|
||||||
|
operation.status = (
|
||||||
|
"partial"
|
||||||
|
if any(status in {"success", "skipped"} for status in statuses)
|
||||||
|
else "failed"
|
||||||
|
)
|
||||||
|
operation.finished_at = now
|
||||||
|
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
|
||||||
|
db.add(operation)
|
||||||
|
db.commit()
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from app.db.session import SessionLocal
|
||||||
|
from app.models import AoiOperation
|
||||||
|
from app.services.aoi_operation_executor import AoiOperationExecutor
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger("geointel.aoi_worker")
|
||||||
|
|
||||||
|
|
||||||
|
class AoiOperationWorker:
|
||||||
|
@staticmethod
|
||||||
|
def run_once() -> int:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
rows = db.query(AoiOperation).filter(AoiOperation.status.in_(("queued", "running"))).order_by(AoiOperation.created_at).limit(10).all()
|
||||||
|
for operation in rows:
|
||||||
|
try:
|
||||||
|
AoiOperationExecutor.execute_next(db, operation.project_id, operation.id)
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
logger.exception("AOI partition execution failed operation_id=%s", operation.id)
|
||||||
|
return len(rows)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
|
||||||
|
while not stop_event.is_set():
|
||||||
|
processed = await asyncio.to_thread(AoiOperationWorker.run_once)
|
||||||
|
if processed == 0:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
|
||||||
|
except TimeoutError:
|
||||||
|
pass
|
||||||
@@ -7,12 +7,90 @@ from geoalchemy2.shape import from_shape, to_shape
|
|||||||
from shapely.geometry import mapping
|
from shapely.geometry import mapping
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Project
|
from app.models import Area, Dataset, Project, VectorFeature
|
||||||
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
|
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
|
||||||
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon
|
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326
|
||||||
|
|
||||||
|
|
||||||
class AreaService:
|
class AreaService:
|
||||||
|
@staticmethod
|
||||||
|
def _municipality_dataset(db: Session, project_id: uuid.UUID) -> Dataset | None:
|
||||||
|
return (
|
||||||
|
db.query(Dataset)
|
||||||
|
.filter(
|
||||||
|
Dataset.project_id == project_id,
|
||||||
|
Dataset.reference_layer_name == "belgium_municipalities",
|
||||||
|
Dataset.status == "ready",
|
||||||
|
)
|
||||||
|
.order_by(Dataset.created_at.desc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _filter_municipality_properties(properties_items: list[dict], query: str, limit: int) -> tuple[list[dict], int]:
|
||||||
|
normalized = query.strip().casefold()
|
||||||
|
matches: list[dict] = []
|
||||||
|
for properties in properties_items:
|
||||||
|
names = [str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger")]
|
||||||
|
niscode = str(properties.get("niscode") or "").strip()
|
||||||
|
if normalized and normalized not in " ".join([niscode, *names]).casefold():
|
||||||
|
continue
|
||||||
|
display_name = next((name for name in names if name), niscode)
|
||||||
|
matches.append({
|
||||||
|
"niscode": niscode,
|
||||||
|
"name": display_name,
|
||||||
|
"name_nl": names[0] or None,
|
||||||
|
"name_fr": names[1] or None,
|
||||||
|
"name_de": names[2] or None,
|
||||||
|
})
|
||||||
|
matches.sort(key=lambda item: (item["name"].casefold(), item["niscode"]))
|
||||||
|
return matches[:limit], len(matches)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def search_municipalities(db: Session, project_id: uuid.UUID, query: str, limit: int = 20) -> tuple[list[dict], int]:
|
||||||
|
dataset = AreaService._municipality_dataset(db, project_id)
|
||||||
|
if dataset is None:
|
||||||
|
return [], 0
|
||||||
|
property_rows = (
|
||||||
|
db.query(VectorFeature.properties_json)
|
||||||
|
.filter(VectorFeature.dataset_id == dataset.id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
properties_items = [row[0] for row in property_rows if isinstance(row[0], dict)]
|
||||||
|
return AreaService._filter_municipality_properties(properties_items, query, limit)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def activate_municipality(db: Session, project_id: uuid.UUID, niscode: str) -> Area:
|
||||||
|
normalized_code = niscode.strip()
|
||||||
|
dataset = AreaService._municipality_dataset(db, project_id)
|
||||||
|
if dataset is None:
|
||||||
|
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
|
||||||
|
feature = (
|
||||||
|
db.query(VectorFeature)
|
||||||
|
.filter(
|
||||||
|
VectorFeature.dataset_id == dataset.id,
|
||||||
|
VectorFeature.properties_json["niscode"].as_string() == normalized_code,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if feature is not None:
|
||||||
|
properties = feature.properties_json if isinstance(feature.properties_json, dict) else {}
|
||||||
|
display_name = next(
|
||||||
|
(str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger") if str(properties.get(key) or "").strip()),
|
||||||
|
normalized_code,
|
||||||
|
)
|
||||||
|
area_name = f"Gemeente {display_name} - NIS {normalized_code}"
|
||||||
|
existing = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
geometry = to_shape(feature.geometry)
|
||||||
|
return AreaService.create_area(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
AreaCreate(name=area_name, geometry=mapping(geometry), crs="EPSG:4326"),
|
||||||
|
)
|
||||||
|
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def serialize_area(area: Area) -> dict:
|
def serialize_area(area: Area) -> dict:
|
||||||
geometry = to_shape(area.geometry) if area.geometry else None
|
geometry = to_shape(area.geometry) if area.geometry else None
|
||||||
@@ -48,7 +126,11 @@ class AreaService:
|
|||||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
multipolygon = normalize_to_multipolygon(payload.geometry)
|
multipolygon, original_crs = normalize_area_to_epsg4326(
|
||||||
|
payload.geometry,
|
||||||
|
payload.crs or "EPSG:4326",
|
||||||
|
)
|
||||||
|
metric_area = area_m2(multipolygon)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
||||||
|
|
||||||
@@ -56,8 +138,8 @@ class AreaService:
|
|||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name=payload.name.strip() or "Unnamed area",
|
name=payload.name.strip() or "Unnamed area",
|
||||||
geometry=from_shape(multipolygon, srid=4326),
|
geometry=from_shape(multipolygon, srid=4326),
|
||||||
original_crs=payload.crs or "EPSG:4326",
|
original_crs=original_crs,
|
||||||
area_m2=area_m2(multipolygon),
|
area_m2=metric_area,
|
||||||
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
|
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
|
||||||
)
|
)
|
||||||
db.add(area)
|
db.add(area)
|
||||||
@@ -79,11 +161,28 @@ class AreaService:
|
|||||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||||
|
|
||||||
changed = False
|
changed = False
|
||||||
if payload.name:
|
if payload.name is not None and payload.name.strip():
|
||||||
area.name = payload.name.strip() or area.name
|
area.name = payload.name.strip() or area.name
|
||||||
changed = True
|
changed = True
|
||||||
if payload.crs:
|
if payload.crs is not None and payload.geometry is None:
|
||||||
area.original_crs = payload.crs
|
raise AppError(
|
||||||
|
code="INVALID_AREA_CRS_UPDATE",
|
||||||
|
message="crs can only be supplied together with replacement geometry",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
if payload.geometry is not None:
|
||||||
|
try:
|
||||||
|
multipolygon, original_crs = normalize_area_to_epsg4326(
|
||||||
|
payload.geometry,
|
||||||
|
payload.crs or "EPSG:4326",
|
||||||
|
)
|
||||||
|
metric_area = area_m2(multipolygon)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
||||||
|
area.geometry = from_shape(multipolygon, srid=4326)
|
||||||
|
area.original_crs = original_crs
|
||||||
|
area.area_m2 = metric_area
|
||||||
|
area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
|
||||||
changed = True
|
changed = True
|
||||||
if not changed:
|
if not changed:
|
||||||
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
|
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
|
||||||
|
|||||||
@@ -0,0 +1,260 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
import secrets
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import deque
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Literal, cast
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.core.config import Settings
|
||||||
|
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AuthPrincipal:
|
||||||
|
username: str
|
||||||
|
expires_at: int
|
||||||
|
session_id: str = field(default_factory=lambda: secrets.token_urlsafe(12))
|
||||||
|
role: Literal["operator", "guest"] = "operator"
|
||||||
|
project_id: UUID | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class AuthService:
|
||||||
|
HASH_NAME = "pbkdf2_sha256"
|
||||||
|
HASH_ITERATIONS = 600_000
|
||||||
|
MAX_FAILURES = 5
|
||||||
|
FAILURE_WINDOW_SECONDS = 300
|
||||||
|
_failures: dict[str, deque[float]] = {}
|
||||||
|
_failure_lock = threading.Lock()
|
||||||
|
_guest_requests: dict[str, deque[float]] = {}
|
||||||
|
_guest_request_lock = threading.Lock()
|
||||||
|
_active_guest_compute = 0
|
||||||
|
_guest_compute_lock = threading.Lock()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _b64_encode(value: bytes) -> str:
|
||||||
|
return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _b64_decode(value: str) -> bytes:
|
||||||
|
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def hash_password(
|
||||||
|
cls,
|
||||||
|
password: str,
|
||||||
|
*,
|
||||||
|
salt: bytes | None = None,
|
||||||
|
iterations: int | None = None,
|
||||||
|
) -> str:
|
||||||
|
resolved_salt = salt or secrets.token_bytes(18)
|
||||||
|
resolved_iterations = iterations or cls.HASH_ITERATIONS
|
||||||
|
digest = hashlib.pbkdf2_hmac(
|
||||||
|
"sha256",
|
||||||
|
password.encode("utf-8"),
|
||||||
|
resolved_salt,
|
||||||
|
resolved_iterations,
|
||||||
|
)
|
||||||
|
return "$".join(
|
||||||
|
(
|
||||||
|
cls.HASH_NAME,
|
||||||
|
str(resolved_iterations),
|
||||||
|
cls._b64_encode(resolved_salt),
|
||||||
|
cls._b64_encode(digest),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def verify_password(cls, password: str, encoded: str) -> bool:
|
||||||
|
try:
|
||||||
|
algorithm, iterations_raw, salt_raw, expected_raw = encoded.split("$", 3)
|
||||||
|
if algorithm != cls.HASH_NAME:
|
||||||
|
return False
|
||||||
|
iterations = int(iterations_raw)
|
||||||
|
if iterations < 100_000 or iterations > 2_000_000:
|
||||||
|
return False
|
||||||
|
salt = cls._b64_decode(salt_raw)
|
||||||
|
expected = cls._b64_decode(expected_raw)
|
||||||
|
actual = hashlib.pbkdf2_hmac(
|
||||||
|
"sha256",
|
||||||
|
password.encode("utf-8"),
|
||||||
|
salt,
|
||||||
|
iterations,
|
||||||
|
)
|
||||||
|
return hmac.compare_digest(actual, expected)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def credentials_match(cls, username: str, password: str, settings: Settings) -> bool:
|
||||||
|
expected_username = settings.auth_username or ""
|
||||||
|
expected_password_hash = settings.auth_password_hash or ""
|
||||||
|
username_matches = hmac.compare_digest(
|
||||||
|
username.encode("utf-8"),
|
||||||
|
expected_username.encode("utf-8"),
|
||||||
|
)
|
||||||
|
password_matches = cls.verify_password(password, expected_password_hash)
|
||||||
|
return username_matches and password_matches
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create_session_token(
|
||||||
|
cls,
|
||||||
|
username: str,
|
||||||
|
settings: Settings,
|
||||||
|
*,
|
||||||
|
role: Literal["operator", "guest"] = "operator",
|
||||||
|
project_id: UUID | None = None,
|
||||||
|
ttl_seconds: int | None = None,
|
||||||
|
now: int | None = None,
|
||||||
|
) -> str:
|
||||||
|
issued_at = int(time.time() if now is None else now)
|
||||||
|
if role == "guest" and project_id is None:
|
||||||
|
raise ValueError("Guest sessions must be scoped to a demo project")
|
||||||
|
resolved_ttl = ttl_seconds if ttl_seconds is not None else (
|
||||||
|
settings.guest_session_ttl_seconds if role == "guest" else settings.auth_session_ttl_seconds
|
||||||
|
)
|
||||||
|
payload = {
|
||||||
|
"exp": issued_at + resolved_ttl,
|
||||||
|
"iat": issued_at,
|
||||||
|
"jti": secrets.token_urlsafe(12),
|
||||||
|
"role": role,
|
||||||
|
"sub": username,
|
||||||
|
"v": 2,
|
||||||
|
}
|
||||||
|
if project_id is not None:
|
||||||
|
payload["project_id"] = str(project_id)
|
||||||
|
encoded_payload = cls._b64_encode(
|
||||||
|
json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||||
|
)
|
||||||
|
signature = hmac.new(
|
||||||
|
(settings.auth_session_secret or "").encode("utf-8"),
|
||||||
|
encoded_payload.encode("ascii"),
|
||||||
|
hashlib.sha256,
|
||||||
|
).digest()
|
||||||
|
return f"{encoded_payload}.{cls._b64_encode(signature)}"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def verify_session_token(
|
||||||
|
cls,
|
||||||
|
token: str | None,
|
||||||
|
settings: Settings,
|
||||||
|
*,
|
||||||
|
now: int | None = None,
|
||||||
|
) -> AuthPrincipal | None:
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
encoded_payload, encoded_signature = token.split(".", 1)
|
||||||
|
expected_signature = hmac.new(
|
||||||
|
(settings.auth_session_secret or "").encode("utf-8"),
|
||||||
|
encoded_payload.encode("ascii"),
|
||||||
|
hashlib.sha256,
|
||||||
|
).digest()
|
||||||
|
supplied_signature = cls._b64_decode(encoded_signature)
|
||||||
|
if not hmac.compare_digest(expected_signature, supplied_signature):
|
||||||
|
return None
|
||||||
|
payload = json.loads(cls._b64_decode(encoded_payload))
|
||||||
|
username = str(payload.get("sub") or "")
|
||||||
|
expires_at = int(payload.get("exp") or 0)
|
||||||
|
issued_at = int(payload.get("iat") or 0)
|
||||||
|
version = int(payload.get("v") or 0)
|
||||||
|
role_value = str(payload.get("role") or "operator")
|
||||||
|
session_id = str(payload.get("jti") or "")
|
||||||
|
current = int(time.time() if now is None else now)
|
||||||
|
if version not in {1, 2} or role_value not in {"operator", "guest"} or not session_id:
|
||||||
|
return None
|
||||||
|
role = cast(Literal["operator", "guest"], role_value)
|
||||||
|
if issued_at <= 0 or issued_at > current + 60 or expires_at <= current:
|
||||||
|
return None
|
||||||
|
if role == "operator":
|
||||||
|
if username != settings.auth_username:
|
||||||
|
return None
|
||||||
|
max_ttl = settings.auth_session_ttl_seconds
|
||||||
|
project_id = None
|
||||||
|
else:
|
||||||
|
if not settings.guest_access_enabled or username != settings.guest_display_name:
|
||||||
|
return None
|
||||||
|
max_ttl = settings.guest_session_ttl_seconds
|
||||||
|
raw_project_id = payload.get("project_id")
|
||||||
|
if not raw_project_id:
|
||||||
|
return None
|
||||||
|
project_id = UUID(str(raw_project_id))
|
||||||
|
if project_id != PUBLIC_DEMO_PROJECT_ID:
|
||||||
|
return None
|
||||||
|
if expires_at - issued_at > max_ttl:
|
||||||
|
return None
|
||||||
|
return AuthPrincipal(
|
||||||
|
username=username,
|
||||||
|
expires_at=expires_at,
|
||||||
|
session_id=session_id,
|
||||||
|
role=role,
|
||||||
|
project_id=project_id,
|
||||||
|
)
|
||||||
|
except (TypeError, ValueError, json.JSONDecodeError, UnicodeDecodeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def retry_after_seconds(cls, key: str, *, now: float | None = None) -> int:
|
||||||
|
current = time.monotonic() if now is None else now
|
||||||
|
with cls._failure_lock:
|
||||||
|
attempts = cls._failures.setdefault(key, deque())
|
||||||
|
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
|
||||||
|
attempts.popleft()
|
||||||
|
if len(attempts) < cls.MAX_FAILURES:
|
||||||
|
if not attempts:
|
||||||
|
cls._failures.pop(key, None)
|
||||||
|
return 0
|
||||||
|
return max(1, int(cls.FAILURE_WINDOW_SECONDS - (current - attempts[0])))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def record_failure(cls, key: str, *, now: float | None = None) -> None:
|
||||||
|
current = time.monotonic() if now is None else now
|
||||||
|
with cls._failure_lock:
|
||||||
|
attempts = cls._failures.setdefault(key, deque())
|
||||||
|
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
|
||||||
|
attempts.popleft()
|
||||||
|
attempts.append(current)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def clear_failures(cls, key: str) -> None:
|
||||||
|
with cls._failure_lock:
|
||||||
|
cls._failures.pop(key, None)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def consume_guest_request(
|
||||||
|
cls,
|
||||||
|
key: str,
|
||||||
|
*,
|
||||||
|
max_requests: int,
|
||||||
|
window_seconds: int = 60,
|
||||||
|
now: float | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Record a guest action and return Retry-After seconds when limited."""
|
||||||
|
current = time.monotonic() if now is None else now
|
||||||
|
with cls._guest_request_lock:
|
||||||
|
attempts = cls._guest_requests.setdefault(key, deque())
|
||||||
|
while attempts and current - attempts[0] >= window_seconds:
|
||||||
|
attempts.popleft()
|
||||||
|
if len(attempts) >= max_requests:
|
||||||
|
return max(1, int(window_seconds - (current - attempts[0])))
|
||||||
|
attempts.append(current)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def try_acquire_guest_compute(cls, *, max_concurrency: int) -> bool:
|
||||||
|
with cls._guest_compute_lock:
|
||||||
|
if cls._active_guest_compute >= max_concurrency:
|
||||||
|
return False
|
||||||
|
cls._active_guest_compute += 1
|
||||||
|
return True
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def release_guest_compute(cls) -> None:
|
||||||
|
with cls._guest_compute_lock:
|
||||||
|
cls._active_guest_compute = max(0, cls._active_guest_compute - 1)
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import secrets
|
||||||
|
from typing import Any
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
from urllib.parse import urlencode, urlsplit
|
||||||
|
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||||
|
|
||||||
|
import jwt
|
||||||
|
from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
|
||||||
|
|
||||||
|
from app.core.config import Settings
|
||||||
|
|
||||||
|
|
||||||
|
MAX_OIDC_JSON_BYTES = 1_048_576
|
||||||
|
|
||||||
|
|
||||||
|
class _RejectRedirects(HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ANN201
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class AuthentikOidcService:
|
||||||
|
def __init__(self, settings: Settings):
|
||||||
|
self.settings = settings
|
||||||
|
self.issuer = (settings.authentik_issuer or "").rstrip("/")
|
||||||
|
self.serializer = URLSafeTimedSerializer(
|
||||||
|
settings.auth_session_secret or "",
|
||||||
|
salt="geointel-authentik-v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self) -> bool:
|
||||||
|
return bool(
|
||||||
|
self.issuer
|
||||||
|
and self.settings.authentik_client_id
|
||||||
|
and self.settings.authentik_client_secret
|
||||||
|
and self.settings.authentik_allowed_email
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def redirect_uri(self) -> str:
|
||||||
|
return (
|
||||||
|
f"{self.settings.public_base_url.rstrip('/')}"
|
||||||
|
f"{self.settings.api_prefix}/auth/authentik/callback"
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _origin(url: str) -> tuple[str, str, int]:
|
||||||
|
parsed = urlsplit(url)
|
||||||
|
if parsed.scheme != "https" or not parsed.hostname:
|
||||||
|
raise ValueError("OIDC URLs must use absolute HTTPS URLs")
|
||||||
|
return parsed.scheme, parsed.hostname.casefold(), parsed.port or 443
|
||||||
|
|
||||||
|
def _validate_endpoint(self, url: str) -> str:
|
||||||
|
parsed = urlsplit(url)
|
||||||
|
if (
|
||||||
|
self._origin(url) != self._origin(self.issuer)
|
||||||
|
or parsed.username
|
||||||
|
or parsed.password
|
||||||
|
or parsed.fragment
|
||||||
|
):
|
||||||
|
raise ValueError("OIDC endpoint is outside the configured issuer origin")
|
||||||
|
return url
|
||||||
|
|
||||||
|
def _fetch_json(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
data: dict[str, str] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
self._validate_endpoint(url)
|
||||||
|
encoded = urlencode(data).encode("utf-8") if data is not None else None
|
||||||
|
headers = {"Accept": "application/json"}
|
||||||
|
if encoded is not None:
|
||||||
|
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||||
|
request = Request(url, data=encoded, headers=headers)
|
||||||
|
try:
|
||||||
|
with build_opener(_RejectRedirects()).open(request, timeout=10) as response:
|
||||||
|
declared_length = response.headers.get("Content-Length")
|
||||||
|
if declared_length and int(declared_length) > MAX_OIDC_JSON_BYTES:
|
||||||
|
raise ValueError("OIDC response exceeds the configured size limit")
|
||||||
|
raw = response.read(MAX_OIDC_JSON_BYTES + 1)
|
||||||
|
except HTTPError as exc:
|
||||||
|
raise ValueError("OIDC endpoint returned an HTTP error or redirect") from exc
|
||||||
|
if len(raw) > MAX_OIDC_JSON_BYTES:
|
||||||
|
raise ValueError("OIDC response exceeds the configured size limit")
|
||||||
|
payload = json.loads(raw)
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise ValueError("OIDC endpoint did not return a JSON object")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
def _discovery(self) -> dict[str, Any]:
|
||||||
|
document = self._fetch_json(
|
||||||
|
f"{self.issuer}/.well-known/openid-configuration"
|
||||||
|
)
|
||||||
|
if str(document.get("issuer", "")).rstrip("/") != self.issuer:
|
||||||
|
raise ValueError("OIDC issuer mismatch")
|
||||||
|
for key in ("authorization_endpoint", "token_endpoint", "jwks_uri"):
|
||||||
|
endpoint = document.get(key)
|
||||||
|
if not isinstance(endpoint, str):
|
||||||
|
raise ValueError(f"OIDC discovery is missing {key}")
|
||||||
|
self._validate_endpoint(endpoint)
|
||||||
|
return document
|
||||||
|
|
||||||
|
def start(self) -> tuple[str, str]:
|
||||||
|
if not self.enabled:
|
||||||
|
raise ValueError("Authentik is not configured")
|
||||||
|
state = secrets.token_urlsafe(32)
|
||||||
|
nonce = secrets.token_urlsafe(32)
|
||||||
|
verifier = secrets.token_urlsafe(48)
|
||||||
|
flow = self.serializer.dumps(
|
||||||
|
{"state": state, "nonce": nonce, "verifier": verifier}
|
||||||
|
)
|
||||||
|
challenge = (
|
||||||
|
base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
|
||||||
|
.rstrip(b"=")
|
||||||
|
.decode()
|
||||||
|
)
|
||||||
|
discovery = self._discovery()
|
||||||
|
query = urlencode(
|
||||||
|
{
|
||||||
|
"client_id": self.settings.authentik_client_id,
|
||||||
|
"redirect_uri": self.redirect_uri,
|
||||||
|
"response_type": "code",
|
||||||
|
"scope": "openid email profile",
|
||||||
|
"state": state,
|
||||||
|
"nonce": nonce,
|
||||||
|
"code_challenge": challenge,
|
||||||
|
"code_challenge_method": "S256",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return f"{discovery['authorization_endpoint']}?{query}", flow
|
||||||
|
|
||||||
|
def finish(self, *, code: str, state: str, flow_cookie: str) -> dict[str, Any]:
|
||||||
|
if not self.enabled or not code:
|
||||||
|
raise ValueError("OIDC flow is incomplete")
|
||||||
|
try:
|
||||||
|
flow = self.serializer.loads(flow_cookie, max_age=600)
|
||||||
|
except (BadSignature, SignatureExpired) as exc:
|
||||||
|
raise ValueError("Invalid OIDC flow") from exc
|
||||||
|
if not isinstance(flow, dict):
|
||||||
|
raise ValueError("Invalid OIDC flow payload")
|
||||||
|
if not state or not secrets.compare_digest(state, str(flow.get("state", ""))):
|
||||||
|
raise ValueError("OIDC state mismatch")
|
||||||
|
verifier = str(flow.get("verifier", ""))
|
||||||
|
nonce = str(flow.get("nonce", ""))
|
||||||
|
if not verifier or not nonce:
|
||||||
|
raise ValueError("OIDC flow payload is incomplete")
|
||||||
|
|
||||||
|
discovery = self._discovery()
|
||||||
|
token_response = self._fetch_json(
|
||||||
|
str(discovery["token_endpoint"]),
|
||||||
|
{
|
||||||
|
"grant_type": "authorization_code",
|
||||||
|
"code": code,
|
||||||
|
"redirect_uri": self.redirect_uri,
|
||||||
|
"client_id": self.settings.authentik_client_id or "",
|
||||||
|
"client_secret": self.settings.authentik_client_secret or "",
|
||||||
|
"code_verifier": verifier,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
token = str(token_response.get("id_token", ""))
|
||||||
|
if not token:
|
||||||
|
raise ValueError("OIDC token response has no ID token")
|
||||||
|
header = jwt.get_unverified_header(token)
|
||||||
|
if header.get("alg") != "RS256" or not header.get("kid"):
|
||||||
|
raise ValueError("OIDC ID token uses an unsupported signing header")
|
||||||
|
jwks = self._fetch_json(str(discovery["jwks_uri"]))
|
||||||
|
matching_keys = [
|
||||||
|
key
|
||||||
|
for key in jwks.get("keys", [])
|
||||||
|
if isinstance(key, dict) and key.get("kid") == header["kid"]
|
||||||
|
]
|
||||||
|
if len(matching_keys) != 1:
|
||||||
|
raise ValueError("OIDC signing key is missing or ambiguous")
|
||||||
|
signing_key = jwt.PyJWK.from_dict(matching_keys[0]).key
|
||||||
|
claims = jwt.decode(
|
||||||
|
token,
|
||||||
|
signing_key,
|
||||||
|
algorithms=["RS256"],
|
||||||
|
audience=self.settings.authentik_client_id,
|
||||||
|
issuer=discovery["issuer"],
|
||||||
|
options={
|
||||||
|
"require": [
|
||||||
|
"exp",
|
||||||
|
"iat",
|
||||||
|
"iss",
|
||||||
|
"aud",
|
||||||
|
"sub",
|
||||||
|
"nonce",
|
||||||
|
"email",
|
||||||
|
"email_verified",
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if not secrets.compare_digest(str(claims.get("nonce", "")), nonce):
|
||||||
|
raise ValueError("OIDC nonce mismatch")
|
||||||
|
email = str(claims.get("email", "")).strip().casefold()
|
||||||
|
allowed = str(self.settings.authentik_allowed_email or "").strip().casefold()
|
||||||
|
if claims.get("email_verified") is not True or not secrets.compare_digest(
|
||||||
|
email, allowed
|
||||||
|
):
|
||||||
|
raise ValueError("OIDC identity is not authorized")
|
||||||
|
return claims
|
||||||
@@ -7,7 +7,7 @@ import math
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
@@ -15,6 +15,7 @@ from shapely.geometry import Point, box, mapping
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.services.outbound_request_guard import guarded_opener
|
||||||
from app.models import Area, Dataset, DatasetVersion, Project
|
from app.models import Area, Dataset, DatasetVersion, Project
|
||||||
from app.schemas.bathymetry import (
|
from app.schemas.bathymetry import (
|
||||||
BathymetryPartitionFinalizeRequest,
|
BathymetryPartitionFinalizeRequest,
|
||||||
@@ -91,20 +92,21 @@ class BathymetryProfileAcquisitionService:
|
|||||||
"authority_level": "authoritative",
|
"authority_level": "authoritative",
|
||||||
"geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen",
|
"geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen",
|
||||||
"data_kind": "bodemhoogteraster en XYZ-puntenwolk",
|
"data_kind": "bodemhoogteraster en XYZ-puntenwolk",
|
||||||
"query_modes": ["download", "arcgis_map_service"],
|
"query_modes": ["operator_archive", "bounded_raster", "arcgis_map_service"],
|
||||||
"vertical_reference": "mDNG",
|
"vertical_reference": "mDNG",
|
||||||
"horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden",
|
"horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden",
|
||||||
"native_resolution": "0,5 m",
|
"native_resolution": "0,5 m",
|
||||||
"integration_status": "available_not_integrated",
|
"integration_status": "operational",
|
||||||
"acquisition_supported": False,
|
"acquisition_supported": True,
|
||||||
"configured": False,
|
"configured": True,
|
||||||
"service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer",
|
"service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer",
|
||||||
"catalog_url": "https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html",
|
"catalog_url": "https://geoportail.wallonie.be/catalogue/0a544b42-0b30-4c8e-85e7-38149b99eae0.html",
|
||||||
"attribution": "Service public de Wallonie",
|
"attribution": "Service public de Wallonie",
|
||||||
"license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.",
|
"license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.",
|
||||||
"limitation_message": (
|
"limitation_message": (
|
||||||
"Dekking en meetjaar verschillen per vaarweg of reservoir. Integratie vereist een beheerde "
|
"De gepinde officiële release kan begrensd als raster worden geïmporteerd via de operator. "
|
||||||
"download- en mosaïekstroom plus expliciete omzetting van mDNG."
|
"Dekking verschilt per vaarweg; de waarden zijn bodemhoogtes in mDNG uit 2019-2022, "
|
||||||
|
"zonder stilzwijgende datumconversie of afleiding van actuele waterdiepte."
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -133,8 +135,37 @@ class BathymetryProfileAcquisitionService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def list_sources() -> list[dict[str, Any]]:
|
def list_sources(settings=None) -> list[dict[str, Any]]:
|
||||||
return [BathymetrySourceRead(**item).model_dump() for item in BathymetryProfileAcquisitionService._SOURCES]
|
from app.core.config import get_settings
|
||||||
|
|
||||||
|
resolved_settings = settings or get_settings()
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
for source in BathymetryProfileAcquisitionService._SOURCES:
|
||||||
|
item = dict(source)
|
||||||
|
if item["key"] == "mdk_bcp_bathymetry":
|
||||||
|
mdk_configured = bool(
|
||||||
|
resolved_settings.mdk_bathymetry_acquisition_enabled
|
||||||
|
and (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
|
||||||
|
)
|
||||||
|
item["acquisition_supported"] = True
|
||||||
|
item["configured"] = mdk_configured
|
||||||
|
if mdk_configured:
|
||||||
|
item["integration_status"] = "operational"
|
||||||
|
item["limitation_message"] = (
|
||||||
|
"Begrensde WCS-acquisitie is expliciet ingeschakeld en draait alleen wanneer de "
|
||||||
|
"live readiness-probe bereikbaar is en het geconfigureerde coverage-id door de "
|
||||||
|
"capabilities wordt geadverteerd. Dieptes blijven LAT-gerefereerd; watervolume "
|
||||||
|
"blijft zonder compatibel wateroppervlak niet ondersteund."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
item["limitation_message"] = (
|
||||||
|
"Begrensde WCS-acquisitie bestaat maar staat uit. Zet "
|
||||||
|
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true en configureer MDK_BATHYMETRY_COVERAGE_ID "
|
||||||
|
"pas nadat de readiness-probe live 'reachable' rapporteert. Er wordt nooit "
|
||||||
|
"onbeveiligd of ongevalideerd gedownload."
|
||||||
|
)
|
||||||
|
items.append(item)
|
||||||
|
return [BathymetrySourceRead(**item).model_dump() for item in items]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
|
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
|
||||||
@@ -194,7 +225,7 @@ class BathymetryProfileAcquisitionService:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
with (opener or urlopen)(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response:
|
with (opener or guarded_opener(url))(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response:
|
||||||
limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024
|
limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024
|
||||||
content = response.read(limit + 1)
|
content = response.read(limit + 1)
|
||||||
except HTTPError as exc:
|
except HTTPError as exc:
|
||||||
@@ -245,6 +276,45 @@ class BathymetryProfileAcquisitionService:
|
|||||||
"spatialRel": "esriSpatialRelIntersects",
|
"spatialRel": "esriSpatialRelIntersects",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _unseen_records(
|
||||||
|
page_features: list[Any],
|
||||||
|
seen_object_ids: set[str],
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Every page must bring records the earlier pages did not.
|
||||||
|
|
||||||
|
An ArcGIS layer without ``supportsPagination`` accepts ``resultOffset``
|
||||||
|
and ignores it, answering every page with the first one. Advancing the
|
||||||
|
offset by the page length still reaches the announced count, so the
|
||||||
|
completeness check below passed while the dataset held N copies of page
|
||||||
|
one — a silent substitution of the source data, which is the one thing
|
||||||
|
bounded acquisition exists to prevent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
fresh: list[dict[str, Any]] = []
|
||||||
|
for item in page_features:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
attributes = item.get("attributes")
|
||||||
|
object_id = attributes.get("OBJECTID") if isinstance(attributes, dict) else None
|
||||||
|
if object_id is None:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||||
|
message="VHA profile record has no OBJECTID, so pagination cannot be verified",
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
key = str(object_id)
|
||||||
|
if key in seen_object_ids:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
|
||||||
|
message="VHA profile pagination repeated a record; the layer is not honouring resultOffset",
|
||||||
|
details={"object_id": key},
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
seen_object_ids.add(key)
|
||||||
|
fresh.append(item)
|
||||||
|
return fresh
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _fetch_profiles(
|
def _fetch_profiles(
|
||||||
bbox_values: tuple[float, float, float, float],
|
bbox_values: tuple[float, float, float, float],
|
||||||
@@ -276,8 +346,16 @@ class BathymetryProfileAcquisitionService:
|
|||||||
features: list[dict[str, Any]] = []
|
features: list[dict[str, Any]] = []
|
||||||
response_hashes: list[str] = []
|
response_hashes: list[str] = []
|
||||||
request_urls: list[str] = [count_url]
|
request_urls: list[str] = [count_url]
|
||||||
|
seen_object_ids: set[str] = set()
|
||||||
offset = 0
|
offset = 0
|
||||||
while offset < candidate_count:
|
while offset < candidate_count:
|
||||||
|
if len(request_urls) > settings.bathymetry_profiles_max_pages:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_SCOPE_TOO_LARGE",
|
||||||
|
message="VHA profile pagination exceeded the configured page limit; acquire smaller area partitions",
|
||||||
|
details={"max_pages": settings.bathymetry_profiles_max_pages},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
page_url = BathymetryProfileAcquisitionService._query_url(
|
page_url = BathymetryProfileAcquisitionService._query_url(
|
||||||
base,
|
base,
|
||||||
{
|
{
|
||||||
@@ -297,11 +375,13 @@ class BathymetryProfileAcquisitionService:
|
|||||||
message="VHA profile response does not contain a feature list",
|
message="VHA profile response does not contain a feature list",
|
||||||
status_code=502,
|
status_code=502,
|
||||||
)
|
)
|
||||||
features.extend(item for item in page_features if isinstance(item, dict))
|
|
||||||
response_hashes.append(page_sha)
|
response_hashes.append(page_sha)
|
||||||
request_urls.append(page_url)
|
request_urls.append(page_url)
|
||||||
if not page_features:
|
if not page_features:
|
||||||
break
|
break
|
||||||
|
features.extend(
|
||||||
|
BathymetryProfileAcquisitionService._unseen_records(page_features, seen_object_ids)
|
||||||
|
)
|
||||||
offset += len(page_features)
|
offset += len(page_features)
|
||||||
if len(features) != candidate_count:
|
if len(features) != candidate_count:
|
||||||
raise AppError(
|
raise AppError(
|
||||||
@@ -332,7 +412,15 @@ class BathymetryProfileAcquisitionService:
|
|||||||
for start in range(0, len(ordered_codes), 100):
|
for start in range(0, len(ordered_codes), 100):
|
||||||
chunk = ordered_codes[start : start + 100]
|
chunk = ordered_codes[start : start + 100]
|
||||||
offset = 0
|
offset = 0
|
||||||
|
seen_page_hashes: set[str] = set()
|
||||||
while True:
|
while True:
|
||||||
|
if len(seen_page_hashes) >= settings.bathymetry_profiles_max_pages:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_SCOPE_TOO_LARGE",
|
||||||
|
message="VHA watercourse pagination exceeded the configured page limit",
|
||||||
|
details={"max_pages": settings.bathymetry_profiles_max_pages},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
url = BathymetryProfileAcquisitionService._query_url(
|
url = BathymetryProfileAcquisitionService._query_url(
|
||||||
base,
|
base,
|
||||||
{
|
{
|
||||||
@@ -355,6 +443,16 @@ class BathymetryProfileAcquisitionService:
|
|||||||
message="VHA watercourse response does not contain a feature list",
|
message="VHA watercourse response does not contain a feature list",
|
||||||
status_code=502,
|
status_code=502,
|
||||||
)
|
)
|
||||||
|
if response_sha in seen_page_hashes:
|
||||||
|
# The names themselves deduplicate by code, so a stuck
|
||||||
|
# provider produced no visible change while the loop, which
|
||||||
|
# ended only on exceededTransferLimit, kept requesting.
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
|
||||||
|
message="VHA watercourse pagination returned the same page again",
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
seen_page_hashes.add(response_sha)
|
||||||
for feature in page_features:
|
for feature in page_features:
|
||||||
attributes = feature.get("attributes") if isinstance(feature, dict) else None
|
attributes = feature.get("attributes") if isinstance(feature, dict) else None
|
||||||
if not isinstance(attributes, dict):
|
if not isinstance(attributes, dict):
|
||||||
|
|||||||
@@ -0,0 +1,366 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import math
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from geoalchemy2.shape import to_shape
|
||||||
|
from pyproj import Transformer
|
||||||
|
from shapely.geometry import box, mapping
|
||||||
|
from shapely.ops import transform as shapely_transform
|
||||||
|
|
||||||
|
from app.core.config import Settings, get_settings
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.services.raster_cell_selection import select_cells
|
||||||
|
from app.models import Area, Dataset
|
||||||
|
from app.schemas.bathymetry import (
|
||||||
|
BathymetryRasterMetric,
|
||||||
|
BathymetryRasterSelectionRequest,
|
||||||
|
BathymetryRasterSelectionResponse,
|
||||||
|
BathymetryRasterSelectionSummary,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetryRasterAnalysisService:
|
||||||
|
SOURCE_NAME = "spw_bathymetry"
|
||||||
|
PRODUCT_KEY = "spw_bathymetry_50cm_mdng"
|
||||||
|
UNSUPPORTED_METRICS = [
|
||||||
|
"current_water_depth_m",
|
||||||
|
"water_volume_m3",
|
||||||
|
"vertical_datum_conversion",
|
||||||
|
]
|
||||||
|
LIMITATION = (
|
||||||
|
"De rasterwaarden zijn waterbodemhoogtes in mDNG uit een samengestelde SPW-opmeting "
|
||||||
|
"(2019-2022). Zonder een gelijktijdig waterpeil zijn actuele waterdiepte en watervolume "
|
||||||
|
"niet berekenbaar. mDNG wordt niet stilzwijgend naar TAW, LAT of een ander verticaal datum omgezet."
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if not dataset or dataset.project_id != project_id:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
if dataset.dataset_type != "raster" or dataset.source_name != BathymetryRasterAnalysisService.SOURCE_NAME:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_BATHYMETRY_RASTER_DATASET",
|
||||||
|
message="Bathymetry analysis requires a governed SPW bathymetry raster dataset",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_FILE_MISSING",
|
||||||
|
message="Persisted bathymetry raster file is unavailable",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _metadata(dataset: Dataset) -> dict:
|
||||||
|
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||||
|
if (
|
||||||
|
metadata.get("product_key") != BathymetryRasterAnalysisService.PRODUCT_KEY
|
||||||
|
or metadata.get("theme") != "bathymetry"
|
||||||
|
or metadata.get("value_semantics") != "bed_elevation"
|
||||||
|
or metadata.get("vertical_reference") != "mDNG"
|
||||||
|
or metadata.get("source_crs") != "EPSG:3812"
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_BATHYMETRY_RASTER_METADATA",
|
||||||
|
message="Bathymetry raster provenance or value semantics are incomplete",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _selection_geometry(db, project_id: UUID, payload: BathymetryRasterSelectionRequest):
|
||||||
|
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
|
||||||
|
if payload.area_id is None:
|
||||||
|
return selection
|
||||||
|
area = db.get(Area, payload.area_id)
|
||||||
|
if not area:
|
||||||
|
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||||
|
if area.project_id != project_id:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_DATASET_SCOPE",
|
||||||
|
message="Area does not belong to this project",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
selection = selection.intersection(to_shape(area.geometry))
|
||||||
|
if selection.is_empty or selection.area <= 0:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_SELECTION_OUTSIDE_AREA",
|
||||||
|
message="Selection does not overlap the selected work area",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return selection
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def analyze(
|
||||||
|
db,
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
payload: BathymetryRasterSelectionRequest,
|
||||||
|
*,
|
||||||
|
settings: Settings | None = None,
|
||||||
|
) -> dict:
|
||||||
|
resolved_settings = settings or get_settings()
|
||||||
|
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
|
||||||
|
source_metadata = BathymetryRasterAnalysisService._metadata(dataset)
|
||||||
|
selection_4326 = BathymetryRasterAnalysisService._selection_geometry(db, project_id, payload)
|
||||||
|
try:
|
||||||
|
import numpy as np
|
||||||
|
import rasterio
|
||||||
|
from rasterio.mask import mask
|
||||||
|
except ImportError as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="RASTER_PROCESSING_UNAVAILABLE",
|
||||||
|
message="Rasterio and numpy are required for bathymetry analysis",
|
||||||
|
status_code=503,
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
with rasterio.open(dataset.storage_path) as source:
|
||||||
|
if source.crs is None or source.crs.to_epsg() != 3812:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_DATASET_CRS",
|
||||||
|
message="SPW bathymetry raster CRS must be EPSG:3812",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
if source.count != 1:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_BATHYMETRY_RASTER_BANDS",
|
||||||
|
message="SPW bathymetry requires one bed-elevation band",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
|
||||||
|
selection_metric = shapely_transform(transformer.transform, selection_4326)
|
||||||
|
analysis_geometry = selection_metric.intersection(box(*source.bounds))
|
||||||
|
if analysis_geometry.is_empty or analysis_geometry.area <= 0:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_SELECTION_OUTSIDE_DATASET",
|
||||||
|
message="Selection does not overlap the persisted bathymetry raster",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
min_x, min_y, max_x, max_y = analysis_geometry.bounds
|
||||||
|
expected_cells = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil(
|
||||||
|
(max_y - min_y) / abs(source.res[1])
|
||||||
|
)
|
||||||
|
if expected_cells > resolved_settings.bathymetry_raster_max_pixels:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_SELECTION_TOO_LARGE",
|
||||||
|
message="Bathymetry analysis exceeds the configured raster cell limit",
|
||||||
|
details={
|
||||||
|
"pixel_count": expected_cells,
|
||||||
|
"max_pixels": resolved_settings.bathymetry_raster_max_pixels,
|
||||||
|
},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
# ``all_touched`` keeps the values of cells the selection only
|
||||||
|
# clips, so a selection finer than one cell still has data to
|
||||||
|
# read. Which of those cells actually count is decided by
|
||||||
|
# ``select_cells`` below, so the normal result is unchanged.
|
||||||
|
clipped, clipped_transform = mask(
|
||||||
|
source,
|
||||||
|
[mapping(analysis_geometry)],
|
||||||
|
crop=True,
|
||||||
|
filled=False,
|
||||||
|
indexes=[1],
|
||||||
|
all_touched=True,
|
||||||
|
)
|
||||||
|
band = np.ma.asarray(clipped[0], dtype="float64")
|
||||||
|
raw = band.filled(np.nan)
|
||||||
|
cell_selection = select_cells(
|
||||||
|
analysis_geometry,
|
||||||
|
out_shape=band.shape,
|
||||||
|
transform=clipped_transform,
|
||||||
|
cell_area_m2=abs(float(source.res[0])) * abs(float(source.res[1])),
|
||||||
|
)
|
||||||
|
selected_cells = cell_selection.mask
|
||||||
|
valid_cells = selected_cells & ~np.ma.getmaskarray(band) & np.isfinite(raw)
|
||||||
|
if source.nodata is not None:
|
||||||
|
valid_cells &= ~np.isclose(raw, float(source.nodata))
|
||||||
|
values = raw[valid_cells]
|
||||||
|
if values.size == 0:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_NO_VALID_DATA",
|
||||||
|
message="No surveyed waterbed cells occur in this selection",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
resolution_x = abs(float(source.res[0]))
|
||||||
|
resolution_y = abs(float(source.res[1]))
|
||||||
|
cell_area_m2 = resolution_x * resolution_y
|
||||||
|
except AppError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_ANALYSIS_FAILED",
|
||||||
|
message="The persisted bathymetry raster could not be analysed",
|
||||||
|
details={"reason": str(exc)},
|
||||||
|
status_code=500,
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def metric(key: str, label: str, value: float, unit: str, method: str) -> BathymetryRasterMetric:
|
||||||
|
return BathymetryRasterMetric(
|
||||||
|
metric_key=key,
|
||||||
|
metric_label=label,
|
||||||
|
metric_value=round(float(value), 4),
|
||||||
|
metric_unit=unit,
|
||||||
|
aggregation_method=method,
|
||||||
|
)
|
||||||
|
|
||||||
|
selected_cell_count = int(selected_cells.sum())
|
||||||
|
valid_cell_count = int(values.size)
|
||||||
|
vertical_unit = str(source_metadata["vertical_reference"])
|
||||||
|
coverage_ratio = valid_cell_count / max(1, selected_cell_count)
|
||||||
|
metrics = [
|
||||||
|
metric(
|
||||||
|
"bed_elevation_mean_m",
|
||||||
|
"Gemiddelde waterbodemhoogte",
|
||||||
|
values.mean(),
|
||||||
|
f"m {vertical_unit}",
|
||||||
|
"mean_valid_source_cells",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"bed_elevation_min_m",
|
||||||
|
"Laagste waterbodemhoogte",
|
||||||
|
values.min(),
|
||||||
|
f"m {vertical_unit}",
|
||||||
|
"minimum_valid_source_cells",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"bed_elevation_max_m",
|
||||||
|
"Hoogste waterbodemhoogte",
|
||||||
|
values.max(),
|
||||||
|
f"m {vertical_unit}",
|
||||||
|
"maximum_valid_source_cells",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"bed_elevation_p10_m",
|
||||||
|
"10e percentiel waterbodemhoogte",
|
||||||
|
np.percentile(values, 10),
|
||||||
|
f"m {vertical_unit}",
|
||||||
|
"percentile_10_valid_source_cells",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"bed_elevation_p90_m",
|
||||||
|
"90e percentiel waterbodemhoogte",
|
||||||
|
np.percentile(values, 90),
|
||||||
|
f"m {vertical_unit}",
|
||||||
|
"percentile_90_valid_source_cells",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"surveyed_bed_surface_ha",
|
||||||
|
"Oppervlakte met gemeten waterbodem",
|
||||||
|
valid_cell_count * cell_area_m2 / 10_000.0,
|
||||||
|
"ha",
|
||||||
|
"valid_source_cells_times_cell_area",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"bathymetry_coverage_pct",
|
||||||
|
"Dekking waterbodemmeting",
|
||||||
|
coverage_ratio * 100.0,
|
||||||
|
"%",
|
||||||
|
"valid_source_cells_divided_by_selected_cells",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
primary = metrics[0]
|
||||||
|
response = BathymetryRasterSelectionResponse(
|
||||||
|
dataset_id=dataset.id,
|
||||||
|
product_key=BathymetryRasterAnalysisService.PRODUCT_KEY,
|
||||||
|
selection_bbox=payload.bbox,
|
||||||
|
selection_area_id=payload.area_id,
|
||||||
|
selected_cell_count=selected_cell_count,
|
||||||
|
valid_cell_count=valid_cell_count,
|
||||||
|
coverage_ratio=round(coverage_ratio, 6),
|
||||||
|
cell_selection_warning=cell_selection.warning,
|
||||||
|
resolution_m=round(max(resolution_x, resolution_y), 4),
|
||||||
|
vertical_reference=vertical_unit,
|
||||||
|
survey_period=str(source_metadata.get("survey_period") or "2019-2022"),
|
||||||
|
summary=BathymetryRasterSelectionSummary(
|
||||||
|
metric_label=primary.metric_label,
|
||||||
|
metric_value=primary.metric_value,
|
||||||
|
metric_unit=primary.metric_unit,
|
||||||
|
aggregation_method=primary.aggregation_method,
|
||||||
|
primary_metric_key=primary.metric_key,
|
||||||
|
metrics=metrics,
|
||||||
|
),
|
||||||
|
unsupported_metrics=BathymetryRasterAnalysisService.UNSUPPORTED_METRICS,
|
||||||
|
limitation_message=BathymetryRasterAnalysisService.LIMITATION,
|
||||||
|
generated_at=datetime.now(UTC).isoformat(),
|
||||||
|
)
|
||||||
|
return response.model_dump(mode="json")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
|
||||||
|
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
|
||||||
|
BathymetryRasterAnalysisService._metadata(dataset)
|
||||||
|
try:
|
||||||
|
import numpy as np
|
||||||
|
import rasterio
|
||||||
|
from PIL import Image
|
||||||
|
from rasterio.enums import Resampling
|
||||||
|
except ImportError as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="RASTER_PROCESSING_UNAVAILABLE",
|
||||||
|
message="Rasterio, numpy and Pillow are required for bathymetry rendering",
|
||||||
|
status_code=503,
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
with rasterio.open(dataset.storage_path) as source:
|
||||||
|
scale = min(1.0, max_dimension / max(source.width, source.height))
|
||||||
|
width = max(1, round(source.width * scale))
|
||||||
|
height = max(1, round(source.height * scale))
|
||||||
|
data = source.read(
|
||||||
|
1,
|
||||||
|
out_shape=(height, width),
|
||||||
|
masked=True,
|
||||||
|
resampling=Resampling.bilinear,
|
||||||
|
)
|
||||||
|
values = np.asarray(data.filled(np.nan), dtype="float64")
|
||||||
|
valid = np.isfinite(values) & ~np.ma.getmaskarray(data)
|
||||||
|
if source.nodata is not None:
|
||||||
|
valid &= ~np.isclose(values, float(source.nodata))
|
||||||
|
if not valid.any():
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_NO_VALID_DATA",
|
||||||
|
message="Bathymetry raster contains no renderable cells",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
low, high = np.percentile(values[valid], [2, 98])
|
||||||
|
if high <= low:
|
||||||
|
high = low + 1.0
|
||||||
|
normalized = np.clip((values - low) / (high - low), 0.0, 1.0)
|
||||||
|
normalized = np.where(valid, normalized, 0.0)
|
||||||
|
stops = np.asarray([0.0, 0.35, 0.7, 1.0])
|
||||||
|
colors = np.asarray(
|
||||||
|
[
|
||||||
|
[8, 47, 73],
|
||||||
|
[15, 118, 140],
|
||||||
|
[103, 190, 170],
|
||||||
|
[236, 224, 163],
|
||||||
|
],
|
||||||
|
dtype="float64",
|
||||||
|
)
|
||||||
|
rgba = np.zeros((height, width, 4), dtype="uint8")
|
||||||
|
for channel in range(3):
|
||||||
|
rgba[:, :, channel] = np.interp(
|
||||||
|
normalized,
|
||||||
|
stops,
|
||||||
|
colors[:, channel],
|
||||||
|
).astype("uint8")
|
||||||
|
rgba[:, :, 3] = np.where(valid, 220, 0).astype("uint8")
|
||||||
|
output = io.BytesIO()
|
||||||
|
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
|
||||||
|
return output.getvalue()
|
||||||
|
except AppError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PREVIEW_FAILED",
|
||||||
|
message="The persisted bathymetry raster could not be rendered",
|
||||||
|
details={"reason": str(exc)},
|
||||||
|
status_code=500,
|
||||||
|
) from exc
|
||||||
@@ -4,10 +4,12 @@ from datetime import datetime, timezone
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import from_shape, to_shape
|
||||||
from shapely.geometry import mapping
|
from shapely.geometry import mapping
|
||||||
from shapely.geometry.base import BaseGeometry
|
from shapely.geometry.base import BaseGeometry
|
||||||
|
from shapely.strtree import STRtree
|
||||||
from shapely.validation import make_valid
|
from shapely.validation import make_valid
|
||||||
|
from sqlalchemy import func
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
@@ -28,97 +30,271 @@ class ChangeDetectionService:
|
|||||||
target_dataset_id: UUID,
|
target_dataset_id: UUID,
|
||||||
iou_threshold: float = 0.8,
|
iou_threshold: float = 0.8,
|
||||||
include_unchanged: bool = True,
|
include_unchanged: bool = True,
|
||||||
|
modified_threshold: float = 0.3,
|
||||||
|
bbox: dict[str, Any] | None = None,
|
||||||
|
area_id: UUID | None = None,
|
||||||
|
preview_limit: int = 2_000,
|
||||||
) -> ChangeDetectionSummary:
|
) -> ChangeDetectionSummary:
|
||||||
if source_dataset_id == target_dataset_id:
|
if source_dataset_id == target_dataset_id:
|
||||||
raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400)
|
raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400)
|
||||||
if iou_threshold < 0 or iou_threshold > 1:
|
if iou_threshold < 0 or iou_threshold > 1:
|
||||||
raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400)
|
raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400)
|
||||||
|
if modified_threshold < 0 or modified_threshold > iou_threshold:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_PARAMETERS",
|
||||||
|
message="modified_threshold must be between 0 and iou_threshold",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
|
||||||
source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source")
|
source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source")
|
||||||
target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target")
|
target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target")
|
||||||
|
|
||||||
source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset)
|
selection_geometry = ChangeDetectionService._selection_geometry(db, project_id, bbox=bbox, area_id=area_id)
|
||||||
target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset)
|
|
||||||
|
source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset, selection_geometry)
|
||||||
|
target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset, selection_geometry)
|
||||||
|
|
||||||
if not source_features:
|
if not source_features:
|
||||||
raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422)
|
raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422)
|
||||||
if not target_features:
|
if not target_features:
|
||||||
raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422)
|
raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422)
|
||||||
|
|
||||||
matched_target_indices: set[int] = set()
|
source_features = ChangeDetectionService.restrict_to_selection(source_features, selection_geometry, label="Source")
|
||||||
unchanged: list[dict[str, Any]] = []
|
target_features = ChangeDetectionService.restrict_to_selection(target_features, selection_geometry, label="Target")
|
||||||
removed: list[dict[str, Any]] = []
|
|
||||||
|
|
||||||
for source_feature in source_features:
|
classified = ChangeDetectionService._classify_features(
|
||||||
best_iou = 0.0
|
source_features,
|
||||||
best_index: int | None = None
|
target_features,
|
||||||
for target_index, target_feature in enumerate(target_features):
|
iou_threshold=iou_threshold,
|
||||||
if target_index in matched_target_indices:
|
modified_threshold=modified_threshold,
|
||||||
continue
|
)
|
||||||
candidate_iou = ChangeDetectionService._iou(source_feature["geometry"], target_feature["geometry"])
|
|
||||||
if candidate_iou > best_iou:
|
|
||||||
best_iou = candidate_iou
|
|
||||||
best_index = target_index
|
|
||||||
|
|
||||||
if best_index is not None and best_iou >= iou_threshold:
|
buckets: dict[str, list[dict[str, Any]]] = {"added": [], "removed": [], "modified": [], "unchanged": []}
|
||||||
matched_target_indices.add(best_index)
|
for item in classified:
|
||||||
if include_unchanged:
|
buckets[item["change_type"]].append(
|
||||||
unchanged.append(
|
|
||||||
ChangeDetectionService._feature(
|
ChangeDetectionService._feature(
|
||||||
geometry=source_feature["geometry"],
|
geometry=item["geometry"],
|
||||||
change_type="unchanged",
|
change_type=item["change_type"],
|
||||||
source_dataset_id=source_dataset_id,
|
source_dataset_id=source_dataset_id,
|
||||||
target_dataset_id=target_dataset_id,
|
target_dataset_id=target_dataset_id,
|
||||||
source_feature_id=source_feature["feature_id"],
|
source_feature_id=item["source_feature_id"],
|
||||||
target_feature_id=target_features[best_index]["feature_id"],
|
target_feature_id=item["target_feature_id"],
|
||||||
iou=best_iou,
|
iou=item["iou"],
|
||||||
properties=source_feature["properties"],
|
properties=item["properties"],
|
||||||
)
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
removed.append(
|
|
||||||
ChangeDetectionService._feature(
|
|
||||||
geometry=source_feature["geometry"],
|
|
||||||
change_type="removed",
|
|
||||||
source_dataset_id=source_dataset_id,
|
|
||||||
target_dataset_id=target_dataset_id,
|
|
||||||
source_feature_id=source_feature["feature_id"],
|
|
||||||
target_feature_id=None,
|
|
||||||
iou=best_iou if best_iou > 0 else None,
|
|
||||||
properties=source_feature["properties"],
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
added = [
|
unchanged_count = len(buckets["unchanged"])
|
||||||
ChangeDetectionService._feature(
|
if not include_unchanged:
|
||||||
geometry=target_feature["geometry"],
|
buckets["unchanged"] = []
|
||||||
change_type="added",
|
|
||||||
source_dataset_id=source_dataset_id,
|
|
||||||
target_dataset_id=target_dataset_id,
|
|
||||||
source_feature_id=None,
|
|
||||||
target_feature_id=target_feature["feature_id"],
|
|
||||||
iou=None,
|
|
||||||
properties=target_feature["properties"],
|
|
||||||
)
|
|
||||||
for target_index, target_feature in enumerate(target_features)
|
|
||||||
if target_index not in matched_target_indices
|
|
||||||
]
|
|
||||||
|
|
||||||
geojson_features = added + removed + unchanged
|
geojson_features, preview_truncated = ChangeDetectionService.limit_preview(
|
||||||
|
buckets["added"] + buckets["removed"] + buckets["modified"] + buckets["unchanged"],
|
||||||
|
limit=preview_limit,
|
||||||
|
)
|
||||||
|
warnings = source_warnings + target_warnings
|
||||||
|
edge_count = sum(
|
||||||
|
1 for feature in source_features + target_features if feature.get("partially_covered")
|
||||||
|
)
|
||||||
|
if edge_count:
|
||||||
|
warnings.append(
|
||||||
|
f"{edge_count} objecten liggen deels buiten de selectie. Ze zijn volledig vergeleken, zodat de "
|
||||||
|
"selectierand zelf geen wijziging veroorzaakt."
|
||||||
|
)
|
||||||
|
if preview_truncated:
|
||||||
|
warnings.append(
|
||||||
|
f"De tellingen gelden voor de volledige selectie; de kaart toont maximaal {preview_limit} objecten, "
|
||||||
|
"wijzigingen eerst."
|
||||||
|
)
|
||||||
return ChangeDetectionSummary(
|
return ChangeDetectionSummary(
|
||||||
source_dataset_id=source_dataset_id,
|
source_dataset_id=source_dataset_id,
|
||||||
target_dataset_id=target_dataset_id,
|
target_dataset_id=target_dataset_id,
|
||||||
source_feature_count=len(source_features),
|
source_feature_count=len(source_features),
|
||||||
target_feature_count=len(target_features),
|
target_feature_count=len(target_features),
|
||||||
added_count=len(added),
|
added_count=len(buckets["added"]),
|
||||||
removed_count=len(removed),
|
removed_count=len(buckets["removed"]),
|
||||||
unchanged_count=len(unchanged) if include_unchanged else len(matched_target_indices),
|
modified_count=len(buckets["modified"]),
|
||||||
|
unchanged_count=unchanged_count,
|
||||||
iou_threshold=iou_threshold,
|
iou_threshold=iou_threshold,
|
||||||
warnings=source_warnings + target_warnings,
|
modified_iou_threshold=modified_threshold,
|
||||||
|
selection_area_id=area_id,
|
||||||
|
preview_limit=preview_limit,
|
||||||
|
preview_truncated=preview_truncated,
|
||||||
|
warnings=warnings,
|
||||||
generated_at=datetime.now(timezone.utc),
|
generated_at=datetime.now(timezone.utc),
|
||||||
geojson={"type": "FeatureCollection", "features": geojson_features},
|
geojson={"type": "FeatureCollection", "features": geojson_features},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _selection_geometry(
|
||||||
|
db: Session,
|
||||||
|
project_id: UUID,
|
||||||
|
*,
|
||||||
|
bbox: dict[str, Any] | None,
|
||||||
|
area_id: UUID | None,
|
||||||
|
) -> BaseGeometry | None:
|
||||||
|
"""Resolve the drawn rectangle against the named work area, if any."""
|
||||||
|
|
||||||
|
from app.models import Area
|
||||||
|
from shapely.geometry import box as shapely_box
|
||||||
|
|
||||||
|
selection = None
|
||||||
|
if bbox:
|
||||||
|
selection = shapely_box(
|
||||||
|
float(bbox["min_x"]), float(bbox["min_y"]), float(bbox["max_x"]), float(bbox["max_y"])
|
||||||
|
)
|
||||||
|
if area_id is None:
|
||||||
|
return selection
|
||||||
|
|
||||||
|
area = db.get(Area, area_id)
|
||||||
|
if area is None or area.project_id != project_id:
|
||||||
|
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||||
|
area_geometry = to_shape(area.geometry)
|
||||||
|
if selection is None:
|
||||||
|
return area_geometry
|
||||||
|
intersection = selection.intersection(area_geometry)
|
||||||
|
if intersection.is_empty or intersection.area <= 0:
|
||||||
|
raise AppError(
|
||||||
|
code="CHANGE_DETECTION_SELECTION_OUTSIDE_AREA",
|
||||||
|
message="Selection does not overlap the selected work area",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return intersection
|
||||||
|
|
||||||
|
# Order the preview spends its budget in. An operator asking what changed
|
||||||
|
# is not helped by a cap filled with unchanged footprints.
|
||||||
|
PREVIEW_PRIORITY = {"modified": 0, "added": 1, "removed": 2, "unchanged": 3}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def restrict_to_selection(
|
||||||
|
features: list[dict[str, Any]],
|
||||||
|
selection_geometry: BaseGeometry | None,
|
||||||
|
*,
|
||||||
|
label: str = "Dataset",
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Keep the features a drawn selection reaches, and say which it cuts.
|
||||||
|
|
||||||
|
Geometry is deliberately *not* clipped. A change class describes a whole
|
||||||
|
object: comparing a clipped 2020 footprint against an unclipped 2024 one
|
||||||
|
would manufacture "modified" along the selection edge. Clipping is right
|
||||||
|
for an area metric and wrong for an identity comparison.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if selection_geometry is None:
|
||||||
|
return features
|
||||||
|
|
||||||
|
kept: list[dict[str, Any]] = []
|
||||||
|
for feature in features:
|
||||||
|
geometry = feature["geometry"]
|
||||||
|
if not geometry.intersects(selection_geometry):
|
||||||
|
continue
|
||||||
|
kept.append({**feature, "partially_covered": not selection_geometry.covers(geometry)})
|
||||||
|
|
||||||
|
if not kept:
|
||||||
|
raise AppError(
|
||||||
|
code="CHANGE_DETECTION_SELECTION_EMPTY",
|
||||||
|
message=f"{label} dataset has no features inside this selection",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return kept
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def limit_preview(
|
||||||
|
features: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
limit: int,
|
||||||
|
) -> tuple[list[dict[str, Any]], bool]:
|
||||||
|
"""Cap the returned geometry without capping the counts.
|
||||||
|
|
||||||
|
``include_unchanged`` defaulted to true and nothing bounded the result,
|
||||||
|
so a regional comparison returned a FeatureCollection holding both
|
||||||
|
datasets in full. The counts describe the whole selection; the preview
|
||||||
|
describes what a map can usefully draw.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if limit <= 0 or len(features) <= limit:
|
||||||
|
return features, False
|
||||||
|
ordered = sorted(
|
||||||
|
features,
|
||||||
|
key=lambda item: ChangeDetectionService.PREVIEW_PRIORITY.get(item["change_type"], 9),
|
||||||
|
)
|
||||||
|
return ordered[:limit], True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _classify_features(
|
||||||
|
source_features: list[dict[str, Any]],
|
||||||
|
target_features: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
iou_threshold: float,
|
||||||
|
modified_threshold: float,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Pair source with target footprints and label how each one changed.
|
||||||
|
|
||||||
|
Matching is indexed rather than a full cross product: comparing two
|
||||||
|
municipal building layers is otherwise hundreds of millions of geometry
|
||||||
|
intersections. Sources are considered largest first so a big footprint
|
||||||
|
is not left over after a small neighbour claimed its counterpart.
|
||||||
|
"""
|
||||||
|
|
||||||
|
target_geometries = [feature["geometry"] for feature in target_features]
|
||||||
|
tree = STRtree(target_geometries) if target_geometries else None
|
||||||
|
claimed: set[int] = set()
|
||||||
|
classified: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
order = sorted(
|
||||||
|
range(len(source_features)),
|
||||||
|
key=lambda index: (-source_features[index]["geometry"].area, str(source_features[index]["feature_id"])),
|
||||||
|
)
|
||||||
|
for source_index in order:
|
||||||
|
source_feature = source_features[source_index]
|
||||||
|
geometry = source_feature["geometry"]
|
||||||
|
best_iou = 0.0
|
||||||
|
best_index: int | None = None
|
||||||
|
candidates = [] if tree is None else sorted(int(value) for value in tree.query(geometry))
|
||||||
|
for target_index in candidates:
|
||||||
|
if target_index in claimed:
|
||||||
|
continue
|
||||||
|
candidate_iou = ChangeDetectionService._iou(geometry, target_geometries[target_index])
|
||||||
|
if candidate_iou > best_iou:
|
||||||
|
best_iou = candidate_iou
|
||||||
|
best_index = target_index
|
||||||
|
|
||||||
|
if best_index is not None and best_iou >= iou_threshold:
|
||||||
|
claimed.add(best_index)
|
||||||
|
change_type = "unchanged"
|
||||||
|
elif best_index is not None and best_iou >= modified_threshold:
|
||||||
|
# The same object, redrawn: an annexe, a demolition of one wing,
|
||||||
|
# or a resurvey. Reporting it as removed + added would hide it.
|
||||||
|
claimed.add(best_index)
|
||||||
|
change_type = "modified"
|
||||||
|
else:
|
||||||
|
change_type = "removed"
|
||||||
|
|
||||||
|
classified.append(
|
||||||
|
{
|
||||||
|
"change_type": change_type,
|
||||||
|
"geometry": geometry if change_type != "modified" else target_geometries[best_index],
|
||||||
|
"source_feature_id": source_feature["feature_id"],
|
||||||
|
"target_feature_id": target_features[best_index]["feature_id"] if change_type != "removed" else None,
|
||||||
|
"iou": best_iou if best_iou > 0 else None,
|
||||||
|
"properties": source_feature["properties"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
classified.extend(
|
||||||
|
{
|
||||||
|
"change_type": "added",
|
||||||
|
"geometry": target_feature["geometry"],
|
||||||
|
"source_feature_id": None,
|
||||||
|
"target_feature_id": target_feature["feature_id"],
|
||||||
|
"iou": None,
|
||||||
|
"properties": target_feature["properties"],
|
||||||
|
}
|
||||||
|
for target_index, target_feature in enumerate(target_features)
|
||||||
|
if target_index not in claimed
|
||||||
|
)
|
||||||
|
return classified
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset:
|
def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset:
|
||||||
dataset = db.get(Dataset, dataset_id)
|
dataset = db.get(Dataset, dataset_id)
|
||||||
@@ -130,8 +306,25 @@ class ChangeDetectionService:
|
|||||||
return dataset
|
return dataset
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _load_features(db: Session, dataset: Dataset) -> tuple[list[dict[str, Any]], list[str]]:
|
def _load_features(
|
||||||
rows = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id).all()
|
db: Session,
|
||||||
|
dataset: Dataset,
|
||||||
|
selection_geometry: BaseGeometry | None = None,
|
||||||
|
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||||
|
query = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id)
|
||||||
|
if selection_geometry is not None and hasattr(query, "filter"):
|
||||||
|
# Bound the load in the database. Pulling a regional building layer
|
||||||
|
# into Python to then discard most of it costs memory and time for
|
||||||
|
# nothing, and the fallback below has no such option.
|
||||||
|
try:
|
||||||
|
query = query.filter(
|
||||||
|
func.ST_Intersects(VectorFeature.geometry, from_shape(selection_geometry, srid=4326))
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Lightweight unit-test sessions do not implement every spatial
|
||||||
|
# predicate; restrict_to_selection still bounds the population.
|
||||||
|
pass
|
||||||
|
rows = query.all()
|
||||||
warnings: list[str] = []
|
warnings: list[str] = []
|
||||||
if rows:
|
if rows:
|
||||||
return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings
|
return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Dataset, Project
|
from app.models import Area, Dataset, Project
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.schemas.coverage import (
|
from app.schemas.coverage import (
|
||||||
CoverageBBox,
|
CoverageBBox,
|
||||||
CoverageCatalogResponse,
|
CoverageCatalogResponse,
|
||||||
@@ -79,6 +80,7 @@ class _SourceDefinition:
|
|||||||
contract: CoverageSourceContract
|
contract: CoverageSourceContract
|
||||||
materialized_layer_names: tuple[str, ...] = ()
|
materialized_layer_names: tuple[str, ...] = ()
|
||||||
materialized_source_names: tuple[str, ...] = ()
|
materialized_source_names: tuple[str, ...] = ()
|
||||||
|
operational_themes: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
def _contract(
|
def _contract(
|
||||||
@@ -98,6 +100,7 @@ def _contract(
|
|||||||
limitation_message: str,
|
limitation_message: str,
|
||||||
materialized_layer_names: tuple[str, ...] = (),
|
materialized_layer_names: tuple[str, ...] = (),
|
||||||
materialized_source_names: tuple[str, ...] = (),
|
materialized_source_names: tuple[str, ...] = (),
|
||||||
|
operational_themes: tuple[str, ...] = (),
|
||||||
) -> _SourceDefinition:
|
) -> _SourceDefinition:
|
||||||
return _SourceDefinition(
|
return _SourceDefinition(
|
||||||
contract=CoverageSourceContract(
|
contract=CoverageSourceContract(
|
||||||
@@ -117,6 +120,7 @@ def _contract(
|
|||||||
),
|
),
|
||||||
materialized_layer_names=materialized_layer_names,
|
materialized_layer_names=materialized_layer_names,
|
||||||
materialized_source_names=materialized_source_names or (source_name,),
|
materialized_source_names=materialized_source_names or (source_name,),
|
||||||
|
operational_themes=operational_themes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -156,12 +160,17 @@ SOURCE_DEFINITIONS = (
|
|||||||
themes=("admin", "population"),
|
themes=("admin", "population"),
|
||||||
native_layers=("statistical_sectors", "population_statistics"),
|
native_layers=("statistical_sectors", "population_statistics"),
|
||||||
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
|
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
|
||||||
acquisition_mode="catalog_only",
|
acquisition_mode="operator_archive",
|
||||||
integration_status="not_configured",
|
integration_status="operational",
|
||||||
source_url="https://statbel.fgov.be/en/open-data",
|
source_url="https://statbel.fgov.be/en/open-data",
|
||||||
attribution="Statbel",
|
attribution="Statbel",
|
||||||
license_note="Consult the license of the selected Statbel release.",
|
license_note="Consult the license of the selected Statbel release.",
|
||||||
limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.",
|
limitation_message=(
|
||||||
|
"National editions require the governed plan-stage-review-apply operator; "
|
||||||
|
"population in partially selected sectors is area-weighted."
|
||||||
|
),
|
||||||
|
materialized_layer_names=("population",),
|
||||||
|
operational_themes=("population",),
|
||||||
),
|
),
|
||||||
_contract(
|
_contract(
|
||||||
source_name="digitaal_vlaanderen",
|
source_name="digitaal_vlaanderen",
|
||||||
@@ -200,6 +209,25 @@ SOURCE_DEFINITIONS = (
|
|||||||
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
_contract(
|
||||||
|
source_name="vmm_vha_bathymetry_profiles",
|
||||||
|
display_name="VHA historische dwarsprofielen",
|
||||||
|
authority_level="authoritative",
|
||||||
|
coverage_zones=("flanders",),
|
||||||
|
themes=("bathymetry",),
|
||||||
|
native_layers=("digitale_atlas_profile_points",),
|
||||||
|
geometry_types=("Point",),
|
||||||
|
acquisition_mode="bounded_api",
|
||||||
|
integration_status="operational",
|
||||||
|
source_url="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
|
||||||
|
attribution="Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas",
|
||||||
|
license_note="Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata.",
|
||||||
|
limitation_message=(
|
||||||
|
"Historische puntmetingen met bronafhankelijke meetdatum en verticale referentie; "
|
||||||
|
"geen continue actuele bodemkaart en zonder gelijktijdig waterpeil geen watervolume."
|
||||||
|
),
|
||||||
|
materialized_source_names=("vmm_vha_bathymetry_profiles",),
|
||||||
|
),
|
||||||
_contract(
|
_contract(
|
||||||
source_name="spw_geoportail",
|
source_name="spw_geoportail",
|
||||||
display_name="SPW Geoportail Wallonie",
|
display_name="SPW Geoportail Wallonie",
|
||||||
@@ -219,12 +247,17 @@ SOURCE_DEFINITIONS = (
|
|||||||
),
|
),
|
||||||
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
|
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
|
||||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||||
acquisition_mode="catalog_only",
|
acquisition_mode="bounded_api",
|
||||||
integration_status="not_configured",
|
integration_status="operational",
|
||||||
source_url="https://geoportail.wallonie.be/catalogue",
|
source_url="https://geoportail.wallonie.be/catalogue",
|
||||||
attribution="Service public de Wallonie",
|
attribution="Service public de Wallonie",
|
||||||
license_note="Consult the license of each Geoportail Wallonie product.",
|
license_note="Consult the license of each Geoportail Wallonie product.",
|
||||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
limitation_message=(
|
||||||
|
"Bounded PICC buildings, road axes and hydrography, the legally current flood-hazard polygons, "
|
||||||
|
"operator-imported SPW bathymetry and bounded SPW MNT terrain are operational; other Walloon themes remain separately governed."
|
||||||
|
),
|
||||||
|
materialized_source_names=("spw_picc", "spw_flood_hazard", "spw_walous_land_cover", "spw_bathymetry", "spw_terrain"),
|
||||||
|
operational_themes=("buildings", "roads", "surface_water", "land_cover_use", "elevation", "flood_climate", "bathymetry"),
|
||||||
),
|
),
|
||||||
_contract(
|
_contract(
|
||||||
source_name="urbis",
|
source_name="urbis",
|
||||||
@@ -234,12 +267,17 @@ SOURCE_DEFINITIONS = (
|
|||||||
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
|
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
|
||||||
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
|
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
|
||||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||||
acquisition_mode="catalog_only",
|
acquisition_mode="bounded_api",
|
||||||
integration_status="not_configured",
|
integration_status="operational",
|
||||||
source_url="https://datastore.brussels",
|
source_url="https://datastore.brussels",
|
||||||
attribution="Brussels UrbIS",
|
attribution="Brussels UrbIS",
|
||||||
license_note="Consult the license of the selected UrbIS dataset.",
|
license_note="Consult the license of the selected UrbIS dataset.",
|
||||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
limitation_message=(
|
||||||
|
"Bounded UrbIS buildings, cadastral parcels, street axes and Land Cover blocks are operational. "
|
||||||
|
"Permanent water uses the official WB block class; no separate hydrography network is inferred."
|
||||||
|
),
|
||||||
|
materialized_source_names=("urbis",),
|
||||||
|
operational_themes=("buildings", "parcels", "roads", "surface_water", "land_cover_use"),
|
||||||
),
|
),
|
||||||
_contract(
|
_contract(
|
||||||
source_name="rbins_marine_reporting_units",
|
source_name="rbins_marine_reporting_units",
|
||||||
@@ -304,7 +342,10 @@ SOURCE_DEFINITIONS = (
|
|||||||
source_url="https://www.vlaanderen.be/datavindplaats",
|
source_url="https://www.vlaanderen.be/datavindplaats",
|
||||||
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
|
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
|
||||||
license_note="Consult the official product license before acquisition.",
|
license_note="Consult the official product license before acquisition.",
|
||||||
limitation_message="Strict-TLS acquisition and vertical datum evidence are not yet sufficient; no depths are synthesized.",
|
limitation_message=(
|
||||||
|
"Bounded strict-TLS WCS acquisition is implemented but stays disabled until the operator enables it "
|
||||||
|
"with a live-validated coverage id; no depths are synthesized."
|
||||||
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -330,6 +371,27 @@ FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
|
|||||||
"flood_climate": {"vmm_flood_hazard": ()},
|
"flood_climate": {"vmm_flood_hazard": ()},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = {
|
||||||
|
"spw_geoportail": {
|
||||||
|
"buildings": {"spw_picc": ("buildings",)},
|
||||||
|
"roads": {"spw_picc": ("roads",)},
|
||||||
|
"surface_water": {"spw_picc": ("water",)},
|
||||||
|
"land_cover_use": {"spw_walous_land_cover": ()},
|
||||||
|
"elevation": {"spw_terrain": ()},
|
||||||
|
"flood_climate": {"spw_flood_hazard": ("flood_hazard",)},
|
||||||
|
"bathymetry": {"spw_bathymetry": ()},
|
||||||
|
"orthophoto": {"spw_orthophoto": ()},
|
||||||
|
},
|
||||||
|
"urbis": {
|
||||||
|
"buildings": {"urbis": ("buildings",)},
|
||||||
|
"parcels": {"urbis": ("parcels",)},
|
||||||
|
"roads": {"urbis": ("roads",)},
|
||||||
|
"surface_water": {"urbis": ("water",)},
|
||||||
|
"land_cover_use": {"urbis": ("space_occupation", "forest")},
|
||||||
|
"orthophoto": {"urbis_orthophoto": ()},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class CoverageRegistryService:
|
class CoverageRegistryService:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -392,26 +454,60 @@ class CoverageRegistryService:
|
|||||||
definition: _SourceDefinition,
|
definition: _SourceDefinition,
|
||||||
theme: str,
|
theme: str,
|
||||||
zone: str,
|
zone: str,
|
||||||
) -> list[Dataset]:
|
selection: Any,
|
||||||
|
) -> tuple[list[Dataset], bool]:
|
||||||
|
if definition.operational_themes and theme not in definition.operational_themes:
|
||||||
|
return [], False
|
||||||
matches: list[Dataset] = []
|
matches: list[Dataset] = []
|
||||||
|
bounded_scopes: list[Any] = []
|
||||||
|
zone_scoped_materialization = False
|
||||||
for dataset in datasets:
|
for dataset in datasets:
|
||||||
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
||||||
continue
|
continue
|
||||||
|
# A source-name claim alone must not cause an unsafe artifact to
|
||||||
|
# appear as operational authoritative coverage.
|
||||||
|
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
|
||||||
|
continue
|
||||||
layer_names = definition.materialized_layer_names
|
layer_names = definition.materialized_layer_names
|
||||||
if definition.contract.source_name == "digitaal_vlaanderen":
|
if definition.contract.source_name == "digitaal_vlaanderen":
|
||||||
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
||||||
if dataset.source_name not in theme_sources:
|
if dataset.source_name not in theme_sources:
|
||||||
continue
|
continue
|
||||||
layer_names = theme_sources[dataset.source_name]
|
layer_names = theme_sources[dataset.source_name]
|
||||||
|
elif definition.contract.source_name in REGIONAL_THEME_DATASETS:
|
||||||
|
theme_sources = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme, {})
|
||||||
|
if dataset.source_name not in theme_sources:
|
||||||
|
continue
|
||||||
|
layer_names = theme_sources[dataset.source_name]
|
||||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||||
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
|
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
|
||||||
if isinstance(coverage_zones, str):
|
if isinstance(coverage_zones, str):
|
||||||
coverage_zones = [coverage_zones]
|
coverage_zones = [coverage_zones]
|
||||||
|
acquired_bbox = metadata.get("bbox_epsg4326")
|
||||||
|
if (
|
||||||
|
definition.contract.acquisition_mode == "bounded_api"
|
||||||
|
and isinstance(acquired_bbox, list)
|
||||||
|
and len(acquired_bbox) == 4
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
acquired_scope = box(*(float(value) for value in acquired_bbox))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if not acquired_scope.is_valid or not acquired_scope.intersects(selection):
|
||||||
|
continue
|
||||||
|
bounded_scopes.append(acquired_scope)
|
||||||
|
elif definition.contract.acquisition_mode == "bounded_api" and coverage_zones:
|
||||||
|
zone_scoped_materialization = zone in coverage_zones or "belgium" in coverage_zones
|
||||||
layer_matches = not layer_names or dataset.reference_layer_name in layer_names
|
layer_matches = not layer_names or dataset.reference_layer_name in layer_names
|
||||||
zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones
|
zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones
|
||||||
if layer_matches and zone_matches:
|
if layer_matches and zone_matches:
|
||||||
matches.append(dataset)
|
matches.append(dataset)
|
||||||
return matches
|
if not matches:
|
||||||
|
return [], False
|
||||||
|
fully_covered = True
|
||||||
|
if definition.contract.acquisition_mode == "bounded_api":
|
||||||
|
fully_covered = zone_scoped_materialization or (bool(bounded_scopes) and unary_union(bounded_scopes).covers(selection))
|
||||||
|
return matches, fully_covered
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _resolve_item(
|
def _resolve_item(
|
||||||
@@ -419,6 +515,7 @@ class CoverageRegistryService:
|
|||||||
zone: str,
|
zone: str,
|
||||||
theme: str,
|
theme: str,
|
||||||
datasets: list[Dataset],
|
datasets: list[Dataset],
|
||||||
|
selection: Any,
|
||||||
) -> CoverageResolutionItem:
|
) -> CoverageResolutionItem:
|
||||||
definitions = [
|
definitions = [
|
||||||
definition
|
definition
|
||||||
@@ -436,22 +533,51 @@ class CoverageRegistryService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
materialized: list[Dataset] = []
|
materialized: list[Dataset] = []
|
||||||
|
evidence: list[dict[str, Any]] = []
|
||||||
source_statuses: list[str] = []
|
source_statuses: list[str] = []
|
||||||
limitations: list[str] = []
|
limitations: list[str] = []
|
||||||
for definition in definitions:
|
for definition in definitions:
|
||||||
matches = CoverageRegistryService._matching_datasets(
|
matches, fully_covered = CoverageRegistryService._matching_datasets(
|
||||||
datasets,
|
datasets,
|
||||||
definition,
|
definition,
|
||||||
theme,
|
theme,
|
||||||
zone,
|
zone,
|
||||||
|
selection,
|
||||||
)
|
)
|
||||||
materialized.extend(matches)
|
materialized.extend(matches)
|
||||||
if matches:
|
for dataset in matches:
|
||||||
|
metadata = dataset.source_metadata if isinstance(getattr(dataset, "source_metadata", None), dict) else {}
|
||||||
|
observed_at = getattr(dataset, "observed_at", None)
|
||||||
|
published_at = metadata.get("published_at") or metadata.get("publication_date") or metadata.get("published_on")
|
||||||
|
evidence.append({
|
||||||
|
"dataset_id": dataset.id,
|
||||||
|
"source_name": str(dataset.source_name or definition.contract.source_name),
|
||||||
|
"authority_level": definition.contract.authority_level,
|
||||||
|
"source_version": getattr(dataset, "source_version", None),
|
||||||
|
"observed_at": observed_at.isoformat() if hasattr(observed_at, "isoformat") else (str(observed_at) if observed_at else None),
|
||||||
|
"published_at": str(published_at) if published_at else None,
|
||||||
|
"crs": getattr(dataset, "crs", None) or metadata.get("source_crs"),
|
||||||
|
"resolution": getattr(dataset, "resolution_json", None),
|
||||||
|
"coverage_bbox_epsg4326": metadata.get("bbox_epsg4326"),
|
||||||
|
"attribution": metadata.get("attribution") or definition.contract.attribution,
|
||||||
|
"license_note": metadata.get("license_note") or definition.contract.license_note,
|
||||||
|
"checksum_sha256": getattr(dataset, "checksum_sha256", None),
|
||||||
|
})
|
||||||
|
if matches and fully_covered:
|
||||||
source_statuses.append("operational")
|
source_statuses.append("operational")
|
||||||
elif definition.contract.integration_status == "operational":
|
elif matches:
|
||||||
|
source_statuses.append("partial")
|
||||||
|
elif (
|
||||||
|
definition.contract.integration_status == "operational"
|
||||||
|
and (not definition.operational_themes or theme in definition.operational_themes)
|
||||||
|
):
|
||||||
source_statuses.append("partial")
|
source_statuses.append("partial")
|
||||||
else:
|
else:
|
||||||
source_statuses.append(definition.contract.integration_status)
|
source_statuses.append(
|
||||||
|
"not_configured"
|
||||||
|
if definition.contract.integration_status == "operational"
|
||||||
|
else definition.contract.integration_status
|
||||||
|
)
|
||||||
limitations.append(definition.contract.limitation_message)
|
limitations.append(definition.contract.limitation_message)
|
||||||
|
|
||||||
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
|
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
|
||||||
@@ -461,6 +587,7 @@ class CoverageRegistryService:
|
|||||||
status=best_status,
|
status=best_status,
|
||||||
source_names=[definition.contract.source_name for definition in definitions],
|
source_names=[definition.contract.source_name for definition in definitions],
|
||||||
materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)),
|
materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)),
|
||||||
|
evidence=list({str(item["dataset_id"]): item for item in evidence}.values()),
|
||||||
limitation_message=" ".join(dict.fromkeys(limitations)),
|
limitation_message=" ".join(dict.fromkeys(limitations)),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -491,7 +618,12 @@ class CoverageRegistryService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
items = [
|
items = [
|
||||||
CoverageRegistryService._resolve_item(zone=zone, theme=theme, datasets=datasets)
|
CoverageRegistryService._resolve_item(
|
||||||
|
zone=zone,
|
||||||
|
theme=theme,
|
||||||
|
datasets=datasets,
|
||||||
|
selection=selection,
|
||||||
|
)
|
||||||
for zone in zones
|
for zone in zones
|
||||||
for theme in requested_themes
|
for theme in requested_themes
|
||||||
]
|
]
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,160 @@
|
|||||||
|
"""Fail-closed quarantine decisions for validated data assets.
|
||||||
|
|
||||||
|
Persistence is intentionally delegated to the caller's transaction. This
|
||||||
|
module derives stable decisions from immutable validation reports and blocks a
|
||||||
|
quarantined or failed asset from training, production inference and derived
|
||||||
|
processing until an explicit, separately persisted release action exists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import StrEnum
|
||||||
|
from hashlib import sha256
|
||||||
|
from typing import Any
|
||||||
|
import json
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.services.data_contract_validation import QuarantineStatus, ValidationReport, ValidationStatus
|
||||||
|
|
||||||
|
|
||||||
|
class AssetUse(StrEnum):
|
||||||
|
TRAINING = "training"
|
||||||
|
PRODUCTION_INFERENCE = "production_inference"
|
||||||
|
DERIVED_PROCESSING = "derived_processing"
|
||||||
|
EXPORT = "export"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class QuarantineDecision:
|
||||||
|
"""A deterministic, auditable quarantine decision for one report."""
|
||||||
|
|
||||||
|
asset_id: str
|
||||||
|
data_contract_key: str
|
||||||
|
data_contract_version: str
|
||||||
|
validation_report_sha256: str
|
||||||
|
validation_status: ValidationStatus
|
||||||
|
quarantine_status: QuarantineStatus
|
||||||
|
reason_codes: tuple[str, ...]
|
||||||
|
idempotency_key: str
|
||||||
|
requires_explicit_release: bool = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def eligible_for_use(self) -> bool:
|
||||||
|
return self.validation_status == ValidationStatus.PASSED and self.quarantine_status == QuarantineStatus.NOT_QUARANTINED
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"asset_id": self.asset_id,
|
||||||
|
"data_contract_key": self.data_contract_key,
|
||||||
|
"data_contract_version": self.data_contract_version,
|
||||||
|
"validation_report_sha256": self.validation_report_sha256,
|
||||||
|
"validation_status": self.validation_status.value,
|
||||||
|
"quarantine_status": self.quarantine_status.value,
|
||||||
|
"reason_codes": list(self.reason_codes),
|
||||||
|
"idempotency_key": self.idempotency_key,
|
||||||
|
"requires_explicit_release": self.requires_explicit_release,
|
||||||
|
"eligible_for_use": self.eligible_for_use,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class DataQuarantineService:
|
||||||
|
"""Derive and enforce safe use decisions from validation results."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def decide(
|
||||||
|
report: ValidationReport,
|
||||||
|
*,
|
||||||
|
previous: QuarantineDecision | None = None,
|
||||||
|
) -> QuarantineDecision:
|
||||||
|
"""Create a stable decision without silently releasing old quarantines.
|
||||||
|
|
||||||
|
A fresh passing validation report can be persisted as a new validated
|
||||||
|
version by the import transaction. It cannot automatically release an
|
||||||
|
existing quarantined record: the caller must explicitly record that
|
||||||
|
reviewed state transition against the new report/version.
|
||||||
|
"""
|
||||||
|
|
||||||
|
is_quarantined = report.validation_status == ValidationStatus.FAILED or report.quarantine_status == QuarantineStatus.QUARANTINED
|
||||||
|
failure_codes = tuple(
|
||||||
|
sorted(
|
||||||
|
{
|
||||||
|
issue.code
|
||||||
|
for issue in report.issues
|
||||||
|
if issue.severity.value == "error" or is_quarantined
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
requires_explicit_release = False
|
||||||
|
reason_codes = failure_codes
|
||||||
|
status = QuarantineStatus.QUARANTINED if is_quarantined else QuarantineStatus.NOT_QUARANTINED
|
||||||
|
|
||||||
|
if previous is not None and previous.quarantine_status == QuarantineStatus.QUARANTINED and not is_quarantined:
|
||||||
|
status = QuarantineStatus.QUARANTINED
|
||||||
|
requires_explicit_release = True
|
||||||
|
reason_codes = ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
|
||||||
|
|
||||||
|
idempotency_key = _decision_key(
|
||||||
|
asset_id=report.asset_id,
|
||||||
|
contract_key=report.data_contract_key,
|
||||||
|
contract_version=report.data_contract_version,
|
||||||
|
report_sha256=report.report_sha256,
|
||||||
|
quarantine_status=status,
|
||||||
|
reason_codes=reason_codes,
|
||||||
|
requires_explicit_release=requires_explicit_release,
|
||||||
|
)
|
||||||
|
return QuarantineDecision(
|
||||||
|
asset_id=report.asset_id,
|
||||||
|
data_contract_key=report.data_contract_key,
|
||||||
|
data_contract_version=report.data_contract_version,
|
||||||
|
validation_report_sha256=report.report_sha256,
|
||||||
|
validation_status=report.validation_status,
|
||||||
|
quarantine_status=status,
|
||||||
|
reason_codes=reason_codes,
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
requires_explicit_release=requires_explicit_release,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def require_eligible(decision: QuarantineDecision, *, use: AssetUse) -> None:
|
||||||
|
"""Raise a typed error before a non-eligible artifact reaches a pipeline."""
|
||||||
|
|
||||||
|
if decision.eligible_for_use:
|
||||||
|
return
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_QUARANTINED",
|
||||||
|
message="Dataset is quarantined or failed validation and cannot enter this pipeline.",
|
||||||
|
status_code=409,
|
||||||
|
details={
|
||||||
|
"asset_id": decision.asset_id,
|
||||||
|
"use": use.value,
|
||||||
|
"quarantine_status": decision.quarantine_status.value,
|
||||||
|
"validation_status": decision.validation_status.value,
|
||||||
|
"reason_codes": list(decision.reason_codes),
|
||||||
|
"validation_report_sha256": decision.validation_report_sha256,
|
||||||
|
"idempotency_key": decision.idempotency_key,
|
||||||
|
"requires_explicit_release": decision.requires_explicit_release,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_key(
|
||||||
|
*,
|
||||||
|
asset_id: str,
|
||||||
|
contract_key: str,
|
||||||
|
contract_version: str,
|
||||||
|
report_sha256: str,
|
||||||
|
quarantine_status: QuarantineStatus,
|
||||||
|
reason_codes: tuple[str, ...],
|
||||||
|
requires_explicit_release: bool,
|
||||||
|
) -> str:
|
||||||
|
payload = {
|
||||||
|
"asset_id": asset_id,
|
||||||
|
"contract_key": contract_key,
|
||||||
|
"contract_version": contract_version,
|
||||||
|
"report_sha256": report_sha256,
|
||||||
|
"quarantine_status": quarantine_status.value,
|
||||||
|
"reason_codes": list(reason_codes),
|
||||||
|
"requires_explicit_release": requires_explicit_release,
|
||||||
|
}
|
||||||
|
return sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
|
||||||
@@ -0,0 +1,440 @@
|
|||||||
|
"""Fail-closed provenance gates at data-consumption boundaries.
|
||||||
|
|
||||||
|
Import validation protects newly staged assets, but a persisted record can
|
||||||
|
subsequently become quarantined or have its provenance marked incomplete. The
|
||||||
|
callers of this service therefore re-check the durable Dataset state directly
|
||||||
|
before production inference, QA, derived processing, export, or authoritative
|
||||||
|
coverage reporting.
|
||||||
|
|
||||||
|
The only legacy relaxation is deliberately narrow: an *explicitly tagged*
|
||||||
|
fixture with no Phase-2 state can be used for fixture QA. A caller-provided
|
||||||
|
``fixture_mode`` flag alone never creates that trust claim. Fixture data can
|
||||||
|
never become a production inference, derived-processing, export or
|
||||||
|
authoritative-coverage input, and it never relaxes a recorded failed,
|
||||||
|
incomplete, or quarantined state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import re
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from sqlalchemy import inspect as sa_inspect
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import Dataset
|
||||||
|
|
||||||
|
|
||||||
|
DatasetConsumptionPurpose = Literal[
|
||||||
|
"production_inference",
|
||||||
|
"quality_assessment",
|
||||||
|
"reference_validation",
|
||||||
|
"derived_processing",
|
||||||
|
"authoritative_coverage",
|
||||||
|
"export",
|
||||||
|
]
|
||||||
|
|
||||||
|
_FIXTURE_SOURCE_KEYS = {"fixture", "test", "test_fixture", "test-fixture", "unit-test-fixture"}
|
||||||
|
_UNTRUSTED_SOURCE_KEYS = {"manual", "fixture", "experimental", "legacy_unknown"}
|
||||||
|
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
|
||||||
|
_CONSUMABLE_SNAPSHOT_FRESHNESS = {"current", "not_applicable"}
|
||||||
|
_VALID_PURPOSES = {
|
||||||
|
"production_inference",
|
||||||
|
"quality_assessment",
|
||||||
|
"reference_validation",
|
||||||
|
"derived_processing",
|
||||||
|
"authoritative_coverage",
|
||||||
|
"export",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DatasetConsumptionDecision:
|
||||||
|
"""Auditable decision returned by the consumption gate."""
|
||||||
|
|
||||||
|
eligible: bool
|
||||||
|
purpose: DatasetConsumptionPurpose
|
||||||
|
fixture_legacy_exception: bool
|
||||||
|
reasons: tuple[str, ...]
|
||||||
|
evidence: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetConsumptionGate:
|
||||||
|
"""Evaluate durable provenance before a Dataset is consumed downstream."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _value(dataset: Any, field: str, default: Any = None) -> Any:
|
||||||
|
if isinstance(dataset, Mapping):
|
||||||
|
return dataset.get(field, default)
|
||||||
|
return getattr(dataset, field, default)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _mapping(value: Any) -> dict[str, Any]:
|
||||||
|
return dict(value) if isinstance(value, Mapping) else {}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _normalise(cls, value: Any) -> str:
|
||||||
|
return str(value or "").strip().lower()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_explicit_fixture(cls, dataset: Any) -> bool:
|
||||||
|
source = cls._normalise(cls._value(dataset, "source"))
|
||||||
|
source_name = cls._normalise(cls._value(dataset, "source_name"))
|
||||||
|
metadata = cls._mapping(cls._value(dataset, "metadata_json"))
|
||||||
|
source_metadata = cls._mapping(cls._value(dataset, "source_metadata"))
|
||||||
|
provenance = cls._mapping(cls._value(dataset, "provenance_metadata"))
|
||||||
|
return bool(
|
||||||
|
source in _FIXTURE_SOURCE_KEYS
|
||||||
|
or source_name in _FIXTURE_SOURCE_KEYS
|
||||||
|
or metadata.get("fixture") is True
|
||||||
|
or metadata.get("fixture_mode") is True
|
||||||
|
or source_metadata.get("fixture") is True
|
||||||
|
or source_metadata.get("fixture_mode") is True
|
||||||
|
or provenance.get("fixture") is True
|
||||||
|
or provenance.get("fixture_mode") is True
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _phase2_state_is_absent(cls, dataset: Any) -> bool:
|
||||||
|
fields = (
|
||||||
|
"data_contract_key",
|
||||||
|
"data_contract_version",
|
||||||
|
"validation_status",
|
||||||
|
"provenance_status",
|
||||||
|
"lineage_status",
|
||||||
|
"quarantine_status",
|
||||||
|
"source_registry_id",
|
||||||
|
"source_snapshot_id",
|
||||||
|
)
|
||||||
|
return all(cls._value(dataset, field) in {None, ""} for field in fields)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_transient_orm_dataset(dataset: Any) -> bool:
|
||||||
|
"""Recognize only unpersisted ORM fixtures, never database rows."""
|
||||||
|
|
||||||
|
if not isinstance(dataset, Dataset):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return bool(sa_inspect(dataset).transient)
|
||||||
|
except Exception: # pragma: no cover - defensive for unusual test doubles
|
||||||
|
return False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _evidence(
|
||||||
|
cls,
|
||||||
|
dataset: Any,
|
||||||
|
purpose: DatasetConsumptionPurpose,
|
||||||
|
reference_task: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
source_registry = cls._value(dataset, "source_registry")
|
||||||
|
source_snapshot = cls._value(dataset, "source_snapshot")
|
||||||
|
source_policy = cls._mapping(cls._value(source_registry, "usage_policy_json"))
|
||||||
|
validation_authority = cls._mapping(source_policy.get("validation_authority"))
|
||||||
|
reference_approvals = cls._mapping(source_policy.get("reference_validation_approvals"))
|
||||||
|
return {
|
||||||
|
"dataset_id": str(cls._value(dataset, "id") or ""),
|
||||||
|
"purpose": purpose,
|
||||||
|
"dataset_status": cls._normalise(cls._value(dataset, "status")),
|
||||||
|
"source": cls._normalise(cls._value(dataset, "source")),
|
||||||
|
"source_name": cls._normalise(cls._value(dataset, "source_name")),
|
||||||
|
"data_contract_key": cls._value(dataset, "data_contract_key"),
|
||||||
|
"data_contract_version": cls._value(dataset, "data_contract_version"),
|
||||||
|
"validation_status": cls._normalise(cls._value(dataset, "validation_status")),
|
||||||
|
"provenance_status": cls._normalise(cls._value(dataset, "provenance_status")),
|
||||||
|
"lineage_status": cls._normalise(cls._value(dataset, "lineage_status")),
|
||||||
|
"quarantine_status": cls._normalise(cls._value(dataset, "quarantine_status")),
|
||||||
|
"checksum_sha256": cls._value(dataset, "checksum_sha256"),
|
||||||
|
"source_registry_id": str(cls._value(dataset, "source_registry_id") or ""),
|
||||||
|
"source_snapshot_id": str(cls._value(dataset, "source_snapshot_id") or ""),
|
||||||
|
"source_classification": cls._normalise(cls._value(source_registry, "classification")),
|
||||||
|
"source_key": cls._normalise(cls._value(source_registry, "source_key")),
|
||||||
|
"source_ground_truth_allowed": source_policy.get("ground_truth_allowed") is True,
|
||||||
|
"source_validation_authority": validation_authority,
|
||||||
|
"source_reference_validation_approvals": reference_approvals,
|
||||||
|
"source_authority_scope": cls._mapping(cls._value(source_registry, "authority_scope_json")),
|
||||||
|
"reference_task": cls._normalise(reference_task),
|
||||||
|
"snapshot_source_registry_id": str(cls._value(source_snapshot, "source_registry_id") or ""),
|
||||||
|
"snapshot_ingest_status": cls._normalise(cls._value(source_snapshot, "ingest_status")),
|
||||||
|
"snapshot_freshness_status": cls._normalise(cls._value(source_snapshot, "freshness_status")),
|
||||||
|
"snapshot_checksum_sha256": cls._value(source_snapshot, "checksum_sha256"),
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def evaluate(
|
||||||
|
cls,
|
||||||
|
dataset: Any,
|
||||||
|
*,
|
||||||
|
purpose: DatasetConsumptionPurpose,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
reference_task: str | None = None,
|
||||||
|
) -> DatasetConsumptionDecision:
|
||||||
|
"""Return a stable decision without mutating the Dataset.
|
||||||
|
|
||||||
|
``fixture_mode`` can be supplied only by an explicitly fixture-only
|
||||||
|
caller. It is evidence for a QA fixture path, never a relaxation for
|
||||||
|
a production boundary or an explicit unsafe state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if purpose not in _VALID_PURPOSES:
|
||||||
|
raise ValueError(f"Unsupported dataset-consumption purpose: {purpose}")
|
||||||
|
|
||||||
|
evidence = cls._evidence(dataset, purpose, reference_task)
|
||||||
|
reasons: list[str] = []
|
||||||
|
explicit_fixture = cls.is_explicit_fixture(dataset)
|
||||||
|
phase2_absent = cls._phase2_state_is_absent(dataset)
|
||||||
|
|
||||||
|
# These are irrevocable safety states. They are checked before a
|
||||||
|
# fixture exception, so fixture rows cannot hide a bad recorded state.
|
||||||
|
if evidence["dataset_status"] in {"failed", "quarantined"}:
|
||||||
|
reasons.append("dataset_status_unsafe")
|
||||||
|
if evidence["quarantine_status"] == "quarantined":
|
||||||
|
reasons.append("dataset_quarantined")
|
||||||
|
if evidence["validation_status"] == "failed":
|
||||||
|
reasons.append("validation_failed")
|
||||||
|
if evidence["provenance_status"] == "incomplete":
|
||||||
|
reasons.append("provenance_incomplete")
|
||||||
|
if evidence["lineage_status"] == "incomplete":
|
||||||
|
reasons.append("lineage_incomplete")
|
||||||
|
if evidence["snapshot_ingest_status"] in {"failed", "quarantined"}:
|
||||||
|
reasons.append("source_snapshot_unsafe")
|
||||||
|
# A source family can be authoritative while an individual snapshot
|
||||||
|
# remains too old or insufficiently described to trust. Historical
|
||||||
|
# data that is intentionally valid needs an explicit
|
||||||
|
# ``not_applicable`` contract policy; an omitted, due or stale status
|
||||||
|
# cannot silently enter a production boundary.
|
||||||
|
if evidence["source_snapshot_id"] and evidence["snapshot_freshness_status"] not in _CONSUMABLE_SNAPSHOT_FRESHNESS:
|
||||||
|
reasons.append("source_snapshot_freshness_not_eligible")
|
||||||
|
|
||||||
|
if reasons:
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=False,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=False,
|
||||||
|
reasons=tuple(sorted(set(reasons))),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fixtures are evidence for tests and QA only. They cannot become
|
||||||
|
# production inference, derived processing or export inputs merely by
|
||||||
|
# presenting a fixture flag at a public service boundary.
|
||||||
|
if phase2_absent and explicit_fixture:
|
||||||
|
if purpose == "quality_assessment":
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=True,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=True,
|
||||||
|
reasons=(),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
if purpose == "authoritative_coverage":
|
||||||
|
reasons.append("fixture_not_authoritative_coverage")
|
||||||
|
else:
|
||||||
|
reasons.append("fixture_qa_only")
|
||||||
|
elif phase2_absent and fixture_mode:
|
||||||
|
# `fixture_mode` is a caller flag, not a source trust claim. A
|
||||||
|
# manual/unknown production dataset must never self-designate as a
|
||||||
|
# fixture merely by supplying this parameter.
|
||||||
|
reasons.append("fixture_source_required")
|
||||||
|
|
||||||
|
# Existing service tests construct transient SQLAlchemy Dataset objects
|
||||||
|
# directly rather than retrieving a persisted row. A production
|
||||||
|
# `db.get()` result is persistent and never enters this branch. This
|
||||||
|
# compatibility path is intentionally unavailable to coverage, where
|
||||||
|
# a fixture must never appear authoritative.
|
||||||
|
if (
|
||||||
|
phase2_absent
|
||||||
|
and not reasons
|
||||||
|
and cls._is_transient_orm_dataset(dataset)
|
||||||
|
and purpose == "quality_assessment"
|
||||||
|
):
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=True,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=True,
|
||||||
|
reasons=(),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
# QA unit tests intentionally use projection objects
|
||||||
|
# instead of persisted ORM Datasets. Those projections cannot enter an
|
||||||
|
# application API boundary; keep the exception isolated to read-only
|
||||||
|
# candidate verification. Inference, reference validation, derived
|
||||||
|
# processing, export and coverage never accept a projection.
|
||||||
|
if phase2_absent and not reasons and not isinstance(dataset, Dataset) and purpose == "quality_assessment":
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=True,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=True,
|
||||||
|
reasons=(),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
if phase2_absent:
|
||||||
|
reasons.append("phase2_provenance_missing")
|
||||||
|
if evidence["dataset_status"] != "ready":
|
||||||
|
reasons.append("dataset_not_ready")
|
||||||
|
if evidence["validation_status"] != "passed":
|
||||||
|
reasons.append("validation_not_passed")
|
||||||
|
if evidence["provenance_status"] != "complete":
|
||||||
|
reasons.append("provenance_not_complete")
|
||||||
|
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
|
||||||
|
reasons.append("lineage_not_complete")
|
||||||
|
if evidence["quarantine_status"] != "not_quarantined":
|
||||||
|
reasons.append("quarantine_status_not_clear")
|
||||||
|
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
|
||||||
|
reasons.append("data_contract_not_versioned")
|
||||||
|
if not _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")):
|
||||||
|
reasons.append("dataset_checksum_invalid")
|
||||||
|
if not evidence["source_registry_id"]:
|
||||||
|
reasons.append("source_registry_missing")
|
||||||
|
if not evidence["source_snapshot_id"]:
|
||||||
|
reasons.append("source_snapshot_missing")
|
||||||
|
if evidence["source_registry_id"] and not evidence["source_classification"]:
|
||||||
|
reasons.append("source_registry_unresolved")
|
||||||
|
if evidence["source_snapshot_id"] and evidence["snapshot_ingest_status"] != "ingested":
|
||||||
|
reasons.append("source_snapshot_not_ingested")
|
||||||
|
if evidence["source_snapshot_id"] and not _CHECKSUM_SHA256.fullmatch(
|
||||||
|
str(evidence["snapshot_checksum_sha256"] or "")
|
||||||
|
):
|
||||||
|
reasons.append("source_snapshot_checksum_invalid")
|
||||||
|
if (
|
||||||
|
evidence["source_registry_id"]
|
||||||
|
and evidence["source_snapshot_id"]
|
||||||
|
and evidence["snapshot_source_registry_id"]
|
||||||
|
and evidence["source_registry_id"] != evidence["snapshot_source_registry_id"]
|
||||||
|
):
|
||||||
|
reasons.append("source_snapshot_registry_mismatch")
|
||||||
|
if (
|
||||||
|
_CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or ""))
|
||||||
|
and _CHECKSUM_SHA256.fullmatch(str(evidence["snapshot_checksum_sha256"] or ""))
|
||||||
|
and str(evidence["checksum_sha256"]).lower() != str(evidence["snapshot_checksum_sha256"]).lower()
|
||||||
|
):
|
||||||
|
reasons.append("source_snapshot_checksum_mismatch")
|
||||||
|
|
||||||
|
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
|
||||||
|
reasons.append("source_registry_identity_mismatch")
|
||||||
|
|
||||||
|
# Contextual, corroborative, authoritative and properly derived
|
||||||
|
# sources can serve their declared non-ground-truth roles once the
|
||||||
|
# full governed contract passes. Experimental/manual sources cannot
|
||||||
|
# cross a production boundary; only an explicitly marked fixture may
|
||||||
|
# participate in candidate QA.
|
||||||
|
experimental_source = (
|
||||||
|
evidence["source_classification"] == "experimental"
|
||||||
|
or evidence["source_key"] in _UNTRUSTED_SOURCE_KEYS
|
||||||
|
)
|
||||||
|
if experimental_source:
|
||||||
|
if purpose == "quality_assessment" and explicit_fixture:
|
||||||
|
pass
|
||||||
|
elif purpose == "quality_assessment":
|
||||||
|
reasons.append("experimental_source_requires_fixture_qa")
|
||||||
|
else:
|
||||||
|
reasons.append("experimental_source_not_allowed_for_purpose")
|
||||||
|
|
||||||
|
if purpose == "authoritative_coverage":
|
||||||
|
if evidence["source_classification"] != "authoritative":
|
||||||
|
reasons.append("coverage_source_not_authoritative")
|
||||||
|
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
|
||||||
|
reasons.append("coverage_source_identity_mismatch")
|
||||||
|
|
||||||
|
if purpose == "reference_validation":
|
||||||
|
if cls._normalise(cls._value(dataset, "dataset_role")) != "reference":
|
||||||
|
reasons.append("reference_dataset_role_required")
|
||||||
|
if evidence["source_classification"] != "authoritative":
|
||||||
|
reasons.append("reference_source_not_authoritative")
|
||||||
|
if evidence["source_ground_truth_allowed"] is not True:
|
||||||
|
reasons.append("reference_source_not_ground_truth_allowed")
|
||||||
|
if not evidence["reference_task"]:
|
||||||
|
reasons.append("reference_validation_task_required")
|
||||||
|
elif not cls._reference_task_is_approved(evidence):
|
||||||
|
reasons.append("reference_task_authority_not_approved")
|
||||||
|
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=not reasons,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=False,
|
||||||
|
reasons=tuple(sorted(set(reasons))),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def assert_eligible(
|
||||||
|
cls,
|
||||||
|
dataset: Any,
|
||||||
|
*,
|
||||||
|
purpose: DatasetConsumptionPurpose,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
reference_task: str | None = None,
|
||||||
|
) -> DatasetConsumptionDecision:
|
||||||
|
decision = cls.evaluate(
|
||||||
|
dataset,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
reference_task=reference_task,
|
||||||
|
)
|
||||||
|
if decision.eligible:
|
||||||
|
return decision
|
||||||
|
code = "DATASET_QUARANTINED" if any(
|
||||||
|
reason in {"dataset_status_unsafe", "dataset_quarantined", "validation_failed", "source_snapshot_unsafe"}
|
||||||
|
for reason in decision.reasons
|
||||||
|
) else "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
raise AppError(
|
||||||
|
code=code,
|
||||||
|
message="Dataset cannot be consumed until its provenance and validation gates are satisfied.",
|
||||||
|
status_code=409,
|
||||||
|
details={
|
||||||
|
"dataset_id": decision.evidence["dataset_id"],
|
||||||
|
"purpose": purpose,
|
||||||
|
"reasons": list(decision.reasons),
|
||||||
|
"fixture_legacy_exception": decision.fixture_legacy_exception,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def eligible_for_authoritative_coverage(cls, dataset: Any) -> bool:
|
||||||
|
"""Return false instead of raising so coverage can report a gap safely."""
|
||||||
|
|
||||||
|
return cls.evaluate(dataset, purpose="authoritative_coverage").eligible
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _reference_task_is_approved(evidence: Mapping[str, Any]) -> bool:
|
||||||
|
"""Require task-specific primary authority or an explicit zone approval.
|
||||||
|
|
||||||
|
``classification=authoritative`` is deliberately not a blanket
|
||||||
|
permission to serve as building truth. A source may be authoritative
|
||||||
|
for an address lifecycle or elevation product while remaining only
|
||||||
|
corroborative for footprint QA. A regional product that is marked
|
||||||
|
``*_pending_contract`` similarly remains blocked until an operator
|
||||||
|
records a narrow product-and-zone approval in its server-owned policy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
task = str(evidence.get("reference_task") or "").strip().lower()
|
||||||
|
authority = DatasetConsumptionGate._normalise(
|
||||||
|
DatasetConsumptionGate._mapping(evidence.get("source_validation_authority")).get(task)
|
||||||
|
)
|
||||||
|
if authority == "primary":
|
||||||
|
return True
|
||||||
|
if authority not in {"approved", "approved_product_zone"}:
|
||||||
|
return False
|
||||||
|
|
||||||
|
approvals = DatasetConsumptionGate._mapping(evidence.get("source_reference_validation_approvals"))
|
||||||
|
approval = DatasetConsumptionGate._mapping(approvals.get(task))
|
||||||
|
if approval.get("approved") is not True:
|
||||||
|
return False
|
||||||
|
|
||||||
|
source_key = str(evidence.get("source_key") or "").strip().lower()
|
||||||
|
source_scope = DatasetConsumptionGate._mapping(evidence.get("source_authority_scope"))
|
||||||
|
source_zone = str(source_scope.get("zone") or source_scope.get("scope") or "").strip()
|
||||||
|
approved_keys = approval.get("source_keys")
|
||||||
|
approved_zones = approval.get("zones")
|
||||||
|
if not isinstance(approved_keys, list) or source_key not in {
|
||||||
|
str(value).strip().lower() for value in approved_keys
|
||||||
|
}:
|
||||||
|
return False
|
||||||
|
if not isinstance(approved_zones, list) or source_zone not in {
|
||||||
|
str(value).strip() for value in approved_zones
|
||||||
|
}:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -10,8 +10,15 @@ from uuid import UUID, uuid4
|
|||||||
from geoalchemy2.shape import from_shape
|
from geoalchemy2.shape import from_shape
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.core.public_demo import (
|
||||||
|
PUBLIC_DEMO_PROJECT_ID,
|
||||||
|
PUBLIC_DEMO_PROJECT_MARKER,
|
||||||
|
PUBLIC_DEMO_PROJECT_NAME,
|
||||||
|
)
|
||||||
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
||||||
from app.schemas.demo import DemoWorkflowResponse
|
from app.schemas.demo import DemoWorkflowResponse
|
||||||
|
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||||
from app.services.geojson_service import parse_geojson_payload
|
from app.services.geojson_service import parse_geojson_payload
|
||||||
from app.services.qa_service import QaService
|
from app.services.qa_service import QaService
|
||||||
from app.services.quality_service import QualityService
|
from app.services.quality_service import QualityService
|
||||||
@@ -22,7 +29,8 @@ from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_mult
|
|||||||
|
|
||||||
|
|
||||||
class DemoWorkflowService:
|
class DemoWorkflowService:
|
||||||
PROJECT_NAME = "GeoIntel Demo - Building QA"
|
PROJECT_ID = PUBLIC_DEMO_PROJECT_ID
|
||||||
|
PROJECT_NAME = PUBLIC_DEMO_PROJECT_NAME
|
||||||
AREA_NAME = "Demo AOI - Geel buildings"
|
AREA_NAME = "Demo AOI - Geel buildings"
|
||||||
REFERENCE_FILENAME = "demo_reference_buildings.geojson"
|
REFERENCE_FILENAME = "demo_reference_buildings.geojson"
|
||||||
CANDIDATE_FILENAME = "demo_predicted_buildings.geojson"
|
CANDIDATE_FILENAME = "demo_predicted_buildings.geojson"
|
||||||
@@ -30,9 +38,8 @@ class DemoWorkflowService:
|
|||||||
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _add_initial_version(db: Session, dataset: Dataset) -> None:
|
def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
|
||||||
db.add(
|
version = DatasetVersion(
|
||||||
DatasetVersion(
|
|
||||||
dataset_id=dataset.id,
|
dataset_id=dataset.id,
|
||||||
version=1,
|
version=1,
|
||||||
storage_path=dataset.storage_path,
|
storage_path=dataset.storage_path,
|
||||||
@@ -40,7 +47,8 @@ class DemoWorkflowService:
|
|||||||
source_metadata=dataset.source_metadata,
|
source_metadata=dataset.source_metadata,
|
||||||
provenance_metadata=dataset.provenance_metadata,
|
provenance_metadata=dataset.provenance_metadata,
|
||||||
)
|
)
|
||||||
)
|
db.add(version)
|
||||||
|
return version
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _repo_root() -> Path:
|
def _repo_root() -> Path:
|
||||||
@@ -90,17 +98,19 @@ class DemoWorkflowService:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _find_existing_project(db: Session) -> Project | None:
|
def _find_existing_project(db: Session) -> Project | None:
|
||||||
projects = (
|
project = db.get(Project, DemoWorkflowService.PROJECT_ID)
|
||||||
db.query(Project)
|
if project is None:
|
||||||
.filter(Project.name == DemoWorkflowService.PROJECT_NAME)
|
return None
|
||||||
.filter(Project.status != "deleted")
|
if (
|
||||||
.order_by(Project.created_at.asc())
|
project.name != DemoWorkflowService.PROJECT_NAME
|
||||||
.all()
|
or project.description != PUBLIC_DEMO_PROJECT_MARKER
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="PUBLIC_DEMO_IDENTITY_CONFLICT",
|
||||||
|
message="The reserved public-demo project identity is already in use.",
|
||||||
|
status_code=409,
|
||||||
)
|
)
|
||||||
for project in projects:
|
|
||||||
if DemoWorkflowService._has_complete_demo_state(db, project.id):
|
|
||||||
return project
|
return project
|
||||||
return projects[0] if projects else None
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _activate_explicit_demo_project(db: Session, project: Project | None) -> Project | None:
|
def _activate_explicit_demo_project(db: Session, project: Project | None) -> Project | None:
|
||||||
@@ -233,24 +243,34 @@ class DemoWorkflowService:
|
|||||||
crs=metadata.get("crs"),
|
crs=metadata.get("crs"),
|
||||||
bounds_json=metadata.get("bounds_json"),
|
bounds_json=metadata.get("bounds_json"),
|
||||||
metadata_json=metadata,
|
metadata_json=metadata,
|
||||||
status="ready",
|
status="validating",
|
||||||
)
|
)
|
||||||
db.add(dataset)
|
db.add(dataset)
|
||||||
DemoWorkflowService._add_initial_version(db, dataset)
|
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||||
db.commit()
|
is_ready = DerivedDatasetGovernanceService.govern_vector(
|
||||||
db.refresh(dataset)
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=version,
|
||||||
|
feature_collection=payload,
|
||||||
|
source_key="fixture",
|
||||||
|
operation="demo.fixture_vector",
|
||||||
|
operation_parameters={"fixture_name": filename, "role": role},
|
||||||
|
)
|
||||||
|
if is_ready:
|
||||||
VectorFeatureService.persist_geojson_features(
|
VectorFeatureService.persist_geojson_features(
|
||||||
db=db,
|
db=db,
|
||||||
dataset_id=dataset.id,
|
dataset_id=dataset.id,
|
||||||
payload=payload,
|
payload=payload,
|
||||||
feature_class=reference_layer_name or "building",
|
feature_class=reference_layer_name or "building",
|
||||||
|
commit=False,
|
||||||
)
|
)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(dataset)
|
||||||
return dataset
|
return dataset
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _create_demo_raster_bytes() -> bytes:
|
def _create_demo_raster_bytes() -> bytes:
|
||||||
numpy = importlib.import_module("numpy")
|
numpy = importlib.import_module("numpy")
|
||||||
rasterio = importlib.import_module("rasterio")
|
|
||||||
rasterio_io = importlib.import_module("rasterio.io")
|
rasterio_io = importlib.import_module("rasterio.io")
|
||||||
rasterio_transform = importlib.import_module("rasterio.transform")
|
rasterio_transform = importlib.import_module("rasterio.transform")
|
||||||
|
|
||||||
@@ -318,10 +338,19 @@ class DemoWorkflowService:
|
|||||||
crs=metadata.get("crs"),
|
crs=metadata.get("crs"),
|
||||||
bounds_json=bounds_json,
|
bounds_json=bounds_json,
|
||||||
metadata_json=metadata,
|
metadata_json=metadata,
|
||||||
status="ready",
|
status="validating",
|
||||||
)
|
)
|
||||||
db.add(dataset)
|
db.add(dataset)
|
||||||
DemoWorkflowService._add_initial_version(db, dataset)
|
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||||
|
DerivedDatasetGovernanceService.govern_raster(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=version,
|
||||||
|
raster_metadata=metadata,
|
||||||
|
source_key="fixture",
|
||||||
|
operation="demo.fixture_raster",
|
||||||
|
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(dataset)
|
db.refresh(dataset)
|
||||||
return dataset
|
return dataset
|
||||||
@@ -467,9 +496,9 @@ class DemoWorkflowService:
|
|||||||
created = True
|
created = True
|
||||||
else:
|
else:
|
||||||
project = Project(
|
project = Project(
|
||||||
id=uuid4(),
|
id=DemoWorkflowService.PROJECT_ID,
|
||||||
name=DemoWorkflowService.PROJECT_NAME,
|
name=DemoWorkflowService.PROJECT_NAME,
|
||||||
description="Offline fixture workflow: reference buildings, predicted buildings and persisted QA metrics.",
|
description=PUBLIC_DEMO_PROJECT_MARKER,
|
||||||
region="Kempen",
|
region="Kempen",
|
||||||
status="active",
|
status="active",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,516 @@
|
|||||||
|
"""Governance for persisted derived and fixture datasets.
|
||||||
|
|
||||||
|
Dataset importers own raw-source ingestion. This small service owns the
|
||||||
|
other persistence boundary: artifacts produced inside the workbench (vector
|
||||||
|
and raster operations) and the explicitly local demo fixtures. It is kept
|
||||||
|
separate from :mod:`dataset_service` so an operation can never create a ready
|
||||||
|
dataset without a source registry binding, immutable snapshot, validation
|
||||||
|
report and, for derived results, a durable lineage edge.
|
||||||
|
|
||||||
|
The ``Session.query`` capability check deliberately preserves lightweight
|
||||||
|
unit-test doubles used by pre-Phase-2 tests. Real SQLAlchemy sessions always
|
||||||
|
take the governed branch; the compatibility branch is not reachable in the
|
||||||
|
application runtime.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.models import Dataset, DatasetVersion
|
||||||
|
from app.services.data_contract_validation import (
|
||||||
|
IssueSeverity,
|
||||||
|
LineageStatus,
|
||||||
|
LineageEvidence,
|
||||||
|
ProvenanceStatus,
|
||||||
|
QuarantineStatus,
|
||||||
|
TransformationEvidence,
|
||||||
|
ValidationIssue,
|
||||||
|
ValidationReport,
|
||||||
|
ValidationStatus,
|
||||||
|
build_raster_ingest_input,
|
||||||
|
build_vector_ingest_input,
|
||||||
|
validate_registered_asset,
|
||||||
|
)
|
||||||
|
from app.services.data_quarantine_service import DataQuarantineService
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate
|
||||||
|
from app.services.source_registry_service import SourceRegistryService
|
||||||
|
|
||||||
|
|
||||||
|
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
_CANONICAL_VECTOR_CRS = "EPSG:4326"
|
||||||
|
|
||||||
|
|
||||||
|
class DerivedDatasetGovernanceService:
|
||||||
|
"""Apply Phase-2 provenance rules to non-importer Dataset creation.
|
||||||
|
|
||||||
|
Callers add the dataset and first immutable version, then call one of the
|
||||||
|
``govern_*`` methods before committing. A failed contract deliberately
|
||||||
|
leaves the artifact and its durable quarantine record in the transaction;
|
||||||
|
it is never silently promoted to ``ready``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def persistence_available(db: Session) -> bool:
|
||||||
|
"""Return whether this is a real ORM persistence session.
|
||||||
|
|
||||||
|
Historical unit tests use minimal fakes with ``add``/``commit`` only.
|
||||||
|
Keeping that explicitly isolated avoids pretending a fake test store
|
||||||
|
has source-registry guarantees while production remains fail-closed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def govern_vector(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
dataset: Dataset,
|
||||||
|
dataset_version: DatasetVersion,
|
||||||
|
feature_collection: Mapping[str, Any],
|
||||||
|
source_key: str,
|
||||||
|
operation: str,
|
||||||
|
parent_dataset: Dataset | None = None,
|
||||||
|
operation_parameters: Mapping[str, Any] | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Validate and bind a vector result, returning ``True`` when ready."""
|
||||||
|
|
||||||
|
if not cls.persistence_available(db):
|
||||||
|
# Explicit compatibility for historical minimal test fixtures.
|
||||||
|
# Production sessions always have query/flush and never take this
|
||||||
|
# branch.
|
||||||
|
dataset.status = "ready"
|
||||||
|
return True
|
||||||
|
|
||||||
|
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
|
||||||
|
metadata = dict(dataset.metadata_json or {})
|
||||||
|
output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS)
|
||||||
|
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
|
||||||
|
snapshot = cls._record_snapshot(
|
||||||
|
db,
|
||||||
|
source_key=source_key,
|
||||||
|
dataset=dataset,
|
||||||
|
operation=operation,
|
||||||
|
source_crs=output_crs,
|
||||||
|
spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"},
|
||||||
|
geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json},
|
||||||
|
observed_schema={
|
||||||
|
"dataset_type": "vector",
|
||||||
|
"geometry_types": metadata.get("geometry_types") or [],
|
||||||
|
"feature_count": metadata.get("feature_count"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
|
||||||
|
report = validate_registered_asset(
|
||||||
|
build_vector_ingest_input(
|
||||||
|
asset_id=str(dataset.id),
|
||||||
|
source_crs=output_crs,
|
||||||
|
storage_crs=output_crs,
|
||||||
|
feature_collection=feature_collection,
|
||||||
|
checksum_sha256=dataset.checksum_sha256,
|
||||||
|
computed_checksum_sha256=dataset.checksum_sha256,
|
||||||
|
source_registry_id=str(source.id),
|
||||||
|
source_snapshot_id=str(snapshot.id),
|
||||||
|
imported_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||||
|
metadata=cls._contract_metadata(metadata, source.license_name),
|
||||||
|
observed_at=dataset.observed_at,
|
||||||
|
valid_from=dataset.valid_from,
|
||||||
|
valid_to=dataset.valid_to,
|
||||||
|
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
|
||||||
|
source_version=dataset.source_version,
|
||||||
|
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
|
||||||
|
lineage=lineage,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if parent_gate is not None and not parent_gate.eligible:
|
||||||
|
report = cls._with_parent_gate_failure(report, parent_gate)
|
||||||
|
return cls._apply(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
report=report,
|
||||||
|
stage="derived_vector_validation",
|
||||||
|
parent_dataset=parent_dataset,
|
||||||
|
operation=operation,
|
||||||
|
operation_parameters=operation_parameters,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def govern_raster(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
dataset: Dataset,
|
||||||
|
dataset_version: DatasetVersion,
|
||||||
|
raster_metadata: Mapping[str, Any],
|
||||||
|
source_key: str,
|
||||||
|
operation: str,
|
||||||
|
parent_dataset: Dataset | None = None,
|
||||||
|
operation_parameters: Mapping[str, Any] | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Validate and bind a raster result, returning ``True`` when ready."""
|
||||||
|
|
||||||
|
if not cls.persistence_available(db):
|
||||||
|
dataset.status = "ready"
|
||||||
|
return True
|
||||||
|
|
||||||
|
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
|
||||||
|
metadata = dict(raster_metadata or {})
|
||||||
|
output_crs = str(metadata.get("crs") or dataset.crs or "") or None
|
||||||
|
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
|
||||||
|
snapshot = cls._record_snapshot(
|
||||||
|
db,
|
||||||
|
source_key=source_key,
|
||||||
|
dataset=dataset,
|
||||||
|
operation=operation,
|
||||||
|
source_crs=output_crs,
|
||||||
|
spatial_resolution=cls._raster_resolution(metadata, output_crs),
|
||||||
|
geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json},
|
||||||
|
observed_schema={
|
||||||
|
"dataset_type": "raster",
|
||||||
|
"width": metadata.get("width"),
|
||||||
|
"height": metadata.get("height"),
|
||||||
|
"band_count": metadata.get("band_count"),
|
||||||
|
"dtype": metadata.get("dtype"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
|
||||||
|
report = validate_registered_asset(
|
||||||
|
build_raster_ingest_input(
|
||||||
|
asset_id=str(dataset.id),
|
||||||
|
source_crs=output_crs,
|
||||||
|
storage_crs=output_crs,
|
||||||
|
raster_profile=metadata,
|
||||||
|
bounds=metadata.get("bounds") or dataset.bounds_json,
|
||||||
|
resolution=cls._raster_resolution(metadata, output_crs),
|
||||||
|
checksum_sha256=dataset.checksum_sha256,
|
||||||
|
computed_checksum_sha256=dataset.checksum_sha256,
|
||||||
|
source_registry_id=str(source.id),
|
||||||
|
source_snapshot_id=str(snapshot.id),
|
||||||
|
imported_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||||
|
metadata=cls._contract_metadata(metadata, source.license_name),
|
||||||
|
observed_at=dataset.observed_at,
|
||||||
|
valid_from=dataset.valid_from,
|
||||||
|
valid_to=dataset.valid_to,
|
||||||
|
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
|
||||||
|
source_version=dataset.source_version,
|
||||||
|
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
|
||||||
|
lineage=lineage,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if parent_gate is not None and not parent_gate.eligible:
|
||||||
|
report = cls._with_parent_gate_failure(report, parent_gate)
|
||||||
|
return cls._apply(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
report=report,
|
||||||
|
stage="derived_raster_validation",
|
||||||
|
parent_dataset=parent_dataset,
|
||||||
|
operation=operation,
|
||||||
|
operation_parameters=operation_parameters,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _record_snapshot(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
source_key: str,
|
||||||
|
dataset: Dataset,
|
||||||
|
operation: str,
|
||||||
|
source_crs: str | None,
|
||||||
|
spatial_resolution: Mapping[str, Any] | None,
|
||||||
|
geographic_coverage: Mapping[str, Any] | None,
|
||||||
|
observed_schema: Mapping[str, Any] | None,
|
||||||
|
):
|
||||||
|
checksum = cls._checksum_or_placeholder(dataset.checksum_sha256)
|
||||||
|
# A derived artifact has a distinct creation event even when its bytes
|
||||||
|
# equal a prior output. Include the immutable dataset id so source
|
||||||
|
# snapshots never collide on a different fetched_at timestamp.
|
||||||
|
snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}"
|
||||||
|
return SourceRegistryService.record_snapshot(
|
||||||
|
db,
|
||||||
|
source_key=source_key,
|
||||||
|
snapshot_key=snapshot_key,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_version=dataset.source_version or f"{operation}:1.0.0",
|
||||||
|
snapshot_at=dataset.observed_at,
|
||||||
|
fetched_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||||
|
crs=source_crs,
|
||||||
|
units=cls._units_for_crs(source_crs),
|
||||||
|
spatial_resolution=dict(spatial_resolution or {"status": "unknown"}),
|
||||||
|
temporal_coverage={
|
||||||
|
"observed_at": cls._datetime_value(dataset.observed_at),
|
||||||
|
"valid_from": cls._datetime_value(dataset.valid_from),
|
||||||
|
"valid_to": cls._datetime_value(dataset.valid_to),
|
||||||
|
},
|
||||||
|
geographic_coverage=dict(geographic_coverage or {"status": "unknown"}),
|
||||||
|
observed_schema=dict(observed_schema or {"status": "unknown"}),
|
||||||
|
# A transform cannot establish source freshness. With no source
|
||||||
|
# observation it is not applicable; with one it still needs an
|
||||||
|
# explicit policy review rather than a fabricated "current" flag.
|
||||||
|
freshness_status="not_applicable" if dataset.observed_at is None else "review_required",
|
||||||
|
ingest_status="ingested",
|
||||||
|
known_limitations=[
|
||||||
|
"Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.",
|
||||||
|
],
|
||||||
|
snapshot_metadata={
|
||||||
|
"operation": operation,
|
||||||
|
"dataset_id": str(dataset.id),
|
||||||
|
"storage_path": dataset.storage_path,
|
||||||
|
"artifact_checksum_sha256": dataset.checksum_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _apply(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
dataset: Dataset,
|
||||||
|
dataset_version: DatasetVersion,
|
||||||
|
source: Any,
|
||||||
|
snapshot: Any,
|
||||||
|
report: ValidationReport,
|
||||||
|
stage: str,
|
||||||
|
parent_dataset: Dataset | None,
|
||||||
|
operation: str,
|
||||||
|
operation_parameters: Mapping[str, Any] | None,
|
||||||
|
) -> bool:
|
||||||
|
fields = report.persistence_fields()
|
||||||
|
dataset.validation_report_json = fields["validation_report_json"]
|
||||||
|
dataset_version.validation_report_json = fields["validation_report_json"]
|
||||||
|
SourceRegistryService.bind_dataset_provenance(
|
||||||
|
dataset,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
data_contract_key=fields["data_contract_key"],
|
||||||
|
data_contract_version=fields["data_contract_version"],
|
||||||
|
validation_status=fields["validation_status"],
|
||||||
|
provenance_status=fields["provenance_status"],
|
||||||
|
lineage_status=fields["lineage_status"],
|
||||||
|
)
|
||||||
|
SourceRegistryService.bind_dataset_version_provenance(
|
||||||
|
dataset_version,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
data_contract_key=fields["data_contract_key"],
|
||||||
|
data_contract_version=fields["data_contract_version"],
|
||||||
|
validation_status=fields["validation_status"],
|
||||||
|
provenance_status=fields["provenance_status"],
|
||||||
|
lineage_status=fields["lineage_status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# IDs for DatasetVersion defaults exist only after the caller adds and
|
||||||
|
# flushes both rows. The operation services call this before commit.
|
||||||
|
db.flush()
|
||||||
|
if parent_dataset is not None:
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
db,
|
||||||
|
parent_dataset_id=parent_dataset.id,
|
||||||
|
child_dataset_id=dataset.id,
|
||||||
|
parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset),
|
||||||
|
child_dataset_version_id=dataset_version.id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name=operation,
|
||||||
|
transformation_version="1.0.0",
|
||||||
|
parameters=dict(operation_parameters or {}),
|
||||||
|
input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256),
|
||||||
|
output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = DataQuarantineService.decide(report)
|
||||||
|
if decision.eligible_for_use:
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
return True
|
||||||
|
|
||||||
|
SourceRegistryService.quarantine_dataset(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
source_snapshot=snapshot,
|
||||||
|
stage=stage,
|
||||||
|
reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"),
|
||||||
|
details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()},
|
||||||
|
artifact_path=dataset.storage_path,
|
||||||
|
artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]:
|
||||||
|
values = dict(metadata)
|
||||||
|
values.setdefault("license", license_name)
|
||||||
|
return values
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _lineage_evidence(
|
||||||
|
cls,
|
||||||
|
parent_dataset: Dataset | None,
|
||||||
|
operation: str,
|
||||||
|
operation_parameters: Mapping[str, Any] | None,
|
||||||
|
) -> LineageEvidence:
|
||||||
|
upstream_ids: tuple[str, ...] = ()
|
||||||
|
upstream_checksums: tuple[str, ...] = ()
|
||||||
|
if parent_dataset is not None:
|
||||||
|
upstream_ids = (str(parent_dataset.id),)
|
||||||
|
# An invalid/missing parent checksum intentionally fails the
|
||||||
|
# derived contract rather than inventing traceability. The same
|
||||||
|
# applies to a legacy/unvalidated parent: it may remain visible
|
||||||
|
# as evidence, but cannot create a new ready derived asset.
|
||||||
|
parent_is_governed = (
|
||||||
|
parent_dataset.status == "ready"
|
||||||
|
and parent_dataset.quarantine_status == "not_quarantined"
|
||||||
|
and parent_dataset.validation_status == "passed"
|
||||||
|
and parent_dataset.provenance_status == "complete"
|
||||||
|
and parent_dataset.source_registry_id is not None
|
||||||
|
and parent_dataset.source_snapshot_id is not None
|
||||||
|
)
|
||||||
|
upstream_checksums = (
|
||||||
|
parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed",
|
||||||
|
)
|
||||||
|
transform_checksum = cls._stable_hash(
|
||||||
|
{"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})}
|
||||||
|
)
|
||||||
|
return LineageEvidence(
|
||||||
|
upstream_asset_ids=upstream_ids,
|
||||||
|
upstream_checksums_sha256=upstream_checksums,
|
||||||
|
transformations=(
|
||||||
|
TransformationEvidence(
|
||||||
|
name=operation,
|
||||||
|
version="1.0.0",
|
||||||
|
checksum_sha256=transform_checksum,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None:
|
||||||
|
"""Evaluate the durable parent boundary before creating a ready child.
|
||||||
|
|
||||||
|
We deliberately turn a rejected parent into a child validation failure
|
||||||
|
(instead of simply raising): the output is then persisted with its
|
||||||
|
source snapshot, validation evidence and durable quarantine record.
|
||||||
|
This makes an attempted derivation from a manual or experimental
|
||||||
|
dataset observable and prevents a caller from bypassing the boundary
|
||||||
|
by invoking the governance service directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if parent_dataset is None:
|
||||||
|
return None
|
||||||
|
return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _with_parent_gate_failure(
|
||||||
|
report: ValidationReport,
|
||||||
|
decision: DatasetConsumptionDecision,
|
||||||
|
) -> ValidationReport:
|
||||||
|
"""Attach an auditable, fail-closed lineage failure to a report."""
|
||||||
|
|
||||||
|
issue = ValidationIssue(
|
||||||
|
code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING",
|
||||||
|
category="lineage",
|
||||||
|
field="lineage.parent_dataset",
|
||||||
|
message="Parent dataset failed the governed derived-processing consumption gate.",
|
||||||
|
severity=IssueSeverity.ERROR,
|
||||||
|
expected="eligible governed parent dataset",
|
||||||
|
observed={
|
||||||
|
"dataset_id": decision.evidence.get("dataset_id"),
|
||||||
|
"reasons": list(decision.reasons),
|
||||||
|
"source_key": decision.evidence.get("source_key"),
|
||||||
|
"source_classification": decision.evidence.get("source_classification"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return ValidationReport(
|
||||||
|
asset_id=report.asset_id,
|
||||||
|
data_contract_key=report.data_contract_key,
|
||||||
|
data_contract_version=report.data_contract_version,
|
||||||
|
contract_fingerprint_sha256=report.contract_fingerprint_sha256,
|
||||||
|
validation_status=ValidationStatus.FAILED,
|
||||||
|
provenance_status=(
|
||||||
|
ProvenanceStatus.INCOMPLETE
|
||||||
|
if report.provenance_status == ProvenanceStatus.COMPLETE
|
||||||
|
else report.provenance_status
|
||||||
|
),
|
||||||
|
lineage_status=LineageStatus.INCOMPLETE,
|
||||||
|
quarantine_status=QuarantineStatus.QUARANTINED,
|
||||||
|
validation_scope=report.validation_scope,
|
||||||
|
checked_at=report.checked_at,
|
||||||
|
issues=(*report.issues, issue),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _latest_parent_version_id(db: Session, dataset: Dataset):
|
||||||
|
version = (
|
||||||
|
db.query(DatasetVersion)
|
||||||
|
.filter(DatasetVersion.dataset_id == dataset.id)
|
||||||
|
.order_by(DatasetVersion.version.desc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
return version.id if version is not None else None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None:
|
||||||
|
values = metadata.get("resolution")
|
||||||
|
if not isinstance(values, (list, tuple)) or len(values) < 2:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)}
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolution_unit(crs: str | None) -> str:
|
||||||
|
return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _units_for_crs(crs: str | None) -> str:
|
||||||
|
return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _temporal_unknown_reason(dataset: Dataset) -> str | None:
|
||||||
|
if dataset.observed_at is not None:
|
||||||
|
return None
|
||||||
|
return "Derived or fixture artifact inherits no precise observation timestamp from its input."
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _source_version_unknown_reason(dataset: Dataset) -> str | None:
|
||||||
|
if dataset.source_version:
|
||||||
|
return None
|
||||||
|
return "Derived or fixture artifact has no source edition; the transform version is recorded separately."
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _datetime_value(value: datetime | None) -> str | None:
|
||||||
|
return value.astimezone(timezone.utc).isoformat() if value is not None else None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _stable_hash(value: Mapping[str, Any]) -> str:
|
||||||
|
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _valid_checksum(value: str | None) -> str | None:
|
||||||
|
normalized = str(value or "").strip().lower()
|
||||||
|
return normalized if _SHA256.fullmatch(normalized) else None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _checksum_or_placeholder(cls, value: str | None) -> str:
|
||||||
|
checksum = cls._valid_checksum(value)
|
||||||
|
if checksum is not None:
|
||||||
|
return checksum
|
||||||
|
# The contract receives the original invalid/missing checksum and
|
||||||
|
# quarantines it. A deterministic placeholder only permits storing
|
||||||
|
# the rejected snapshot without fabricating a valid artifact hash.
|
||||||
|
return cls._stable_hash({"invalid_dataset_checksum": value or "missing"})
|
||||||
@@ -0,0 +1,200 @@
|
|||||||
|
"""Placing two detection runs side by side, honestly.
|
||||||
|
|
||||||
|
The workbench ranks model variants by a stored F1, each measured at that
|
||||||
|
variant's own confidence threshold. That number says as much about the
|
||||||
|
threshold as about the model: a conservatively calibrated detector looks worse
|
||||||
|
than a liberal one without detecting anything differently. Average precision
|
||||||
|
describes the whole ranking the model produced and is the comparable figure.
|
||||||
|
|
||||||
|
Comparability comes first, though. Two runs over different rasters, scored
|
||||||
|
against different references, or covering different ground are not two answers
|
||||||
|
to one question, and no metric makes them so.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionComparisonService:
|
||||||
|
# A run whose inference footprint was never established cannot be placed
|
||||||
|
# beside one that was: their recalls have different denominators.
|
||||||
|
PROVEN_COVERAGE_MODES = ("persisted_tile_manifest_union",)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def assess_comparability(entries: list[dict[str, Any]]) -> dict[str, Any]:
|
||||||
|
"""Say whether these runs answer the same question, and why not if they don't."""
|
||||||
|
|
||||||
|
if len(entries) < 2:
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_COMPARISON_NEEDS_TWO_RUNS",
|
||||||
|
message="Comparing detection models requires at least two runs",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
|
||||||
|
reasons: list[str] = []
|
||||||
|
source_rasters = {str(entry.get("dataset_id")) for entry in entries}
|
||||||
|
if len(source_rasters) > 1:
|
||||||
|
reasons.append("different_source_raster")
|
||||||
|
|
||||||
|
references = {str(entry.get("reference_dataset_id")) for entry in entries}
|
||||||
|
if len(references) > 1:
|
||||||
|
reasons.append("different_reference_dataset")
|
||||||
|
|
||||||
|
if any(
|
||||||
|
str(entry.get("coverage_mode")) not in DetectionComparisonService.PROVEN_COVERAGE_MODES
|
||||||
|
for entry in entries
|
||||||
|
):
|
||||||
|
reasons.append("coverage_not_proven")
|
||||||
|
|
||||||
|
populations = {int(entry.get("reference_evaluated_count") or 0) for entry in entries}
|
||||||
|
if len(populations) > 1:
|
||||||
|
reasons.append("different_evaluated_population")
|
||||||
|
|
||||||
|
# Two runs that suppressed duplicates differently produced different
|
||||||
|
# candidate sets from the same model output, so their scores describe
|
||||||
|
# different pipelines. Runs from before these values were recorded
|
||||||
|
# carry none; absence is not a difference.
|
||||||
|
post_processing = {
|
||||||
|
(
|
||||||
|
entry.get("containment_suppression_threshold"),
|
||||||
|
entry.get("duplicate_iou_threshold"),
|
||||||
|
)
|
||||||
|
for entry in entries
|
||||||
|
}
|
||||||
|
if len(post_processing) > 1:
|
||||||
|
reasons.append("different_post_processing")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"comparable": not reasons,
|
||||||
|
"blocking_reasons": reasons,
|
||||||
|
"source_raster_count": len(source_rasters),
|
||||||
|
"reference_dataset_count": len(references),
|
||||||
|
"evaluated_population_counts": sorted(populations),
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def rank(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
"""Order runs by average precision, stating the margin and any tie.
|
||||||
|
|
||||||
|
Ranking on the F1 each run happened to be read at would order the
|
||||||
|
thresholds, not the models.
|
||||||
|
"""
|
||||||
|
|
||||||
|
ordered = sorted(
|
||||||
|
rows,
|
||||||
|
key=lambda row: (-(row.get("average_precision") or 0.0), str(row.get("model_asset_id") or "")),
|
||||||
|
)
|
||||||
|
if not ordered:
|
||||||
|
return []
|
||||||
|
|
||||||
|
leader = ordered[0].get("average_precision") or 0.0
|
||||||
|
tied_count = sum(1 for row in ordered if (row.get("average_precision") or 0.0) == leader)
|
||||||
|
runner_up = (ordered[1].get("average_precision") or 0.0) if len(ordered) > 1 else leader
|
||||||
|
|
||||||
|
ranked: list[dict[str, Any]] = []
|
||||||
|
for row in ordered:
|
||||||
|
average_precision = row.get("average_precision") or 0.0
|
||||||
|
is_leader = average_precision == leader
|
||||||
|
ranked.append(
|
||||||
|
{
|
||||||
|
**row,
|
||||||
|
"rank": 1 if is_leader else 1 + sum(
|
||||||
|
1 for other in ordered if (other.get("average_precision") or 0.0) > average_precision
|
||||||
|
),
|
||||||
|
# Distance behind the best run; zero for the leader itself.
|
||||||
|
"average_precision_gap": leader - average_precision,
|
||||||
|
"tied": is_leader and tied_count > 1,
|
||||||
|
# Only the leader has a lead; stating it on every row would
|
||||||
|
# invite reading a follower's gap as an advantage.
|
||||||
|
"lead_over_next": (leader - runner_up) if is_leader and tied_count == 1 else None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return ranked
|
||||||
|
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def compare_runs(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
analysis_run_ids: list,
|
||||||
|
reference_dataset_id,
|
||||||
|
iou_threshold: float = 0.5,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Score several runs against one reference and rank them on AP.
|
||||||
|
|
||||||
|
Each run is scored through the same QA path the workbench uses, so the
|
||||||
|
comparison and the persisted quality checks cannot drift apart.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Lazy: detection_service imports this module's siblings at load.
|
||||||
|
from app.models import AnalysisRun
|
||||||
|
from app.services.detection_service import DetectionService
|
||||||
|
|
||||||
|
if len(set(analysis_run_ids)) < 2:
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_COMPARISON_NEEDS_TWO_RUNS",
|
||||||
|
message="Comparing detection models requires at least two distinct runs",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
|
||||||
|
entries: list[dict[str, Any]] = []
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
for analysis_run_id in analysis_run_ids:
|
||||||
|
result = DetectionService.compare_detections_with_reference(
|
||||||
|
db,
|
||||||
|
analysis_run_id=analysis_run_id,
|
||||||
|
reference_dataset_id=reference_dataset_id,
|
||||||
|
iou_threshold=iou_threshold,
|
||||||
|
)
|
||||||
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
|
parameters = (run.parameters_json if run and isinstance(run.parameters_json, dict) else {}) or {}
|
||||||
|
run_result = (run.result_json if run and isinstance(run.result_json, dict) else {}) or {}
|
||||||
|
coverage = result.get("coverage") if isinstance(result.get("coverage"), dict) else {}
|
||||||
|
curve = result.get("precision_recall_curve") or {}
|
||||||
|
|
||||||
|
entries.append(
|
||||||
|
{
|
||||||
|
"analysis_run_id": analysis_run_id,
|
||||||
|
"dataset_id": getattr(run, "dataset_id", None),
|
||||||
|
"model_id": getattr(run, "model_name", None),
|
||||||
|
"model_asset_id": parameters.get("model_asset_id"),
|
||||||
|
"reference_dataset_id": reference_dataset_id,
|
||||||
|
"coverage_mode": coverage.get("mode"),
|
||||||
|
"reference_evaluated_count": coverage.get("reference_evaluated_count"),
|
||||||
|
# Recorded on the run itself, so two runs that suppressed
|
||||||
|
# duplicates differently cannot be ranked against each other.
|
||||||
|
"containment_suppression_threshold": run_result.get("containment_suppression_threshold"),
|
||||||
|
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"analysis_run_id": str(analysis_run_id),
|
||||||
|
"quality_check_id": result.get("quality_check_id"),
|
||||||
|
"model_id": getattr(run, "model_name", None),
|
||||||
|
"model_asset_id": parameters.get("model_asset_id"),
|
||||||
|
"run_confidence_threshold": parameters.get("confidence_threshold"),
|
||||||
|
"average_precision": curve.get("average_precision"),
|
||||||
|
"best_f1": curve.get("best_f1"),
|
||||||
|
"best_f1_threshold": curve.get("best_f1_threshold"),
|
||||||
|
# The figure the workbench used to rank on, kept visible so
|
||||||
|
# the difference between the two readings is auditable.
|
||||||
|
"f1_at_run_threshold": result.get("f1_score"),
|
||||||
|
"precision_at_run_threshold": result.get("precision"),
|
||||||
|
"recall_at_run_threshold": result.get("recall"),
|
||||||
|
"containment_suppression_threshold": run_result.get("containment_suppression_threshold"),
|
||||||
|
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
comparability = DetectionComparisonService.assess_comparability(entries)
|
||||||
|
return {
|
||||||
|
"reference_dataset_id": str(reference_dataset_id),
|
||||||
|
"iou_threshold": iou_threshold,
|
||||||
|
"comparability": comparability,
|
||||||
|
"ranking_metric": "average_precision",
|
||||||
|
"rows": DetectionComparisonService.rank(rows),
|
||||||
|
}
|
||||||
@@ -8,6 +8,24 @@ from shapely.geometry import Polygon
|
|||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
|
||||||
|
|
||||||
|
def _require_source_crs(crs: str | None, tile: dict[str, Any]) -> str:
|
||||||
|
"""Resolve the CRS a pixel coordinate is measured in, or fail.
|
||||||
|
|
||||||
|
Falling back to EPSG:4326 turned a missing manifest field into geometry
|
||||||
|
that sits in the wrong place while still looking like a valid polygon on
|
||||||
|
the map. A georeferenced result without a known CRS is not a result.
|
||||||
|
"""
|
||||||
|
|
||||||
|
for candidate in (crs, tile.get("crs"), tile.get("source_crs")):
|
||||||
|
if isinstance(candidate, str) and candidate.strip():
|
||||||
|
return candidate.strip()
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_TILE_CRS_REQUIRED",
|
||||||
|
message="Georeferencing a tile requires explicit CRS metadata",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs: str | None = None) -> Polygon:
|
def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs: str | None = None) -> Polygon:
|
||||||
if len(bbox) != 4:
|
if len(bbox) != 4:
|
||||||
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must contain four pixel coordinates", status_code=422)
|
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must contain four pixel coordinates", status_code=422)
|
||||||
@@ -28,7 +46,7 @@ def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs:
|
|||||||
else:
|
else:
|
||||||
corners = _corners_from_bounds(bbox=[x_min, y_min, x_max, y_max], tile=tile)
|
corners = _corners_from_bounds(bbox=[x_min, y_min, x_max, y_max], tile=tile)
|
||||||
|
|
||||||
source_crs = crs or tile.get("crs") or tile.get("source_crs") or "EPSG:4326"
|
source_crs = _require_source_crs(crs, tile)
|
||||||
if str(source_crs).upper() not in {"EPSG:4326", "4326"}:
|
if str(source_crs).upper() not in {"EPSG:4326", "4326"}:
|
||||||
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
|
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
|
||||||
corners = [transformer.transform(x, y) for x, y in corners]
|
corners = [transformer.transform(x, y) for x, y in corners]
|
||||||
@@ -39,6 +57,75 @@ def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs:
|
|||||||
return polygon
|
return polygon
|
||||||
|
|
||||||
|
|
||||||
|
def pixel_points_to_epsg4326_polygon(points: list[list[float]], tile: dict[str, Any], crs: str | None = None) -> Polygon:
|
||||||
|
if not isinstance(points, list) or len(points) < 3:
|
||||||
|
raise AppError(
|
||||||
|
code="SEGMENTATION_INVALID_MASK",
|
||||||
|
message="Segmentation mask polygon must contain at least three pixel points",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
pixel_points = [(float(point[0]), float(point[1])) for point in points]
|
||||||
|
except (TypeError, ValueError, IndexError) as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="SEGMENTATION_INVALID_MASK",
|
||||||
|
message="Segmentation mask polygon points must be numeric [x, y] pairs",
|
||||||
|
status_code=422,
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
transform = tile.get("transform")
|
||||||
|
if isinstance(transform, list) and len(transform) >= 6:
|
||||||
|
coordinates = [_apply_gdal_transform(transform, x, y) for x, y in pixel_points]
|
||||||
|
else:
|
||||||
|
coordinates = [_project_pixel_with_bounds(tile, x, y) for x, y in pixel_points]
|
||||||
|
|
||||||
|
source_crs = _require_source_crs(crs, tile)
|
||||||
|
if str(source_crs).upper() not in {"EPSG:4326", "4326"}:
|
||||||
|
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
|
||||||
|
coordinates = [transformer.transform(x, y) for x, y in coordinates]
|
||||||
|
|
||||||
|
if coordinates[0] != coordinates[-1]:
|
||||||
|
coordinates.append(coordinates[0])
|
||||||
|
polygon = Polygon(coordinates)
|
||||||
|
if not polygon.is_valid:
|
||||||
|
from shapely.validation import make_valid
|
||||||
|
|
||||||
|
repaired = make_valid(polygon)
|
||||||
|
polygon = _largest_polygon(repaired)
|
||||||
|
if polygon is None or polygon.is_empty or not polygon.is_valid or polygon.area <= 0:
|
||||||
|
raise AppError(
|
||||||
|
code="SEGMENTATION_INVALID_GEOMETRY",
|
||||||
|
message="Georeferenced segmentation geometry is invalid",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return polygon
|
||||||
|
|
||||||
|
|
||||||
|
def _largest_polygon(geometry: Any) -> Polygon | None:
|
||||||
|
if isinstance(geometry, Polygon):
|
||||||
|
return geometry
|
||||||
|
candidates = [geom for geom in getattr(geometry, "geoms", []) if isinstance(geom, Polygon) and geom.area > 0]
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
return max(candidates, key=lambda geom: geom.area)
|
||||||
|
|
||||||
|
|
||||||
|
def _project_pixel_with_bounds(tile: dict[str, Any], px: float, py: float) -> tuple[float, float]:
|
||||||
|
bounds = tile.get("bounds")
|
||||||
|
pixel_window = tile.get("pixel_window")
|
||||||
|
if not (isinstance(bounds, list) and len(bounds) == 4 and isinstance(pixel_window, list) and len(pixel_window) == 4):
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_TILE_MANIFEST_INVALID",
|
||||||
|
message="Tile manifest entries require transform or bounds plus pixel_window for georeferencing",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
left, bottom, right, top = [float(value) for value in bounds]
|
||||||
|
_, _, width, height = [float(value) for value in pixel_window]
|
||||||
|
if width <= 0 or height <= 0:
|
||||||
|
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile pixel_window must have positive size", status_code=422)
|
||||||
|
return (left + (px / width) * (right - left), top - (py / height) * (top - bottom))
|
||||||
|
|
||||||
|
|
||||||
def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]:
|
def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]:
|
||||||
c, a, b, f, d, e = [float(value) for value in transform[:6]]
|
c, a, b, f, d, e = [float(value) for value in transform[:6]]
|
||||||
return (a * x + b * y + c, d * x + e * y + f)
|
return (a * x + b * y + c, d * x + e * y + f)
|
||||||
|
|||||||
@@ -0,0 +1,235 @@
|
|||||||
|
"""Threshold-independent detection metrics.
|
||||||
|
|
||||||
|
A single precision/recall/F1 triple describes one operating point. Which point
|
||||||
|
that is depends on the confidence threshold the operator typed, so two models
|
||||||
|
cannot be compared from it: a conservatively calibrated model looks worse at a
|
||||||
|
low cut and better at a high one without detecting anything differently.
|
||||||
|
|
||||||
|
This service produces the standard alternative — the full precision/recall
|
||||||
|
curve over every confidence value that occurs in the run, the average
|
||||||
|
precision derived from it, and the threshold where F1 actually peaks — using
|
||||||
|
the same greedy IoU matching rule as the rest of QA so the numbers stay
|
||||||
|
comparable with the persisted quality checks.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from shapely.geometry.base import BaseGeometry
|
||||||
|
from shapely.strtree import STRtree
|
||||||
|
|
||||||
|
from app.services.qa_service import QaService
|
||||||
|
|
||||||
|
|
||||||
|
class DetectionMetricsService:
|
||||||
|
SUPPORTED_GEOMETRY_TYPES = QaService.SUPPORTED_GEOMETRY_TYPES
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _rank_candidates(
|
||||||
|
candidates: list[tuple[dict[str, Any], BaseGeometry]],
|
||||||
|
) -> list[tuple[float, str, dict[str, Any], BaseGeometry]]:
|
||||||
|
"""Order candidates by confidence, highest first, deterministically."""
|
||||||
|
|
||||||
|
ranked: list[tuple[float, str, dict[str, Any], BaseGeometry]] = []
|
||||||
|
for index, (feature, geometry) in enumerate(candidates):
|
||||||
|
if geometry.is_empty or geometry.geom_type not in DetectionMetricsService.SUPPORTED_GEOMETRY_TYPES:
|
||||||
|
continue
|
||||||
|
if geometry.area <= 0:
|
||||||
|
continue
|
||||||
|
confidence = QaService._feature_confidence(feature)
|
||||||
|
identifier = QaService._feature_identifier(feature, "candidate", index)
|
||||||
|
ranked.append((float(confidence if confidence is not None else 0.0), identifier, feature, geometry))
|
||||||
|
ranked.sort(key=lambda item: (-item[0], item[1]))
|
||||||
|
return ranked
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _greedy_hits(
|
||||||
|
ranked: list[tuple[float, str, dict[str, Any], BaseGeometry]],
|
||||||
|
references: list[tuple[dict[str, Any], BaseGeometry]],
|
||||||
|
iou_threshold: float,
|
||||||
|
) -> tuple[list[bool], int]:
|
||||||
|
"""Mark each ranked candidate as a hit or a miss, best score first.
|
||||||
|
|
||||||
|
Walking the ranking once and consuming references as they are claimed
|
||||||
|
is exactly the COCO/PASCAL rule, and it is what makes the result
|
||||||
|
independent of the order rows came out of the database.
|
||||||
|
"""
|
||||||
|
|
||||||
|
supported = [
|
||||||
|
(index, geometry)
|
||||||
|
for index, (_, geometry) in enumerate(references)
|
||||||
|
if geometry.geom_type in DetectionMetricsService.SUPPORTED_GEOMETRY_TYPES
|
||||||
|
and not geometry.is_empty
|
||||||
|
and geometry.area > 0
|
||||||
|
]
|
||||||
|
reference_count = len(supported)
|
||||||
|
if not supported:
|
||||||
|
return [False] * len(ranked), 0
|
||||||
|
|
||||||
|
tree = STRtree([geometry for _, geometry in supported])
|
||||||
|
claimed: set[int] = set()
|
||||||
|
hits: list[bool] = []
|
||||||
|
|
||||||
|
for _, _, _, geometry in ranked:
|
||||||
|
best_iou = 0.0
|
||||||
|
best_index: int | None = None
|
||||||
|
for position in sorted(int(value) for value in tree.query(geometry)):
|
||||||
|
if position in claimed:
|
||||||
|
continue
|
||||||
|
_, reference_geometry = supported[position]
|
||||||
|
intersection_area = geometry.intersection(reference_geometry).area
|
||||||
|
if intersection_area <= 0:
|
||||||
|
continue
|
||||||
|
union_area = geometry.area + reference_geometry.area - intersection_area
|
||||||
|
if union_area <= 0:
|
||||||
|
continue
|
||||||
|
iou = intersection_area / union_area
|
||||||
|
if iou > best_iou:
|
||||||
|
best_iou = iou
|
||||||
|
best_index = position
|
||||||
|
if best_index is not None and best_iou >= iou_threshold:
|
||||||
|
claimed.add(best_index)
|
||||||
|
hits.append(True)
|
||||||
|
else:
|
||||||
|
hits.append(False)
|
||||||
|
|
||||||
|
return hits, reference_count
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _average_precision(points: list[dict[str, Any]]) -> float:
|
||||||
|
"""Area under the precision/recall curve, with precision made monotone.
|
||||||
|
|
||||||
|
Interpolating precision to its running maximum from the right is the
|
||||||
|
VOC/COCO convention; without it the sawtooth from individual false
|
||||||
|
positives shows up as noise in the score.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not points:
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
recalls = [0.0] + [point["recall"] for point in points]
|
||||||
|
precisions = [1.0] + [point["precision"] for point in points]
|
||||||
|
for index in range(len(precisions) - 2, -1, -1):
|
||||||
|
precisions[index] = max(precisions[index], precisions[index + 1])
|
||||||
|
|
||||||
|
area = 0.0
|
||||||
|
for index in range(1, len(recalls)):
|
||||||
|
area += (recalls[index] - recalls[index - 1]) * precisions[index]
|
||||||
|
return area
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def operating_point(curve: dict[str, Any], *, min_confidence: float) -> dict[str, Any]:
|
||||||
|
"""The metrics that hold when detections below ``min_confidence`` are dropped.
|
||||||
|
|
||||||
|
Read off the curve rather than recomputed: the curve already walked the
|
||||||
|
ranking once, and every threshold is a prefix of that walk. Running the
|
||||||
|
model again per threshold would spend N GPU passes to reproduce numbers
|
||||||
|
that are already here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
points = curve.get("points") or []
|
||||||
|
reference_count = int(curve.get("reference_count") or 0)
|
||||||
|
admitted = [point for point in points if point["confidence_threshold"] >= min_confidence]
|
||||||
|
# Points are cumulative down the ranking, so the last admitted one is
|
||||||
|
# the complete tally at this cut.
|
||||||
|
tally = admitted[-1] if admitted else None
|
||||||
|
|
||||||
|
true_positives = int(tally["true_positives"]) if tally else 0
|
||||||
|
false_positives = int(tally["false_positives"]) if tally else 0
|
||||||
|
false_negatives = max(0, reference_count - true_positives)
|
||||||
|
candidate_count = true_positives + false_positives
|
||||||
|
|
||||||
|
precision = true_positives / candidate_count if candidate_count else None
|
||||||
|
recall = true_positives / reference_count if reference_count else None
|
||||||
|
f1_score = None
|
||||||
|
if precision is not None and recall is not None:
|
||||||
|
f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"min_confidence": min_confidence,
|
||||||
|
"confidence_threshold": tally["confidence_threshold"] if tally else None,
|
||||||
|
"candidate_count": candidate_count,
|
||||||
|
"true_positives": true_positives,
|
||||||
|
"false_positives": false_positives,
|
||||||
|
"false_negatives": false_negatives,
|
||||||
|
"precision": precision,
|
||||||
|
"recall": recall,
|
||||||
|
"f1_score": f1_score,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def calibration_sweep(curve: dict[str, Any], *, thresholds: list[float]) -> list[dict[str, Any]]:
|
||||||
|
"""Every requested operating point, strictest first, from one curve.
|
||||||
|
|
||||||
|
Recall cannot fall as the cut loosens — that monotonicity is exactly
|
||||||
|
why a single run answers the whole sweep.
|
||||||
|
"""
|
||||||
|
|
||||||
|
ordered = sorted({float(value) for value in thresholds}, reverse=True)
|
||||||
|
rows = [
|
||||||
|
DetectionMetricsService.operating_point(curve, min_confidence=value) for value in ordered
|
||||||
|
]
|
||||||
|
best_f1 = max((row["f1_score"] or 0.0) for row in rows) if rows else 0.0
|
||||||
|
marked = False
|
||||||
|
for row in rows:
|
||||||
|
is_best = not marked and (row["f1_score"] or 0.0) == best_f1
|
||||||
|
row["best_f1_in_sweep"] = is_best
|
||||||
|
marked = marked or is_best
|
||||||
|
return rows
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def precision_recall_curve(
|
||||||
|
candidates: list[tuple[dict[str, Any], BaseGeometry]],
|
||||||
|
references: list[tuple[dict[str, Any], BaseGeometry]],
|
||||||
|
*,
|
||||||
|
iou_threshold: float,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Sweep every confidence value present and describe the whole curve."""
|
||||||
|
|
||||||
|
ranked = DetectionMetricsService._rank_candidates(candidates)
|
||||||
|
hits, reference_count = DetectionMetricsService._greedy_hits(ranked, references, iou_threshold)
|
||||||
|
|
||||||
|
points: list[dict[str, Any]] = []
|
||||||
|
true_positives = 0
|
||||||
|
for position, hit in enumerate(hits):
|
||||||
|
if hit:
|
||||||
|
true_positives += 1
|
||||||
|
false_positives = position + 1 - true_positives
|
||||||
|
precision = true_positives / (position + 1)
|
||||||
|
recall = true_positives / reference_count if reference_count else 0.0
|
||||||
|
f1 = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
|
||||||
|
points.append(
|
||||||
|
{
|
||||||
|
"confidence_threshold": ranked[position][0],
|
||||||
|
"candidate_count": position + 1,
|
||||||
|
"true_positives": true_positives,
|
||||||
|
"false_positives": false_positives,
|
||||||
|
"false_negatives": max(0, reference_count - true_positives),
|
||||||
|
"precision": precision,
|
||||||
|
"recall": recall,
|
||||||
|
"f1_score": f1,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Keep one point per distinct threshold: the last one, which is the
|
||||||
|
# complete tally for everything at or above that confidence.
|
||||||
|
deduplicated: list[dict[str, Any]] = []
|
||||||
|
for point in points:
|
||||||
|
if deduplicated and deduplicated[-1]["confidence_threshold"] == point["confidence_threshold"]:
|
||||||
|
deduplicated[-1] = point
|
||||||
|
else:
|
||||||
|
deduplicated.append(point)
|
||||||
|
|
||||||
|
best = max(deduplicated, key=lambda point: (point["f1_score"], point["confidence_threshold"]), default=None)
|
||||||
|
return {
|
||||||
|
"iou_threshold": iou_threshold,
|
||||||
|
"reference_count": reference_count,
|
||||||
|
"candidate_count": len(ranked),
|
||||||
|
"average_precision": DetectionMetricsService._average_precision(points),
|
||||||
|
"best_f1": best["f1_score"] if best else 0.0,
|
||||||
|
"best_f1_threshold": best["confidence_threshold"] if best else None,
|
||||||
|
"best_f1_precision": best["precision"] if best else None,
|
||||||
|
"best_f1_recall": best["recall"] if best else None,
|
||||||
|
"points": deduplicated,
|
||||||
|
}
|
||||||
@@ -168,24 +168,61 @@ class DetectionQaService:
|
|||||||
clipped_boundary_count=clipped_boundary_count,
|
clipped_boundary_count=clipped_boundary_count,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# A polygon whose area is within this fraction of its own bounding box is
|
||||||
|
# an axis-aligned rectangle for practical purposes.
|
||||||
|
RECTANGULAR_AREA_RATIO = 0.99
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def candidate_geometry_mode(geometries: list[tuple[dict[str, Any], BaseGeometry]]) -> str:
|
||||||
|
"""Say whether the candidates are detector boxes or true footprints.
|
||||||
|
|
||||||
|
It matters for reading the score. An axis-aligned box can never reach
|
||||||
|
IoU 1 against a rotated or L-shaped building footprint, so a strict
|
||||||
|
footprint IoU understates a box detector by a fixed amount that has
|
||||||
|
nothing to do with whether it found the building.
|
||||||
|
"""
|
||||||
|
|
||||||
|
polygonal = [
|
||||||
|
geometry
|
||||||
|
for _, geometry in geometries
|
||||||
|
if geometry.geom_type in {"Polygon", "MultiPolygon"} and geometry.area > 0
|
||||||
|
]
|
||||||
|
if not polygonal:
|
||||||
|
return "unknown"
|
||||||
|
rectangular = sum(
|
||||||
|
1
|
||||||
|
for geometry in polygonal
|
||||||
|
if geometry.area / geometry.envelope.area >= DetectionQaService.RECTANGULAR_AREA_RATIO
|
||||||
|
)
|
||||||
|
return "axis_aligned_boxes" if rectangular == len(polygonal) else "footprint_polygons"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def box_to_footprint_diagnostics(
|
def box_to_footprint_diagnostics(
|
||||||
strict_evidence: QaMatchEvidence,
|
strict_evidence: QaMatchEvidence,
|
||||||
envelope_evidence: QaMatchEvidence,
|
envelope_evidence: QaMatchEvidence,
|
||||||
*,
|
*,
|
||||||
iou_threshold: float,
|
iou_threshold: float,
|
||||||
|
candidate_geometry_mode: str = "unknown",
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
envelope_metrics = DetectionQaService._metrics(envelope_evidence)
|
envelope_metrics = DetectionQaService._metrics(envelope_evidence)
|
||||||
return {
|
diagnostics = {
|
||||||
"diagnostic_only": True,
|
"diagnostic_only": True,
|
||||||
"canonical_method": "candidate_polygon_vs_reference_footprint_iou",
|
"canonical_method": "candidate_polygon_vs_reference_footprint_iou",
|
||||||
"diagnostic_method": "candidate_polygon_vs_reference_envelope_iou",
|
"diagnostic_method": "candidate_polygon_vs_reference_envelope_iou",
|
||||||
"iou_threshold": iou_threshold,
|
"iou_threshold": iou_threshold,
|
||||||
|
"candidate_geometry_mode": candidate_geometry_mode,
|
||||||
"strict_matches": strict_evidence.matches,
|
"strict_matches": strict_evidence.matches,
|
||||||
"envelope_matches": envelope_evidence.matches,
|
"envelope_matches": envelope_evidence.matches,
|
||||||
"possible_box_to_footprint_mismatch_count": max(0, envelope_evidence.matches - strict_evidence.matches),
|
"possible_box_to_footprint_mismatch_count": max(0, envelope_evidence.matches - strict_evidence.matches),
|
||||||
**envelope_metrics,
|
**envelope_metrics,
|
||||||
}
|
}
|
||||||
|
if candidate_geometry_mode == "axis_aligned_boxes":
|
||||||
|
diagnostics["interpretation"] = (
|
||||||
|
"Candidates are axis-aligned detector boxes. The strict footprint IoU therefore has a "
|
||||||
|
"ceiling below 1 for rotated or non-rectangular buildings; the envelope figures isolate "
|
||||||
|
"detection quality from that shape mismatch."
|
||||||
|
)
|
||||||
|
return diagnostics
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _metrics(evidence: QaMatchEvidence) -> dict[str, Any]:
|
def _metrics(evidence: QaMatchEvidence) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.services.reviewed_metrics_service import ReviewedMetricsService
|
||||||
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
|
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
|
||||||
from app.schemas.detection_review import (
|
from app.schemas.detection_review import (
|
||||||
DetectionReviewList,
|
DetectionReviewList,
|
||||||
@@ -80,7 +80,11 @@ class DetectionReviewService:
|
|||||||
return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
|
return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary:
|
def _summary(
|
||||||
|
evidence: list[dict[str, str]],
|
||||||
|
reviews: dict[tuple[str, str], DetectionReview],
|
||||||
|
quality_check: QualityCheck | None = None,
|
||||||
|
) -> DetectionReviewSummary:
|
||||||
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
|
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
|
||||||
decisions = Counter(
|
decisions = Counter(
|
||||||
reviews[key].decision if key in reviews else "unreviewed"
|
reviews[key].decision if key in reviews else "unreviewed"
|
||||||
@@ -96,6 +100,43 @@ class DetectionReviewService:
|
|||||||
false_positive_total=false_positive_total,
|
false_positive_total=false_positive_total,
|
||||||
false_negative_total=false_negative_total,
|
false_negative_total=false_negative_total,
|
||||||
decision_counts=dict(sorted(decisions.items())),
|
decision_counts=dict(sorted(decisions.items())),
|
||||||
|
reviewed_metrics=DetectionReviewService._reviewed_metrics(evidence_keys, reviews, quality_check),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _reviewed_metrics(
|
||||||
|
evidence_keys: set[tuple[str, str]],
|
||||||
|
reviews: dict[tuple[str, str], DetectionReview],
|
||||||
|
quality_check: QualityCheck | None,
|
||||||
|
) -> dict | None:
|
||||||
|
"""The score with the operator's verdicts applied.
|
||||||
|
|
||||||
|
Without this the panel shows a precision the operator has already
|
||||||
|
disproved: a false positive adjudicated as a reference gap is not the
|
||||||
|
model's error, and the raw number keeps counting it as one.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if quality_check is None:
|
||||||
|
return None
|
||||||
|
findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
|
||||||
|
matches = findings.get("matches")
|
||||||
|
false_positives = findings.get("false_positives")
|
||||||
|
false_negatives = findings.get("false_negatives")
|
||||||
|
if not all(isinstance(value, int) for value in (matches, false_positives, false_negatives)):
|
||||||
|
return None
|
||||||
|
|
||||||
|
per_role: dict[str, Counter] = {"false_positive": Counter(), "false_negative": Counter()}
|
||||||
|
for role, feature_id in evidence_keys:
|
||||||
|
review = reviews.get((role, feature_id))
|
||||||
|
if review is not None and role in per_role:
|
||||||
|
per_role[role][review.decision] += 1
|
||||||
|
|
||||||
|
return ReviewedMetricsService.adjudicate(
|
||||||
|
matches=int(matches),
|
||||||
|
false_positives=int(false_positives),
|
||||||
|
false_negatives=int(false_negatives),
|
||||||
|
false_positive_decisions=dict(per_role["false_positive"]),
|
||||||
|
false_negative_decisions=dict(per_role["false_negative"]),
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -180,7 +221,7 @@ class DetectionReviewService:
|
|||||||
total=len(filtered),
|
total=len(filtered),
|
||||||
limit=limit,
|
limit=limit,
|
||||||
offset=offset,
|
offset=offset,
|
||||||
summary=DetectionReviewService._summary(evidence, reviews),
|
summary=DetectionReviewService._summary(evidence, reviews, quality_check),
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -9,21 +9,31 @@ from typing import Any
|
|||||||
from typing import Type
|
from typing import Type
|
||||||
|
|
||||||
from geoalchemy2.shape import from_shape, to_shape
|
from geoalchemy2.shape import from_shape, to_shape
|
||||||
|
from pyproj import Transformer
|
||||||
|
from shapely.geometry import box as shapely_box
|
||||||
from shapely.geometry import mapping, shape
|
from shapely.geometry import mapping, shape
|
||||||
|
from shapely.ops import transform as shapely_transform
|
||||||
|
from shapely.strtree import STRtree
|
||||||
from sqlalchemy import func
|
from sqlalchemy import func
|
||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.core.request_context import get_request_id
|
from app.core.request_context import get_request_id
|
||||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project, VectorFeature
|
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, VectorFeature
|
||||||
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
|
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
|
||||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||||
|
from app.services.detection_metrics_service import DetectionMetricsService
|
||||||
from app.services.detection_qa_service import DetectionQaService
|
from app.services.detection_qa_service import DetectionQaService
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
|
from app.services.model_validation_scope_service import ModelValidationScopeService
|
||||||
from app.services.qa_service import QaService
|
from app.services.qa_service import QaService
|
||||||
|
from app.services.storage_service import StorageService
|
||||||
from app.services.quality_service import QualityService
|
from app.services.quality_service import QualityService
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
||||||
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
||||||
|
from app.services.tile_manifest_service import TileManifestService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
|
|
||||||
|
|
||||||
@@ -48,22 +58,11 @@ class DetectionService:
|
|||||||
parameters_json: dict[str, Any] | None = None,
|
parameters_json: dict[str, Any] | None = None,
|
||||||
settings: Settings | None = None,
|
settings: Settings | None = None,
|
||||||
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
|
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
|
||||||
|
existing_job: Job | None = None,
|
||||||
) -> DetectionRunResponse:
|
) -> DetectionRunResponse:
|
||||||
parameters = dict(parameters_json or {})
|
parameters = dict(parameters_json or {})
|
||||||
resolved_settings = settings or get_settings()
|
resolved_settings = settings or get_settings()
|
||||||
project = db.get(Project, project_id)
|
dataset = DetectionService._validate_run_request(db, project_id=project_id, dataset_id=dataset_id)
|
||||||
if not project:
|
|
||||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
|
||||||
dataset = db.get(Dataset, dataset_id)
|
|
||||||
if not dataset or dataset.project_id != project_id:
|
|
||||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
|
||||||
if dataset.dataset_type != "raster":
|
|
||||||
raise AppError(
|
|
||||||
code="INVALID_DATASET_TYPE",
|
|
||||||
message="Detection requires a raster dataset",
|
|
||||||
details={"dataset_type": dataset.dataset_type},
|
|
||||||
status_code=400,
|
|
||||||
)
|
|
||||||
TemporalCompatibilityService.ensure_detection_source_supported(dataset)
|
TemporalCompatibilityService.ensure_detection_source_supported(dataset)
|
||||||
|
|
||||||
selected_model_asset = None
|
selected_model_asset = None
|
||||||
@@ -90,6 +89,24 @@ class DetectionService:
|
|||||||
message="Configured YOLO inference requires an existing raster tile manifest path",
|
message="Configured YOLO inference requires an existing raster tile manifest path",
|
||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
|
requested_classes = {DetectionService._canonical_class_name(value) for value in (class_filter or [])}
|
||||||
|
unsupported_classes = sorted(requested_classes - set(model.supported_classes))
|
||||||
|
if unsupported_classes:
|
||||||
|
raise AppError(code="DETECTION_CLASS_NOT_VALIDATED", message="The selected model is not validated for one or more requested classes", details={"unsupported_classes": unsupported_classes, "supported_classes": model.supported_classes}, status_code=422)
|
||||||
|
if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope:
|
||||||
|
DetectionService._validate_model_area_scope(db, dataset, resolved_settings)
|
||||||
|
|
||||||
|
# Never enter a production inference path with a persisted dataset
|
||||||
|
# that has failed validation, incomplete provenance, or an active
|
||||||
|
# quarantine. Fixture detection is a separate QA/test-only path.
|
||||||
|
if model.model_id == "manual-fixture-detector":
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
dataset,
|
||||||
|
purpose="quality_assessment",
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
elif model.model_id == resolved_settings.yolo_model_id and model.configured:
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||||
|
|
||||||
run_parameters = {
|
run_parameters = {
|
||||||
"model_id": model.model_id,
|
"model_id": model.model_id,
|
||||||
@@ -101,7 +118,7 @@ class DetectionService:
|
|||||||
"tile_manifest_path": tile_manifest_path,
|
"tile_manifest_path": tile_manifest_path,
|
||||||
"parameters_json": parameters,
|
"parameters_json": parameters,
|
||||||
}
|
}
|
||||||
job = DetectionService._create_job(db, project_id, dataset_id, run_parameters)
|
job = DetectionService._create_job(db, project_id, dataset_id, run_parameters, existing_job=existing_job)
|
||||||
analysis_run = DetectionService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters)
|
analysis_run = DetectionService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters)
|
||||||
logger.info(
|
logger.info(
|
||||||
"detection_started request_id=%s project_id=%s dataset_id=%s job_id=%s analysis_run_id=%s model_id=%s",
|
"detection_started request_id=%s project_id=%s dataset_id=%s job_id=%s analysis_run_id=%s model_id=%s",
|
||||||
@@ -130,6 +147,7 @@ class DetectionService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if model.model_id == "manual-fixture-detector":
|
if model.model_id == "manual-fixture-detector":
|
||||||
|
try:
|
||||||
detections = DetectionService._persist_fixture_detections(
|
detections = DetectionService._persist_fixture_detections(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
@@ -142,6 +160,10 @@ class DetectionService:
|
|||||||
confidence_threshold=confidence_threshold,
|
confidence_threshold=confidence_threshold,
|
||||||
class_filter=class_filter or [],
|
class_filter=class_filter or [],
|
||||||
)
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
# A rejected fixture payload must never leave the run stuck in "running".
|
||||||
|
DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR")
|
||||||
|
raise
|
||||||
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
|
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
|
||||||
return DetectionRunResponse(
|
return DetectionRunResponse(
|
||||||
analysis_run_id=analysis_run.id,
|
analysis_run_id=analysis_run.id,
|
||||||
@@ -183,6 +205,10 @@ class DetectionService:
|
|||||||
error_code=exc.code,
|
error_code=exc.code,
|
||||||
message=exc.message,
|
message=exc.message,
|
||||||
)
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
# An unexpected inference error must never leave the run stuck in "running".
|
||||||
|
DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR")
|
||||||
|
raise
|
||||||
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary)
|
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary)
|
||||||
return DetectionRunResponse(
|
return DetectionRunResponse(
|
||||||
analysis_run_id=analysis_run.id,
|
analysis_run_id=analysis_run.id,
|
||||||
@@ -195,8 +221,54 @@ class DetectionService:
|
|||||||
message="YOLO detections persisted.",
|
message="YOLO detections persisted.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
DetectionService._mark_failed(
|
||||||
|
db,
|
||||||
|
analysis_run,
|
||||||
|
job,
|
||||||
|
code="DETECTION_MODEL_UNAVAILABLE",
|
||||||
|
message="Detection model is unavailable",
|
||||||
|
)
|
||||||
raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503)
|
raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validate_model_area_scope(db, dataset: Dataset, settings: Settings) -> None:
|
||||||
|
area = db.get(Area, dataset.area_id) if dataset.area_id else None
|
||||||
|
if area is None or area.geometry is None:
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_VALIDATION_SCOPE_UNAVAILABLE",
|
||||||
|
message="Configured YOLO inference requires a persisted Dataset area geometry.",
|
||||||
|
details={"dataset_id": str(dataset.id)},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
area_geometry = to_shape(area.geometry)
|
||||||
|
except Exception as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_VALIDATION_SCOPE_UNAVAILABLE",
|
||||||
|
message="The persisted Dataset area geometry cannot be validated for model inference.",
|
||||||
|
details={"dataset_id": str(dataset.id), "error_type": type(exc).__name__},
|
||||||
|
status_code=422,
|
||||||
|
) from exc
|
||||||
|
ModelValidationScopeService.assert_area_covered(
|
||||||
|
area_geometry=area_geometry,
|
||||||
|
manifest_path=settings.yolo_validation_scope_manifest_path,
|
||||||
|
expected_manifest_sha256=settings.yolo_validation_scope_manifest_sha256,
|
||||||
|
model_id=settings.yolo_model_id,
|
||||||
|
model_path=settings.yolo_model_path,
|
||||||
|
)
|
||||||
|
@staticmethod
|
||||||
|
def _fail_run_after_exception(db, analysis_run: AnalysisRun, job: Job, exc: Exception, fallback_code: str) -> None:
|
||||||
|
try:
|
||||||
|
db.rollback()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
code = getattr(exc, "code", None) or fallback_code
|
||||||
|
message = getattr(exc, "message", None) or "Unexpected internal error during analysis run"
|
||||||
|
try:
|
||||||
|
DetectionService._mark_failed(db, analysis_run, job, code=str(code), message=str(message))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead:
|
def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead:
|
||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
@@ -210,6 +282,8 @@ class DetectionService:
|
|||||||
*,
|
*,
|
||||||
project_id: uuid.UUID | None = None,
|
project_id: uuid.UUID | None = None,
|
||||||
dataset_id: uuid.UUID | None = None,
|
dataset_id: uuid.UUID | None = None,
|
||||||
|
limit: int | None = None,
|
||||||
|
offset: int = 0,
|
||||||
) -> DetectionRunListResponse:
|
) -> DetectionRunListResponse:
|
||||||
query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "detection")
|
query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "detection")
|
||||||
if project_id is not None:
|
if project_id is not None:
|
||||||
@@ -217,7 +291,16 @@ class DetectionService:
|
|||||||
if dataset_id is not None:
|
if dataset_id is not None:
|
||||||
query = query.filter(AnalysisRun.dataset_id == dataset_id)
|
query = query.filter(AnalysisRun.dataset_id == dataset_id)
|
||||||
rows = query.order_by(AnalysisRun.created_at.desc()).all()
|
rows = query.order_by(AnalysisRun.created_at.desc()).all()
|
||||||
return DetectionRunListResponse(items=[DetectionRunRead.model_validate(row) for row in rows], total=len(rows))
|
# Runs accumulate with every analysis; the panel draws the recent ones.
|
||||||
|
resolved_limit = DetectionService.DEFAULT_RUN_LIST_LIMIT if limit is None else int(limit)
|
||||||
|
page, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=offset)
|
||||||
|
return DetectionRunListResponse(
|
||||||
|
items=[DetectionRunRead.model_validate(row) for row in page],
|
||||||
|
total=total,
|
||||||
|
limit=resolved_limit,
|
||||||
|
offset=max(0, int(offset)),
|
||||||
|
truncated=truncated,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def list_detections(
|
def list_detections(
|
||||||
@@ -227,6 +310,8 @@ class DetectionService:
|
|||||||
dataset_id: uuid.UUID | None = None,
|
dataset_id: uuid.UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int | None = None,
|
||||||
|
offset: int = 0,
|
||||||
) -> DetectionListResponse:
|
) -> DetectionListResponse:
|
||||||
if analysis_run_id is not None:
|
if analysis_run_id is not None:
|
||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
@@ -239,8 +324,15 @@ class DetectionService:
|
|||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
min_confidence=min_confidence,
|
min_confidence=min_confidence,
|
||||||
)
|
)
|
||||||
items = [DetectionRead.model_validate(row) for row in rows]
|
resolved_limit = DetectionService.DEFAULT_RESULT_LIMIT if limit is None else int(limit)
|
||||||
return DetectionListResponse(items=items, total=len(items))
|
page, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=offset)
|
||||||
|
return DetectionListResponse(
|
||||||
|
items=[DetectionRead.model_validate(row) for row in page],
|
||||||
|
total=total,
|
||||||
|
limit=resolved_limit,
|
||||||
|
offset=max(0, int(offset)),
|
||||||
|
truncated=truncated,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_detection(db, detection_id: uuid.UUID) -> DetectionRead:
|
def get_detection(db, detection_id: uuid.UUID) -> DetectionRead:
|
||||||
@@ -257,16 +349,27 @@ class DetectionService:
|
|||||||
dataset_id: uuid.UUID | None = None,
|
dataset_id: uuid.UUID | None = None,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
limit: int | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
detections = DetectionService._query_detection_rows(
|
rows = DetectionService._query_detection_rows(
|
||||||
db,
|
db,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
class_name=class_name,
|
class_name=class_name,
|
||||||
min_confidence=min_confidence,
|
min_confidence=min_confidence,
|
||||||
)
|
)
|
||||||
|
resolved_limit = DetectionService.DEFAULT_RESULT_LIMIT if limit is None else int(limit)
|
||||||
|
# Rows arrive ranked by confidence, so a capped overlay draws the
|
||||||
|
# strongest detections rather than an arbitrary slice.
|
||||||
|
detections, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=0)
|
||||||
return {
|
return {
|
||||||
"type": "FeatureCollection",
|
"type": "FeatureCollection",
|
||||||
|
"geointel_result_window": {
|
||||||
|
"feature_count": len(detections),
|
||||||
|
"total_feature_count": total,
|
||||||
|
"limit": resolved_limit,
|
||||||
|
"truncated": truncated,
|
||||||
|
},
|
||||||
"features": [
|
"features": [
|
||||||
{
|
{
|
||||||
"type": "Feature",
|
"type": "Feature",
|
||||||
@@ -286,6 +389,7 @@ class DetectionService:
|
|||||||
iou_threshold: float = 0.5,
|
iou_threshold: float = 0.5,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
calibration_thresholds: list[float] | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
if not run or run.analysis_type != "detection":
|
if not run or run.analysis_type != "detection":
|
||||||
@@ -305,15 +409,6 @@ class DetectionService:
|
|||||||
reference_dataset,
|
reference_dataset,
|
||||||
)
|
)
|
||||||
|
|
||||||
detections = DetectionService._query_detection_rows(
|
|
||||||
db,
|
|
||||||
analysis_run_id=analysis_run_id,
|
|
||||||
dataset_id=run.dataset_id,
|
|
||||||
class_name=class_name,
|
|
||||||
min_confidence=min_confidence,
|
|
||||||
)
|
|
||||||
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
|
|
||||||
candidate_geometries = raw_candidate_geometries
|
|
||||||
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
||||||
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
|
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
|
||||||
resolved_settings = get_settings()
|
resolved_settings = get_settings()
|
||||||
@@ -328,9 +423,48 @@ class DetectionService:
|
|||||||
status_code=422,
|
status_code=422,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
fixture_parameters = run_parameters.get("parameters_json")
|
||||||
|
fixture_mode = bool(
|
||||||
|
run.model_name == "manual-fixture-detector"
|
||||||
|
and isinstance(fixture_parameters, dict)
|
||||||
|
and fixture_parameters.get("fixture_mode") is True
|
||||||
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
candidate_dataset,
|
||||||
|
purpose="quality_assessment",
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference_dataset,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
|
detections = DetectionService._query_detection_rows(
|
||||||
|
db,
|
||||||
|
analysis_run_id=analysis_run_id,
|
||||||
|
dataset_id=run.dataset_id,
|
||||||
|
class_name=class_name,
|
||||||
|
min_confidence=min_confidence,
|
||||||
|
)
|
||||||
|
raw_candidate_geometries = [
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"id": str(row.id),
|
||||||
|
"class_name": row.class_name,
|
||||||
|
# Confidence lets the matcher rank candidates the way
|
||||||
|
# detection benchmarks do instead of by row order.
|
||||||
|
"confidence": row.confidence,
|
||||||
|
},
|
||||||
|
to_shape(row.geometry),
|
||||||
|
)
|
||||||
|
for row in detections
|
||||||
|
]
|
||||||
|
candidate_geometries = raw_candidate_geometries
|
||||||
|
|
||||||
coverage = None
|
coverage = None
|
||||||
if manifest_path:
|
if manifest_path:
|
||||||
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
|
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles, resolved_settings)
|
||||||
coverage = DetectionQaService.build_tile_coverage(
|
coverage = DetectionQaService.build_tile_coverage(
|
||||||
manifest,
|
manifest,
|
||||||
manifest_path=manifest_path,
|
manifest_path=manifest_path,
|
||||||
@@ -416,10 +550,40 @@ class DetectionService:
|
|||||||
reference_envelopes,
|
reference_envelopes,
|
||||||
iou_threshold,
|
iou_threshold,
|
||||||
)
|
)
|
||||||
|
candidate_geometry_mode = DetectionQaService.candidate_geometry_mode(candidate_geometries)
|
||||||
box_to_footprint_diagnostics = DetectionQaService.box_to_footprint_diagnostics(
|
box_to_footprint_diagnostics = DetectionQaService.box_to_footprint_diagnostics(
|
||||||
evidence,
|
evidence,
|
||||||
envelope_evidence,
|
envelope_evidence,
|
||||||
iou_threshold=iou_threshold,
|
iou_threshold=iou_threshold,
|
||||||
|
candidate_geometry_mode=candidate_geometry_mode,
|
||||||
|
)
|
||||||
|
box_to_footprint_diagnostics["envelope_precision_recall_curve"] = (
|
||||||
|
DetectionMetricsService.precision_recall_curve(
|
||||||
|
candidate_geometries,
|
||||||
|
reference_envelopes,
|
||||||
|
iou_threshold=iou_threshold,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if candidate_geometry_mode == "axis_aligned_boxes":
|
||||||
|
coverage_warnings.append(
|
||||||
|
"Candidates are axis-aligned detector boxes; strict footprint IoU cannot reach 1 for "
|
||||||
|
"rotated or non-rectangular buildings. See box_to_footprint_diagnostics."
|
||||||
|
)
|
||||||
|
# Threshold-independent view of the same populations, so the run can be
|
||||||
|
# compared with another model instead of only with itself.
|
||||||
|
precision_recall_curve = DetectionMetricsService.precision_recall_curve(
|
||||||
|
candidate_geometries,
|
||||||
|
reference_geometries,
|
||||||
|
iou_threshold=iou_threshold,
|
||||||
|
)
|
||||||
|
# Every requested confidence cut, answered from that one matching pass.
|
||||||
|
# Re-running inference per threshold spends N GPU passes to reproduce
|
||||||
|
# numbers already present here: suppression walks candidates in
|
||||||
|
# descending confidence, so the kept set above a cut does not depend on
|
||||||
|
# the threshold the run itself used.
|
||||||
|
calibration_sweep = DetectionMetricsService.calibration_sweep(
|
||||||
|
precision_recall_curve,
|
||||||
|
thresholds=list(calibration_thresholds or []),
|
||||||
)
|
)
|
||||||
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
|
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
|
||||||
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
|
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
|
||||||
@@ -455,6 +619,8 @@ class DetectionService:
|
|||||||
"coverage": coverage_summary,
|
"coverage": coverage_summary,
|
||||||
"temporal_compatibility": temporal_compatibility,
|
"temporal_compatibility": temporal_compatibility,
|
||||||
"box_to_footprint_diagnostics": box_to_footprint_diagnostics,
|
"box_to_footprint_diagnostics": box_to_footprint_diagnostics,
|
||||||
|
"precision_recall_curve": precision_recall_curve,
|
||||||
|
"calibration_sweep": calibration_sweep,
|
||||||
"match_evidence": evidence.match_evidence,
|
"match_evidence": evidence.match_evidence,
|
||||||
"false_positive_evidence": evidence.false_positive_evidence,
|
"false_positive_evidence": evidence.false_positive_evidence,
|
||||||
"false_negative_evidence": evidence.false_negative_evidence,
|
"false_negative_evidence": evidence.false_negative_evidence,
|
||||||
@@ -466,6 +632,9 @@ class DetectionService:
|
|||||||
"mean_iou": mean_iou,
|
"mean_iou": mean_iou,
|
||||||
"false_positive_count": evidence.false_positives,
|
"false_positive_count": evidence.false_positives,
|
||||||
"false_negative_count": evidence.false_negatives,
|
"false_negative_count": evidence.false_negatives,
|
||||||
|
"average_precision": precision_recall_curve["average_precision"],
|
||||||
|
"best_f1": precision_recall_curve["best_f1"],
|
||||||
|
"best_f1_threshold": precision_recall_curve["best_f1_threshold"],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -500,13 +669,33 @@ class DetectionService:
|
|||||||
"coverage": coverage_summary,
|
"coverage": coverage_summary,
|
||||||
"temporal_compatibility": temporal_compatibility,
|
"temporal_compatibility": temporal_compatibility,
|
||||||
"box_to_footprint_diagnostics": box_to_footprint_diagnostics,
|
"box_to_footprint_diagnostics": box_to_footprint_diagnostics,
|
||||||
|
"precision_recall_curve": precision_recall_curve,
|
||||||
|
"calibration_sweep": calibration_sweep,
|
||||||
"match_evidence": evidence.match_evidence,
|
"match_evidence": evidence.match_evidence,
|
||||||
"false_positive_evidence": evidence.false_positive_evidence,
|
"false_positive_evidence": evidence.false_positive_evidence,
|
||||||
"false_negative_evidence": evidence.false_negative_evidence,
|
"false_negative_evidence": evidence.false_negative_evidence,
|
||||||
}
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _create_job(db, project_id: uuid.UUID, dataset_id: uuid.UUID, parameters: dict[str, Any]) -> Job:
|
def _create_job(
|
||||||
|
db,
|
||||||
|
project_id: uuid.UUID,
|
||||||
|
dataset_id: uuid.UUID,
|
||||||
|
parameters: dict[str, Any],
|
||||||
|
existing_job: Job | None = None,
|
||||||
|
) -> Job:
|
||||||
|
if existing_job is not None:
|
||||||
|
# A queued job already represents this run; reuse it so the client
|
||||||
|
# keeps polling one identifier from request to result.
|
||||||
|
existing_job.status = "running"
|
||||||
|
existing_job.dataset_id = dataset_id
|
||||||
|
existing_job.input_dataset_id = dataset_id
|
||||||
|
existing_job.parameters_json = {**(existing_job.parameters_json or {}), **parameters}
|
||||||
|
existing_job.started_at = DetectionService._now()
|
||||||
|
db.add(existing_job)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(existing_job)
|
||||||
|
return existing_job
|
||||||
job = Job(
|
job = Job(
|
||||||
id=uuid.uuid4(),
|
id=uuid.uuid4(),
|
||||||
job_type="detection.run",
|
job_type="detection.run",
|
||||||
@@ -522,6 +711,99 @@ class DetectionService:
|
|||||||
db.refresh(job)
|
db.refresh(job)
|
||||||
return job
|
return job
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def enqueue_detection(
|
||||||
|
db,
|
||||||
|
project_id: uuid.UUID,
|
||||||
|
dataset_id: uuid.UUID,
|
||||||
|
model_id: str,
|
||||||
|
confidence_threshold: float,
|
||||||
|
model_asset_id: str | None = None,
|
||||||
|
class_filter: list[str] | None = None,
|
||||||
|
tile_manifest_path: str | None = None,
|
||||||
|
parameters_json: dict[str, Any] | None = None,
|
||||||
|
) -> Job:
|
||||||
|
"""Accept a detection run for background execution.
|
||||||
|
|
||||||
|
Everything cheap enough to answer inside the request is checked here,
|
||||||
|
so an operator learns about a missing dataset or an unvalidated class
|
||||||
|
immediately rather than from a job that fails minutes later.
|
||||||
|
"""
|
||||||
|
|
||||||
|
DetectionService._validate_run_request(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
)
|
||||||
|
job = Job(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
job_type="detection.run",
|
||||||
|
status="queued",
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
input_dataset_id=dataset_id,
|
||||||
|
parameters_json={
|
||||||
|
"project_id": str(project_id),
|
||||||
|
"dataset_id": str(dataset_id),
|
||||||
|
"model_id": model_id,
|
||||||
|
"model_asset_id": model_asset_id,
|
||||||
|
"confidence_threshold": confidence_threshold,
|
||||||
|
"class_filter": class_filter or [],
|
||||||
|
"tile_manifest_path": tile_manifest_path,
|
||||||
|
"parameters_json": dict(parameters_json or {}),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
db.add(job)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(job)
|
||||||
|
logger.info(
|
||||||
|
"detection_queued request_id=%s project_id=%s dataset_id=%s job_id=%s model_id=%s",
|
||||||
|
get_request_id(),
|
||||||
|
project_id,
|
||||||
|
dataset_id,
|
||||||
|
job.id,
|
||||||
|
model_id,
|
||||||
|
)
|
||||||
|
return job
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validate_run_request(db, *, project_id: uuid.UUID, dataset_id: uuid.UUID) -> Dataset:
|
||||||
|
project = db.get(Project, project_id)
|
||||||
|
if not project:
|
||||||
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if not dataset or dataset.project_id != project_id:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
if dataset.dataset_type != "raster":
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_DATASET_TYPE",
|
||||||
|
message="Detection requires a raster dataset",
|
||||||
|
details={"dataset_type": dataset.dataset_type},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
# A regional run holds tens of thousands of detections; the results table
|
||||||
|
# and the map overlay both read them after every run.
|
||||||
|
DEFAULT_RESULT_LIMIT = 2_000
|
||||||
|
DEFAULT_RUN_LIST_LIMIT = 200
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def paginate(rows: list[Any], *, limit: int, offset: int) -> tuple[list[Any], int, bool]:
|
||||||
|
"""Slice a result population, keeping the total intact.
|
||||||
|
|
||||||
|
``limit <= 0`` means "everything", for callers that genuinely need the
|
||||||
|
whole population and know what they are asking for.
|
||||||
|
"""
|
||||||
|
|
||||||
|
total = len(rows)
|
||||||
|
start = max(0, int(offset))
|
||||||
|
if limit <= 0:
|
||||||
|
return rows[start:], total, False
|
||||||
|
page = rows[start : start + int(limit)]
|
||||||
|
# Truncated means: this page is not the whole population.
|
||||||
|
return page, total, len(page) < total
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _query_detection_rows(
|
def _query_detection_rows(
|
||||||
db,
|
db,
|
||||||
@@ -540,7 +822,14 @@ class DetectionService:
|
|||||||
query = query.filter(Detection.class_name == class_name)
|
query = query.filter(Detection.class_name == class_name)
|
||||||
if min_confidence is not None:
|
if min_confidence is not None:
|
||||||
query = query.filter(Detection.confidence >= min_confidence)
|
query = query.filter(Detection.confidence >= min_confidence)
|
||||||
return query.order_by(Detection.created_at.desc()).all()
|
# ``created_at`` defaults to the transaction timestamp, so every
|
||||||
|
# detection in a run shares one value and ordering by it alone leaves
|
||||||
|
# the row order undefined. Confidence first, id as a stable tiebreak.
|
||||||
|
return query.order_by(
|
||||||
|
Detection.confidence.desc(),
|
||||||
|
Detection.created_at.desc(),
|
||||||
|
Detection.id.asc(),
|
||||||
|
).all()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _detection_properties(detection: Detection) -> dict[str, Any]:
|
def _detection_properties(detection: Detection) -> dict[str, Any]:
|
||||||
@@ -679,16 +968,51 @@ class DetectionService:
|
|||||||
settings: Settings,
|
settings: Settings,
|
||||||
yolo_adapter_class: Type[YoloDetectionAdapter],
|
yolo_adapter_class: Type[YoloDetectionAdapter],
|
||||||
) -> tuple[list[Detection], dict[str, Any]]:
|
) -> tuple[list[Detection], dict[str, Any]]:
|
||||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if dataset is None:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
manifest_binding = TileManifestService.validate_for_inference(
|
||||||
|
db,
|
||||||
|
dataset,
|
||||||
|
manifest,
|
||||||
|
manifest_path=tile_manifest_path or "",
|
||||||
|
settings=settings,
|
||||||
|
error_prefix="DETECTION",
|
||||||
|
)
|
||||||
|
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
|
||||||
model_path = Path(settings.yolo_model_path or "").expanduser()
|
model_path = Path(settings.yolo_model_path or "").expanduser()
|
||||||
|
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
|
db=db,
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=model_name,
|
||||||
|
task_type="object_detection",
|
||||||
|
expected_model_version=model_version,
|
||||||
|
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
DetectionService._attach_runtime_model_provenance(
|
||||||
|
analysis_run,
|
||||||
|
job,
|
||||||
|
runtime_model_provenance,
|
||||||
|
)
|
||||||
adapter = yolo_adapter_class(settings)
|
adapter = yolo_adapter_class(settings)
|
||||||
model = adapter.load_model(model_path)
|
model = adapter.load_model(model_path)
|
||||||
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
||||||
candidates: list[dict[str, Any]] = []
|
candidates: list[dict[str, Any]] = []
|
||||||
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326"
|
manifest_crs = DetectionService._require_manifest_crs(manifest)
|
||||||
for tile in manifest["tiles"]:
|
raster_bounds = DetectionService._bounds_to_epsg4326(manifest.get("bounds"), manifest_crs)
|
||||||
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser())
|
tiles = list(manifest["tiles"])
|
||||||
for raw in adapter.predict_tile(model, tile_path, confidence_threshold):
|
tile_paths = [
|
||||||
|
DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser(), settings) for tile in tiles
|
||||||
|
]
|
||||||
|
# Batched so the GPU is not idle between tiles; each tile keeps its own
|
||||||
|
# transform for georeferencing, so results stay per tile and in order.
|
||||||
|
detections_per_tile = adapter.predict_tiles(model, tile_paths, confidence_threshold)
|
||||||
|
for tile, tile_path, raw_detections in zip(tiles, tile_paths, detections_per_tile):
|
||||||
|
tile_crs = tile.get("crs") or manifest_crs
|
||||||
|
tile_bounds_4326 = DetectionService._bounds_to_epsg4326(tile.get("bounds"), tile_crs)
|
||||||
|
tile_edge_tolerance = DetectionService._tile_edge_tolerance(tile, tile_bounds_4326)
|
||||||
|
for raw in raw_detections:
|
||||||
model_class_name = str(raw.get("class_name") or "").strip()
|
model_class_name = str(raw.get("class_name") or "").strip()
|
||||||
class_name = DetectionService._canonical_class_name(model_class_name)
|
class_name = DetectionService._canonical_class_name(model_class_name)
|
||||||
confidence = float(raw.get("confidence", 0.0))
|
confidence = float(raw.get("confidence", 0.0))
|
||||||
@@ -699,7 +1023,7 @@ class DetectionService:
|
|||||||
bbox = raw.get("bbox")
|
bbox = raw.get("bbox")
|
||||||
if not isinstance(bbox, list):
|
if not isinstance(bbox, list):
|
||||||
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO adapter returned a detection without bbox", status_code=422)
|
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO adapter returned a detection without bbox", status_code=422)
|
||||||
geometry = pixel_bbox_to_epsg4326_polygon(bbox=bbox, tile=tile, crs=tile.get("crs") or manifest_crs)
|
geometry = pixel_bbox_to_epsg4326_polygon(bbox=bbox, tile=tile, crs=tile_crs)
|
||||||
properties = dict(raw.get("properties") or {})
|
properties = dict(raw.get("properties") or {})
|
||||||
if model_class_name and model_class_name != class_name:
|
if model_class_name and model_class_name != class_name:
|
||||||
properties.setdefault("model_class_name", model_class_name)
|
properties.setdefault("model_class_name", model_class_name)
|
||||||
@@ -711,11 +1035,21 @@ class DetectionService:
|
|||||||
"bbox": bbox,
|
"bbox": bbox,
|
||||||
"source_tile_path": str(tile_path),
|
"source_tile_path": str(tile_path),
|
||||||
"properties": {**properties, "tile_index": tile.get("index")},
|
"properties": {**properties, "tile_index": tile.get("index")},
|
||||||
|
"tile_bounds": tile_bounds_4326,
|
||||||
|
"tile_edge_tolerance": tile_edge_tolerance,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
filtered_candidates = DetectionService._suppress_duplicate_candidates(
|
edge_filtered_candidates = candidates
|
||||||
|
if settings.yolo_suppress_tile_edge_detections:
|
||||||
|
edge_filtered_candidates = DetectionService._drop_tile_edge_truncations(
|
||||||
candidates,
|
candidates,
|
||||||
|
raster_bounds=raster_bounds,
|
||||||
|
tolerance=0.0,
|
||||||
|
)
|
||||||
|
filtered_candidates = DetectionService._suppress_duplicate_candidates(
|
||||||
|
edge_filtered_candidates,
|
||||||
iou_threshold=float(settings.yolo_duplicate_iou_threshold),
|
iou_threshold=float(settings.yolo_duplicate_iou_threshold),
|
||||||
|
containment_threshold=float(settings.yolo_containment_nms_threshold),
|
||||||
)
|
)
|
||||||
persisted: list[Detection] = []
|
persisted: list[Detection] = []
|
||||||
for candidate in filtered_candidates:
|
for candidate in filtered_candidates:
|
||||||
@@ -738,7 +1072,10 @@ class DetectionService:
|
|||||||
"y_max": float(bbox[3]),
|
"y_max": float(bbox[3]),
|
||||||
},
|
},
|
||||||
source_tile_path=candidate["source_tile_path"],
|
source_tile_path=candidate["source_tile_path"],
|
||||||
properties_json=candidate["properties"],
|
properties_json={
|
||||||
|
**candidate["properties"],
|
||||||
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
db.add(detection)
|
db.add(detection)
|
||||||
persisted.append(detection)
|
persisted.append(detection)
|
||||||
@@ -748,28 +1085,140 @@ class DetectionService:
|
|||||||
return persisted, {
|
return persisted, {
|
||||||
"raw_detection_count": len(candidates),
|
"raw_detection_count": len(candidates),
|
||||||
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
|
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
|
||||||
|
"tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates),
|
||||||
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
||||||
|
"containment_suppression_threshold": float(settings.yolo_containment_nms_threshold),
|
||||||
|
"tile_manifest_binding": manifest_binding,
|
||||||
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _attach_tile_manifest_binding(
|
||||||
|
analysis_run: AnalysisRun,
|
||||||
|
job: Job,
|
||||||
|
binding: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||||
|
analysis_parameters["tile_manifest_binding"] = dict(binding)
|
||||||
|
analysis_run.parameters_json = analysis_parameters
|
||||||
|
job_parameters = dict(job.parameters_json or {})
|
||||||
|
job_parameters["tile_manifest_binding"] = dict(binding)
|
||||||
|
job.parameters_json = job_parameters
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _attach_runtime_model_provenance(
|
||||||
|
analysis_run: AnalysisRun,
|
||||||
|
job: Job,
|
||||||
|
provenance: RuntimeModelProvenance,
|
||||||
|
) -> None:
|
||||||
|
"""Persist byte-bound model evidence with the run before adapter loading.
|
||||||
|
|
||||||
|
Individual detections retain the same evidence in ``properties_json``;
|
||||||
|
this run-level copy is the compact audit root for a complete inference.
|
||||||
|
Assigning fresh dictionaries matters for SQLAlchemy JSON change tracking.
|
||||||
|
"""
|
||||||
|
|
||||||
|
evidence = provenance.as_dict()
|
||||||
|
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||||
|
analysis_parameters["runtime_model_provenance"] = evidence
|
||||||
|
analysis_run.parameters_json = analysis_parameters
|
||||||
|
job_parameters = dict(job.parameters_json or {})
|
||||||
|
job_parameters["runtime_model_provenance"] = evidence
|
||||||
|
job.parameters_json = job_parameters
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _canonical_class_name(value: Any) -> str:
|
def _canonical_class_name(value: Any) -> str:
|
||||||
return str(value or "").strip().casefold()
|
return str(value or "").strip().casefold()
|
||||||
|
|
||||||
|
# An object wider than the tile overlap is truncated by both tiles, so the
|
||||||
|
# two halves barely intersect and IoU alone never suppresses them. Overlap
|
||||||
|
# measured against the smaller box catches that case; the threshold is
|
||||||
|
# deliberately strict so that terraced houses stay separate detections.
|
||||||
|
# Fallback only. The served value is configuration, so a promoted model can
|
||||||
|
# be run at the threshold its evaluation froze.
|
||||||
|
CONTAINMENT_SUPPRESSION_THRESHOLD = 0.85
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _suppress_duplicate_candidates(candidates: list[dict[str, Any]], iou_threshold: float) -> list[dict[str, Any]]:
|
def _suppress_duplicate_candidates(
|
||||||
|
candidates: list[dict[str, Any]],
|
||||||
|
iou_threshold: float,
|
||||||
|
containment_threshold: float | None = None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
if iou_threshold <= 0 or len(candidates) < 2:
|
if iou_threshold <= 0 or len(candidates) < 2:
|
||||||
return candidates
|
return candidates
|
||||||
|
if containment_threshold is None:
|
||||||
|
containment_threshold = DetectionService.CONTAINMENT_SUPPRESSION_THRESHOLD
|
||||||
|
|
||||||
|
ordered = sorted(
|
||||||
|
candidates,
|
||||||
|
key=lambda item: (-float(item["confidence"]), str(item.get("source_tile_path") or "")),
|
||||||
|
)
|
||||||
kept: list[dict[str, Any]] = []
|
kept: list[dict[str, Any]] = []
|
||||||
for candidate in sorted(candidates, key=lambda item: float(item["confidence"]), reverse=True):
|
kept_geometries: list[Any] = []
|
||||||
|
tree = None
|
||||||
|
|
||||||
|
for candidate in ordered:
|
||||||
|
geometry = candidate["geometry"]
|
||||||
duplicate = False
|
duplicate = False
|
||||||
for kept_candidate in kept:
|
# Only geometries that actually touch this candidate can suppress
|
||||||
|
# it, so an index keeps a dense AOI from turning into an O(n^2) scan.
|
||||||
|
neighbour_indexes = range(len(kept)) if tree is None else (int(index) for index in tree.query(geometry))
|
||||||
|
for index in neighbour_indexes:
|
||||||
|
kept_candidate = kept[index]
|
||||||
if candidate["class_name"] != kept_candidate["class_name"]:
|
if candidate["class_name"] != kept_candidate["class_name"]:
|
||||||
continue
|
continue
|
||||||
if DetectionService._geometry_iou(candidate["geometry"], kept_candidate["geometry"]) >= iou_threshold:
|
other = kept_geometries[index]
|
||||||
|
if DetectionService._geometry_iou(geometry, other) >= iou_threshold:
|
||||||
|
duplicate = True
|
||||||
|
break
|
||||||
|
if DetectionService._geometry_containment(geometry, other) >= containment_threshold:
|
||||||
duplicate = True
|
duplicate = True
|
||||||
break
|
break
|
||||||
if not duplicate:
|
if not duplicate:
|
||||||
kept.append(candidate)
|
kept.append(candidate)
|
||||||
|
kept_geometries.append(geometry)
|
||||||
|
tree = STRtree(kept_geometries)
|
||||||
|
return kept
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _drop_tile_edge_truncations(
|
||||||
|
candidates: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
raster_bounds: tuple[float, float, float, float] | None,
|
||||||
|
tolerance: float,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Discard boxes cut off by an interior tile edge.
|
||||||
|
|
||||||
|
Such a box describes only the part of the object that fell inside its
|
||||||
|
tile. Because tiles overlap, the neighbouring tile saw the object whole
|
||||||
|
and contributed the box worth keeping. A box against the outer raster
|
||||||
|
edge has no such neighbour and is kept.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if raster_bounds is None or tolerance <= 0:
|
||||||
|
return candidates
|
||||||
|
|
||||||
|
raster_left, raster_bottom, raster_right, raster_top = raster_bounds
|
||||||
|
kept: list[dict[str, Any]] = []
|
||||||
|
for candidate in candidates:
|
||||||
|
tile_bounds = candidate.get("tile_bounds")
|
||||||
|
if not tile_bounds or len(tuple(tile_bounds)) != 4:
|
||||||
|
kept.append(candidate)
|
||||||
|
continue
|
||||||
|
tile_left, tile_bottom, tile_right, tile_top = (float(value) for value in tile_bounds)
|
||||||
|
left, bottom, right, top = candidate["geometry"].bounds
|
||||||
|
# A pixel-sized tolerance per tile: a fixed degree value would be
|
||||||
|
# wrong for both a 10 cm orthophoto and a coarse thematic raster.
|
||||||
|
tolerance = float(candidate.get("tile_edge_tolerance") or 0.0) or tolerance
|
||||||
|
|
||||||
|
touches_interior_edge = (
|
||||||
|
(abs(left - tile_left) <= tolerance and abs(tile_left - raster_left) > tolerance)
|
||||||
|
or (abs(right - tile_right) <= tolerance and abs(tile_right - raster_right) > tolerance)
|
||||||
|
or (abs(bottom - tile_bottom) <= tolerance and abs(tile_bottom - raster_bottom) > tolerance)
|
||||||
|
or (abs(top - tile_top) <= tolerance and abs(tile_top - raster_top) > tolerance)
|
||||||
|
)
|
||||||
|
if not touches_interior_edge:
|
||||||
|
kept.append(candidate)
|
||||||
return kept
|
return kept
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -785,14 +1234,90 @@ class DetectionService:
|
|||||||
return intersection_area / union_area
|
return intersection_area / union_area
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]:
|
def _geometry_containment(left, right) -> float:
|
||||||
|
"""Intersection over the smaller of the two areas."""
|
||||||
|
|
||||||
|
if left.is_empty or right.is_empty:
|
||||||
|
return 0.0
|
||||||
|
smaller_area = min(left.area, right.area)
|
||||||
|
if smaller_area <= 0:
|
||||||
|
return 0.0
|
||||||
|
intersection_area = left.intersection(right).area
|
||||||
|
if intersection_area <= 0:
|
||||||
|
return 0.0
|
||||||
|
return intersection_area / smaller_area
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _require_manifest_crs(manifest: dict[str, Any]) -> str:
|
||||||
|
"""Refuse to georeference inference output against a guessed CRS.
|
||||||
|
|
||||||
|
Detection QA already rejects a tile without explicit CRS metadata.
|
||||||
|
Silently assuming EPSG:4326 on the inference side produced geometry
|
||||||
|
that looks plausible on a map but sits in the wrong place.
|
||||||
|
"""
|
||||||
|
|
||||||
|
raw_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
|
||||||
|
if not isinstance(raw_crs, str) or not raw_crs.strip():
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_TILE_MANIFEST_INVALID",
|
||||||
|
message="Raster tile manifest requires explicit CRS metadata for georeferencing",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return raw_crs.strip()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _bounds_to_epsg4326(bounds: Any, crs: str | None) -> tuple[float, float, float, float] | None:
|
||||||
|
if not isinstance(bounds, (list, tuple)) or len(bounds) != 4:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
left, bottom, right, top = (float(value) for value in bounds)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if left >= right or bottom >= top:
|
||||||
|
return None
|
||||||
|
if not crs or str(crs).strip().upper() in {"EPSG:4326", "4326"}:
|
||||||
|
return (left, bottom, right, top)
|
||||||
|
try:
|
||||||
|
transformer = Transformer.from_crs(crs, "EPSG:4326", always_xy=True)
|
||||||
|
# Transform the whole rectangle, not just two corners: a projected
|
||||||
|
# box does not stay axis-aligned after reprojection.
|
||||||
|
projected = shapely_transform(transformer.transform, shapely_box(left, bottom, right, top))
|
||||||
|
return projected.bounds
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _tile_edge_tolerance(tile: dict[str, Any], tile_bounds_4326: tuple[float, float, float, float] | None) -> float:
|
||||||
|
"""One and a half pixels, expressed in the degrees the boxes live in."""
|
||||||
|
|
||||||
|
if tile_bounds_4326 is None:
|
||||||
|
return 0.0
|
||||||
|
pixel_window = tile.get("pixel_window")
|
||||||
|
if not (isinstance(pixel_window, (list, tuple)) and len(pixel_window) == 4):
|
||||||
|
return 0.0
|
||||||
|
try:
|
||||||
|
width = float(pixel_window[2])
|
||||||
|
height = float(pixel_window[3])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0.0
|
||||||
|
if width <= 0 or height <= 0:
|
||||||
|
return 0.0
|
||||||
|
left, bottom, right, top = tile_bounds_4326
|
||||||
|
return 1.5 * max((right - left) / width, (top - bottom) / height)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int, settings: Settings | None = None) -> dict[str, Any]:
|
||||||
if not tile_manifest_path:
|
if not tile_manifest_path:
|
||||||
raise AppError(
|
raise AppError(
|
||||||
code="DETECTION_TILE_MANIFEST_REQUIRED",
|
code="DETECTION_TILE_MANIFEST_REQUIRED",
|
||||||
message="Configured YOLO inference requires an existing raster tile manifest path",
|
message="Configured YOLO inference requires an existing raster tile manifest path",
|
||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
manifest_path = Path(tile_manifest_path).expanduser()
|
# The path arrives in the request, so it must name a governed artifact
|
||||||
|
# rather than an arbitrary file on the host.
|
||||||
|
manifest_path = StorageService.assert_within_storage_root(
|
||||||
|
tile_manifest_path, label="tile manifest", settings=settings
|
||||||
|
)
|
||||||
if not manifest_path.exists() or not manifest_path.is_file():
|
if not manifest_path.exists() or not manifest_path.is_file():
|
||||||
raise AppError(
|
raise AppError(
|
||||||
code="DETECTION_TILE_MANIFEST_NOT_FOUND",
|
code="DETECTION_TILE_MANIFEST_NOT_FOUND",
|
||||||
@@ -817,13 +1342,16 @@ class DetectionService:
|
|||||||
return manifest
|
return manifest
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path) -> Path:
|
def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path, settings: Settings | None = None) -> Path:
|
||||||
raw_path = tile.get("path")
|
raw_path = tile.get("path")
|
||||||
if not isinstance(raw_path, str) or not raw_path:
|
if not isinstance(raw_path, str) or not raw_path:
|
||||||
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile manifest entries require a path", status_code=422)
|
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile manifest entries require a path", status_code=422)
|
||||||
tile_path = Path(raw_path).expanduser()
|
tile_path = Path(raw_path).expanduser()
|
||||||
if not tile_path.is_absolute():
|
if not tile_path.is_absolute():
|
||||||
tile_path = manifest_path.parent / tile_path
|
tile_path = manifest_path.parent / tile_path
|
||||||
|
# A manifest entry may name an absolute path; it is still only allowed
|
||||||
|
# to point at a tile the runtime itself produced.
|
||||||
|
tile_path = StorageService.assert_within_storage_root(tile_path, label="raster tile", settings=settings)
|
||||||
if not tile_path.exists() or not tile_path.is_file():
|
if not tile_path.exists() or not tile_path.is_file():
|
||||||
raise AppError(
|
raise AppError(
|
||||||
code="DETECTION_TILE_NOT_FOUND",
|
code="DETECTION_TILE_NOT_FOUND",
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
@@ -22,6 +22,7 @@ from shapely.ops import transform as shapely_transform
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.services.outbound_request_guard import guarded_opener
|
||||||
from app.models import Area, Dataset, Project
|
from app.models import Area, Dataset, Project
|
||||||
from app.schemas.dhmv import DhmvAcquireRequest, DhmvAcquisitionResult, DhmvProductRead
|
from app.schemas.dhmv import DhmvAcquireRequest, DhmvAcquisitionResult, DhmvProductRead
|
||||||
from app.services.dataset_service import DatasetService
|
from app.services.dataset_service import DatasetService
|
||||||
@@ -294,7 +295,7 @@ class DhmvAcquisitionService:
|
|||||||
)
|
)
|
||||||
max_bytes = settings.dhmv_max_response_mb * 1024 * 1024
|
max_bytes = settings.dhmv_max_response_mb * 1024 * 1024
|
||||||
try:
|
try:
|
||||||
with (opener or urlopen)(request, timeout=settings.dhmv_timeout_seconds) as response:
|
with (opener or guarded_opener(request_url))(request, timeout=settings.dhmv_timeout_seconds) as response:
|
||||||
content_type = str(response.headers.get("Content-Type", ""))
|
content_type = str(response.headers.get("Content-Type", ""))
|
||||||
content_length = response.headers.get("Content-Length")
|
content_length = response.headers.get("Content-Length")
|
||||||
if content_length and int(content_length) > max_bytes:
|
if content_length and int(content_length) > max_bytes:
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import uuid
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
from html import escape
|
from html import escape
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -23,6 +24,7 @@ from app.schemas.flood_hazard import FloodHazardPartitionSelectionRequest, Flood
|
|||||||
from app.schemas.temporal import TemporalComparisonRequest
|
from app.schemas.temporal import TemporalComparisonRequest
|
||||||
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
|
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
|
||||||
from app.services.dataset_service import DatasetService
|
from app.services.dataset_service import DatasetService
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||||
from app.services.segmentation_service import SegmentationService
|
from app.services.segmentation_service import SegmentationService
|
||||||
@@ -34,12 +36,194 @@ from app.services.vector_feature_service import VectorFeatureService
|
|||||||
|
|
||||||
|
|
||||||
class ExportService:
|
class ExportService:
|
||||||
|
# RFC 7946 allows foreign members on a FeatureCollection and requires
|
||||||
|
# parsers to ignore ones they do not know, so the provenance travels with
|
||||||
|
# the file without breaking QGIS, ogr2ogr or any other reader.
|
||||||
|
PROVENANCE_MEMBER = "geointel_provenance"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def provenance_member(
|
||||||
|
*,
|
||||||
|
source: str,
|
||||||
|
project_id: uuid.UUID,
|
||||||
|
dataset_id: uuid.UUID | None = None,
|
||||||
|
analysis_run_id: uuid.UUID | None = None,
|
||||||
|
source_name: str | None = None,
|
||||||
|
source_version: str | None = None,
|
||||||
|
observed_at: Any = None,
|
||||||
|
selection_bbox: dict[str, Any] | None = None,
|
||||||
|
selection_area_id: uuid.UUID | None = None,
|
||||||
|
feature_count: int | None = None,
|
||||||
|
total_feature_count: int | None = None,
|
||||||
|
truncated: bool = False,
|
||||||
|
warnings: list[str] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Describe an exported FeatureCollection inside the file itself.
|
||||||
|
|
||||||
|
A capped export previously recorded ``truncated`` on the export record
|
||||||
|
only, so the downloaded file looked complete. Completeness is derived
|
||||||
|
from the counts as well as the flag: a caller that forgets to pass the
|
||||||
|
flag cannot produce a file that claims to hold everything.
|
||||||
|
"""
|
||||||
|
|
||||||
|
complete = not truncated
|
||||||
|
if feature_count is not None and total_feature_count is not None:
|
||||||
|
complete = complete and feature_count >= total_feature_count
|
||||||
|
|
||||||
|
member: dict[str, Any] = {
|
||||||
|
"source": source,
|
||||||
|
"project_id": str(project_id),
|
||||||
|
"exported_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"complete": complete,
|
||||||
|
"completeness_note": None,
|
||||||
|
}
|
||||||
|
if dataset_id is not None:
|
||||||
|
member["dataset_id"] = str(dataset_id)
|
||||||
|
if analysis_run_id is not None:
|
||||||
|
member["analysis_run_id"] = str(analysis_run_id)
|
||||||
|
if source_name:
|
||||||
|
member["source_name"] = source_name
|
||||||
|
if source_version:
|
||||||
|
member["source_version"] = source_version
|
||||||
|
if observed_at is not None:
|
||||||
|
member["observed_at"] = observed_at.isoformat() if hasattr(observed_at, "isoformat") else str(observed_at)
|
||||||
|
if selection_bbox is not None:
|
||||||
|
member["selection_bbox"] = selection_bbox
|
||||||
|
if selection_area_id is not None:
|
||||||
|
member["selection_area_id"] = str(selection_area_id)
|
||||||
|
if feature_count is not None:
|
||||||
|
member["feature_count"] = feature_count
|
||||||
|
if total_feature_count is not None:
|
||||||
|
member["total_feature_count"] = total_feature_count
|
||||||
|
if warnings:
|
||||||
|
member["warnings"] = list(warnings)
|
||||||
|
|
||||||
|
if not complete:
|
||||||
|
written = feature_count if feature_count is not None else "?"
|
||||||
|
available = total_feature_count if total_feature_count is not None else "?"
|
||||||
|
member["completeness_note"] = (
|
||||||
|
f"Dit bestand bevat {written} van {available} objecten uit de selectie. Het is een "
|
||||||
|
"begrensde uitsnede, geen volledige export."
|
||||||
|
)
|
||||||
|
return member
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def attach_provenance(feature_collection: dict[str, Any], member: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
feature_collection[ExportService.PROVENANCE_MEMBER] = member
|
||||||
|
return feature_collection
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _detection_export_trust(db: Session, run: AnalysisRun) -> dict[str, Any]:
|
||||||
|
"""Classify persisted AI output without turning confidence into truth."""
|
||||||
|
|
||||||
|
checks = (
|
||||||
|
db.query(QualityCheck)
|
||||||
|
.filter(
|
||||||
|
QualityCheck.analysis_run_id == run.id,
|
||||||
|
QualityCheck.check_type == "detections_vs_reference",
|
||||||
|
)
|
||||||
|
.order_by(QualityCheck.created_at.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
quality_check = checks[0] if checks else None
|
||||||
|
reasons: list[str] = []
|
||||||
|
reference_dataset = None
|
||||||
|
if quality_check is None:
|
||||||
|
reasons.append("authoritative_qa_missing")
|
||||||
|
else:
|
||||||
|
findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
|
||||||
|
coverage = findings.get("coverage") if isinstance(findings.get("coverage"), dict) else {}
|
||||||
|
temporal = findings.get("temporal_compatibility") if isinstance(findings.get("temporal_compatibility"), dict) else {}
|
||||||
|
warnings = findings.get("warnings") if isinstance(findings.get("warnings"), list) else []
|
||||||
|
if quality_check.status != "ok":
|
||||||
|
reasons.append("quality_check_not_ok")
|
||||||
|
if findings.get("unsupported_geometry") is True:
|
||||||
|
reasons.append("unsupported_geometry")
|
||||||
|
false_positives = findings.get("false_positives")
|
||||||
|
false_negatives = findings.get("false_negatives")
|
||||||
|
if (
|
||||||
|
isinstance(false_positives, bool)
|
||||||
|
or not isinstance(false_positives, (int, float))
|
||||||
|
or false_positives != 0
|
||||||
|
):
|
||||||
|
reasons.append("false_positives_present")
|
||||||
|
if (
|
||||||
|
isinstance(false_negatives, bool)
|
||||||
|
or not isinstance(false_negatives, (int, float))
|
||||||
|
or false_negatives != 0
|
||||||
|
):
|
||||||
|
reasons.append("false_negatives_present")
|
||||||
|
if warnings:
|
||||||
|
reasons.append("quality_warnings_present")
|
||||||
|
if coverage.get("applied") is not True:
|
||||||
|
reasons.append("inference_coverage_not_proven")
|
||||||
|
if temporal.get("status") != "compatible":
|
||||||
|
reasons.append("temporal_compatibility_not_proven")
|
||||||
|
reference_dataset = db.get(Dataset, quality_check.reference_dataset_id)
|
||||||
|
if reference_dataset is None:
|
||||||
|
reasons.append("reference_dataset_missing")
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference_dataset,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
except AppError:
|
||||||
|
reasons.append("reference_not_authoritative_for_buildings")
|
||||||
|
|
||||||
|
operational_use_allowed = not reasons
|
||||||
|
return {
|
||||||
|
"schema_version": "geointel.result-trust/v1",
|
||||||
|
"classification": (
|
||||||
|
"authoritative_reference_checked_ai_output"
|
||||||
|
if operational_use_allowed
|
||||||
|
else "unverified_ai_review_output"
|
||||||
|
),
|
||||||
|
"authoritative": False,
|
||||||
|
"operational_use_allowed": operational_use_allowed,
|
||||||
|
"operator_review_required": True,
|
||||||
|
"quality_check_id": str(quality_check.id) if quality_check else None,
|
||||||
|
"reference_dataset_id": str(reference_dataset.id) if reference_dataset else None,
|
||||||
|
"blocking_reasons": sorted(set(reasons)),
|
||||||
|
"limitation": (
|
||||||
|
"AI output is not ground truth. Operational use is bounded to the exact source, AOI, model and reference QA evidence."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset:
|
||||||
|
"""Block an output export when its persisted source dataset is unsafe."""
|
||||||
|
|
||||||
|
if not run.dataset_id:
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_PROVENANCE_INCOMPLETE",
|
||||||
|
message="Analysis output cannot be exported without a persisted source dataset.",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
dataset = db.get(Dataset, run.dataset_id)
|
||||||
|
if not dataset or dataset.project_id != run.project_id:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Analysis source dataset not found", status_code=404)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
|
return dataset
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def export_map_result(
|
def export_map_result(
|
||||||
db: Session,
|
db: Session,
|
||||||
payload: MapResultExportRequest,
|
payload: MapResultExportRequest,
|
||||||
) -> ExportCreateResponse:
|
) -> ExportCreateResponse:
|
||||||
if payload.mode == "evolution":
|
if payload.mode == "evolution":
|
||||||
|
earlier_dataset = db.get(Dataset, payload.earlier_dataset_id)
|
||||||
|
later_dataset = db.get(Dataset, payload.later_dataset_id)
|
||||||
|
if (
|
||||||
|
not earlier_dataset
|
||||||
|
or not later_dataset
|
||||||
|
or earlier_dataset.project_id != payload.project_id
|
||||||
|
or later_dataset.project_id != payload.project_id
|
||||||
|
):
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Temporal export dataset not found", status_code=404)
|
||||||
|
DatasetConsumptionGate.assert_eligible(earlier_dataset, purpose="export")
|
||||||
|
DatasetConsumptionGate.assert_eligible(later_dataset, purpose="export")
|
||||||
comparison = TemporalAnalysisService.compare(
|
comparison = TemporalAnalysisService.compare(
|
||||||
db,
|
db,
|
||||||
project_id=payload.project_id,
|
project_id=payload.project_id,
|
||||||
@@ -86,6 +270,7 @@ class ExportService:
|
|||||||
dataset = db.get(Dataset, payload.dataset_id)
|
dataset = db.get(Dataset, payload.dataset_id)
|
||||||
if not dataset or dataset.project_id != payload.project_id:
|
if not dataset or dataset.project_id != payload.project_id:
|
||||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
if dataset.dataset_type in DatasetService.VECTOR_TYPES:
|
if dataset.dataset_type in DatasetService.VECTOR_TYPES:
|
||||||
if payload.partitioned:
|
if payload.partitioned:
|
||||||
return ExportService.export_partitioned_vector_selection_geojson(
|
return ExportService.export_partitioned_vector_selection_geojson(
|
||||||
@@ -219,6 +404,7 @@ class ExportService:
|
|||||||
limit: int = 1000,
|
limit: int = 1000,
|
||||||
name: str | None = None,
|
name: str | None = None,
|
||||||
) -> ExportCreateResponse:
|
) -> ExportCreateResponse:
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
|
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
|
||||||
raise AppError(
|
raise AppError(
|
||||||
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
|
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
|
||||||
@@ -308,6 +494,7 @@ class ExportService:
|
|||||||
details={"dataset_type": dataset.dataset_type},
|
details={"dataset_type": dataset.dataset_type},
|
||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
|
|
||||||
selection_kwargs: dict[str, Any] = {
|
selection_kwargs: dict[str, Any] = {
|
||||||
"dataset_id": dataset_id,
|
"dataset_id": dataset_id,
|
||||||
@@ -337,6 +524,28 @@ class ExportService:
|
|||||||
preclipped_partition_filter=preclipped_partition_filter,
|
preclipped_partition_filter=preclipped_partition_filter,
|
||||||
)
|
)
|
||||||
selection = VectorFeatureService.select_features_by_bbox(db, **selection_kwargs)
|
selection = VectorFeatureService.select_features_by_bbox(db, **selection_kwargs)
|
||||||
|
summary = selection.get("summary") if isinstance(selection.get("summary"), dict) else {}
|
||||||
|
ExportService.attach_provenance(
|
||||||
|
selection["geojson"],
|
||||||
|
ExportService.provenance_member(
|
||||||
|
source="vector_selection",
|
||||||
|
project_id=dataset.project_id,
|
||||||
|
dataset_id=dataset.id,
|
||||||
|
source_name=dataset.source_name,
|
||||||
|
source_version=dataset.source_version,
|
||||||
|
observed_at=dataset.observed_at,
|
||||||
|
selection_bbox=selection["selection_bbox"],
|
||||||
|
selection_area_id=area_id,
|
||||||
|
feature_count=selection["feature_count"],
|
||||||
|
total_feature_count=selection.get("total_feature_count"),
|
||||||
|
truncated=selection["truncated"],
|
||||||
|
warnings=[
|
||||||
|
warning
|
||||||
|
for warning in (summary.get("selection_edge_warning"), summary.get("warning"))
|
||||||
|
if warning
|
||||||
|
],
|
||||||
|
),
|
||||||
|
)
|
||||||
filename = ExportService._filename(name, f"{dataset.id}-selection.geojson", ".geojson")
|
filename = ExportService._filename(name, f"{dataset.id}-selection.geojson", ".geojson")
|
||||||
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
|
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
|
||||||
metadata = {
|
metadata = {
|
||||||
@@ -374,8 +583,21 @@ class ExportService:
|
|||||||
details={"dataset_type": dataset.dataset_type},
|
details={"dataset_type": dataset.dataset_type},
|
||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
|
|
||||||
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
|
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
|
||||||
|
ExportService.attach_provenance(
|
||||||
|
feature_collection,
|
||||||
|
ExportService.provenance_member(
|
||||||
|
source="dataset",
|
||||||
|
project_id=dataset.project_id,
|
||||||
|
dataset_id=dataset.id,
|
||||||
|
source_name=dataset.source_name,
|
||||||
|
source_version=dataset.source_version,
|
||||||
|
observed_at=dataset.observed_at,
|
||||||
|
feature_count=len(feature_collection.get("features", [])),
|
||||||
|
),
|
||||||
|
)
|
||||||
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
|
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
|
||||||
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
|
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
|
||||||
metadata = {
|
metadata = {
|
||||||
@@ -397,12 +619,43 @@ class ExportService:
|
|||||||
return ExportService._create_response(export)
|
return ExportService._create_response(export)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def export_detection_run_geojson(db: Session, analysis_run_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse:
|
def export_detection_run_geojson(
|
||||||
|
db: Session,
|
||||||
|
analysis_run_id: uuid.UUID,
|
||||||
|
name: str | None = None,
|
||||||
|
*,
|
||||||
|
intended_use: str = "review",
|
||||||
|
) -> ExportCreateResponse:
|
||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
if not run or run.analysis_type != "detection":
|
if not run or run.analysis_type != "detection":
|
||||||
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
|
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
|
||||||
|
ExportService._assert_run_source_dataset_exportable(db, run)
|
||||||
|
|
||||||
|
trust = ExportService._detection_export_trust(db, run)
|
||||||
|
if intended_use == "operational" and not trust["operational_use_allowed"]:
|
||||||
|
raise AppError(
|
||||||
|
code="DETECTION_OPERATIONAL_EXPORT_BLOCKED",
|
||||||
|
message="Operational detection export requires complete authoritative QA with no remaining errors or warnings.",
|
||||||
|
details=trust,
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
|
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
|
||||||
|
feature_collection["geointel_result"] = trust
|
||||||
|
ExportService.attach_provenance(
|
||||||
|
feature_collection,
|
||||||
|
ExportService.provenance_member(
|
||||||
|
source="detection_run",
|
||||||
|
project_id=run.project_id,
|
||||||
|
dataset_id=run.dataset_id,
|
||||||
|
analysis_run_id=run.id,
|
||||||
|
feature_count=len(feature_collection.get("features", [])),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for feature in feature_collection.get("features", []):
|
||||||
|
properties = feature.get("properties") if isinstance(feature, dict) else None
|
||||||
|
if isinstance(properties, dict):
|
||||||
|
properties["result_classification"] = trust["classification"]
|
||||||
|
properties["authoritative"] = False
|
||||||
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
|
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
|
||||||
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
|
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
|
||||||
metadata = {
|
metadata = {
|
||||||
@@ -411,6 +664,8 @@ class ExportService:
|
|||||||
"project_id": str(run.project_id),
|
"project_id": str(run.project_id),
|
||||||
"dataset_id": str(run.dataset_id) if run.dataset_id else None,
|
"dataset_id": str(run.dataset_id) if run.dataset_id else None,
|
||||||
"feature_count": len(feature_collection.get("features", [])),
|
"feature_count": len(feature_collection.get("features", [])),
|
||||||
|
"intended_use": intended_use,
|
||||||
|
"result_trust": trust,
|
||||||
}
|
}
|
||||||
export = ExportService._write_json_export(
|
export = ExportService._write_json_export(
|
||||||
db,
|
db,
|
||||||
@@ -428,8 +683,19 @@ class ExportService:
|
|||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
if not run or run.analysis_type != "segmentation":
|
if not run or run.analysis_type != "segmentation":
|
||||||
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
||||||
|
ExportService._assert_run_source_dataset_exportable(db, run)
|
||||||
|
|
||||||
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
|
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
|
||||||
|
ExportService.attach_provenance(
|
||||||
|
feature_collection,
|
||||||
|
ExportService.provenance_member(
|
||||||
|
source="segmentation_run",
|
||||||
|
project_id=run.project_id,
|
||||||
|
dataset_id=run.dataset_id,
|
||||||
|
analysis_run_id=run.id,
|
||||||
|
feature_count=len(feature_collection.get("features", [])),
|
||||||
|
),
|
||||||
|
)
|
||||||
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
|
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
|
||||||
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
|
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
|
||||||
metadata = {
|
metadata = {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
from xml.etree import ElementTree
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
@@ -23,6 +23,7 @@ from shapely.ops import transform as shapely_transform
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.services.outbound_request_guard import guarded_opener
|
||||||
from app.models import Area, Dataset, Project
|
from app.models import Area, Dataset, Project
|
||||||
from app.schemas.flood_hazard import FloodHazardAcquireRequest, FloodHazardAcquisitionResult, FloodHazardProductRead
|
from app.schemas.flood_hazard import FloodHazardAcquireRequest, FloodHazardAcquisitionResult, FloodHazardProductRead
|
||||||
from app.services.dataset_service import DatasetService
|
from app.services.dataset_service import DatasetService
|
||||||
@@ -256,7 +257,7 @@ class FloodHazardAcquisitionService:
|
|||||||
request = Request(request_url, headers={"Accept": "*/*", "User-Agent": "GeoIntel/0.1 bounded-vmm-flood-hazard-acquisition"})
|
request = Request(request_url, headers={"Accept": "*/*", "User-Agent": "GeoIntel/0.1 bounded-vmm-flood-hazard-acquisition"})
|
||||||
max_bytes = settings.flood_hazard_max_response_mb * 1024 * 1024
|
max_bytes = settings.flood_hazard_max_response_mb * 1024 * 1024
|
||||||
try:
|
try:
|
||||||
with (opener or urlopen)(request, timeout=settings.flood_hazard_timeout_seconds) as response:
|
with (opener or guarded_opener(request_url))(request, timeout=settings.flood_hazard_timeout_seconds) as response:
|
||||||
content_type = str(response.headers.get("Content-Type", ""))
|
content_type = str(response.headers.get("Content-Type", ""))
|
||||||
content_length = response.headers.get("Content-Length")
|
content_length = response.headers.get("Content-Length")
|
||||||
if content_length and int(content_length) > max_bytes:
|
if content_length and int(content_length) > max_bytes:
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import io
|
import io
|
||||||
import math
|
import math
|
||||||
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
@@ -13,6 +15,7 @@ from shapely.ops import transform as shapely_transform
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.services.raster_cell_selection import select_cells
|
||||||
from app.models import Area, Dataset
|
from app.models import Area, Dataset
|
||||||
from app.schemas.flood_hazard import (
|
from app.schemas.flood_hazard import (
|
||||||
FloodHazardMetric,
|
FloodHazardMetric,
|
||||||
@@ -25,6 +28,83 @@ from app.services.flood_hazard_acquisition_service import FloodHazardAcquisition
|
|||||||
from app.services.raster_partition_analysis_service import RasterPartitionAnalysisService
|
from app.services.raster_partition_analysis_service import RasterPartitionAnalysisService
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FloodHazardCellStatistics:
|
||||||
|
"""Cell populations behind one flood-hazard selection.
|
||||||
|
|
||||||
|
Three populations, deliberately kept apart:
|
||||||
|
|
||||||
|
``selected``
|
||||||
|
every cell whose centre falls inside the drawn selection;
|
||||||
|
``valid``
|
||||||
|
the subset the VMM raster actually models — finite, not nodata;
|
||||||
|
``inundated``
|
||||||
|
the subset of valid cells with a positive modelled depth.
|
||||||
|
|
||||||
|
Risk is a share of what was modelled. Dividing by the selected cells
|
||||||
|
instead silently reports "no data" as "no risk", which for a selection
|
||||||
|
reaching past the modelled extent understates the hazard by whatever
|
||||||
|
fraction of the rectangle the model never covered.
|
||||||
|
"""
|
||||||
|
|
||||||
|
selected_cell_count: int
|
||||||
|
valid_cell_count: int
|
||||||
|
inundated_cell_count: int
|
||||||
|
depth_values: Any
|
||||||
|
|
||||||
|
@property
|
||||||
|
def no_data_cell_count(self) -> int:
|
||||||
|
return max(0, self.selected_cell_count - self.valid_cell_count)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def data_coverage_ratio(self) -> float:
|
||||||
|
if self.selected_cell_count <= 0:
|
||||||
|
return 0.0
|
||||||
|
return self.valid_cell_count / self.selected_cell_count
|
||||||
|
|
||||||
|
@property
|
||||||
|
def inundated_fraction(self) -> float | None:
|
||||||
|
"""``None`` when nothing was modelled: absence of data is not a zero."""
|
||||||
|
|
||||||
|
if self.valid_cell_count <= 0:
|
||||||
|
return None
|
||||||
|
return self.inundated_cell_count / self.valid_cell_count
|
||||||
|
|
||||||
|
def inundated_area_ha(self, cell_area_m2: float) -> float:
|
||||||
|
return self.inundated_cell_count * cell_area_m2 / 10_000.0
|
||||||
|
|
||||||
|
def analysed_area_ha(self, cell_area_m2: float) -> float:
|
||||||
|
"""Area the model actually covers inside the selection."""
|
||||||
|
|
||||||
|
return self.valid_cell_count * cell_area_m2 / 10_000.0
|
||||||
|
|
||||||
|
def selected_area_ha(self, cell_area_m2: float) -> float:
|
||||||
|
"""Area of the selection as rasterised, model coverage aside."""
|
||||||
|
|
||||||
|
return self.selected_cell_count * cell_area_m2 / 10_000.0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_cells(cls, values: Any, selected: Any, *, nodata: float | None) -> "FloodHazardCellStatistics":
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
raw = np.asarray(values, dtype="float64")
|
||||||
|
selected_mask = np.asarray(selected, dtype=bool)
|
||||||
|
|
||||||
|
has_data = selected_mask & np.isfinite(raw)
|
||||||
|
if nodata is not None:
|
||||||
|
has_data &= ~np.isclose(raw, float(nodata))
|
||||||
|
# A modelled zero or negative depth is data: it says "dry here", which
|
||||||
|
# is a different statement from "not modelled here".
|
||||||
|
inundated = has_data & (raw > 0.0)
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
selected_cell_count=int(selected_mask.sum()),
|
||||||
|
valid_cell_count=int(has_data.sum()),
|
||||||
|
inundated_cell_count=int(inundated.sum()),
|
||||||
|
depth_values=raw[inundated],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FloodHazardAnalysisService:
|
class FloodHazardAnalysisService:
|
||||||
UNSUPPORTED_METRICS = [
|
UNSUPPORTED_METRICS = [
|
||||||
"bathymetry_depth_m",
|
"bathymetry_depth_m",
|
||||||
@@ -36,6 +116,77 @@ class FloodHazardAnalysisService:
|
|||||||
"gemodelleerde maxima op en is geen gelijktijdig opgeslagen watervolume, actuele waterstand of bathymetrie."
|
"gemodelleerde maxima op en is geen gelijktijdig opgeslagen watervolume, actuele waterstand of bathymetrie."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _coverage_metrics(stats: "FloodHazardCellStatistics", cell_area_m2: float, metric) -> list[FloodHazardMetric]:
|
||||||
|
"""Headline metrics, each stating which population it is a share of.
|
||||||
|
|
||||||
|
The analysed area is reported next to the drawn area so an operator can
|
||||||
|
see immediately how much of the rectangle the flood model covers. A
|
||||||
|
selection with no model data reports 0% coverage rather than 0% risk.
|
||||||
|
"""
|
||||||
|
|
||||||
|
metrics = [
|
||||||
|
metric(
|
||||||
|
"modelled_inundated_area_ha",
|
||||||
|
"Gemodelleerd overstroomd oppervlak",
|
||||||
|
stats.inundated_area_ha(cell_area_m2),
|
||||||
|
"ha",
|
||||||
|
"positive_depth_cells_times_cell_area",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"modelled_inundated_share_pct",
|
||||||
|
"Aandeel gemodelleerd gebied met diepte",
|
||||||
|
0.0 if stats.inundated_fraction is None else stats.inundated_fraction * 100.0,
|
||||||
|
"%",
|
||||||
|
"positive_depth_cells_divided_by_modelled_cells",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"modelled_area_ha",
|
||||||
|
"Oppervlak met overstromingsmodel",
|
||||||
|
stats.analysed_area_ha(cell_area_m2),
|
||||||
|
"ha",
|
||||||
|
"modelled_cells_times_cell_area",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"selection_area_ha",
|
||||||
|
"Oppervlak van de selectie",
|
||||||
|
stats.selected_area_ha(cell_area_m2),
|
||||||
|
"ha",
|
||||||
|
"selected_cells_times_cell_area",
|
||||||
|
),
|
||||||
|
metric(
|
||||||
|
"model_coverage_pct",
|
||||||
|
"Deel van de selectie met een model",
|
||||||
|
stats.data_coverage_ratio * 100.0,
|
||||||
|
"%",
|
||||||
|
"modelled_cells_divided_by_selected_cells",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
return metrics
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _combined_warning(stats: "FloodHazardCellStatistics", cell_selection_warning: str | None) -> str | None:
|
||||||
|
parts = [
|
||||||
|
part
|
||||||
|
for part in (cell_selection_warning, FloodHazardAnalysisService._coverage_warning(stats))
|
||||||
|
if part
|
||||||
|
]
|
||||||
|
return " ".join(parts) if parts else None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _coverage_warning(stats: "FloodHazardCellStatistics") -> str | None:
|
||||||
|
if stats.valid_cell_count <= 0:
|
||||||
|
return (
|
||||||
|
"Voor deze selectie bestaat geen VMM-overstromingsmodel. Er is dus geen overstromingsrisico "
|
||||||
|
"gemeten; dit is geen bevestiging dat het risico nul is."
|
||||||
|
)
|
||||||
|
if stats.data_coverage_ratio < 0.999:
|
||||||
|
return (
|
||||||
|
f"Het VMM-model dekt {stats.data_coverage_ratio * 100:.1f}% van deze selectie. Percentages gelden "
|
||||||
|
"voor het gemodelleerde deel, niet voor de volledige selectie."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
|
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
|
||||||
dataset = db.get(Dataset, dataset_id)
|
dataset = db.get(Dataset, dataset_id)
|
||||||
@@ -81,7 +232,6 @@ class FloodHazardAnalysisService:
|
|||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import rasterio
|
import rasterio
|
||||||
from rasterio.features import geometry_mask
|
|
||||||
from rasterio.mask import mask
|
from rasterio.mask import mask
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc
|
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc
|
||||||
@@ -110,17 +260,32 @@ class FloodHazardAnalysisService:
|
|||||||
details={"pixel_count": expected_cells, "max_pixels": resolved_settings.flood_hazard_max_pixels},
|
details={"pixel_count": expected_cells, "max_pixels": resolved_settings.flood_hazard_max_pixels},
|
||||||
status_code=422,
|
status_code=422,
|
||||||
)
|
)
|
||||||
clipped, clipped_transform = mask(source, [mapping(analysis_geometry)], crop=True, filled=False, indexes=[1])
|
# ``all_touched`` keeps the values of cells the selection only
|
||||||
|
# clips, so a selection finer than one cell still has data to
|
||||||
|
# read. Which of those cells actually count is decided by
|
||||||
|
# ``select_cells`` below, so the normal result is unchanged.
|
||||||
|
clipped, clipped_transform = mask(
|
||||||
|
source,
|
||||||
|
[mapping(analysis_geometry)],
|
||||||
|
crop=True,
|
||||||
|
filled=False,
|
||||||
|
indexes=[1],
|
||||||
|
all_touched=True,
|
||||||
|
)
|
||||||
depth = np.ma.asarray(clipped[0], dtype="float64")
|
depth = np.ma.asarray(clipped[0], dtype="float64")
|
||||||
raw = depth.filled(np.nan)
|
raw = depth.filled(np.nan)
|
||||||
selected_cells = geometry_mask([mapping(analysis_geometry)], out_shape=depth.shape, transform=clipped_transform, invert=True)
|
cell_selection = select_cells(
|
||||||
nodata = source.nodata
|
analysis_geometry,
|
||||||
valid = selected_cells & ~np.ma.getmaskarray(depth) & np.isfinite(raw) & (raw > 0.0)
|
out_shape=depth.shape,
|
||||||
if nodata is not None:
|
transform=clipped_transform,
|
||||||
valid &= raw != float(nodata)
|
cell_area_m2=abs(float(source.res[0])) * abs(float(source.res[1])),
|
||||||
values = raw[valid]
|
)
|
||||||
selected_cell_count = int(selected_cells.sum())
|
selected_cells = cell_selection.mask
|
||||||
inundated_cell_count = int(values.size)
|
# A masked cell carries no model value, so fold the mask into
|
||||||
|
# the raw array before the populations are separated.
|
||||||
|
raw = np.where(np.ma.getmaskarray(depth), np.nan, raw)
|
||||||
|
stats = FloodHazardCellStatistics.from_cells(raw, selected_cells, nodata=source.nodata)
|
||||||
|
values = stats.depth_values
|
||||||
resolution_x = abs(float(source.res[0]))
|
resolution_x = abs(float(source.res[0]))
|
||||||
resolution_y = abs(float(source.res[1]))
|
resolution_y = abs(float(source.res[1]))
|
||||||
cell_area_m2 = resolution_x * resolution_y
|
cell_area_m2 = resolution_x * resolution_y
|
||||||
@@ -143,18 +308,8 @@ class FloodHazardAnalysisService:
|
|||||||
aggregation_method=method,
|
aggregation_method=method,
|
||||||
)
|
)
|
||||||
|
|
||||||
inundated_area_ha = inundated_cell_count * cell_area_m2 / 10_000.0
|
metrics = FloodHazardAnalysisService._coverage_metrics(stats, cell_area_m2, metric)
|
||||||
metrics = [
|
if stats.inundated_cell_count:
|
||||||
metric("modelled_inundated_area_ha", "Gemodelleerd overstroomd oppervlak", inundated_area_ha, "ha", "positive_depth_cells_times_cell_area"),
|
|
||||||
metric(
|
|
||||||
"modelled_inundated_share_pct",
|
|
||||||
"Aandeel selectie met gemodelleerde diepte",
|
|
||||||
inundated_cell_count / max(1, selected_cell_count) * 100.0,
|
|
||||||
"%",
|
|
||||||
"positive_depth_cells_divided_by_selected_cells",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
if inundated_cell_count:
|
|
||||||
metrics.extend(
|
metrics.extend(
|
||||||
[
|
[
|
||||||
metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"),
|
metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"),
|
||||||
@@ -181,9 +336,14 @@ class FloodHazardAnalysisService:
|
|||||||
return_period_years=product.return_period_years,
|
return_period_years=product.return_period_years,
|
||||||
selection_bbox=payload.bbox,
|
selection_bbox=payload.bbox,
|
||||||
selection_area_id=payload.area_id,
|
selection_area_id=payload.area_id,
|
||||||
selected_cell_count=selected_cell_count,
|
selected_cell_count=stats.selected_cell_count,
|
||||||
inundated_cell_count=inundated_cell_count,
|
valid_cell_count=stats.valid_cell_count,
|
||||||
inundated_fraction=round(inundated_cell_count / max(1, selected_cell_count), 6),
|
no_data_cell_count=stats.no_data_cell_count,
|
||||||
|
data_coverage_ratio=round(stats.data_coverage_ratio, 6),
|
||||||
|
inundated_cell_count=stats.inundated_cell_count,
|
||||||
|
inundated_fraction=(
|
||||||
|
None if stats.inundated_fraction is None else round(stats.inundated_fraction, 6)
|
||||||
|
),
|
||||||
resolution_m=round(max(resolution_x, resolution_y), 4),
|
resolution_m=round(max(resolution_x, resolution_y), 4),
|
||||||
summary=FloodHazardSelectionSummary(
|
summary=FloodHazardSelectionSummary(
|
||||||
metric_label=primary.metric_label,
|
metric_label=primary.metric_label,
|
||||||
@@ -193,6 +353,7 @@ class FloodHazardAnalysisService:
|
|||||||
primary_metric_key=primary.metric_key,
|
primary_metric_key=primary.metric_key,
|
||||||
metrics=metrics,
|
metrics=metrics,
|
||||||
),
|
),
|
||||||
|
coverage_warning=FloodHazardAnalysisService._combined_warning(stats, cell_selection.warning),
|
||||||
unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS,
|
unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS,
|
||||||
limitation_message=FloodHazardAnalysisService.LIMITATION,
|
limitation_message=FloodHazardAnalysisService.LIMITATION,
|
||||||
generated_at=datetime.now(UTC).isoformat(),
|
generated_at=datetime.now(UTC).isoformat(),
|
||||||
@@ -225,6 +386,7 @@ class FloodHazardAnalysisService:
|
|||||||
selection_geometry_4326=selection_4326,
|
selection_geometry_4326=selection_4326,
|
||||||
nodata=FloodHazardAcquisitionService.NODATA,
|
nodata=FloodHazardAcquisitionService.NODATA,
|
||||||
max_pixels=resolved_settings.flood_hazard_max_pixels,
|
max_pixels=resolved_settings.flood_hazard_max_pixels,
|
||||||
|
dataset_ids=payload.dataset_ids,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -235,16 +397,12 @@ class FloodHazardAnalysisService:
|
|||||||
status_code=503,
|
status_code=503,
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
raw = partition.values
|
stats = FloodHazardCellStatistics.from_cells(
|
||||||
valid = (
|
partition.values,
|
||||||
partition.selected_cells
|
partition.selected_cells,
|
||||||
& np.isfinite(raw)
|
nodata=FloodHazardAcquisitionService.NODATA,
|
||||||
& (raw != FloodHazardAcquisitionService.NODATA)
|
|
||||||
& (raw > 0.0)
|
|
||||||
)
|
)
|
||||||
values = raw[valid]
|
values = stats.depth_values
|
||||||
selected_cell_count = int(partition.selected_cells.sum())
|
|
||||||
inundated_cell_count = int(values.size)
|
|
||||||
cell_area_m2 = partition.resolution_x * partition.resolution_y
|
cell_area_m2 = partition.resolution_x * partition.resolution_y
|
||||||
|
|
||||||
def metric(key: str, label: str, value: float, unit: str, method: str) -> FloodHazardMetric:
|
def metric(key: str, label: str, value: float, unit: str, method: str) -> FloodHazardMetric:
|
||||||
@@ -256,24 +414,8 @@ class FloodHazardAnalysisService:
|
|||||||
aggregation_method=method,
|
aggregation_method=method,
|
||||||
)
|
)
|
||||||
|
|
||||||
inundated_area_ha = inundated_cell_count * cell_area_m2 / 10_000.0
|
metrics = FloodHazardAnalysisService._coverage_metrics(stats, cell_area_m2, metric)
|
||||||
metrics = [
|
if stats.inundated_cell_count:
|
||||||
metric(
|
|
||||||
"modelled_inundated_area_ha",
|
|
||||||
"Gemodelleerd overstroomd oppervlak",
|
|
||||||
inundated_area_ha,
|
|
||||||
"ha",
|
|
||||||
"positive_depth_cells_times_cell_area",
|
|
||||||
),
|
|
||||||
metric(
|
|
||||||
"modelled_inundated_share_pct",
|
|
||||||
"Aandeel selectie met gemodelleerde diepte",
|
|
||||||
inundated_cell_count / max(1, selected_cell_count) * 100.0,
|
|
||||||
"%",
|
|
||||||
"positive_depth_cells_divided_by_selected_cells",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
if inundated_cell_count:
|
|
||||||
metrics.extend(
|
metrics.extend(
|
||||||
[
|
[
|
||||||
metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"),
|
metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"),
|
||||||
@@ -301,9 +443,14 @@ class FloodHazardAnalysisService:
|
|||||||
return_period_years=product.return_period_years,
|
return_period_years=product.return_period_years,
|
||||||
selection_bbox=payload.bbox,
|
selection_bbox=payload.bbox,
|
||||||
selection_area_id=payload.area_id,
|
selection_area_id=payload.area_id,
|
||||||
selected_cell_count=selected_cell_count,
|
selected_cell_count=stats.selected_cell_count,
|
||||||
inundated_cell_count=inundated_cell_count,
|
valid_cell_count=stats.valid_cell_count,
|
||||||
inundated_fraction=round(inundated_cell_count / max(1, selected_cell_count), 6),
|
no_data_cell_count=stats.no_data_cell_count,
|
||||||
|
data_coverage_ratio=round(stats.data_coverage_ratio, 6),
|
||||||
|
inundated_cell_count=stats.inundated_cell_count,
|
||||||
|
inundated_fraction=(
|
||||||
|
None if stats.inundated_fraction is None else round(stats.inundated_fraction, 6)
|
||||||
|
),
|
||||||
resolution_m=round(max(partition.resolution_x, partition.resolution_y), 4),
|
resolution_m=round(max(partition.resolution_x, partition.resolution_y), 4),
|
||||||
summary=FloodHazardSelectionSummary(
|
summary=FloodHazardSelectionSummary(
|
||||||
metric_label=primary.metric_label,
|
metric_label=primary.metric_label,
|
||||||
@@ -313,6 +460,7 @@ class FloodHazardAnalysisService:
|
|||||||
primary_metric_key=primary.metric_key,
|
primary_metric_key=primary.metric_key,
|
||||||
metrics=metrics,
|
metrics=metrics,
|
||||||
),
|
),
|
||||||
|
coverage_warning=FloodHazardAnalysisService._combined_warning(stats, partition.cell_selection_warning),
|
||||||
unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS,
|
unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS,
|
||||||
limitation_message=(
|
limitation_message=(
|
||||||
f"{FloodHazardAnalysisService.LIMITATION} De selectie werd exact berekend over "
|
f"{FloodHazardAnalysisService.LIMITATION} De selectie werd exact berekend over "
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from app.core.errors import AppError
|
|||||||
from app.models import Area, Dataset, Project
|
from app.models import Area, Dataset, Project
|
||||||
from app.schemas.assistant import (
|
from app.schemas.assistant import (
|
||||||
AssistantContextMetric,
|
AssistantContextMetric,
|
||||||
|
AssistantEstimateDisclosure,
|
||||||
AssistantModelRead,
|
AssistantModelRead,
|
||||||
AssistantQueryRequest,
|
AssistantQueryRequest,
|
||||||
AssistantQueryResponse,
|
AssistantQueryResponse,
|
||||||
@@ -112,6 +113,44 @@ class GeoAssistantService:
|
|||||||
}
|
}
|
||||||
return themes or None
|
return themes or None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def estimate_disclosures(
|
||||||
|
cls,
|
||||||
|
metrics: list[AssistantContextMetric],
|
||||||
|
) -> list[AssistantEstimateDisclosure]:
|
||||||
|
"""List every estimated value behind the answer, straight from metadata.
|
||||||
|
|
||||||
|
``ensure_estimate_disclosure`` can only add a caveat when it recognises
|
||||||
|
the phrasing the model produced, which makes the guarantee dependent on
|
||||||
|
generated text. This derives the same statement from the source
|
||||||
|
metadata, so it holds regardless of how the answer was written.
|
||||||
|
"""
|
||||||
|
|
||||||
|
seen: set[tuple[str, UUID]] = set()
|
||||||
|
disclosures: list[AssistantEstimateDisclosure] = []
|
||||||
|
for metric in sorted(metrics, key=lambda item: (item.theme, item.label)):
|
||||||
|
if not metric.is_estimate:
|
||||||
|
continue
|
||||||
|
key = (metric.theme, metric.dataset_id)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
topic = cls.ESTIMATE_TOPIC_LABELS.get(metric.theme, metric.label)
|
||||||
|
disclosures.append(
|
||||||
|
AssistantEstimateDisclosure(
|
||||||
|
theme=metric.theme,
|
||||||
|
label=metric.label,
|
||||||
|
unit=metric.unit,
|
||||||
|
source=metric.source,
|
||||||
|
dataset_id=metric.dataset_id,
|
||||||
|
reason=(
|
||||||
|
f"De bronmetadata van {metric.source} markeert {topic} als schatting, "
|
||||||
|
"geen exacte telling."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return disclosures
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def ensure_estimate_disclosure(
|
def ensure_estimate_disclosure(
|
||||||
cls,
|
cls,
|
||||||
@@ -702,6 +741,7 @@ class GeoAssistantService:
|
|||||||
scope_label=scope_label,
|
scope_label=scope_label,
|
||||||
context_metrics=metrics,
|
context_metrics=metrics,
|
||||||
temporal_series=series,
|
temporal_series=series,
|
||||||
|
estimate_disclosures=self.estimate_disclosures(metrics),
|
||||||
source_dataset_ids=dataset_ids,
|
source_dataset_ids=dataset_ids,
|
||||||
warnings=warnings,
|
warnings=warnings,
|
||||||
generated_at=datetime.now(timezone.utc),
|
generated_at=datetime.now(timezone.utc),
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
|
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
|
||||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
@@ -20,20 +20,12 @@ from shapely.validation import make_valid
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.services.outbound_request_guard import guarded_opener
|
||||||
from app.models import Area, Dataset, Project
|
from app.models import Area, Dataset, Project
|
||||||
from app.schemas.grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
from app.schemas.grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
||||||
from app.services.dataset_service import DatasetService
|
from app.services.dataset_service import DatasetService
|
||||||
|
|
||||||
|
|
||||||
class _RejectRedirects(HTTPRedirectHandler):
|
|
||||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
||||||
del req, fp, code, msg, headers, newurl
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
_NO_REDIRECT_OPENER = build_opener(_RejectRedirects())
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class GrbCollection:
|
class GrbCollection:
|
||||||
name: str
|
name: str
|
||||||
@@ -368,7 +360,7 @@ class GrbAcquisitionService:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
with (opener or _NO_REDIRECT_OPENER.open)(
|
with (opener or guarded_opener(url, allow_redirect=False))(
|
||||||
request,
|
request,
|
||||||
timeout=settings.grb_timeout_seconds,
|
timeout=settings.grb_timeout_seconds,
|
||||||
) as response:
|
) as response:
|
||||||
|
|||||||
@@ -76,6 +76,22 @@ class JobService:
|
|||||||
result_json["output_dataset_id"] = str(result_json["output_dataset_id"])
|
result_json["output_dataset_id"] = str(result_json["output_dataset_id"])
|
||||||
payload["result_json"] = result_json
|
payload["result_json"] = result_json
|
||||||
raise
|
raise
|
||||||
|
except Exception:
|
||||||
|
# An unexpected error must never leave the job stuck in "running".
|
||||||
|
try:
|
||||||
|
db.rollback()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
JobService.mark_failed(
|
||||||
|
db,
|
||||||
|
created.id,
|
||||||
|
error_message="Unexpected internal error during synchronous job execution",
|
||||||
|
details={"code": "JOB_INTERNAL_ERROR"},
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]:
|
def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -0,0 +1,335 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any, Callable
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
|
from urllib.request import Request
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.core.config import Settings, get_settings
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.services.outbound_request_guard import guarded_opener
|
||||||
|
from app.models import Dataset
|
||||||
|
from app.schemas.bathymetry import MdkBathymetryAcquireRequest, MdkBathymetryAcquisitionResult
|
||||||
|
from app.services.dataset_service import DatasetService
|
||||||
|
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
||||||
|
|
||||||
|
|
||||||
|
class MdkBathymetryAcquisitionService:
|
||||||
|
"""Bounded, fail-closed GetCoverage acquisition for the MDK Belgian North Sea depth model.
|
||||||
|
|
||||||
|
Acquisition only runs when:
|
||||||
|
|
||||||
|
- the operator explicitly enabled acquisition and configured a coverage id,
|
||||||
|
- the live strict-TLS readiness probe reports ``reachable``,
|
||||||
|
- the configured coverage id is advertised by the live capabilities document,
|
||||||
|
- the requested EPSG:4326 bbox stays within the configured size bound.
|
||||||
|
|
||||||
|
No depth values are ever synthesized, no insecure TLS fallback exists and the
|
||||||
|
LAT vertical reference is persisted with every artifact so it can never be
|
||||||
|
silently compared with TAW or mDNG data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
PROVIDER = "mdk_bcp_bathymetry"
|
||||||
|
VERTICAL_REFERENCE = "LAT"
|
||||||
|
NATIVE_RESOLUTION_M = 20.0
|
||||||
|
MAX_PIXELS_PER_SIDE = 4096
|
||||||
|
LIMITATION = (
|
||||||
|
"Dieptewaarden zijn LAT-gerefereerd en gelden voor de bemonsterde survey-periode van het officiële "
|
||||||
|
"MDK-model. LAT mag nooit zonder gedocumenteerde datumtransformatie met TAW- of mDNG-gegevens worden "
|
||||||
|
"vergeleken; watervolume blijft zonder compatibel wateroppervlak niet ondersteund."
|
||||||
|
)
|
||||||
|
ATTRIBUTION = "Agentschap Maritieme Dienstverlening en Kust (MDK)"
|
||||||
|
LICENSE_NOTE = "Consult the official MDK product license before redistribution."
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def acquire(
|
||||||
|
db,
|
||||||
|
project_id: UUID,
|
||||||
|
payload: MdkBathymetryAcquireRequest,
|
||||||
|
*,
|
||||||
|
settings: Settings | None = None,
|
||||||
|
opener: Callable[..., Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
resolved_settings = settings or get_settings()
|
||||||
|
if not resolved_settings.mdk_bathymetry_acquisition_enabled:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_ACQUISITION_DISABLED",
|
||||||
|
message=(
|
||||||
|
"MDK bathymetry acquisition is disabled. Enable it explicitly with "
|
||||||
|
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true after the readiness probe reports reachable."
|
||||||
|
),
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
coverage_id = (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
|
||||||
|
if not coverage_id:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED",
|
||||||
|
message="MDK_BATHYMETRY_COVERAGE_ID is not configured; GeoIntel will not guess coverage identifiers.",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
|
||||||
|
bbox = MdkBathymetryAcquisitionService._validated_bbox(payload, resolved_settings)
|
||||||
|
|
||||||
|
probe = MdkBathymetryProbeService.probe(settings=resolved_settings, opener=opener)
|
||||||
|
if probe.get("status") != "reachable":
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_ENDPOINT_NOT_READY",
|
||||||
|
message="The live MDK readiness probe does not report a reachable, TLS-verified WCS endpoint.",
|
||||||
|
details={"probe_status": probe.get("status"), "probe_message": probe.get("message")},
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
if coverage_id not in (probe.get("coverage_identifiers") or []):
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED",
|
||||||
|
message="The configured coverage id is not advertised by the live MDK capabilities document.",
|
||||||
|
details={
|
||||||
|
"configured_coverage_id": coverage_id,
|
||||||
|
"advertised_coverage_identifiers": probe.get("coverage_identifiers") or [],
|
||||||
|
},
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
|
||||||
|
request_url = MdkBathymetryAcquisitionService._get_coverage_url(resolved_settings, coverage_id, bbox)
|
||||||
|
request_hash = hashlib.sha256(request_url.encode("utf-8")).hexdigest()
|
||||||
|
filename = f"mdk_bathymetry_{request_hash[:12]}.tif"
|
||||||
|
|
||||||
|
if not payload.force_refresh:
|
||||||
|
cached = MdkBathymetryAcquisitionService._cached_dataset(db, project_id, filename)
|
||||||
|
if cached is not None:
|
||||||
|
return MdkBathymetryAcquisitionResult(
|
||||||
|
output_dataset_id=cached.id,
|
||||||
|
reused=True,
|
||||||
|
provider=MdkBathymetryAcquisitionService.PROVIDER,
|
||||||
|
coverage_id=coverage_id,
|
||||||
|
bbox_epsg4326=bbox,
|
||||||
|
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
|
||||||
|
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
|
||||||
|
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
|
||||||
|
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
|
||||||
|
).model_dump(mode="json")
|
||||||
|
|
||||||
|
content, content_type = MdkBathymetryAcquisitionService._fetch(request_url, resolved_settings, opener)
|
||||||
|
validation = MdkBathymetryAcquisitionService._validate_geotiff(content)
|
||||||
|
acquired_at = datetime.now(UTC)
|
||||||
|
|
||||||
|
dataset = DatasetService.import_raster_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
area_id=payload.area_id,
|
||||||
|
filename=filename,
|
||||||
|
content=content,
|
||||||
|
source=f"MDK Belgian Continental Shelf WCS {coverage_id}",
|
||||||
|
source_name=MdkBathymetryAcquisitionService.PROVIDER,
|
||||||
|
source_metadata={
|
||||||
|
"provider": MdkBathymetryAcquisitionService.PROVIDER,
|
||||||
|
"service": "WCS",
|
||||||
|
"service_version": "1.0.0",
|
||||||
|
"coverage_id": coverage_id,
|
||||||
|
"vertical_reference": MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
|
||||||
|
"native_resolution_m": MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
|
||||||
|
"bbox_epsg4326": bbox,
|
||||||
|
"attribution": MdkBathymetryAcquisitionService.ATTRIBUTION,
|
||||||
|
"license_note": MdkBathymetryAcquisitionService.LICENSE_NOTE,
|
||||||
|
"raster_validation": validation,
|
||||||
|
},
|
||||||
|
provenance_metadata={
|
||||||
|
"acquisition": "explicit_bounded_wcs_get_coverage",
|
||||||
|
"acquired_at": acquired_at.isoformat(),
|
||||||
|
"request_url": request_url,
|
||||||
|
"request_hash": request_hash,
|
||||||
|
"response_content_type": content_type,
|
||||||
|
"coverage_sha256": hashlib.sha256(content).hexdigest(),
|
||||||
|
"probe_status": probe.get("status"),
|
||||||
|
"probe_response_sha256": probe.get("response_sha256"),
|
||||||
|
"probe_checked_at": probe.get("checked_at"),
|
||||||
|
"limitation_message": MdkBathymetryAcquisitionService.LIMITATION,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return MdkBathymetryAcquisitionResult(
|
||||||
|
output_dataset_id=dataset.id,
|
||||||
|
reused=False,
|
||||||
|
provider=MdkBathymetryAcquisitionService.PROVIDER,
|
||||||
|
coverage_id=coverage_id,
|
||||||
|
bbox_epsg4326=bbox,
|
||||||
|
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
|
||||||
|
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
|
||||||
|
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
|
||||||
|
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
|
||||||
|
).model_dump(mode="json")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validated_bbox(payload: MdkBathymetryAcquireRequest, settings: Settings) -> list[float]:
|
||||||
|
bbox = payload.bbox
|
||||||
|
min_x, min_y, max_x, max_y = (
|
||||||
|
float(bbox.min_x),
|
||||||
|
float(bbox.min_y),
|
||||||
|
float(bbox.max_x),
|
||||||
|
float(bbox.max_y),
|
||||||
|
)
|
||||||
|
if max_x <= min_x or max_y <= min_y:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_INVALID_BBOX",
|
||||||
|
message="The requested bbox must have positive width and height in EPSG:4326.",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
area_deg2 = (max_x - min_x) * (max_y - min_y)
|
||||||
|
if area_deg2 > float(settings.mdk_bathymetry_max_bbox_deg2):
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_BBOX_TOO_LARGE",
|
||||||
|
message="The requested bbox exceeds the configured bounded acquisition size.",
|
||||||
|
details={
|
||||||
|
"bbox_area_deg2": area_deg2,
|
||||||
|
"max_bbox_deg2": float(settings.mdk_bathymetry_max_bbox_deg2),
|
||||||
|
},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return [min_x, min_y, max_x, max_y]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_coverage_url(settings: Settings, coverage_id: str, bbox: list[float]) -> str:
|
||||||
|
parsed = urlsplit(settings.mdk_bathymetry_wcs_url.strip())
|
||||||
|
if parsed.scheme.lower() != "https" or not parsed.hostname:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_INVALID_CONFIGURATION",
|
||||||
|
message="MDK bathymetry acquisition requires an absolute HTTPS WCS URL.",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox)
|
||||||
|
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
|
||||||
|
parameters.update(
|
||||||
|
{
|
||||||
|
"service": "WCS",
|
||||||
|
"request": "GetCoverage",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"coverage": coverage_id,
|
||||||
|
"crs": settings.mdk_bathymetry_request_crs,
|
||||||
|
"bbox": ",".join(f"{value:.8f}" for value in bbox),
|
||||||
|
"width": str(width),
|
||||||
|
"height": str(height),
|
||||||
|
"format": "GeoTIFF",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _pixel_dimensions(bbox: list[float]) -> tuple[int, int]:
|
||||||
|
min_x, min_y, max_x, max_y = bbox
|
||||||
|
# Approximate meters per degree near the Belgian North Sea (~51.5N).
|
||||||
|
meters_per_deg_lat = 111_320.0
|
||||||
|
meters_per_deg_lon = 69_400.0
|
||||||
|
width = int((max_x - min_x) * meters_per_deg_lon / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
|
||||||
|
height = int((max_y - min_y) * meters_per_deg_lat / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
|
||||||
|
width = max(1, min(width, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
|
||||||
|
height = max(1, min(height, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
|
||||||
|
return width, height
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
|
||||||
|
request = Request(
|
||||||
|
request_url,
|
||||||
|
headers={
|
||||||
|
"Accept": "image/tiff,*/*;q=0.1",
|
||||||
|
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-bounded-acquisition",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
max_bytes = settings.mdk_bathymetry_acquisition_max_response_mb * 1024 * 1024
|
||||||
|
try:
|
||||||
|
with (opener or guarded_opener(request_url))(request, timeout=settings.mdk_bathymetry_acquisition_timeout_seconds) as response:
|
||||||
|
content_type = str(response.headers.get("Content-Type", "")) if hasattr(response, "headers") else ""
|
||||||
|
content = response.read(max_bytes + 1)
|
||||||
|
except HTTPError as exc:
|
||||||
|
preview = exc.read(300).decode("utf-8", errors="replace")
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
|
||||||
|
message="The MDK WCS could not complete the bounded GetCoverage request.",
|
||||||
|
details={"provider_status_code": int(exc.code), "response_preview": preview},
|
||||||
|
status_code=502,
|
||||||
|
) from exc
|
||||||
|
except (URLError, TimeoutError, OSError) as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
|
||||||
|
message="The MDK WCS could not be reached for the bounded GetCoverage request.",
|
||||||
|
details={"reason": str(exc)},
|
||||||
|
status_code=502,
|
||||||
|
) from exc
|
||||||
|
if len(content) > max_bytes:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_RESPONSE_TOO_LARGE",
|
||||||
|
message="The MDK coverage response exceeds the configured size limit.",
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
if not content.startswith((b"II*\x00", b"MM\x00*")):
|
||||||
|
preview = content[:300].decode("utf-8", errors="replace")
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_INVALID_RESPONSE",
|
||||||
|
message="The MDK WCS did not return a GeoTIFF coverage.",
|
||||||
|
details={"content_type": content_type, "response_preview": preview},
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
return content, content_type
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validate_geotiff(content: bytes) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
import numpy as np
|
||||||
|
from rasterio.io import MemoryFile
|
||||||
|
except ImportError as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="RASTER_PROCESSING_UNAVAILABLE",
|
||||||
|
message="Rasterio is required to validate the MDK bathymetry coverage before persistence.",
|
||||||
|
status_code=503,
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
with MemoryFile(content) as memory, memory.open() as source:
|
||||||
|
if source.count < 1:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_INVALID_RESPONSE",
|
||||||
|
message="The MDK coverage contains no raster bands.",
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
band = source.read(1, masked=True)
|
||||||
|
valid = band.compressed()
|
||||||
|
if valid.size == 0:
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_NO_VALID_DATA",
|
||||||
|
message="The MDK coverage contains no valid depth cells in this selection.",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"crs": str(source.crs) if source.crs else None,
|
||||||
|
"width": int(source.width),
|
||||||
|
"height": int(source.height),
|
||||||
|
"nodata": None if source.nodata is None else float(source.nodata),
|
||||||
|
"valid_cell_count": int(valid.size),
|
||||||
|
"minimum_value": float(np.min(valid)),
|
||||||
|
"maximum_value": float(np.max(valid)),
|
||||||
|
}
|
||||||
|
except AppError:
|
||||||
|
raise
|
||||||
|
except Exception as exc: # rasterio raises many distinct errors for corrupt input
|
||||||
|
raise AppError(
|
||||||
|
code="MDK_BATHYMETRY_INVALID_RESPONSE",
|
||||||
|
message="The MDK coverage could not be opened as a valid GeoTIFF.",
|
||||||
|
details={"reason": str(exc)},
|
||||||
|
status_code=502,
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
candidate = (
|
||||||
|
db.query(Dataset)
|
||||||
|
.filter(
|
||||||
|
Dataset.project_id == project_id,
|
||||||
|
Dataset.name == filename,
|
||||||
|
Dataset.source_name == MdkBathymetryAcquisitionService.PROVIDER,
|
||||||
|
Dataset.status == "ready",
|
||||||
|
)
|
||||||
|
.order_by(Dataset.imported_at.desc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
|
||||||
@@ -6,11 +6,12 @@ import ssl
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from xml.etree import ElementTree
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.schemas.bathymetry import BathymetrySourceProbeRead
|
from app.schemas.bathymetry import BathymetrySourceProbeRead
|
||||||
|
from app.services.outbound_request_guard import guarded_opener
|
||||||
|
|
||||||
|
|
||||||
class MdkBathymetryProbeService:
|
class MdkBathymetryProbeService:
|
||||||
@@ -43,7 +44,7 @@ class MdkBathymetryProbeService:
|
|||||||
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe",
|
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
with (opener or urlopen)(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response:
|
with (opener or guarded_opener(capabilities_url))(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response:
|
||||||
limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024
|
limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024
|
||||||
content = response.read(limit + 1)
|
content = response.read(limit + 1)
|
||||||
if len(content) > limit:
|
if len(content) > limit:
|
||||||
|
|||||||
@@ -24,11 +24,18 @@ class ModelAssetCatalogService:
|
|||||||
if not model_directory.exists() or not model_directory.is_dir():
|
if not model_directory.exists() or not model_directory.is_dir():
|
||||||
return ModelAssetListResponse(items=[], total=0, model_directory=str(model_directory))
|
return ModelAssetListResponse(items=[], total=0, model_directory=str(model_directory))
|
||||||
|
|
||||||
items = [
|
candidate_paths = [
|
||||||
ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path)
|
path
|
||||||
for path in sorted(model_directory.iterdir(), key=lambda item: item.name.lower())
|
for path in sorted(model_directory.iterdir(), key=lambda item: item.name.lower())
|
||||||
if path.is_file() and path.suffix.lower() in ModelAssetCatalogService.SUPPORTED_SUFFIXES
|
if path.is_file() and path.suffix.lower() in ModelAssetCatalogService.SUPPORTED_SUFFIXES
|
||||||
]
|
]
|
||||||
|
if active_model_path is not None:
|
||||||
|
candidate_paths = [path for path in candidate_paths if path.resolve() == active_model_path]
|
||||||
|
|
||||||
|
items = [
|
||||||
|
ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path)
|
||||||
|
for path in candidate_paths
|
||||||
|
]
|
||||||
return ModelAssetListResponse(items=items, total=len(items), model_directory=str(model_directory))
|
return ModelAssetListResponse(items=items, total=len(items), model_directory=str(model_directory))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -72,8 +79,16 @@ class ModelAssetCatalogService:
|
|||||||
size_bytes=path.stat().st_size,
|
size_bytes=path.stat().st_size,
|
||||||
sha256=ModelAssetCatalogService._sha256(path),
|
sha256=ModelAssetCatalogService._sha256(path),
|
||||||
active=active_model_path == resolved_path,
|
active=active_model_path == resolved_path,
|
||||||
status="available",
|
runtime_available=True,
|
||||||
limitation_message="Local runtime model asset. GeoIntel will not download or mutate model weights.",
|
runtime_status="active" if active_model_path == resolved_path else "available",
|
||||||
|
governed_validation_status="not_verified_by_catalog",
|
||||||
|
promotion_status="not_verified_by_catalog",
|
||||||
|
status="runtime_active" if active_model_path == resolved_path else "runtime_available",
|
||||||
|
limitation_message=(
|
||||||
|
"Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion."
|
||||||
|
if active_model_path == resolved_path
|
||||||
|
else "Local runtime model asset. Governed validation and promotion are not established by this catalog."
|
||||||
|
),
|
||||||
will_download_models=False,
|
will_download_models=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,13 @@ from typing import Type
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.schemas.detection import DetectionModelCapability
|
from app.schemas.detection import DetectionModelCapability
|
||||||
|
from app.services.segmentation_adapter import (
|
||||||
|
SamSegmentationAdapter,
|
||||||
|
YoloSegmentationAdapter,
|
||||||
|
)
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
|
from app.core.errors import AppError
|
||||||
|
|
||||||
|
|
||||||
class ModelRegistryService:
|
class ModelRegistryService:
|
||||||
@@ -14,10 +20,16 @@ class ModelRegistryService:
|
|||||||
settings: Settings | None = None,
|
settings: Settings | None = None,
|
||||||
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
|
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
|
||||||
task_type: str = "object_detection",
|
task_type: str = "object_detection",
|
||||||
|
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
|
||||||
|
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
|
||||||
) -> list[DetectionModelCapability]:
|
) -> list[DetectionModelCapability]:
|
||||||
resolved_settings = settings or get_settings()
|
resolved_settings = settings or get_settings()
|
||||||
if task_type == "segmentation":
|
if task_type == "segmentation":
|
||||||
return ModelRegistryService.list_segmentation_model_capabilities()
|
return ModelRegistryService.list_segmentation_model_capabilities(
|
||||||
|
settings=resolved_settings,
|
||||||
|
yolo_seg_adapter_class=yolo_seg_adapter_class,
|
||||||
|
sam_adapter_class=sam_adapter_class,
|
||||||
|
)
|
||||||
if task_type != "object_detection":
|
if task_type != "object_detection":
|
||||||
return []
|
return []
|
||||||
return [
|
return [
|
||||||
@@ -32,7 +44,9 @@ class ModelRegistryService:
|
|||||||
limitation_message="YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed.",
|
limitation_message="YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed.",
|
||||||
version=None,
|
version=None,
|
||||||
),
|
),
|
||||||
ModelRegistryService._configured_yolo_capability(resolved_settings, yolo_adapter_class),
|
ModelRegistryService._configured_yolo_capability(
|
||||||
|
resolved_settings, yolo_adapter_class
|
||||||
|
),
|
||||||
DetectionModelCapability(
|
DetectionModelCapability(
|
||||||
model_id="manual-fixture-detector",
|
model_id="manual-fixture-detector",
|
||||||
display_name="Manual fixture detector",
|
display_name="Manual fixture detector",
|
||||||
@@ -52,15 +66,28 @@ class ModelRegistryService:
|
|||||||
settings: Settings | None = None,
|
settings: Settings | None = None,
|
||||||
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
|
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
|
||||||
task_type: str = "object_detection",
|
task_type: str = "object_detection",
|
||||||
|
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
|
||||||
|
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
|
||||||
) -> DetectionModelCapability | None:
|
) -> DetectionModelCapability | None:
|
||||||
normalized = model_id.strip()
|
normalized = model_id.strip()
|
||||||
for model in ModelRegistryService.list_model_capabilities(settings=settings, yolo_adapter_class=yolo_adapter_class, task_type=task_type):
|
for model in ModelRegistryService.list_model_capabilities(
|
||||||
|
settings=settings,
|
||||||
|
yolo_adapter_class=yolo_adapter_class,
|
||||||
|
task_type=task_type,
|
||||||
|
yolo_seg_adapter_class=yolo_seg_adapter_class,
|
||||||
|
sam_adapter_class=sam_adapter_class,
|
||||||
|
):
|
||||||
if model.model_id == normalized:
|
if model.model_id == normalized:
|
||||||
return model
|
return model
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def list_segmentation_model_capabilities() -> list[DetectionModelCapability]:
|
def list_segmentation_model_capabilities(
|
||||||
|
settings: Settings | None = None,
|
||||||
|
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
|
||||||
|
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
|
||||||
|
) -> list[DetectionModelCapability]:
|
||||||
|
resolved_settings = settings or get_settings()
|
||||||
return [
|
return [
|
||||||
DetectionModelCapability(
|
DetectionModelCapability(
|
||||||
model_id="segmentation-placeholder",
|
model_id="segmentation-placeholder",
|
||||||
@@ -70,7 +97,7 @@ class ModelRegistryService:
|
|||||||
supported_classes=["building", "vegetation", "water", "landuse"],
|
supported_classes=["building", "vegetation", "water", "landuse"],
|
||||||
configured=False,
|
configured=False,
|
||||||
status="not_configured",
|
status="not_configured",
|
||||||
limitation_message="Segmentation inference is not configured in Sprint 9; no SAM/YOLO-seg model is downloaded or executed.",
|
limitation_message="Segmentation inference is not configured for this placeholder; no model is downloaded or executed.",
|
||||||
version=None,
|
version=None,
|
||||||
),
|
),
|
||||||
DetectionModelCapability(
|
DetectionModelCapability(
|
||||||
@@ -84,29 +111,127 @@ class ModelRegistryService:
|
|||||||
limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.",
|
limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.",
|
||||||
version="fixture-v1",
|
version="fixture-v1",
|
||||||
),
|
),
|
||||||
DetectionModelCapability(
|
ModelRegistryService._configured_yolo_seg_capability(
|
||||||
model_id="yolo-seg-configured",
|
resolved_settings, yolo_seg_adapter_class
|
||||||
display_name="Configured YOLO segmentation",
|
),
|
||||||
|
ModelRegistryService._configured_sam_capability(
|
||||||
|
resolved_settings, sam_adapter_class
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _configured_yolo_seg_capability(
|
||||||
|
settings: Settings,
|
||||||
|
adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
|
||||||
|
) -> DetectionModelCapability:
|
||||||
|
configured = False
|
||||||
|
status = "not_configured"
|
||||||
|
limitation = (
|
||||||
|
"YOLO segmentation is disabled. Set YOLO_SEG_ENABLED=true and YOLO_SEG_MODEL_PATH to a local "
|
||||||
|
"segmentation model file to enable inference. GeoIntel never downloads model weights automatically."
|
||||||
|
)
|
||||||
|
model_path = (
|
||||||
|
Path(settings.yolo_seg_model_path).expanduser()
|
||||||
|
if settings.yolo_seg_model_path
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
|
if settings.yolo_seg_enabled:
|
||||||
|
if not adapter_class.dependencies_available():
|
||||||
|
status = "dependency_unavailable"
|
||||||
|
limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
|
||||||
|
elif model_path is None:
|
||||||
|
limitation = "YOLO_SEG_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically."
|
||||||
|
elif not model_path.exists() or not model_path.is_file():
|
||||||
|
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=settings.yolo_seg_model_id,
|
||||||
|
task_type="segmentation",
|
||||||
|
expected_model_version=settings.yolo_seg_model_version,
|
||||||
|
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
status = "contract_incomplete"
|
||||||
|
limitation = (
|
||||||
|
"Configured YOLO segmentation weights are not runnable until their immutable "
|
||||||
|
f"runtime provenance sidecar validates: {exc.message}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
configured = True
|
||||||
|
status = "configured"
|
||||||
|
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
|
||||||
|
|
||||||
|
return DetectionModelCapability(
|
||||||
|
model_id=settings.yolo_seg_model_id,
|
||||||
|
display_name=settings.yolo_seg_model_display_name,
|
||||||
framework="ultralytics/pytorch",
|
framework="ultralytics/pytorch",
|
||||||
task_type="segmentation",
|
task_type="segmentation",
|
||||||
supported_classes=["building", "vegetation", "water", "landuse"],
|
supported_classes=["building", "vegetation", "water", "landuse"],
|
||||||
configured=False,
|
configured=configured,
|
||||||
status="not_configured",
|
status=status,
|
||||||
limitation_message="YOLO-seg is not configured in Sprint 9. GeoIntel will not download segmentation model weights automatically.",
|
limitation_message=limitation,
|
||||||
version=None,
|
version=settings.yolo_seg_model_version,
|
||||||
),
|
)
|
||||||
DetectionModelCapability(
|
|
||||||
model_id="sam-configured",
|
@staticmethod
|
||||||
display_name="Configured SAM segmentation",
|
def _configured_sam_capability(
|
||||||
framework="sam",
|
settings: Settings,
|
||||||
|
adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
|
||||||
|
) -> DetectionModelCapability:
|
||||||
|
configured = False
|
||||||
|
status = "not_configured"
|
||||||
|
limitation = (
|
||||||
|
"SAM is disabled. Set SAM_ENABLED=true and SAM_MODEL_PATH to a local SAM-compatible model file to "
|
||||||
|
"enable class-agnostic segmentation. GeoIntel never downloads model weights automatically."
|
||||||
|
)
|
||||||
|
model_path = (
|
||||||
|
Path(settings.sam_model_path).expanduser()
|
||||||
|
if settings.sam_model_path
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
|
if settings.sam_enabled:
|
||||||
|
if not adapter_class.dependencies_available():
|
||||||
|
status = "dependency_unavailable"
|
||||||
|
limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
|
||||||
|
elif model_path is None:
|
||||||
|
limitation = "SAM_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically."
|
||||||
|
elif not model_path.exists() or not model_path.is_file():
|
||||||
|
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=settings.sam_model_id,
|
||||||
task_type="segmentation",
|
task_type="segmentation",
|
||||||
supported_classes=["building", "vegetation", "water", "landuse"],
|
expected_model_version=settings.sam_model_version,
|
||||||
configured=False,
|
allowed_frameworks=("ultralytics/sam", "sam", "ultralytics", "pytorch"),
|
||||||
status="not_configured",
|
)
|
||||||
limitation_message="SAM is not configured in Sprint 9 and is not installed as a backend dependency.",
|
except AppError as exc:
|
||||||
version=None,
|
status = "contract_incomplete"
|
||||||
),
|
limitation = (
|
||||||
]
|
"Configured SAM weights are not runnable until their immutable runtime provenance "
|
||||||
|
f"sidecar validates: {exc.message}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
configured = True
|
||||||
|
status = "configured"
|
||||||
|
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
|
||||||
|
|
||||||
|
return DetectionModelCapability(
|
||||||
|
model_id=settings.sam_model_id,
|
||||||
|
display_name=settings.sam_model_display_name,
|
||||||
|
framework="ultralytics/sam",
|
||||||
|
task_type="segmentation",
|
||||||
|
supported_classes=["segment"],
|
||||||
|
configured=configured,
|
||||||
|
status=status,
|
||||||
|
limitation_message=limitation,
|
||||||
|
version=settings.sam_model_version,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _configured_yolo_capability(
|
def _configured_yolo_capability(
|
||||||
@@ -116,7 +241,11 @@ class ModelRegistryService:
|
|||||||
configured = False
|
configured = False
|
||||||
status = "not_configured"
|
status = "not_configured"
|
||||||
limitation = "YOLO is disabled. Set YOLO_ENABLED=true and YOLO_MODEL_PATH to a local model file to enable inference."
|
limitation = "YOLO is disabled. Set YOLO_ENABLED=true and YOLO_MODEL_PATH to a local model file to enable inference."
|
||||||
model_path = Path(settings.yolo_model_path).expanduser() if settings.yolo_model_path else None
|
model_path = (
|
||||||
|
Path(settings.yolo_model_path).expanduser()
|
||||||
|
if settings.yolo_model_path
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
if settings.yolo_enabled:
|
if settings.yolo_enabled:
|
||||||
if not yolo_adapter_class.dependencies_available():
|
if not yolo_adapter_class.dependencies_available():
|
||||||
@@ -126,19 +255,49 @@ class ModelRegistryService:
|
|||||||
limitation = "YOLO_MODEL_PATH is not set. GeoIntel will not download model weights automatically."
|
limitation = "YOLO_MODEL_PATH is not set. GeoIntel will not download model weights automatically."
|
||||||
elif not model_path.exists() or not model_path.is_file():
|
elif not model_path.exists() or not model_path.is_file():
|
||||||
limitation = "YOLO_MODEL_PATH does not point to an existing local model file. GeoIntel will not download model weights automatically."
|
limitation = "YOLO_MODEL_PATH does not point to an existing local model file. GeoIntel will not download model weights automatically."
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
validate_runtime = getattr(yolo_adapter_class, "validate_runtime", None)
|
||||||
|
if validate_runtime is not None:
|
||||||
|
yolo_adapter_class(settings).validate_runtime()
|
||||||
|
except AppError as exc:
|
||||||
|
status = "accelerator_unavailable"
|
||||||
|
limitation = exc.message
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=settings.yolo_model_id,
|
||||||
|
task_type="object_detection",
|
||||||
|
expected_model_version=settings.yolo_model_version,
|
||||||
|
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
status = "contract_incomplete"
|
||||||
|
limitation = (
|
||||||
|
"Configured YOLO weights are not runnable until their immutable runtime provenance "
|
||||||
|
f"sidecar validates: {exc.message}"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
configured = True
|
configured = True
|
||||||
status = "configured"
|
status = "configured"
|
||||||
limitation = "Configured for local YOLO inference over an existing raster tile manifest."
|
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
|
||||||
|
|
||||||
return DetectionModelCapability(
|
return DetectionModelCapability(
|
||||||
model_id=settings.yolo_model_id,
|
model_id=settings.yolo_model_id,
|
||||||
display_name=settings.yolo_model_display_name,
|
display_name=settings.yolo_model_display_name,
|
||||||
framework="ultralytics/pytorch",
|
framework="ultralytics/pytorch",
|
||||||
task_type="object_detection",
|
task_type="object_detection",
|
||||||
supported_classes=["building", "road", "water", "landuse"],
|
supported_classes=[value.strip().lower() for value in settings.yolo_model_classes.split(",") if value.strip()],
|
||||||
configured=configured,
|
configured=configured,
|
||||||
status=status,
|
status=status,
|
||||||
limitation_message=limitation,
|
limitation_message=limitation,
|
||||||
version=settings.yolo_model_version,
|
version=settings.yolo_model_version,
|
||||||
|
training_scope=(
|
||||||
|
"Operator-managed local weights; the runtime has no nationally governed training-corpus evidence."
|
||||||
|
),
|
||||||
|
validation_scope="Mol and the Kempen operator evidence; no Belgian national validation matrix is bound.",
|
||||||
|
validated_regions=["flanders_mol_kempen"],
|
||||||
|
nationally_validated=False,
|
||||||
|
operator_review_required=True,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
"""Checksum-bound geographic validation scope for production model inference."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from shapely.geometry import shape
|
||||||
|
from shapely.geometry.base import BaseGeometry
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
|
||||||
|
|
||||||
|
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
|
||||||
|
|
||||||
|
class ModelValidationScopeService:
|
||||||
|
"""Load an immutable, model-bound AOI and prove that an input is covered."""
|
||||||
|
|
||||||
|
SCHEMA_VERSION = "geointel.model-validation-scope/v1"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def assert_area_covered(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
area_geometry: BaseGeometry,
|
||||||
|
manifest_path: str | None,
|
||||||
|
expected_manifest_sha256: str | None,
|
||||||
|
model_id: str,
|
||||||
|
model_path: str | None,
|
||||||
|
) -> dict[str, str]:
|
||||||
|
path = Path(manifest_path).expanduser() if manifest_path else None
|
||||||
|
expected_checksum = (expected_manifest_sha256 or "").strip().lower()
|
||||||
|
if path is None or not expected_checksum:
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_NOT_CONFIGURED",
|
||||||
|
"Configured YOLO inference requires a checksum-bound geographic validation-scope manifest.",
|
||||||
|
)
|
||||||
|
if not _SHA256.fullmatch(expected_checksum):
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_INVALID",
|
||||||
|
"The configured validation-scope checksum must be a lowercase SHA-256 digest.",
|
||||||
|
manifest_path=str(path),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
raw_manifest = path.read_bytes()
|
||||||
|
payload = json.loads(raw_manifest.decode("utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_INVALID",
|
||||||
|
"The configured validation-scope manifest is missing or unreadable.",
|
||||||
|
manifest_path=str(path),
|
||||||
|
error_type=type(exc).__name__,
|
||||||
|
)
|
||||||
|
observed_manifest_sha256 = sha256(raw_manifest).hexdigest()
|
||||||
|
if observed_manifest_sha256 != expected_checksum:
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_CHECKSUM_MISMATCH",
|
||||||
|
"The validation-scope manifest does not match its configured checksum.",
|
||||||
|
manifest_path=str(path),
|
||||||
|
expected=expected_checksum,
|
||||||
|
observed=observed_manifest_sha256,
|
||||||
|
)
|
||||||
|
if not isinstance(payload, dict) or payload.get("schema_version") != cls.SCHEMA_VERSION:
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_INVALID",
|
||||||
|
"The validation-scope manifest has an unsupported schema.",
|
||||||
|
manifest_path=str(path),
|
||||||
|
)
|
||||||
|
if payload.get("model_id") != model_id:
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_MODEL_MISMATCH",
|
||||||
|
"The validation scope is not bound to the selected model identity.",
|
||||||
|
expected_model_id=model_id,
|
||||||
|
observed_model_id=payload.get("model_id"),
|
||||||
|
)
|
||||||
|
configured_model_path = Path(model_path).expanduser() if model_path else None
|
||||||
|
if configured_model_path is None or not configured_model_path.is_file():
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_MODEL_UNAVAILABLE",
|
||||||
|
"The model bytes bound by the validation scope are unavailable.",
|
||||||
|
)
|
||||||
|
declared_model_sha256 = str(payload.get("model_sha256") or "").strip().lower()
|
||||||
|
observed_model_sha256 = cls._file_sha256(configured_model_path)
|
||||||
|
if not _SHA256.fullmatch(declared_model_sha256) or declared_model_sha256 != observed_model_sha256:
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_MODEL_MISMATCH",
|
||||||
|
"The validation scope is not bound to the exact selected model bytes.",
|
||||||
|
expected=declared_model_sha256 or None,
|
||||||
|
observed=observed_model_sha256,
|
||||||
|
)
|
||||||
|
if payload.get("crs") != "EPSG:4326":
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_INVALID",
|
||||||
|
"The validation-scope geometry must explicitly use EPSG:4326.",
|
||||||
|
observed_crs=payload.get("crs"),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
scope_geometry = shape(payload.get("geometry"))
|
||||||
|
except Exception as exc:
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_INVALID",
|
||||||
|
"The validation-scope geometry is not valid GeoJSON.",
|
||||||
|
error_type=type(exc).__name__,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
scope_geometry.is_empty
|
||||||
|
or not scope_geometry.is_valid
|
||||||
|
or scope_geometry.geom_type not in {"Polygon", "MultiPolygon"}
|
||||||
|
):
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_INVALID",
|
||||||
|
"The validation scope must be a non-empty valid Polygon or MultiPolygon.",
|
||||||
|
geometry_type=scope_geometry.geom_type,
|
||||||
|
)
|
||||||
|
if area_geometry.is_empty or not area_geometry.is_valid or not scope_geometry.covers(area_geometry):
|
||||||
|
cls._raise(
|
||||||
|
"DETECTION_VALIDATION_SCOPE_UNAVAILABLE",
|
||||||
|
"Configured YOLO inference is not validated for the complete Dataset area.",
|
||||||
|
scope_key=payload.get("scope_key"),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"scope_key": str(payload.get("scope_key") or "unspecified"),
|
||||||
|
"manifest_path": str(path.resolve()),
|
||||||
|
"manifest_sha256": observed_manifest_sha256,
|
||||||
|
"model_sha256": observed_model_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _file_sha256(path: Path) -> str:
|
||||||
|
digest = sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raise(code: str, message: str, **details: Any) -> None:
|
||||||
|
raise AppError(code=code, message=message, details=details, status_code=422)
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user