5 Commits
Author SHA1 Message Date
Jens ac977bcd4b Merge pull request 'hygiene: finalize GeoIntel public release' (#11) from chore/public-release-finalization-20260902 into main
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 26s
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 2m33s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 14m10s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 21s
2026-09-03 02:01:19 +00:00
NuklearRabbit 50dc6692a0 security: upgrade expat in release image
GeoIntel release gates / Production AI image, SBOM and container scan (pull_request) Successful in 15m25s
Managed validation / Managed repository validation (pull_request) Successful in 2m0s
GeoIntel release gates / Python and npm vulnerability policy (pull_request) Successful in 23s
GeoIntel release gates / Compile, test, contracts and builds (pull_request) Successful in 2m42s
GeoIntel release gates / Deploy exact gated revision to Unraid (pull_request) Skipped
2026-09-03 01:25:54 +02:00
NuklearRabbit 4d37fa0b18 security: update vulnerable browserslist dependency
Managed validation / Managed repository validation (pull_request) Successful in 1m47s
GeoIntel release gates / Compile, test, contracts and builds (pull_request) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (pull_request) Successful in 24s
GeoIntel release gates / Production AI image, SBOM and container scan (pull_request) Failing after 10m54s
GeoIntel release gates / Deploy exact gated revision to Unraid (pull_request) Skipped
2026-09-03 01:07:16 +02:00
NuklearRabbit 3969a35d7a hygiene: finalize GeoIntel public release
Managed validation / Managed repository validation (pull_request) Successful in 1m54s
GeoIntel release gates / Compile, test, contracts and builds (pull_request) Successful in 1m53s
GeoIntel release gates / Python and npm vulnerability policy (pull_request) Failing after 20s
GeoIntel release gates / Production AI image, SBOM and container scan (pull_request) Failing after 12m1s
GeoIntel release gates / Deploy exact gated revision to Unraid (pull_request) Skipped
2026-09-02 23:43:03 +02:00
Jens faeb58ef6d Initial public release
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
2026-08-31 21:56:53 +02:00
765 changed files with 105861 additions and 24278 deletions
+34
View File
@@ -1,19 +1,51 @@
# Build context exclusions for deploy/unraid/Dockerfile.all-in-one.
# Everything listed here is NOT sent to the Docker daemon.
# Keep this aggressive: the all-in-one image only needs
# backend/, frontend/, fixtures/, scripts/, deploy/ and VERSION.
.git .git
.gitea
.github
.venv .venv
venv venv
__pycache__ __pycache__
*.pyc *.pyc
.pytest_cache .pytest_cache
.mypy_cache
.ruff_cache
# Node dependencies at every level (root test harness + frontend).
node_modules
**/node_modules
frontend/node_modules frontend/node_modules
frontend/dist frontend/dist
frontend/*.tsbuildinfo frontend/*.tsbuildinfo
**/*.tsbuildinfo
frontend-src.tar.gz
backend/.pytest_cache backend/.pytest_cache
backend/**/*.pyc backend/**/*.pyc
backend/**/__pycache__ backend/**/__pycache__
# Accidental self-copy of the repository into itself.
# Without this the build context is duplicated and the build stalls.
/geointel
# Local work products, never needed inside the image.
.codex-input
test-results
playwright-report
artifacts
RELEASE_NOTES
adr
docs
checklists
*.log
# Runtime data and secrets: mounted at runtime, never baked in.
storage storage
postgres-data postgres-data
backups
datasets/raw datasets/raw
datasets/processed datasets/processed
datasets/cache datasets/cache
@@ -21,3 +53,5 @@ exports
models models
.env .env
.env.*
!.env.example
+91 -1
View File
@@ -4,9 +4,30 @@ GEOINTEL_API_PREFIX=/api/v1
DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1 DATABASE_URL=postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1
STORAGE_ROOT=./storage STORAGE_ROOT=./storage
MAX_UPLOAD_MB=500 MAX_UPLOAD_MB=500
GEOINTEL_MAX_IN_MEMORY_VECTOR_MB=64
CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202 CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202
# Optional single-operator access gate. Store only a PBKDF2-SHA256 hash and
# a unique 32+ character signing secret. Guest access is opt-in and should be
# enabled only on a dedicated demo installation with bounded public data.
GEOINTEL_AUTH_ENABLED=false
GEOINTEL_AUTH_REQUIRE_HTTPS=false
GEOINTEL_AUTH_USERNAME=
GEOINTEL_AUTH_PASSWORD_HASH=
GEOINTEL_AUTH_SESSION_SECRET=
GEOINTEL_AUTH_SESSION_TTL_SECONDS=43200
GEOINTEL_PUBLIC_BASE_URL=http://localhost:1202
GEOINTEL_AUTHENTIK_ISSUER=
GEOINTEL_AUTHENTIK_CLIENT_ID=
GEOINTEL_AUTHENTIK_CLIENT_SECRET=
GEOINTEL_AUTHENTIK_ALLOWED_EMAIL=
GEOINTEL_GUEST_ACCESS_ENABLED=false
GEOINTEL_GUEST_DISPLAY_NAME=Gast
GEOINTEL_GUEST_SESSION_TTL_SECONDS=7200
ORTHOPHOTO_ENABLED=true ORTHOPHOTO_ENABLED=true
ORTHOPHOTO_WMS_URL=https://geo.api.vlaanderen.be/OMWRGBMRVL/wms ORTHOPHOTO_WMS_URL=https://geo.api.vlaanderen.be/OMWRGBMRVL/wms
SPW_ORTHOPHOTO_WMS_URL=https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_LAST/MapServer/WMSServer
BRUSSELS_ORTHOPHOTO_WMS_URL=https://geoservices-grid.irisnet.be/geoserver/urbisgrid/ows
ORTHOPHOTO_WMS_LAYER=Ortho ORTHOPHOTO_WMS_LAYER=Ortho
ORTHOPHOTO_RESOLUTION_M=1.0 ORTHOPHOTO_RESOLUTION_M=1.0
ORTHOPHOTO_MIN_SIDE_M=128 ORTHOPHOTO_MIN_SIDE_M=128
@@ -28,6 +49,12 @@ GRB_CACHE_TTL_HOURS=24
OFFICIAL_VECTOR_ENABLED=true OFFICIAL_VECTOR_ENABLED=true
BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs
DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs
SPW_PICC_ENABLED=true
SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer
SPW_FLOOD_HAZARD_ENABLED=true
SPW_FLOOD_HAZARD_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer
URBIS_ENABLED=true
URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows
OFFICIAL_VECTOR_MIN_SIDE_M=10 OFFICIAL_VECTOR_MIN_SIDE_M=10
OFFICIAL_VECTOR_MAX_SIDE_M=20000 OFFICIAL_VECTOR_MAX_SIDE_M=20000
OFFICIAL_VECTOR_PAGE_SIZE=1000 OFFICIAL_VECTOR_PAGE_SIZE=1000
@@ -63,6 +90,7 @@ BATHYMETRY_PROFILES_ENABLED=true
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0 BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1 BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
BATHYMETRY_PROFILES_PAGE_SIZE=1000 BATHYMETRY_PROFILES_PAGE_SIZE=1000
BATHYMETRY_PROFILES_MAX_PAGES=200
BATHYMETRY_PROFILES_MAX_FEATURES=50000 BATHYMETRY_PROFILES_MAX_FEATURES=50000
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120 BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32 BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
@@ -70,6 +98,14 @@ MDK_BATHYMETRY_PROBE_ENABLED=true
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20 MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4 MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
# Bounded MDK acquisition stays fail-closed until the readiness probe reports
# "reachable" and an advertised coverage id is configured explicitly.
MDK_BATHYMETRY_ACQUISITION_ENABLED=false
MDK_BATHYMETRY_COVERAGE_ID=
MDK_BATHYMETRY_REQUEST_CRS=EPSG:4326
MDK_BATHYMETRY_MAX_BBOX_DEG2=0.25
MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS=120
MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB=160
THEMATIC_RASTER_ENABLED=true THEMATIC_RASTER_ENABLED=true
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
THEMATIC_RASTER_MIN_SIDE_M=100 THEMATIC_RASTER_MIN_SIDE_M=100
@@ -77,19 +113,62 @@ THEMATIC_RASTER_MAX_SIDE_M=60000
THEMATIC_RASTER_MAX_PIXELS=30000000 THEMATIC_RASTER_MAX_PIXELS=30000000
THEMATIC_RASTER_TIMEOUT_SECONDS=300 THEMATIC_RASTER_TIMEOUT_SECONDS=300
THEMATIC_RASTER_MAX_RESPONSE_MB=160 THEMATIC_RASTER_MAX_RESPONSE_MB=160
WALOUS_ENABLED=true
WALOUS_SOURCE_DIR=/app/storage/source-cache/walous
WALOUS_ANALYSIS_RESOLUTION_M=10
WALOUS_MAX_SIDE_M=60000
WALOUS_MAX_PIXELS=36000000
YOLO_ENABLED=false YOLO_ENABLED=false
YOLO_MODELS_DIR=/app/models YOLO_MODELS_DIR=/app/models
YOLO_MODEL_PATH= YOLO_MODEL_PATH=
YOLO_MODEL_ID=yolo-configured YOLO_MODEL_ID=yolo-configured
YOLO_MODEL_DISPLAY_NAME=Configured YOLO detector YOLO_MODEL_DISPLAY_NAME=Configured YOLO detector
YOLO_MODEL_VERSION= YOLO_MODEL_VERSION=
YOLO_MODEL_CLASSES=building
YOLO_ENFORCE_VALIDATION_SCOPE=false
# Required when scope enforcement is enabled. The manifest is bound to exact
# model bytes and contains the allowed EPSG:4326 validation geometry.
YOLO_VALIDATION_SCOPE_MANIFEST_PATH=
YOLO_VALIDATION_SCOPE_MANIFEST_SHA256=
# Deprecated display metadata; never used as an inference authorization gate.
YOLO_VALIDATED_AREA_NAMES=Mol,Kempen
YOLO_CONFIG_DIR=./storage/ultralytics YOLO_CONFIG_DIR=./storage/ultralytics
YOLO_DEVICE=cpu YOLO_DEVICE=cpu
YOLO_REQUIRE_CUDA=false
YOLO_IMAGE_SIZE=640 YOLO_IMAGE_SIZE=640
YOLO_MAX_TILES=100 YOLO_MAX_TILES=100
YOLO_MAX_DETECTIONS=1000 YOLO_MAX_DETECTIONS=1000
YOLO_DUPLICATE_IOU_THRESHOLD=0.5 YOLO_DUPLICATE_IOU_THRESHOLD=0.5
# Tiles per model.predict call. 1 leaves most of a modern GPU idle on a run of
# a hundred tiles; raise it until VRAM is the limit.
YOLO_BATCH_SIZE=1 YOLO_BATCH_SIZE=1
# Drop boxes truncated by an interior tile edge. Because tiles overlap, the
# neighbouring tile saw the same object whole, so the truncated half is a
# duplicate and a shape error at once. Boxes on the outer raster edge are kept.
YOLO_SUPPRESS_TILE_EDGE_DETECTIONS=true
# Intersection over the smaller box. scripts/evaluate_belgium_building_candidate.py
# freezes this during calibration (--containment-nms) before the protected test.
# Serving a promoted model at a different value means the runtime suppresses
# detections its gate counted, so set this to the value the candidate was gated at.
YOLO_CONTAINMENT_NMS_THRESHOLD=0.85
# Segmentation carries its own value: masks and boxes overlap differently,
# so one threshold need not fit both.
SEGMENTATION_CONTAINMENT_NMS_THRESHOLD=0.85
# Local segmentation models. GeoIntel never downloads model weights
# automatically; point these to existing local files to enable inference.
YOLO_SEG_ENABLED=false
YOLO_SEG_MODEL_PATH=
YOLO_SEG_MODEL_ID=yolo-seg-configured
YOLO_SEG_MODEL_DISPLAY_NAME=Configured YOLO segmentation
YOLO_SEG_MODEL_VERSION=
SAM_ENABLED=false
SAM_MODEL_PATH=
SAM_MODEL_ID=sam-configured
SAM_MODEL_DISPLAY_NAME=Configured SAM segmentation
SAM_MODEL_VERSION=
SEGMENTATION_MAX_MASKS_PER_TILE=300
SEGMENTATION_DUPLICATE_IOU_THRESHOLD=0.5
ENABLE_GRB_WFS=false ENABLE_GRB_WFS=false
GRB_WFS_URL= GRB_WFS_URL=
OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter
@@ -115,5 +194,16 @@ GEOINTEL_POSTGIS_DATA_PATH=./postgres-data
GEOINTEL_POSTGRES_DB=geointel GEOINTEL_POSTGRES_DB=geointel
GEOINTEL_POSTGRES_USER=geointel GEOINTEL_POSTGRES_USER=geointel
GEOINTEL_POSTGRES_PASSWORD=geointel GEOINTEL_POSTGRES_PASSWORD=geointel
GEOINTEL_CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202 GEOINTEL_CORS_ORIGINS=https://geointel.example.com,http://localhost:1202,http://127.0.0.1:1202
GEOINTEL_MAX_UPLOAD_MB=500 GEOINTEL_MAX_UPLOAD_MB=500
GEOINTEL_AOI_WORKER_ENABLED=false
GEOINTEL_AOI_WORKER_POLL_SECONDS=2
# Executes queued detection.run / segmentation.run jobs from POST
# /detection/run-async, so tiled GPU inference never blocks an HTTP request.
GEOINTEL_ANALYSIS_WORKER_ENABLED=false
GEOINTEL_ANALYSIS_WORKER_POLL_SECONDS=2
# Analysis consumes only artifacts under STORAGE_ROOT: a tile manifest path
# arrives in the request and a manifest entry may name an absolute tile
# path, so without this an API field is an unbounded filesystem reference.
# Provisioning workflows that stage tiles elsewhere before ingest can opt out.
GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS=false
+5
View File
@@ -1,4 +1,5 @@
*.sh text eol=lf *.sh text eol=lf
deploy/unraid/gosu-setpriv text eol=lf
*.py text eol=lf *.py text eol=lf
*.yml text eol=lf *.yml text eol=lf
*.yaml text eol=lf *.yaml text eol=lf
@@ -10,3 +11,7 @@ Dockerfile text eol=lf
*.ts text eol=lf *.ts text eol=lf
*.css text eol=lf *.css text eol=lf
*.json text eol=lf *.json text eol=lf
# Retained audit exports preserve source bytes, including line endings and
# tool-emitted whitespace.
artifacts/evidence/accuracy/** -text -whitespace
+89
View File
@@ -0,0 +1,89 @@
name: Managed validation
on:
pull_request:
workflow_dispatch:
inputs:
profile:
description: Allowlisted validation profile
required: true
default: full
type: choice
options: [test, lint, typecheck, build, security, full]
permissions:
contents: read
concurrency:
group: managed-validation-${{ gitea.repository }}-${{ gitea.ref }}
cancel-in-progress: true
jobs:
full:
# Gitea Actions does not consistently evaluate the GitHub-style `||`
# expression for pull-request runs without workflow inputs.
name: Managed repository validation
# Public fork code must never execute automatically on the private runner.
if: ${{ gitea.event_name != 'pull_request' || gitea.event.pull_request.head.repo.full_name == gitea.repository }}
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.11"
cache: pip
cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: "20"
cache: npm
cache-dependency-path: frontend/package-lock.json
- name: Validate the requested profile against the real projects
shell: bash
env:
REQUESTED_PROFILE: ${{ inputs.profile }}
run: |
set -euo pipefail
profile="${REQUESTED_PROFILE:-full}"
case "${profile}" in
test|lint|typecheck|build|security|full) ;;
*) echo "Profile is not allowlisted" >&2; exit 2 ;;
esac
git diff --check
if git grep -nE '^(<<<<<<< |=======$|>>>>>>> )' -- . ':!*.lock' ':!*.patch'; then
echo "Unresolved merge markers detected" >&2
exit 1
fi
python scripts/verify_repository_layout.py
python -m pip install --disable-pip-version-check --require-hashes -r backend/requirements-ci.lock
python -m pip install --disable-pip-version-check --no-deps -e backend
(cd frontend && npm ci)
case "${profile}" in
test)
(cd backend && python -m pytest -W error::DeprecationWarning)
(cd frontend && npm run test:unit)
;;
lint)
python -m ruff check backend scripts tests
(cd frontend && npm run lint --if-present)
;;
typecheck)
(cd frontend && npm run typecheck)
;;
build)
python -m compileall backend/app
(cd frontend && npm run build)
;;
security)
python -m pip install --disable-pip-version-check pip-audit==2.10.1
bash scripts/audit_python_dependencies.sh
(cd frontend && npm audit --audit-level=high)
;;
full)
PYTHON_BIN=python bash scripts/run_readiness_check.sh
;;
esac
+76 -22
View File
@@ -1,9 +1,8 @@
name: GeoIntel release gates name: GeoIntel release gates
on: on:
push:
branches: [main, develop, "codex/**", "build/**"]
pull_request: pull_request:
push:
branches: [main, develop] branches: [main, develop]
workflow_dispatch: workflow_dispatch:
@@ -12,21 +11,32 @@ permissions:
concurrency: concurrency:
group: geointel-release-${{ gitea.ref }} group: geointel-release-${{ gitea.ref }}
cancel-in-progress: true # A cancelled HTTP caller does not terminate the allowlisted controller
# process that already owns the production lock. Queue a newer revision
# instead of orphaning an in-flight backup or deploy.
cancel-in-progress: false
jobs: jobs:
quality: quality:
name: Compile, test, contracts and builds name: Compile, test, contracts and builds
# Public fork code must never execute automatically on the private runner.
if: ${{ gitea.event_name != 'pull_request' || gitea.event.pull_request.head.repo.full_name == gitea.repository }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 45 timeout-minutes: 60
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@v5 - name: Secret scan
run: >-
docker run --rm
--volume "$PWD:/repo:ro"
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
filesystem /repo --only-verified --no-update
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -53,7 +63,9 @@ jobs:
docker compose config > artifacts/docker-compose.resolved.yml docker compose config > artifacts/docker-compose.resolved.yml
- name: Publish quality evidence - name: Publish quality evidence
if: always() if: always()
uses: actions/upload-artifact@v4 # Gitea Actions currently exposes the GHES-compatible artifact API;
# upload-artifact v4 deliberately refuses that API.
uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
with: with:
name: quality-evidence name: quality-evidence
path: | path: |
@@ -64,16 +76,17 @@ jobs:
dependency-audit: dependency-audit:
name: Python and npm vulnerability policy name: Python and npm vulnerability policy
if: ${{ gitea.event_name != 'pull_request' || gitea.event.pull_request.head.repo.full_name == gitea.repository }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@v5 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -90,7 +103,7 @@ jobs:
npm audit --audit-level=high --json > ../artifacts/npm-audit.json npm audit --audit-level=high --json > ../artifacts/npm-audit.json
- name: Publish dependency evidence - name: Publish dependency evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
with: with:
name: dependency-audits name: dependency-audits
path: | path: |
@@ -101,41 +114,82 @@ jobs:
retention-days: 30 retention-days: 30
container: container:
name: GIS image, SBOM and container scan name: Production AI image, SBOM and container scan
if: ${{ gitea.event_name != 'pull_request' || gitea.event.pull_request.head.repo.full_name == gitea.repository }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 120
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Build non-AI release image - name: Build production AI release image
env: env:
RELEASE_SHA: ${{ gitea.sha }} RELEASE_SHA: ${{ gitea.sha }}
run: | run: |
mkdir -p artifacts mkdir -p artifacts
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
docker build \ docker build \
-f deploy/unraid/Dockerfile.all-in-one \ -f deploy/unraid/Dockerfile.all-in-one \
--build-arg GEOINTEL_INSTALL_AI=false \ --build-arg GEOINTEL_INSTALL_AI=true \
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \ --build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \ --build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
-t "geointel-ci:$RELEASE_SHA-gis" \ --build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
-t "geointel-ci:$RELEASE_SHA-ai" \
. .
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
- name: Generate SPDX SBOM - name: Generate SPDX SBOM
env: env:
RELEASE_SHA: ${{ gitea.sha }} RELEASE_SHA: ${{ gitea.sha }}
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis" GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
SYFT_PARALLELISM: "1"
run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
- name: Enforce container vulnerability policy - name: Enforce container vulnerability policy
env: env:
RELEASE_SHA: ${{ gitea.sha }} RELEASE_SHA: ${{ gitea.sha }}
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis" GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/scan_container_image.sh "$IMAGE_ID"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
- name: Remove temporary image archive
if: always()
run: >-
rm -f -- artifacts/geointel-image.tar
artifacts/geointel-image.tar.image-id
artifacts/geointel-image.tar.partial.*
- name: Publish container evidence - name: Publish container evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20
with: with:
name: container-evidence name: container-evidence
path: | path: |
artifacts/image-inspect.json artifacts/image-inspect.json
artifacts/image-id.txt
artifacts/geointel-sbom.spdx.json artifacts/geointel-sbom.spdx.json
artifacts/geointel-container-vulnerabilities.json artifacts/geointel-container-vulnerabilities.json
if-no-files-found: warn if-no-files-found: warn
retention-days: 30 retention-days: 30
deploy:
name: Deploy exact gated revision to Unraid
needs: [quality, dependency-audit, container]
if: ${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' }}
runs-on: unraid-deploy
# The first byte-complete storage snapshot can exceed 100 GiB. Keep the
# gated caller attached for the full conservative backup/build window;
# the controller and deploy script still serialize every mutation.
timeout-minutes: 720
steps:
- name: Deploy only after every release gate is green
run: |
set -euo pipefail
docker exec gitea-deploy-control \
/opt/gitea-deploy/deploy.py deploy \
"${{ gitea.repository }}" "${{ gitea.sha }}"
+47 -18
View File
@@ -20,13 +20,19 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 45 timeout-minutes: 45
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@v5 - name: Secret scan
run: >-
docker run --rm
--volume "$PWD:/repo:ro"
trufflesecurity/trufflehog:3.79.0@sha256:7104dbb84d1ad2f5f6fa1134e92c6aa6f701f0a4ac2efd5a4c5c96225d899fe3
filesystem /repo --only-verified --no-update
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -53,7 +59,7 @@ jobs:
docker compose config > artifacts/docker-compose.resolved.yml docker compose config > artifacts/docker-compose.resolved.yml
- name: Publish quality evidence - name: Publish quality evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: quality-evidence name: quality-evidence
path: | path: |
@@ -67,13 +73,13 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@v5 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with: with:
python-version: "3.11" python-version: "3.11"
cache: pip cache: pip
cache-dependency-path: backend/requirements-ci.lock cache-dependency-path: backend/requirements-ci.lock
- uses: actions/setup-node@v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with: with:
node-version: "20" node-version: "20"
cache: npm cache: npm
@@ -90,7 +96,7 @@ jobs:
npm audit --audit-level=high --json > ../artifacts/npm-audit.json npm audit --audit-level=high --json > ../artifacts/npm-audit.json
- name: Publish dependency evidence - name: Publish dependency evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: dependency-audits name: dependency-audits
path: | path: |
@@ -101,40 +107,63 @@ jobs:
retention-days: 30 retention-days: 30
container: container:
name: GIS image, SBOM and container scan name: Production AI image, SBOM and container scan
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 60 timeout-minutes: 120
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Build non-AI release image - name: Build production AI release image
env: env:
RELEASE_SHA: ${{ github.sha }} RELEASE_SHA: ${{ github.sha }}
run: | run: |
mkdir -p artifacts mkdir -p artifacts
BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
APP_VERSION="$(tr -d '[:space:]' < VERSION)"
docker build \ docker build \
-f deploy/unraid/Dockerfile.all-in-one \ -f deploy/unraid/Dockerfile.all-in-one \
--build-arg GEOINTEL_INSTALL_AI=false \ --build-arg GEOINTEL_INSTALL_AI=true \
--build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \ --build-arg GEOINTEL_BUILD_SHA="$RELEASE_SHA" \
--build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \ --build-arg GEOINTEL_BUILD_TIME="$BUILD_TIME" \
-t "geointel-ci:$RELEASE_SHA-gis" \ --build-arg GEOINTEL_APP_VERSION="$APP_VERSION" \
-t "geointel-ci:$RELEASE_SHA-ai" \
. .
docker image inspect "geointel-ci:$RELEASE_SHA-gis" > artifacts/image-inspect.json IMAGE_ID="$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")"
printf '%s\n' "$IMAGE_ID" > artifacts/image-id.txt
docker image inspect "$IMAGE_ID" > artifacts/image-inspect.json
- name: Generate SPDX SBOM - name: Generate SPDX SBOM
env: env:
RELEASE_SHA: ${{ github.sha }} RELEASE_SHA: ${{ github.sha }}
run: bash scripts/generate_container_sbom.sh "geointel-ci:$RELEASE_SHA-gis" GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
SYFT_PARALLELISM: "1"
run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/generate_container_sbom.sh "$IMAGE_ID"
- name: Enforce container vulnerability policy - name: Enforce container vulnerability policy
env: env:
RELEASE_SHA: ${{ github.sha }} RELEASE_SHA: ${{ github.sha }}
run: bash scripts/scan_container_image.sh "geointel-ci:$RELEASE_SHA-gis" GEOINTEL_IMAGE_ARCHIVE: artifacts/geointel-image.tar
GEOINTEL_KEEP_IMAGE_ARCHIVE: "true"
run: |
IMAGE_ID="$(cat artifacts/image-id.txt)"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
bash scripts/scan_container_image.sh "$IMAGE_ID"
test "$(docker image inspect --format '{{.Id}}' "geointel-ci:$RELEASE_SHA-ai")" = "$IMAGE_ID"
- name: Remove temporary image archive
if: always()
run: >-
rm -f -- artifacts/geointel-image.tar
artifacts/geointel-image.tar.image-id
artifacts/geointel-image.tar.partial.*
- name: Publish container evidence - name: Publish container evidence
if: always() if: always()
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with: with:
name: container-evidence name: container-evidence
path: | path: |
artifacts/image-inspect.json artifacts/image-inspect.json
artifacts/image-id.txt
artifacts/geointel-sbom.spdx.json artifacts/geointel-sbom.spdx.json
artifacts/geointel-container-vulnerabilities.json artifacts/geointel-container-vulnerabilities.json
if-no-files-found: warn if-no-files-found: warn
+13 -1
View File
@@ -4,6 +4,8 @@ __pycache__/
.venv/ .venv/
venv/ venv/
.env .env
.env.*
!.env.example
*.egg-info/ *.egg-info/
.pytest_cache/ .pytest_cache/
.ruff_cache/ .ruff_cache/
@@ -14,7 +16,10 @@ dist/
build/ build/
*.tsbuildinfo *.tsbuildinfo
# Large local data # Local source-transfer archives; preserve them outside version control.
/frontend-src.tar.gz
# Generated evidence, model output and operational snapshots stay outside Git.
/artifacts/ /artifacts/
/.cache/ /.cache/
/datasets/raw/* /datasets/raw/*
@@ -54,3 +59,10 @@ build/
.DS_Store .DS_Store
.vscode/ .vscode/
.idea/ .idea/
# Local investigation scratch is never repository input.
/.codex-input/
/.codex-artifacts/
/.playwright-mcp/
/.mcp.json
/data/
+13
View File
@@ -0,0 +1,13 @@
[extend]
useDefault = true
[[allowlists]]
description = "Public registry identifiers that resemble generic API keys"
regexTarget = "match"
regexes = [
'''key="population_density_2019"''',
'''source_key="inbo_bwk_natura2000"''',
'''product_key": "population_density_2019"''',
'''key="spw_flood_hazard_2021"''',
'''key="phase2-mutation"''',
]
+22 -2897
View File
File diff suppressed because it is too large Load Diff
+12
View File
@@ -0,0 +1,12 @@
# Contributing
Keep GeoIntel changes reproducible, privacy-preserving and free from generated workspace state.
- Branch from the protected default branch.
- Use synthetic or explicitly redistributable imagery, coordinates and datasets in tests and documentation.
- Do not commit `.codex-input` contents, SQLite/WAL/SHM files, archives, contact sheets, model outputs, private maps, credentials or local deployment data.
- Document the origin, license and intended use of every external dataset, model and media asset.
- Add tests for behavioural changes and run the documented backend, frontend, geospatial and managed validation gates.
- Explain data-model, coordinate-system, model, privacy and deployment impact in the pull request.
Report security issues according to `SECURITY.md`.
+201
View File
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2026 Jens / ITWorx.tech
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+225 -232
View File
@@ -1,292 +1,285 @@
# GeoIntel Belgium and the Belgian North Sea # GeoIntel
GeoIntel is a map-first GeoAI Workbench for Belgium and the Belgian North Sea. <p align="center">
It combines governed official-source coverage, raster/vector processing, <img src="frontend/public/geointel-icon-180.png" width="92" alt="GeoIntel logo">
historical comparison, computer vision, QA/QC and geospatial exports. </p>
Mol and the Kempen remain deep regression and model-validation references. The <p align="center"><strong>Evidence-first GeoAI voor België en de Belgische Noordzee.</strong></p>
release scope is all of Belgium plus legally labelled Belgian maritime zones;
source coverage remains explicit per theme and jurisdiction.
GeoIntel is not a generic dashboard or chatbot. The core product is: GeoIntel is een kaartgerichte GeoAI-workbench waarin officiële databronnen,
ruimtelijke analyse, PyTorch-computervisie, QA/QC en export samenkomen. Het
platform bewaart niet alleen een resultaat, maar ook de bron, ruimtelijke
context, modelversie en het bewijs waarmee dat resultaat gecontroleerd kan
worden.
> data → processing → geospatial output → QA/QC → export De release ondersteunt heel België en de juridisch onderscheiden Belgische
Noordzeezones. Regionale bronnen blijven herkenbaar: GRB, PICC, UrbIS en
maritieme datasets worden niet voorgesteld alsof ze dezelfde semantiek,
resolutie of actualiteit hebben.
## Current milestone > **Data → ruimtelijke verwerking → AI-resultaat → kwaliteitsbewijs → export**
**v1.0.0-rc.1 - Belgium/North Sea release candidate** ![GeoIntel landingspagina](docs/assets/portfolio/geointel-landing-hero.png)
The canonical release controls are: ## Waarom GeoIntel?
- `docs/00-start/START_HERE.md` Veel geoportalen tonen lagen. GeoIntel ondersteunt een volledige,
- `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md` controleerbare analyseketen:
- `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`
- `docs/RELEASE_RUNBOOK.md`
- `docs/KNOWN_LIMITATIONS.md`
- `docs/DEFINITION_OF_DONE.md`
Older milestone and sprint handoff files remain historical evidence. They do - selecteer een officiële grens of teken een eigen Area of Interest;
not override the active national/maritime scope freeze or RC roadmap. - combineer vector-, raster- en historische bronnen met expliciete dekking;
- voer GIS- en CUDA-versnelde beeldanalyse uit;
- vergelijk kandidaatresultaten met referentiedata;
- inspecteer false positives, false negatives, overlap en provenance;
- exporteer alleen wat als reproduceerbaar resultaat is vastgelegd.
## Core V1 vertical slice Mol en de Kempen blijven de gouden regressiegebieden. De productgrens is heel
België plus de juridisch benoemde Belgische maritieme zones.
The first implementation target is: ## Product in beeld
1. Project + Area creation. De onderstaande screenshots tonen de huidige applicatie met publieke
2. Dataset registration/upload and metadata extraction. demodata. De gastmodus toont een projectgebonden demowerkruimte
3. Reference building layer loading. met dezelfde kaart-, bron-, model-, analyse-, QA- en exportflow als een
4. Predicted detection layer loading/import. operator. Alleen beheer, instellingen, uploads, bronconfiguratie,
5. QA/QC matching against reference polygons. projectbeheer en evidence-review blijven afgeschermd.
6. Metrics and false positive/false negative outputs.
7. GeoJSON export.
8. Minimal map/workbench UI.
## Primary stack ### Interactieve projectketen
- Frontend: React, TypeScript, MapLibre GL, Deck.gl, Tailwind. De landingspagina vertaalt de technische keten naar vier interactieve
- Backend: FastAPI, Python. schakels. De illustratie reageert op selectie, bronnen, analyse en bewijs en
- Database: PostgreSQL + PostGIS. respecteert automatisch `prefers-reduced-motion`.
- GIS processing: GeoPandas, Shapely, Rasterio, PyProj, GDAL.
- AI: PyTorch, Ultralytics YOLO, SAM-compatible architecture.
- Jobs: Redis + RQ.
- Storage: local filesystem first, MinIO-compatible later.
## Codex instructions ![Interactieve GeoIntel-projectketen](docs/assets/portfolio/geointel-interactive-story.png)
Codex must start with: ![Geanimeerde GeoIntel-projectketen](docs/assets/portfolio/geointel-interactive-project-chain.gif)
1. `docs/00-start/START_HERE.md` ### Kaart als werkruimte
2. `prompts/codex/M11_ARCHITECT_MASTER_PROMPT.md`
Then follow the build order in: De kaart blijft het primaire werkblad. Thema, broncontext, selectie en
resultaten zijn vanuit dezelfde ruimtelijke context bereikbaar.
- `docs/build/BUILD_ORDER_DEPENDENCY_GRAPH.md` ![GeoIntel kaartwerkruimte](docs/assets/portfolio/geointel-workbench-map.png)
- `docs/build/CODEX_OPERATING_SYSTEM.md`
Before every implementation pass, run available preflight/smoke scripts where applicable. Op een breed scherm krijgt de kaart extra ruimte terwijl de themakolom en de
controleerbare analysestappen zichtbaar blijven.
## Repo principle ![GeoIntel brede kaartwerkruimte](docs/assets/portfolio/geointel-workbench-wide.png)
This is a documentation-driven engineering repo. The documentation is not decorative; it is the control system for autonomous implementation. ### Kwaliteit vóór resultaat
## Fastest Day 1 command path QA/QC is een eerste-klas workflow. Bewaarde controles koppelen scores aan
objectbewijs, kandidaat- en referentielagen en technische provenance.
```bash ![GeoIntel kwaliteitscontrole](docs/assets/portfolio/geointel-workbench-quality.png)
make readiness
### Mobiele werkruimte
Dezelfde kaartgerichte workflow blijft bruikbaar op een smal scherm. Thema's,
selectieacties en de kaart worden gestapeld zonder de actieve werkcontext te
verbergen.
![GeoIntel mobiele werkruimte](docs/assets/portfolio/geointel-workbench-mobile.png)
## Kernmogelijkheden
| Domein | Mogelijkheden |
| --- | --- |
| Werkgebieden | Officiële grenzen, vrije AOI-selectie, clipping en buffering |
| Bronnen | GeoJSON, vector, raster, orthofoto, historische en regionale catalogi |
| GIS | CRS-normalisatie, spatial joins, rasterstatistiek, tile- en selectieanalyse |
| GeoAI | PyTorch, Ultralytics YOLO en SAM-compatibele abstrahering |
| Tijd | Immutable DatasetVersions en vergelijking van ruimtelijke snapshots |
| QA/QC | Precision, recall, F1, IoU, false-positive/-negative evidence en reviews |
| Export | Reproduceerbare GeoJSON- en artefactexports met provenance |
| Runtime | DockerMan-native Unraid-container met PostGIS, backend en frontend |
## Architectuur
```mermaid
flowchart LR
UI["React + MapLibre workbench"] --> API["FastAPI contracts"]
API --> GIS["GeoPandas / Shapely / Rasterio"]
API --> JOBS["Job-queue in PostgreSQL"]
JOBS --> AI["PyTorch / YOLO / SAM"]
GIS --> DB[("PostgreSQL + PostGIS")]
AI --> DB
DB --> QA["QA/QC + provenance"]
QA --> UI
QA --> EXPORT["GeoJSON / artifacts"]
``` ```
## Unraid / Tower deployment De publieke endpoints volgen [docs/API_CONTRACTS.md](docs/API_CONTRACTS.md).
PostGIS en de persistentieregels volgen
[docs/DATABASE_IMPLEMENTATION_PLAN.md](docs/DATABASE_IMPLEMENTATION_PLAN.md).
GeoIntel runs on Unraid as an all-in-one DockerMan-native container. The container embeds PostGIS, runs the FastAPI backend internally, and serves the frontend through nginx on one editable web port. ## Stack
Unraid template assets live in: - **Frontend:** React 18, TypeScript, MapLibre GL, Lucide en Vite
- **Backend:** FastAPI en Python
- **Spatial:** GeoPandas, Shapely, Rasterio, PyProj en GDAL
- **AI:** PyTorch, Ultralytics YOLO en SAM-compatibele segmentatie
- **Data:** PostgreSQL/PostGIS, lokale artefactopslag en immutable versions
- **Jobs:** job-tabel in PostgreSQL met achtergrondworkers in het API-proces
- **Deployment:** één DockerMan-native Unraid-container
- `deploy/unraid/geointel.env.example` ## Lokaal starten
- `deploy/unraid/geointel-unraid-template.xml`
- `deploy/unraid/geointel-icon.svg`
- `deploy/unraid/geointel-icon.png`
- `docker-compose.unraid.yml`
Copy the Unraid env template to `.env` in the checkout and edit ports/paths there: Vereisten: Python 3.11+, Node 20.19+ of 22.12+ en PostgreSQL/PostGIS.
```bash
cd /mnt/user/appdata/geointel
cp deploy/unraid/geointel.env.example .env
nano .env
docker build -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest .
bash deploy/unraid/run-dockerman-container.sh
```
Common editable values:
```env
GEOINTEL_FRONTEND_PORT=1202
GEOINTEL_STORAGE_PATH=/mnt/user/appdata/geointel/storage
GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data
```
The backend and PostGIS ports are intentionally not exposed to the LAN in the all-in-one runtime. See `deploy/unraid/README.md` for full setup, port-change and cleanup notes.
On Tower/Unraid, `scripts/deploy_tower.ps1` and `scripts/deploy_tower.sh` validate the Compose reference but build with plain `docker build`, then automatically install the editable DockerMan template as `/boot/config/plugins/dockerMan/templates-user/my-geointel.xml`, install the PNG icon as `/boot/config/plugins/dockerMan/images/geointel-icon.png`, remove any old Compose-owned `geointel` container and start the final container with DockerMan labels.
## Sprint 2 quick start
- Update dependencies:
```bash ```bash
python -m pip install -e backend/.[dev] python -m pip install -e backend/.[dev]
cd frontend && npm install cd frontend
npm install
npm run start
``` ```
- Run full readiness checks (with no scope expansion): Voor de volledige lokale stack:
```bash ```bash
docker compose up --build
```
Open daarna `http://localhost:1202`.
### Gastdemo
Met `GEOINTEL_AUTH_ENABLED=true` en de expliciete opt-in
`GEOINTEL_GUEST_ACCESS_ENABLED=true` biedt de toegangspagina een
kortlevende, projectgebonden demosessie. De gast heeft binnen het ingestelde
demoproject dezelfde kaart-, bronselectie-, modelselectie-, analyse-, QA- en
exportmogelijkheden als een operator. Alleen beheerfuncties zoals instellingen,
uploads, bronconfiguratie, projectbeheer en evidence-review blijven geblokkeerd;
dit is geen multi-user- of tenantisolatie. Gasttoegang staat standaard uit en
hoort alleen op een aparte demo-installatie met publieke data.
Een optionele Authentik OIDC-login kan naast de lokale operatorlogin worden
ingeschakeld. GeoIntel gebruikt daarbij PKCE, state, nonce, issuer-/audience-
controle en één expliciet toegelaten, geverifieerd e-mailadres. De lokale
operatorlogin blijft altijd het herstelpad.
## NVIDIA/CUDA
Productie-AI gebruikt de NVIDIA GPU van de server. De runtime faalt gesloten
wanneer CUDA vereist maar niet beschikbaar is; ontbrekende modelconfiguratie
wordt als `not_configured` getoond en nooit als geslaagde inferentie.
Corpus-, kalibratie-, test- en pure-background-gates blijven gescheiden: een
experimentele label- of modelvariant wordt pas actief nadat alle toepasselijke
kwaliteitscontroles slagen. Exacte runtime- en hardwaregegevens horen bij de
lokale deployment, niet bij de publieke broncode.
Controleer GPU-zichtbaarheid in de container met:
```bash
docker exec geointel nvidia-smi
```
![PyTorch- en NVIDIA-analyseketen](docs/assets/portfolio/geointel-ai-pytorch-pipeline.png)
## Training en labelkwaliteit
GeoIntel behandelt officiële gebouwgrondvlakken niet automatisch als perfecte
daklabels. Voor trainingsdata worden temporele geldigheid, ruimtelijke leakage,
bronklasse, zichtbaarheid en pure-background-gedrag afzonderlijk gecontroleerd.
Een directe polygon-overlay maakt bovendien zichtbaar of de officiële geometrie
op het bijbehorende luchtbeeld aansluit vóór omzetting naar YOLO-boxen. Voor
productresultaten blijft de taakgeschikte officiële bron doorslaggevend; een
AI-detectie is controleerbaar voorstelbewijs zolang een taakgebonden releasegate
niet aantoonbaar anders beslist.
Modelactivatie vereist naast het oudere diagnostische promotierapport ook een
geslaagd, governed Phase-4/5 release-gaterapport dat exact dezelfde candidate
key, model-SHA-256 en benchmarkmanifest-SHA-256 bindt. Zie
[Accuracy, modelgrenzen en bewijs](docs/ACCURACY.md). Er wordt geen
100%-accuratieclaim gemaakt; modeloutput blijft controleerbaar voorstelbewijs.
## Portfolio case study
De presentatieklare case study combineert productvraag, interface, PyTorch- en
NVIDIA-keten, architectuur en resultaten in één visueel document:
- [GeoIntel case study PDF](output/pdf/geointel-case-study.pdf)
- [Donkere cover in hoge resolutie](frontend/public/portfolio/geointel-dark-case-study-cover.png)
- [Architectuurvisual](docs/assets/portfolio/geointel-architecture.png)
De PDF kan reproduceerbaar opnieuw worden opgebouwd met:
```bash
python scripts/create_portfolio_case_study.py
```
## Kwaliteitscontrole
```bash
# Backend
python -m compileall backend/app python -m compileall backend/app
cd backend && python -m pytest cd backend && python -m pytest
cd ../frontend && npm run typecheck && npm run build
bash scripts/run_readiness_check.sh
```
- Raster workflow validation command (backend only): # Frontend
cd frontend
```bash
bash scripts/smoke_backend_import.sh
cd backend && python -c "from app.main import app; print(app.title)"
```
If `rasterio` is not installed, raster metadata endpoints return `RASTER_PROCESSING_UNAVAILABLE` and the frontend displays the
state as failed until the dependency is added.
## Sprint 4 raster foundation
- Raster operations now support:
- raster metadata extraction,
- raster preview generation,
- raster clip by area (with provenance on derived datasets),
- raster tile generation with manifest output.
- Raster services are dependency-aware:
- if `rasterio` is unavailable, endpoints return `RASTER_PROCESSING_UNAVAILABLE`.
- if preview dependencies (`numpy`, `pillow`) are unavailable, preview generation is unavailable with a clear error.
- Enable raster stack explicitly when needed:
```bash
cd backend && python -m pip install -e .[dev,raster]
```
## Sprint 5 raster analytics hardening
- Added raster band statistics (min/max/mean/std, nodata ratio/count, valid pixel count, dtype, optional histograms).
- Added raster reproject workflow with CRS validation and provenance persistence.
- Extended tile manifest expectations (`tile_set_id`, `tile_size`, `overlap`, `bounds`, `source_raster_id`, `tile_paths`, `tile_server`).
- Clarified raster operation availability in frontend/backend docs (`RASTER_PROCESSING_UNAVAILABLE` and invalid-CRS cases).
- Raster workflow command set (where available):
```bash
cd backend
python -m pip install -e .[dev,raster]
python -m pytest
cd ../frontend
npm run typecheck npm run typecheck
npm run test:unit
npm run build npm run build
``` ```
Then give Codex the prompt in: Een wijziging is pas afgerond wanneer de toepasselijke criteria uit
[docs/DEFINITION_OF_DONE.md](docs/DEFINITION_OF_DONE.md) aantoonbaar zijn
afgevinkt.
- `prompts/codex/final/DAY_1_MASTER_PROMPT.md` ## Unraid-deployment
De ondersteunde productieopstelling bundelt PostGIS, backend en frontend in één
Unraid-container. Kopieer de voorbeeldconfiguratie, vul de verplichte
authenticatie- en opslaginstellingen in en gebruik de releaseworkflow met
backup-, healthcheck- en rollbackcontrole.
## M13 Codex optimization Zie [deploy/unraid/README.md](deploy/unraid/README.md) en
[docs/RELEASE_RUNBOOK.md](docs/RELEASE_RUNBOOK.md) voor configuratie,
healthchecks en rollback.
For the first serious Codex build run, use: ## Documentatiekaart
- `prompts/codex/m13/DAY_1_OPTIMIZED_MASTER_PROMPT.md` - [Start hier](docs/00-start/START_HERE.md)
- [Belgium/North Sea scope freeze](docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md)
- [Actieve release-roadmap](docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md)
- [API-contracten](docs/API_CONTRACTS.md)
- [Database-implementatieplan](docs/DATABASE_IMPLEMENTATION_PLAN.md)
- [Bekende beperkingen](docs/KNOWN_LIMITATIONS.md)
- [Release-runbook](docs/RELEASE_RUNBOOK.md)
Codex should also use the relevant reusable skill under `skills/` for each implementation pass. Validate the optimization assets with: ## Portfolio-assets
```bash De map [`docs/assets/portfolio`](docs/assets/portfolio) bevat presentatieklare
make m13 screenshots, de geanimeerde projectketen, architectuur- en AI-visuals. De drie
``` originele campagnebeelden staan in
[`frontend/public/portfolio`](frontend/public/portfolio). Geoptimaliseerde
WebP-varianten worden door de applicatie gebruikt; de PNG-bronnen blijven
beschikbaar voor drukwerk en portfolio-opmaak.
The full readiness path remains: Herkomst en hergebruik van deze bestanden zijn vastgelegd in
[Asset provenance and redistribution](docs/ASSET_PROVENANCE.md).
```bash Actuele rasterassets:
make readiness
```
| Asset | Gebruik |
| --- | --- |
| `geointel-landing-hero.png` | Desktop hero en projectintroductie |
| `geointel-interactive-story.png` | Vierstappenworkflow en bewijsvoering |
| `geointel-workbench-map.png` | Kaartgerichte gastwerkruimte |
| `geointel-workbench-wide.png` | Brede kaartwerkruimte voor desktopportfolio's |
| `geointel-workbench-quality.png` | QA/QC, metrics en objectbewijs |
| `geointel-landing-mobile.png` | Mobiele landing |
| `geointel-workbench-mobile.png` | Mobiele kaartworkflow |
## M14 Build Launch ![GeoIntel mobiele landingspagina](docs/assets/portfolio/geointel-landing-mobile.png)
For the first serious implementation run, use: ## Status
- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` Actieve mijlpaal: **v1.0.0 — Belgium and Belgian North Sea**.
- `docs/40-build-launch/BUILD_SUCCESS_DEFINITION.md`
- `docs/40-build-launch/CODEX_STOP_RULES.md`
- `prompts/codex/m14/CODEX_FIRST_DAY_MASTER_PROMPT.md`
Validate launch assets with: GeoIntel is een project van Jens / ITWorx.tech.
```bash ## Licentie
make m14
```
Full readiness remains: GeoIntel is beschikbaar onder de [Apache License 2.0](LICENSE).
```bash
make readiness
```
## Sprint 1 execution (Sprint 1 only)
From a clean machine:
```bash
cd backend && python -m pip install -e .[dev]
cd ..
make backend-install
make frontend-install
make readiness
```
Copy `.env.example` to `.env` only when you want local overrides. Docker Compose has safe defaults for the local PostGIS/backend/frontend stack and does not require a root `.env` file to exist.
With Docker Compose, open the workbench at `http://localhost:1202`.
The Docker frontend is served by nginx and proxies `/api` and `/health` to the backend container, so browser clients should use the frontend URL only, for example `http://192.168.10.150:1202` on a LAN host.
Runtime containers include healthchecks for PostGIS, backend and frontend. After
startup, inspect them with:
```bash
docker compose ps
```
Verify the browser-facing API proxy after rebuilding Docker images:
```bash
bash scripts/verify_browser_runtime.sh http://localhost:1202 http://localhost:8000/health
```
Verify the Docker GIS runtime after rebuilding the backend image:
```bash
bash scripts/verify_gis_runtime.sh http://localhost:1202
```
On the LAN host use the published browser URL, for example:
```bash
bash scripts/verify_gis_runtime.sh http://192.168.10.150:1202
```
Load the explicit offline demo workflow:
```bash
curl -X POST http://192.168.10.150:1202/api/v1/demo/workflow
```
If `/api/v1/projects` returns frontend HTML instead of a JSON envelope, rebuild
and restart the frontend container.
Useful direct verification commands:
```bash
python -m compileall backend/app
cd backend && python -c "from app.main import app; print(app.title)"
python -m pytest
cd ../frontend && npm run typecheck
cd ../frontend && npm run build
docker compose config
bash scripts/run_readiness_check.sh
```
If `make` or `docker` are unavailable in your shell, run the equivalent script entrypoints directly:
```bash
bash scripts/backend_install.sh
bash scripts/backend_test.sh
bash scripts/frontend_install.sh
bash scripts/frontend_typecheck.sh
bash scripts/frontend_build.sh
bash scripts/run_readiness_check.sh
```
+38
View File
@@ -0,0 +1,38 @@
# Security Policy
## Supported code
Security fixes target the current `main` release line. Historical research,
calibration and development branches are not supported release channels unless
explicitly documented otherwise.
## Reporting vulnerabilities
Report suspected security issues privately to `security@itworx.tech`. Do not put
credentials, access tokens, private infrastructure, precise sensitive
locations, proprietary imagery or datasets, model artifacts, production
database contents, personal data or exploit-sensitive evidence in a public
issue.
Include the affected commit and component, minimal reproduction conditions
using synthetic or public data where possible, expected and observed
behaviour, impact, and whether the issue affects ingestion, geospatial
processing, authentication or authorization, uploads, archive or path handling,
model inference, reports and exports, deployment, data retention or secrets.
## Repository and data boundary
Geospatial inputs and derived outputs can reveal people, assets and sensitive
locations. Treat screenshots, contact sheets, coordinates, source archives,
SQLite/WAL state, model output and exports as potentially sensitive. These
belong in controlled external storage, not the source repository.
Local Codex inputs, calibration databases, runtime data, generated reports and
cross-project scratch are not release source. Never commit live `.env` files,
private keys, production backups or databases, customer or operator data,
unpublished imagery, credentials or private datasets. Public examples and tests
must use synthetic or explicitly redistributable material.
## Disclosure
Coordinate remediation and disclosure with the repository owner before publishing details that would materially increase exploitation risk.
+1 -1
View File
@@ -1 +1 @@
1.0.0-rc.1 1.0.0
+2
View File
@@ -8,3 +8,5 @@ storage
dist dist
node_modules node_modules
.env .env
.env.*
!.env.example
+109 -19
View File
@@ -142,6 +142,7 @@ bash scripts/live_migration_smoke.sh
- `GET /api/v1/detection/models` - `GET /api/v1/detection/models`
- `GET /api/v1/detection/model-assets` - `GET /api/v1/detection/model-assets`
- `POST /api/v1/detection/run` - `POST /api/v1/detection/run`
- `POST /api/v1/detection/run-async` (production browser path)
- `GET /api/v1/detection/runs/{analysis_run_id}` - `GET /api/v1/detection/runs/{analysis_run_id}`
- `GET /api/v1/detection/runs/{analysis_run_id}/detections` - `GET /api/v1/detection/runs/{analysis_run_id}/detections`
- YOLO/PyTorch real inference is not enabled in Sprint 8. - YOLO/PyTorch real inference is not enabled in Sprint 8.
@@ -184,6 +185,7 @@ bash scripts/live_migration_smoke.sh
- Added segmentation endpoints: - Added segmentation endpoints:
- `GET /api/v1/segmentation/models` - `GET /api/v1/segmentation/models`
- `POST /api/v1/segmentation/run` - `POST /api/v1/segmentation/run`
- `POST /api/v1/segmentation/run-async` (production browser path)
- `GET /api/v1/segmentation/runs` - `GET /api/v1/segmentation/runs`
- `GET /api/v1/segmentation/runs/{analysis_run_id}` - `GET /api/v1/segmentation/runs/{analysis_run_id}`
- `GET /api/v1/segmentation/runs/{analysis_run_id}/segmentations` - `GET /api/v1/segmentation/runs/{analysis_run_id}/segmentations`
@@ -191,6 +193,11 @@ bash scripts/live_migration_smoke.sh
- `POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference` - `POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference`
- Real SAM and YOLO-seg inference are not enabled in Sprint 9. - Real SAM and YOLO-seg inference are not enabled in Sprint 9.
- Mask paths are provenance/debug artifacts; persisted PostGIS geometry is authoritative for QA, map display and GeoJSON. - Mask paths are provenance/debug artifacts; persisted PostGIS geometry is authoritative for QA, map display and GeoJSON.
- Current configured detection and segmentation run through the async analysis
worker (`GEOINTEL_ANALYSIS_WORKER_ENABLED`) and are followed through
`GET /api/v1/projects/{project_id}/jobs/{job_id}`. The Unraid profile sets
`YOLO_REQUIRE_CUDA=true`, so both pipelines fail closed instead of silently
falling back from NVIDIA CUDA to CPU.
## Sprint 17 additions ## Sprint 17 additions
- Added export foundation backed by the existing `exports` table. - Added export foundation backed by the existing `exports` table.
@@ -453,7 +460,7 @@ To validate the full configured-YOLO runtime path against Docker/Tower after a
model is mounted and selected, run: model is mounted and selected, run:
```bash ```bash
bash scripts/verify_model_asset_detection_workflow.sh http://192.168.10.150:1202 bash scripts/verify_model_asset_detection_workflow.sh http://192.0.2.10:1202
``` ```
The smoke uses the existing demo raster to generate a tile manifest, selects a The smoke uses the existing demo raster to generate a tile manifest, selects a
@@ -470,7 +477,7 @@ the runtime host, then run:
```bash ```bash
REAL_RASTER_PATH=/mnt/user/appdata/geointel/data/orthophoto.tif \ REAL_RASTER_PATH=/mnt/user/appdata/geointel/data/orthophoto.tif \
REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/data/reference-buildings.geojson \ REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/data/reference-buildings.geojson \
bash scripts/verify_real_data_detection_qa_workflow.sh http://192.168.10.150:1202 bash scripts/verify_real_data_detection_qa_workflow.sh http://192.0.2.10:1202
``` ```
This smoke refuses missing/unsupported inputs, uploads the raster and reference This smoke refuses missing/unsupported inputs, uploads the raster and reference
@@ -549,7 +556,7 @@ For model-quality calibration, run the confidence sweep wrapper:
REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \ REAL_RASTER_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_orthophoto_wms_512.tif \
REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \ REAL_REFERENCE_VECTOR_PATH=/mnt/user/appdata/geointel/storage/operator-data/geel_grb_gbg_buildings.geojson \
CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \ CALIBRATION_THRESHOLDS="0.50 0.35 0.25 0.15" \
bash scripts/run_detection_calibration_sweep.sh http://192.168.10.150:1202 bash scripts/run_detection_calibration_sweep.sh http://192.0.2.10:1202
``` ```
The sweep creates one real persisted workflow run per threshold, fetches the The sweep creates one real persisted workflow run per threshold, fetches the
@@ -569,7 +576,7 @@ QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
QUALITY_TILE_SIZES="512 640" \ QUALITY_TILE_SIZES="512 640" \
QUALITY_TILE_OVERLAPS="64" \ QUALITY_TILE_OVERLAPS="64" \
QUALITY_THRESHOLDS="0.50 0.15" \ QUALITY_THRESHOLDS="0.50 0.15" \
bash scripts/run_detection_quality_matrix.sh http://192.168.10.150:1202 bash scripts/run_detection_quality_matrix.sh http://192.0.2.10:1202
``` ```
The matrix creates one real persisted workflow run per combination and writes The matrix creates one real persisted workflow run per combination and writes
@@ -587,7 +594,7 @@ QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
QUALITY_TILE_SIZES="512 640" \ QUALITY_TILE_SIZES="512 640" \
QUALITY_TILE_OVERLAPS="64" \ QUALITY_TILE_OVERLAPS="64" \
QUALITY_THRESHOLDS="0.50 0.15" \ QUALITY_THRESHOLDS="0.50 0.15" \
bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202 bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.0.2.10:1202
``` ```
The combined `multi_sample_quality_summary.json` reports per-sample and overall The combined `multi_sample_quality_summary.json` reports per-sample and overall
@@ -604,7 +611,7 @@ QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-bu
QUALITY_TILE_SIZES="640" \ QUALITY_TILE_SIZES="640" \
QUALITY_TILE_OVERLAPS="64" \ QUALITY_TILE_OVERLAPS="64" \
QUALITY_THRESHOLDS="0.25 0.15 0.05" \ QUALITY_THRESHOLDS="0.25 0.15 0.05" \
bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.168.10.150:1202 bash scripts/run_operator_hard_negative_detection_matrix.sh http://192.0.2.10:1202
``` ```
This path uploads only background rasters, runs configured-YOLO detection and This path uploads only background rasters, runs configured-YOLO detection and
@@ -617,7 +624,7 @@ evidence bundle:
```bash ```bash
CALIBRATION_SUMMARY_PATH=/mnt/user/appdata/geointel/artifacts/detection-calibration/20260707T002103Z/calibration_summary.json \ CALIBRATION_SUMMARY_PATH=/mnt/user/appdata/geointel/artifacts/detection-calibration/20260707T002103Z/calibration_summary.json \
bash scripts/export_detection_calibration_evidence.sh http://192.168.10.150:1202 bash scripts/export_detection_calibration_evidence.sh http://192.0.2.10:1202
``` ```
The bundle writes combined QA evidence GeoJSON plus a standalone HTML/SVG review The bundle writes combined QA evidence GeoJSON plus a standalone HTML/SVG review
@@ -731,7 +738,7 @@ python scripts/seed_demo_workflow.py --json
In Docker Compose on a LAN host: In Docker Compose on a LAN host:
```bash ```bash
curl -X POST http://192.168.10.150:1202/api/v1/demo/workflow curl -X POST http://192.0.2.10:1202/api/v1/demo/workflow
``` ```
### QA/QC result listing ### QA/QC result listing
@@ -816,7 +823,7 @@ After rebuilding a Docker/LAN deployment, verify the end-to-end demo and export
flow through the browser-facing frontend proxy: flow through the browser-facing frontend proxy:
```bash ```bash
bash scripts/verify_demo_export_workflow.sh http://192.168.10.150:1202 bash scripts/verify_demo_export_workflow.sh http://192.0.2.10:1202
``` ```
The script seeds the explicit demo workflow, verifies persisted QA/QC results, The script seeds the explicit demo workflow, verifies persisted QA/QC results,
@@ -879,7 +886,7 @@ bash scripts/verify_gis_runtime.sh http://localhost:1202
On a NAS or server host, use the published LAN URL: On a NAS or server host, use the published LAN URL:
```bash ```bash
bash scripts/verify_gis_runtime.sh http://192.168.10.150:1202 bash scripts/verify_gis_runtime.sh http://192.0.2.10:1202
``` ```
The script calls `/api/v1/system/capabilities` through the frontend proxy and The script calls `/api/v1/system/capabilities` through the frontend proxy and
@@ -1336,7 +1343,10 @@ not copied into queryable properties.
`GET /api/v1/projects/{project_id}/datasets/orthophoto/products` lists the `GET /api/v1/projects/{project_id}/datasets/orthophoto/products` lists the
governed product allowlist. `POST .../datasets/orthophoto/acquire` accepts an governed product allowlist. `POST .../datasets/orthophoto/acquire` accepts an
explicit EPSG:4326 map rectangle plus `product_key` and stores the official explicit EPSG:4326 map rectangle plus `product_key` and stores the official
Digitaal Vlaanderen WMS response as a canonical EPSG:31370 raster Dataset. The regional WMS response as a canonical EPSG:31370 raster Dataset. Digitaal
Vlaanderen, SPW (`wallonia_latest`) and Paradigm UrbIS (`brussels_latest`) are
allowlisted. The two regional products are bound to persisted Wallonia and
Brussels-Capital Region Areas. The
default safety envelope is 128-1,024 m per side, 1 m/pixel, 32 MiB and a default safety envelope is 128-1,024 m per side, 1 m/pixel, 32 MiB and a
24-hour exact-request cache. It runs synchronously behind the existing Job 24-hour exact-request cache. It runs synchronously behind the existing Job
abstraction and never during startup. abstraction and never during startup.
@@ -1348,11 +1358,16 @@ from configured-YOLO/current-GRB QA. `GET .../datasets/{dataset_id}/raster/image
is the constrained binary PNG endpoint used by the MapLibre image overlay. is the constrained binary PNG endpoint used by the MapLibre image overlay.
Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`, Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`,
`SPW_ORTHOPHOTO_WMS_URL`, `BRUSSELS_ORTHOPHOTO_WMS_URL`,
`ORTHOPHOTO_WMS_LAYER`, `ORTHOPHOTO_RESOLUTION_M`, `ORTHOPHOTO_WMS_LAYER`, `ORTHOPHOTO_RESOLUTION_M`,
`ORTHOPHOTO_MIN_SIDE_M`, `ORTHOPHOTO_MAX_SIDE_M`, `ORTHOPHOTO_MIN_SIDE_M`, `ORTHOPHOTO_MAX_SIDE_M`,
`ORTHOPHOTO_TIMEOUT_SECONDS`, `ORTHOPHOTO_MAX_RESPONSE_MB` and `ORTHOPHOTO_TIMEOUT_SECONDS`, `ORTHOPHOTO_MAX_RESPONSE_MB` and
`ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile `ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile
unless a separately verified deployment/model profile requires a change. unless a separately verified deployment/model profile requires a change.
An explicit bounded request may provide `resolution_m` down to the governed
product's native resolution. This is intended for reviewed training corpora;
the service rejects source oversampling and records rolling-latest observation
time as unknown per pixel rather than equating it with download time.
Before a future `most_recent` source release is allowed into a governed pixel Before a future `most_recent` source release is allowed into a governed pixel
stage, run the metadata-only preflight for the exact intended rectangle: stage, run the metadata-only preflight for the exact intended rectangle:
@@ -1599,6 +1614,53 @@ Datasets. A full-Flanders raster request remains blocked by the same 60 km and
`Gemeente ...` as `coverage_scope=municipality`; regional Area clipping is `Gemeente ...` as `coverage_scope=municipality`; regional Area clipping is
stored as `bounded_selection`. stored as `bounded_selection`.
## Walloon WALOUS land cover and flood hazard
The Wallonia map flow uses bounded PICC vector products, the queryable legal
SPW flood-hazard polygon layer and provisioned official WALOUS land-cover
rasters. Provision the 2018, 2020 and 2023 source editions once in the persistent
storage mount:
```bash
docker exec geointel python /app/scripts/provision_walous_sources.py \
--years 2018 2020 2023 \
--destination /app/storage/source-cache/walous
```
The provisioner verifies advertised archive sizes, safe ZIP structure,
EPSG:3812, one band, 1 m cells, the official non-contiguous class codes
`1,2,3,4,5,6,7,8,9,80,90` and SHA-256 checksums. It does not run at
application startup. `GET .../datasets/walous/products` therefore reports
`source_not_provisioned` for each edition whose source file is absent.
For a bounded Walloon selection the browser persists the latest edition and
all other configured comparable editions. `POST .../raster/walous/select`
returns cell-area hectares; the temporal API compares the same semantic metric
keys for 2018, 2020 and 2023. The 2018 stacked classes use the official visible-
class crosswalk and retain the earlier-method limitation. WALOUS is land cover,
not legal land use, ownership,
tree count, timber volume or water volume.
The class semantics follow the official raster codes, not display-list
positions: 1 artificial ground, 2 above-ground construction, 3 railway, 4 bare
soil, 5 surface water, 6 rotating herbaceous cover, 7 continuous herbaceous
cover, 8/9 trees above 3 m and 80/90 woody cover up to 3 m. Observation ranges
are retained from the SPW metadata rather than replaced by arbitrary year-end
dates.
Settings: `WALOUS_ENABLED`, `WALOUS_SOURCE_DIR`,
`WALOUS_ANALYSIS_RESOLUTION_M`, `WALOUS_MAX_SIDE_M` and
`WALOUS_MAX_PIXELS`. The SPW flood polygon adapter uses
`SPW_FLOOD_HAZARD_ENABLED` and `SPW_FLOOD_HAZARD_MAPSERVER_URL`.
The official Walloon 2021-2022 1 m MNT is an explicit operator asset. Provision
it once with `scripts/provision_spw_terrain_source.py`; the runtime then reads
only bounded windows and persists 5 m analysis derivatives. The full 0.5 m
artifact remains intentionally excluded because it adds no V1 metric and is
about 213 GB. Settings: `SPW_TERRAIN_ENABLED`, `SPW_TERRAIN_SOURCE_DIR`,
`SPW_TERRAIN_ANALYSIS_RESOLUTION_M`, `SPW_TERRAIN_MAX_SIDE_M` and
`SPW_TERRAIN_MAX_PIXELS`.
Provision the official DOV soil polygons for Mol through the existing vector Provision the official DOV soil polygons for Mol through the existing vector
upload path: upload path:
@@ -1806,7 +1868,8 @@ matched name with `--show-names`.
performs a bounded official VHA ArcGIS query, exact persisted-Area clipping, performs a bounded official VHA ArcGIS query, exact persisted-Area clipping,
watercourse-name normalization and ordinary Dataset/VectorFeature persistence. watercourse-name normalization and ordinary Dataset/VectorFeature persistence.
`GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as `GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as
operational and the audited MDK/SPW candidates as unavailable for acquisition. operational, MDK as probe-only and the pinned SPW raster operator as
operational.
Runtime controls are `BATHYMETRY_PROFILES_ENABLED`, Runtime controls are `BATHYMETRY_PROFILES_ENABLED`,
`BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`, `BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`,
@@ -1854,21 +1917,48 @@ readiness state such as TLS or endpoint failure. Runtime controls are
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and `MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled. `MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
## Governed forest, agriculture, nature and soil acquisition ### SPW waterbed raster
The official 2023-05-23 SPW bathymetry ZIP is integrated only through the
bounded operator. Stage the immutable ZIP under persistent storage and run:
```bash
docker exec geointel python /app/scripts/import_spw_bathymetry.py \
--base-url http://127.0.0.1:8000 \
--project-name "Belgium and North Sea Workbench" \
--area "RC Golden - Wallonia urban-rural" \
--bbox 4.85,50.45,4.87,50.47 \
--raw-zip /app/storage/operator-evidence/spw-bathymetry/2023-05-23/raw/BATHY_50CM_ALTITUDE_DNG_GEOTIFF_3812.zip \
--output-dir /app/storage/operator-evidence/spw-bathymetry/2023-05-23/derived
```
The script validates the pinned official checksum, safe archive members,
EPSG:3812, one Float32 band, approximately 0.5 m cells and nodata `-9999`.
It then creates a bounded COG and uploads it through `/datasets/upload`.
`POST .../raster/bathymetry/select` returns waterbed elevation in mDNG,
surveyed surface and coverage. Current depth, volume and datum conversion stay
unavailable without a compatible water-surface source. Selection analysis is
bounded by `BATHYMETRY_RASTER_MAX_PIXELS` (30 million by default).
## Governed regional official-vector acquisition
The thematic raster registry includes forest and agricultural land-use masks The thematic raster registry includes forest and agricultural land-use masks
derived from Landgebruik Vlaanderen 2025 classes 12 and 13/14. They use the derived from Landgebruik Vlaanderen 2025 classes 12 and 13/14. They use the
existing thematic acquisition and selection routes. existing thematic acquisition and selection routes.
Two polygon products are exposed through Eight fixed products are exposed through
`/datasets/official-vector/products` and `/datasets/official-vector/products` and
`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and the DOV `/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and DOV soil
digital soil map. Both require an EPSG:4326 rectangle, optionally intersect it for Flanders; PICC buildings, roads, hydrographic axes and surfaces for
with a persisted Area, clip in EPSG:31370 and persist through Wallonia; and UrbIS buildings and cadastral parcels for Brussels. All require
`DatasetService.import_vector_bytes`. an EPSG:4326 rectangle, clip in a provider-appropriate metric CRS and persist
through `DatasetService.import_vector_bytes`. SPW/PICC and UrbIS additionally
require a persisted exact regional coverage Area and never write directly to
`vector_features`.
Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`, Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`,
`DOV_SOIL_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`, `DOV_SOIL_WFS_URL`, `SPW_PICC_ENABLED`, `SPW_PICC_MAPSERVER_URL`,
`URBIS_ENABLED`, `URBIS_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`,
`OFFICIAL_VECTOR_MAX_SIDE_M`, `OFFICIAL_VECTOR_PAGE_SIZE`, `OFFICIAL_VECTOR_MAX_SIDE_M`, `OFFICIAL_VECTOR_PAGE_SIZE`,
`OFFICIAL_VECTOR_MAX_PAGES`, `OFFICIAL_VECTOR_MAX_FEATURES`, `OFFICIAL_VECTOR_MAX_PAGES`, `OFFICIAL_VECTOR_MAX_FEATURES`,
`OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`, `OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`,
@@ -0,0 +1,65 @@
"""Add resumable AOI parent and partition operations."""
from alembic import op
import sqlalchemy as sa
from geoalchemy2 import Geometry
revision = "202607260001"
down_revision = "202607160001"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"aoi_operations",
sa.Column("id", sa.UUID(), primary_key=True),
sa.Column("project_id", sa.UUID(), sa.ForeignKey("projects.id", ondelete="CASCADE"), nullable=False),
sa.Column("area_id", sa.UUID(), sa.ForeignKey("areas.id", ondelete="SET NULL")),
sa.Column("parent_job_id", sa.UUID(), sa.ForeignKey("jobs.id", ondelete="SET NULL")),
sa.Column("operation_type", sa.String(128), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("geometry", Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False),
sa.Column("request_json", sa.JSON(), nullable=False),
sa.Column("plan_json", sa.JSON(), nullable=False),
sa.Column("result_json", sa.JSON()),
sa.Column("error_message", sa.Text()),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("started_at", sa.DateTime(timezone=True)),
sa.Column("finished_at", sa.DateTime(timezone=True)),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.CheckConstraint("status IN ('queued', 'running', 'partial', 'success', 'failed', 'cancelled')", name="ck_aoi_operations_status"),
)
op.create_index("ix_aoi_operations_project_status", "aoi_operations", ["project_id", "status"])
op.create_index("ix_aoi_operations_geometry", "aoi_operations", ["geometry"], postgresql_using="gist")
op.create_table(
"aoi_operation_partitions",
sa.Column("id", sa.UUID(), primary_key=True),
sa.Column("operation_id", sa.UUID(), sa.ForeignKey("aoi_operations.id", ondelete="CASCADE"), nullable=False),
sa.Column("child_job_id", sa.UUID(), sa.ForeignKey("jobs.id", ondelete="SET NULL")),
sa.Column("partition_key", sa.String(255), nullable=False),
sa.Column("provider_key", sa.String(120), nullable=False),
sa.Column("product_key", sa.String(120), nullable=False),
sa.Column("ordinal", sa.Integer(), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("geometry", Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False),
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
sa.Column("checkpoint_json", sa.JSON()),
sa.Column("result_json", sa.JSON()),
sa.Column("error_message", sa.Text()),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column("started_at", sa.DateTime(timezone=True)),
sa.Column("finished_at", sa.DateTime(timezone=True)),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.CheckConstraint("status IN ('queued', 'running', 'success', 'failed', 'skipped')", name="ck_aoi_operation_partitions_status"),
sa.UniqueConstraint("operation_id", "partition_key", name="uq_aoi_operation_partition_key"),
)
op.create_index("ix_aoi_operation_partitions_operation_status", "aoi_operation_partitions", ["operation_id", "status"])
op.create_index("ix_aoi_operation_partitions_geometry", "aoi_operation_partitions", ["geometry"], postgresql_using="gist")
def downgrade() -> None:
op.drop_table("aoi_operation_partitions")
op.drop_table("aoi_operations")
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,43 @@
"""Configure the immutable model source registry for governed snapshots.
The phase-2 seed intentionally registered model artifacts as unknown. Runtime
model provenance now records exact immutable snapshots, so the server-owned
registry must advertise that configured capability. The write guard is only
disabled for this narrowly-scoped, versioned migration and is restored in the
same transaction.
"""
from alembic import op
revision = "202608230001"
down_revision = "202608010001"
branch_labels = None
depends_on = None
def _set_status(*, freshness_status: str, ingest_status: str) -> None:
op.execute("ALTER TABLE source_registry DISABLE TRIGGER trg_source_registry_write_guard")
op.execute(
f"""
UPDATE source_registry
SET freshness_status = '{freshness_status}',
ingest_status = '{ingest_status}',
registry_metadata_json = (
registry_metadata_json::jsonb ||
'{{"runtime_model_contract": {{"key": "geointel.model.pytorch", "version": "1.0.0"}}}}'::jsonb
)::json,
updated_at = now()
WHERE source_key = 'model'
AND registry_metadata_json ->> 'registry_owner' = 'server'
"""
)
op.execute("ALTER TABLE source_registry ENABLE TRIGGER trg_source_registry_write_guard")
def upgrade() -> None:
_set_status(freshness_status="current", ingest_status="configured")
def downgrade() -> None:
_set_status(freshness_status="unknown", ingest_status="registered")
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
from uuid import UUID
from fastapi import Request
from app.core.errors import AppError
def guest_project_scope(request: Request) -> UUID | None:
principal = getattr(request.state, "auth_principal", None)
if getattr(principal, "role", None) != "guest":
return None
project_id = getattr(principal, "project_id", None)
if isinstance(project_id, UUID):
return project_id
raise AppError(
code="GUEST_PROJECT_SCOPE_REQUIRED",
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
status_code=403,
)
def assert_guest_project_scope(request: Request, project_id: UUID) -> None:
guest_project_id = guest_project_scope(request)
if guest_project_id is not None and project_id != guest_project_id:
raise AppError(
code="GUEST_PROJECT_SCOPE_REQUIRED",
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
status_code=403,
)
def guest_scoped_project_filter(
request: Request,
requested_project_id: UUID | None,
) -> UUID | None:
guest_project_id = guest_project_scope(request)
if guest_project_id is None:
return requested_project_id
if requested_project_id is not None:
assert_guest_project_scope(request, requested_project_id)
return guest_project_id
+15 -1
View File
@@ -1 +1,15 @@
__all__ = ["analysis", "areas", "assistant", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"] __all__ = [
"analysis",
"areas",
"assistant",
"auth",
"datasets",
"exports",
"external",
"health",
"jobs",
"projects",
"qa",
"source_registry",
"temporal",
]
+9 -1
View File
@@ -1,8 +1,9 @@
from __future__ import annotations from __future__ import annotations
from fastapi import APIRouter, Depends from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.api.guest_scope import assert_guest_project_scope
from app.core.errors import AppError from app.core.errors import AppError
from app.db.session import get_db from app.db.session import get_db
from app.models import Dataset from app.models import Dataset
@@ -18,12 +19,15 @@ router = APIRouter(prefix="/analysis", tags=["analysis"])
@router.post("/change-detection", response_model=Envelope[JobRead]) @router.post("/change-detection", response_model=Envelope[JobRead])
def run_change_detection( def run_change_detection(
payload: ChangeDetectionRequest, payload: ChangeDetectionRequest,
request: Request,
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
source_dataset = db.get(Dataset, payload.source_dataset_id) source_dataset = db.get(Dataset, payload.source_dataset_id)
if not source_dataset: if not source_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404) raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
assert_guest_project_scope(request, source_dataset.project_id)
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source") ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target")
job = JobService.run_sync_job( job = JobService.run_sync_job(
db=db, db=db,
project_id=source_dataset.project_id, project_id=source_dataset.project_id,
@@ -36,7 +40,11 @@ def run_change_detection(
source_dataset_id=payload.source_dataset_id, source_dataset_id=payload.source_dataset_id,
target_dataset_id=payload.target_dataset_id, target_dataset_id=payload.target_dataset_id,
iou_threshold=payload.iou_threshold, iou_threshold=payload.iou_threshold,
modified_threshold=payload.modified_threshold,
include_unchanged=payload.include_unchanged, include_unchanged=payload.include_unchanged,
bbox=payload.bbox.model_dump() if payload.bbox is not None else None,
area_id=payload.area_id,
preview_limit=payload.preview_limit,
).model_dump(mode="json"), ).model_dump(mode="json"),
) )
return envelope(job) return envelope(job)
+58
View File
@@ -0,0 +1,58 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from app.db.session import get_db
from app.schemas.aoi_operation import AoiOperationCreate, AoiOperationList, AoiOperationRead, AoiPartitionCheckpoint, AoiPartitionComplete, AoiPartitionFail, AoiPartitionRead
from app.schemas.common import Envelope
from app.services.aoi_operation_service import AoiOperationService
from app.services.aoi_operation_executor import AoiOperationExecutor
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}/aoi-operations", tags=["aoi-operations"])
@router.post("", status_code=201, response_model=Envelope[AoiOperationRead])
def create_operation(project_id: UUID, payload: AoiOperationCreate, db: Session = Depends(get_db)):
return envelope(AoiOperationService.create(db, project_id, payload))
@router.get("", response_model=Envelope[AoiOperationList])
def list_operations(project_id: UUID, limit: int = Query(default=50, ge=1, le=200), db: Session = Depends(get_db)):
return envelope(AoiOperationService.list(db, project_id, limit))
@router.get("/{operation_id}", response_model=Envelope[AoiOperationRead])
def read_operation(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
return envelope(AoiOperationService.read(db, project_id, operation_id))
@router.post("/{operation_id}/partitions/claim", response_model=Envelope[AoiPartitionRead | None])
def claim_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
partition = AoiOperationService.claim_next(db, project_id, operation_id)
return envelope(AoiPartitionRead.model_validate(partition).model_dump() if partition else None)
@router.post("/{operation_id}/execute-next", response_model=Envelope[AoiOperationRead])
def execute_next_partition(project_id: UUID, operation_id: UUID, db: Session = Depends(get_db)):
return envelope(AoiOperationExecutor.execute_next(db, project_id, operation_id))
@router.put("/{operation_id}/partitions/{partition_id}/checkpoint", response_model=Envelope[AoiPartitionRead])
def checkpoint_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionCheckpoint, db: Session = Depends(get_db)):
partition = AoiOperationService.checkpoint(db, project_id, operation_id, partition_id, payload.checkpoint_json)
return envelope(AoiPartitionRead.model_validate(partition).model_dump())
@router.post("/{operation_id}/partitions/{partition_id}/complete", response_model=Envelope[AoiOperationRead])
def complete_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionComplete, db: Session = Depends(get_db)):
return envelope(AoiOperationService.complete(db, project_id, operation_id, partition_id, payload.result_json, payload.skipped))
@router.post("/{operation_id}/partitions/{partition_id}/fail", response_model=Envelope[AoiOperationRead])
def fail_partition(project_id: UUID, operation_id: UUID, partition_id: UUID, payload: AoiPartitionFail, db: Session = Depends(get_db)):
return envelope(AoiOperationService.fail(db, project_id, operation_id, partition_id, payload.error_message, payload.retryable, payload.details))
+18 -1
View File
@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
from app.db.session import get_db from app.db.session import get_db
from app.models import Area from app.models import Area
from app.schemas import Envelope from app.schemas import Envelope
from app.schemas.area import AreaCreate, AreaList, AreaRead, AreaUpdate from app.schemas.area import AreaCreate, AreaList, AreaRead, AreaUpdate, MunicipalitySearchList
from app.services.area_service import AreaService from app.services.area_service import AreaService
from app.utils.response import envelope from app.utils.response import envelope
@@ -33,6 +33,23 @@ def create_area(project_id: UUID, payload: AreaCreate, db: Session = Depends(get
return envelope(AreaService.serialize_area(area)) return envelope(AreaService.serialize_area(area))
@router.get("/municipalities", response_model=Envelope[MunicipalitySearchList])
def search_municipalities(
project_id: UUID,
query: str = Query(default="", max_length=120),
limit: int = Query(default=20, ge=1, le=50),
db: Session = Depends(get_db),
):
items, total = AreaService.search_municipalities(db, project_id, query, limit)
return envelope({"items": items, "total": total})
@router.post("/municipalities/{niscode}/activate", response_model=Envelope[AreaRead])
def activate_municipality(project_id: UUID, niscode: str, db: Session = Depends(get_db)):
area = AreaService.activate_municipality(db, project_id, niscode)
return envelope(AreaService.serialize_area(area))
@router.get("/{area_id}", response_model=Envelope[AreaRead]) @router.get("/{area_id}", response_model=Envelope[AreaRead])
def get_area( def get_area(
project_id: UUID, project_id: UUID,
+326
View File
@@ -0,0 +1,326 @@
from __future__ import annotations
import logging
from datetime import UTC, datetime
from ipaddress import ip_address, ip_network
from fastapi import APIRouter, Depends, Request, Response, status
from fastapi.responses import RedirectResponse
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.core.errors import AppError
from app.db.session import get_db
from app.schemas.auth import AuthLoginRequest, AuthSession, AuthSessionEnvelope
from app.services.auth_service import AuthPrincipal, AuthService
from app.services.authentik_oidc_service import AuthentikOidcService
from app.services.demo_workflow_service import DemoWorkflowService
router = APIRouter(prefix="/auth", tags=["auth"])
COOKIE_NAME = "geointel_session"
OIDC_FLOW_COOKIE_NAME = "geointel_oidc_flow"
logger = logging.getLogger("geointel.auth")
_TRUSTED_PROXY_NETWORKS = (
ip_network("127.0.0.0/8"),
ip_network("::1/128"),
ip_network("172.16.0.0/12"),
)
def _peer_is_trusted_proxy(request: Request) -> bool:
if request.client is None:
return False
try:
peer_address = ip_address(request.client.host)
except ValueError:
return False
return any(peer_address in network for network in _TRUSTED_PROXY_NETWORKS)
def _request_is_https(request: Request) -> bool:
if request.url.scheme == "https":
return True
if not _peer_is_trusted_proxy(request):
return False
forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",", 1)[0].strip().lower()
return forwarded_proto == "https"
def _client_host(request: Request) -> str:
peer = request.client.host if request.client else "unknown"
if not _peer_is_trusted_proxy(request):
return peer
forwarded = request.headers.get("x-real-ip", "").strip()
if not forwarded:
return peer
try:
return str(ip_address(forwarded))
except ValueError:
return peer
def _session_from_principal(
principal: AuthPrincipal,
*,
guest_access_enabled: bool,
authentik_enabled: bool,
) -> AuthSession:
return AuthSession(
authentication_required=True,
authenticated=True,
username=principal.username,
expires_at=datetime.fromtimestamp(principal.expires_at, tz=UTC),
role=principal.role,
guest_access_enabled=guest_access_enabled,
authentik_enabled=authentik_enabled,
guest_project_id=principal.project_id,
)
def _session_payload(request: Request) -> AuthSession:
settings = get_settings()
guest_access_enabled = settings.auth_enabled and settings.guest_access_enabled
authentik_enabled = AuthentikOidcService(settings).enabled
if not settings.auth_enabled:
return AuthSession(
authentication_required=False,
authenticated=True,
guest_access_enabled=False,
authentik_enabled=False,
)
principal = AuthService.verify_session_token(request.cookies.get(COOKIE_NAME), settings)
if principal is None:
return AuthSession(
authentication_required=True,
authenticated=False,
guest_access_enabled=guest_access_enabled,
authentik_enabled=authentik_enabled,
)
return _session_from_principal(
principal,
guest_access_enabled=guest_access_enabled,
authentik_enabled=authentik_enabled,
)
def _set_session_cookie(
*,
request: Request,
response: Response,
token: str,
max_age: int,
) -> None:
response.set_cookie(
key=COOKIE_NAME,
value=token,
max_age=max_age,
httponly=True,
secure=_request_is_https(request),
samesite="strict",
path="/",
)
@router.get("/session", response_model=AuthSessionEnvelope)
def session(request: Request) -> AuthSessionEnvelope:
return AuthSessionEnvelope(data=_session_payload(request))
@router.post("/login", response_model=AuthSessionEnvelope)
def login(payload: AuthLoginRequest, request: Request, response: Response) -> AuthSessionEnvelope:
settings = get_settings()
if not settings.auth_enabled:
raise AppError(
code="AUTHENTICATION_DISABLED",
message="Operator authentication is not enabled on this runtime",
status_code=status.HTTP_409_CONFLICT,
)
if settings.auth_require_https and not _request_is_https(request):
raise AppError(
code="AUTH_HTTPS_REQUIRED",
message="Operator authentication requires HTTPS on this runtime",
status_code=status.HTTP_426_UPGRADE_REQUIRED,
)
client_host = _client_host(request)
throttle_key = f"{client_host}:{payload.username.casefold()}"
retry_after = AuthService.retry_after_seconds(throttle_key)
if retry_after:
raise AppError(
code="LOGIN_RATE_LIMITED",
message="Te veel mislukte aanmeldpogingen. Probeer later opnieuw.",
details={"retry_after_seconds": retry_after},
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
)
if not AuthService.credentials_match(payload.username, payload.password, settings):
AuthService.record_failure(throttle_key)
raise AppError(
code="INVALID_CREDENTIALS",
message="Gebruikersnaam of wachtwoord is onjuist.",
status_code=status.HTTP_401_UNAUTHORIZED,
)
AuthService.clear_failures(throttle_key)
token = AuthService.create_session_token(payload.username, settings)
principal = AuthService.verify_session_token(token, settings)
if principal is None: # pragma: no cover - defensive invariant
raise AppError(
code="SESSION_CREATION_FAILED",
message="De beveiligde sessie kon niet worden aangemaakt.",
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
_set_session_cookie(
request=request,
response=response,
token=token,
max_age=settings.auth_session_ttl_seconds,
)
return AuthSessionEnvelope(
data=_session_from_principal(
principal,
guest_access_enabled=settings.guest_access_enabled,
authentik_enabled=AuthentikOidcService(settings).enabled,
)
)
@router.get("/authentik/start")
def authentik_start(request: Request) -> RedirectResponse:
settings = get_settings()
service = AuthentikOidcService(settings)
try:
location, flow = service.start()
except Exception as exc:
logger.warning("Authentik authorization start failed: %s", type(exc).__name__)
raise AppError(
code="AUTHENTIK_UNAVAILABLE",
message="Authentik is momenteel niet beschikbaar.",
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
) from exc
response = RedirectResponse(location, status_code=status.HTTP_302_FOUND)
response.set_cookie(
OIDC_FLOW_COOKIE_NAME,
flow,
max_age=600,
httponly=True,
secure=True,
samesite="lax",
path=f"{settings.api_prefix}/auth/authentik",
)
return response
@router.get("/authentik/callback")
def authentik_callback(
request: Request,
code: str = "",
state: str = "",
) -> RedirectResponse:
settings = get_settings()
service = AuthentikOidcService(settings)
base_url = settings.public_base_url.rstrip("/")
try:
service.finish(
code=code,
state=state,
flow_cookie=request.cookies.get(OIDC_FLOW_COOKIE_NAME, ""),
)
token = AuthService.create_session_token(
settings.auth_username or "operator",
settings,
)
except Exception as exc:
logger.warning("Authentik callback rejected: %s", type(exc).__name__)
response = RedirectResponse(
f"{base_url}/?authentik=error",
status_code=status.HTTP_302_FOUND,
)
else:
response = RedirectResponse(
f"{base_url}/",
status_code=status.HTTP_302_FOUND,
)
_set_session_cookie(
request=request,
response=response,
token=token,
max_age=settings.auth_session_ttl_seconds,
)
response.delete_cookie(
OIDC_FLOW_COOKIE_NAME,
path=f"{settings.api_prefix}/auth/authentik",
secure=True,
httponly=True,
samesite="lax",
)
return response
@router.post("/guest", response_model=AuthSessionEnvelope)
def guest_login(
request: Request,
response: Response,
db: Session = Depends(get_db),
) -> AuthSessionEnvelope:
settings = get_settings()
if not settings.auth_enabled or not settings.guest_access_enabled:
raise AppError(
code="GUEST_ACCESS_DISABLED",
message="Gasttoegang is niet ingeschakeld op deze GeoIntel-installatie.",
status_code=status.HTTP_403_FORBIDDEN,
)
client_host = _client_host(request)
retry_after = AuthService.consume_guest_request(
f"guest-login:{client_host}",
max_requests=settings.guest_login_requests_per_minute,
)
if retry_after:
raise AppError(
code="GUEST_LOGIN_RATE_LIMITED",
message="Too many guest sessions were requested. Try again later.",
details={"retry_after_seconds": retry_after},
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
)
demo = DemoWorkflowService.seed(db)
token = AuthService.create_session_token(
settings.guest_display_name,
settings,
role="guest",
project_id=demo.project_id,
ttl_seconds=settings.guest_session_ttl_seconds,
)
principal = AuthService.verify_session_token(token, settings)
if principal is None: # pragma: no cover - defensive invariant
raise AppError(
code="SESSION_CREATION_FAILED",
message="De tijdelijke gastensessie kon niet worden aangemaakt.",
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
_set_session_cookie(
request=request,
response=response,
token=token,
max_age=settings.guest_session_ttl_seconds,
)
return AuthSessionEnvelope(
data=_session_from_principal(
principal,
guest_access_enabled=True,
authentik_enabled=AuthentikOidcService(settings).enabled,
)
)
@router.post("/logout", response_model=AuthSessionEnvelope)
def logout(response: Response) -> AuthSessionEnvelope:
settings = get_settings()
response.delete_cookie(key=COOKIE_NAME, path="/", httponly=True, samesite="strict")
return AuthSessionEnvelope(
data=AuthSession(
authentication_required=settings.auth_enabled,
authenticated=not settings.auth_enabled,
guest_access_enabled=settings.auth_enabled and settings.guest_access_enabled,
authentik_enabled=AuthentikOidcService(settings).enabled,
)
)
+154 -6
View File
@@ -5,19 +5,22 @@ from datetime import datetime
from typing import Any from typing import Any
from uuid import UUID from uuid import UUID
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response
from fastapi import UploadFile from fastapi import UploadFile
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.models import Area, Project from app.core.config import get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.core.public_demo import is_public_demo_project
from app.db.session import get_db from app.db.session import get_db
from app.models import Area, Project
from app.schemas import ( from app.schemas import (
BathymetryPartitionFinalizationResult, BathymetryPartitionFinalizationResult,
BathymetrySourceProbeRead, BathymetrySourceProbeRead,
BathymetrySourceRead, BathymetrySourceRead,
DatasetList, DatasetList,
DhmvProductRead, DhmvProductRead,
SpwTerrainAcquireRequest,
SpwTerrainProductRead,
Envelope, Envelope,
FloodHazardProductRead, FloodHazardProductRead,
FloodHazardSelectionResponse, FloodHazardSelectionResponse,
@@ -48,6 +51,9 @@ from app.schemas import (
FloodHazardSelectionRequest, FloodHazardSelectionRequest,
BathymetryPartitionFinalizeRequest, BathymetryPartitionFinalizeRequest,
BathymetryProfileAcquireRequest, BathymetryProfileAcquireRequest,
BathymetryRasterSelectionRequest,
BathymetryRasterSelectionResponse,
MdkBathymetryAcquireRequest,
ThematicRasterAcquireRequest, ThematicRasterAcquireRequest,
ThematicRasterProductRead, ThematicRasterProductRead,
ThematicRasterSelectionResponse, ThematicRasterSelectionResponse,
@@ -85,13 +91,17 @@ from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.spw_terrain_service import SpwTerrainService
from app.services.terrain_analysis_service import TerrainAnalysisService from app.services.terrain_analysis_service import TerrainAnalysisService
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.bathymetry_raster_analysis_service import BathymetryRasterAnalysisService
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
from app.services.walous_land_cover_service import WalousLandCoverService
from app.utils.response import envelope from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"]) router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
@@ -154,6 +164,12 @@ async def upload_dataset(
source_version: str | None = Form(None), source_version: str | None = Form(None),
db: Session = Depends(get_db), db: Session = Depends(get_db),
): ):
if is_public_demo_project(project_id):
raise AppError(
code="PUBLIC_DEMO_UPLOAD_FORBIDDEN",
message="Operator uploads are not accepted in the public demo project.",
status_code=403,
)
if area_id is not None: if area_id is not None:
area = db.get(Area, area_id) area = db.get(Area, area_id)
if not area: if not area:
@@ -237,6 +253,33 @@ def list_dhmv_products(project_id: UUID, db: Session = Depends(get_db)):
return envelope({"items": items, "total": len(items)}) return envelope({"items": items, "total": len(items)})
@router.post("/datasets/spw-terrain/acquire", response_model=Envelope[JobRead])
def acquire_bounded_spw_terrain(
project_id: UUID,
payload: SpwTerrainAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="raster.spw-terrain.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: SpwTerrainService.acquire(db, project_id, payload),
)
return envelope(job)
@router.get(
"/datasets/spw-terrain/products",
response_model=Envelope[ItemList[SpwTerrainProductRead]],
)
def list_spw_terrain_products(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = SpwTerrainService.list_products()
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/grb/acquire", response_model=Envelope[JobRead]) @router.post("/datasets/grb/acquire", response_model=Envelope[JobRead])
def acquire_bounded_grb( def acquire_bounded_grb(
project_id: UUID, project_id: UUID,
@@ -339,6 +382,25 @@ def probe_mdk_bathymetry_readiness(project_id: UUID, db: Session = Depends(get_d
return envelope(MdkBathymetryProbeService.probe()) return envelope(MdkBathymetryProbeService.probe())
@router.post(
"/datasets/bathymetry/mdk/acquire",
response_model=Envelope[JobRead],
)
def acquire_bounded_mdk_bathymetry(
project_id: UUID,
payload: MdkBathymetryAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="raster.mdk_bathymetry.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: MdkBathymetryAcquisitionService.acquire(db, project_id, payload),
)
return envelope(job)
@router.post( @router.post(
"/datasets/bathymetry/profiles/acquire", "/datasets/bathymetry/profiles/acquire",
response_model=Envelope[JobRead], response_model=Envelope[JobRead],
@@ -435,6 +497,33 @@ def list_thematic_raster_products(project_id: UUID, db: Session = Depends(get_db
return envelope({"items": items, "total": len(items)}) return envelope({"items": items, "total": len(items)})
@router.post("/datasets/walous/acquire", response_model=Envelope[JobRead])
def acquire_bounded_walous_land_cover(
project_id: UUID,
payload: ThematicRasterAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="raster.walous.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: WalousLandCoverService.acquire(db, project_id, payload),
)
return envelope(job)
@router.get(
"/datasets/walous/products",
response_model=Envelope[ItemList[ThematicRasterProductRead]],
)
def list_walous_products(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = WalousLandCoverService.list_products()
return envelope({"items": items, "total": len(items)})
@router.get("/datasets", response_model=Envelope[DatasetList]) @router.get("/datasets", response_model=Envelope[DatasetList])
def list_datasets( def list_datasets(
project_id: UUID, project_id: UUID,
@@ -630,16 +719,17 @@ def select_vector_features(
payload.bbox.model_dump(), payload.bbox.model_dump(),
selection_area.geometry, selection_area.geometry,
) )
full_dataset_area = covers_full_area and VectorFeatureService.can_use_full_area_fast_path( dataset_is_preclipped_to_area = VectorFeatureService.can_use_full_area_fast_path(
dataset, dataset,
selection_area.id, selection_area.id,
) )
full_dataset_area = covers_full_area and dataset_is_preclipped_to_area
preclipped_partition_filter = VectorFeatureService.preclipped_partition_filter( preclipped_partition_filter = VectorFeatureService.preclipped_partition_filter(
dataset, dataset,
getattr(selection_area, "name", None), getattr(selection_area, "name", None),
) )
selection_kwargs.update( selection_kwargs.update(
selection_geometry=selection_geometry, selection_geometry=None if dataset_is_preclipped_to_area else selection_geometry,
selection_area_id=selection_area.id, selection_area_id=selection_area.id,
full_dataset_area=full_dataset_area, full_dataset_area=full_dataset_area,
preclipped_partition_filter=preclipped_partition_filter, preclipped_partition_filter=preclipped_partition_filter,
@@ -652,7 +742,7 @@ def select_vector_features(
"total_feature_count": result.get("total_feature_count"), "total_feature_count": result.get("total_feature_count"),
} }
if selection_area is not None: if selection_area is not None:
summary_kwargs["selection_geometry"] = selection_geometry summary_kwargs["selection_geometry"] = None if dataset_is_preclipped_to_area else selection_geometry
summary_kwargs["full_dataset_area"] = full_dataset_area summary_kwargs["full_dataset_area"] = full_dataset_area
summary_kwargs["preclipped_partition_filter"] = preclipped_partition_filter summary_kwargs["preclipped_partition_filter"] = preclipped_partition_filter
result["summary"] = VectorFeatureService.summarize_features_by_bbox(db, **summary_kwargs) result["summary"] = VectorFeatureService.summarize_features_by_bbox(db, **summary_kwargs)
@@ -888,6 +978,33 @@ def raster_terrain_image(
) )
@router.post(
"/datasets/{dataset_id}/raster/bathymetry/select",
response_model=Envelope[BathymetryRasterSelectionResponse],
)
def raster_bathymetry_selection(
project_id: UUID,
dataset_id: UUID,
payload: BathymetryRasterSelectionRequest,
db: Session = Depends(get_db),
):
return envelope(BathymetryRasterAnalysisService.analyze(db, project_id, dataset_id, payload))
@router.get("/datasets/{dataset_id}/raster/bathymetry/image")
def raster_bathymetry_image(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
):
content = BathymetryRasterAnalysisService.render_png(db, project_id, dataset_id)
return Response(
content=content,
media_type="image/png",
headers={"Cache-Control": "private, max-age=86400"},
)
@router.post( @router.post(
"/datasets/{dataset_id}/raster/flood-hazard/select", "/datasets/{dataset_id}/raster/flood-hazard/select",
response_model=Envelope[FloodHazardSelectionResponse], response_model=Envelope[FloodHazardSelectionResponse],
@@ -954,6 +1071,33 @@ def raster_thematic_image(
) )
@router.post(
"/datasets/{dataset_id}/raster/walous/select",
response_model=Envelope[ThematicRasterSelectionResponse],
)
def raster_walous_selection(
project_id: UUID,
dataset_id: UUID,
payload: ThematicRasterSelectionRequest,
db: Session = Depends(get_db),
):
return envelope(WalousLandCoverService.analyze(db, project_id, dataset_id, payload))
@router.get("/datasets/{dataset_id}/raster/walous/image")
def raster_walous_image(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
):
content = WalousLandCoverService.render_png(db, project_id, dataset_id)
return Response(
content=content,
media_type="image/png",
headers={"Cache-Control": "private, max-age=86400"},
)
@router.get( @router.get(
"/datasets/{dataset_id}/raster/stats", "/datasets/{dataset_id}/raster/stats",
response_model=Envelope[RasterStatsResponse], response_model=Envelope[RasterStatsResponse],
@@ -1035,11 +1179,14 @@ def raster_tile_dataset(
project_id: UUID, project_id: UUID,
dataset_id: UUID, dataset_id: UUID,
payload: RasterTileRequest, payload: RasterTileRequest,
request: Request,
db: Session = Depends(get_db), db: Session = Depends(get_db),
): ):
dataset = DatasetService.get_dataset(db, dataset_id) dataset = DatasetService.get_dataset(db, dataset_id)
if dataset.project_id != project_id: if dataset.project_id != project_id:
raise HTTPException(status_code=404, detail="Dataset not found") raise HTTPException(status_code=404, detail="Dataset not found")
principal = getattr(request.state, "auth_principal", None)
guest_max_tiles = get_settings().yolo_max_tiles if getattr(principal, "role", None) == "guest" else None
job = _run_job_sync( job = _run_job_sync(
db=db, db=db,
project_id=project_id, project_id=project_id,
@@ -1052,6 +1199,7 @@ def raster_tile_dataset(
tile_size=payload.tile_size, tile_size=payload.tile_size,
overlap=payload.overlap, overlap=payload.overlap,
output_name=payload.output_name, output_name=payload.output_name,
max_tiles=guest_max_tiles,
), ),
) )
return envelope(job) return envelope(job)
+150 -11
View File
@@ -2,14 +2,21 @@ from __future__ import annotations
from uuid import UUID from uuid import UUID
from fastapi import APIRouter, Depends from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.api.guest_scope import (
assert_guest_project_scope,
guest_project_scope,
guest_scoped_project_filter,
)
from app.db.session import get_db from app.db.session import get_db
from app.schemas import ( from app.schemas import (
AnalysisQaResponse, AnalysisQaResponse,
DetectionListResponse, DetectionListResponse,
DetectionModelsResponse, DetectionModelsResponse,
DetectionComparisonRequest,
DetectionComparisonResponse,
DetectionQaRequest, DetectionQaRequest,
DetectionRead, DetectionRead,
DetectionRunListResponse, DetectionRunListResponse,
@@ -18,9 +25,12 @@ from app.schemas import (
DetectionRunResponse, DetectionRunResponse,
Envelope, Envelope,
GeoJsonFeatureCollection, GeoJsonFeatureCollection,
JobRead,
ModelAssetListResponse, ModelAssetListResponse,
YoloPreflightResponse, YoloPreflightResponse,
) )
from app.services.detection_comparison_service import DetectionComparisonService
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService from app.services.detection_service import DetectionService
from app.services.model_asset_catalog_service import ModelAssetCatalogService from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.model_registry_service import ModelRegistryService from app.services.model_registry_service import ModelRegistryService
@@ -45,18 +55,25 @@ def get_yolo_preflight(
tile_manifest_path: str | None = None, tile_manifest_path: str | None = None,
check_model_load: bool = False, check_model_load: bool = False,
model_asset_id: str | None = None, model_asset_id: str | None = None,
db: Session = Depends(get_db),
) -> dict: ) -> dict:
return envelope( return envelope(
YoloPreflightService.run( YoloPreflightService.run(
tile_manifest_path=tile_manifest_path, tile_manifest_path=tile_manifest_path,
check_model_load=check_model_load, check_model_load=check_model_load,
model_asset_id=model_asset_id, model_asset_id=model_asset_id,
db=db,
) )
) )
@router.post("/run", response_model=Envelope[DetectionRunResponse]) @router.post("/run", response_model=Envelope[DetectionRunResponse])
def run_detection(payload: DetectionRunRequest, db: Session = Depends(get_db)) -> dict: def run_detection(
payload: DetectionRunRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
assert_guest_project_scope(request, payload.project_id)
result = DetectionService.run_detection( result = DetectionService.run_detection(
db=db, db=db,
project_id=payload.project_id, project_id=payload.project_id,
@@ -71,18 +88,60 @@ def run_detection(payload: DetectionRunRequest, db: Session = Depends(get_db)) -
return envelope(result.model_dump()) return envelope(result.model_dump())
@router.get("/runs", response_model=Envelope[DetectionRunListResponse]) @router.post("/run-async", response_model=Envelope[JobRead])
def list_detection_runs( def queue_detection(
project_id: UUID | None = None, payload: DetectionRunRequest,
dataset_id: UUID | None = None, request: Request,
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
return envelope(DetectionService.list_runs(db, project_id=project_id, dataset_id=dataset_id).model_dump()) """Queue a detection run for the background worker.
Tiled GPU inference takes minutes; ``POST /detection/run`` performs it
inside the request and is only appropriate for a handful of tiles. Poll
``GET /jobs/{id}`` for the queued run instead.
"""
assert_guest_project_scope(request, payload.project_id)
job = DetectionService.enqueue_detection(
db=db,
project_id=payload.project_id,
dataset_id=payload.dataset_id,
model_id=payload.model_id,
model_asset_id=payload.model_asset_id,
confidence_threshold=payload.confidence_threshold,
class_filter=payload.class_filter,
tile_manifest_path=payload.tile_manifest_path,
parameters_json=payload.parameters_json,
)
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
@router.get("/runs", response_model=Envelope[DetectionRunListResponse])
def list_detection_runs(
request: Request,
project_id: UUID | None = None,
dataset_id: UUID | None = None,
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
project_id = guest_scoped_project_filter(request, project_id)
return envelope(
DetectionService.list_runs(
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
).model_dump()
)
@router.get("/runs/{analysis_run_id}", response_model=Envelope[DetectionRunRead]) @router.get("/runs/{analysis_run_id}", response_model=Envelope[DetectionRunRead])
def get_detection_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> dict: def get_detection_run(
return envelope(DetectionService.get_run(db, analysis_run_id).model_dump()) analysis_run_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(run.model_dump())
@router.get( @router.get(
@@ -91,11 +150,22 @@ def get_detection_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> d
) )
def list_detection_run_detections( def list_detection_run_detections(
analysis_run_id: UUID, analysis_run_id: UUID,
request: Request,
dataset_id: UUID | None = None, dataset_id: UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope( return envelope(
DetectionService.list_detections( DetectionService.list_detections(
db, db,
@@ -103,6 +173,8 @@ def list_detection_run_detections(
dataset_id=dataset_id, dataset_id=dataset_id,
class_name=class_name, class_name=class_name,
min_confidence=min_confidence, min_confidence=min_confidence,
limit=limit,
offset=offset,
).model_dump() ).model_dump()
) )
@@ -113,11 +185,22 @@ def list_detection_run_detections(
) )
def list_dataset_detections( def list_dataset_detections(
dataset_id: UUID, dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None, analysis_run_id: UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope( return envelope(
DetectionService.list_detections( DetectionService.list_detections(
db, db,
@@ -125,13 +208,21 @@ def list_dataset_detections(
dataset_id=dataset_id, dataset_id=dataset_id,
class_name=class_name, class_name=class_name,
min_confidence=min_confidence, min_confidence=min_confidence,
limit=limit,
offset=offset,
).model_dump() ).model_dump()
) )
@router.get("/detections/{detection_id}", response_model=Envelope[DetectionRead]) @router.get("/detections/{detection_id}", response_model=Envelope[DetectionRead])
def get_detection(detection_id: UUID, db: Session = Depends(get_db)) -> dict: def get_detection(
return envelope(DetectionService.get_detection(db, detection_id).model_dump()) detection_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
detection = DetectionService.get_detection(db, detection_id)
assert_guest_project_scope(request, detection.project_id)
return envelope(detection.model_dump())
@router.get( @router.get(
@@ -140,13 +231,24 @@ def get_detection(detection_id: UUID, db: Session = Depends(get_db)) -> dict:
) )
def get_detection_run_geojson( def get_detection_run_geojson(
analysis_run_id: UUID, analysis_run_id: UUID,
request: Request,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope( return envelope(
DetectionService.detections_to_geojson( DetectionService.detections_to_geojson(
db, db,
limit=limit,
analysis_run_id=analysis_run_id, analysis_run_id=analysis_run_id,
class_name=class_name, class_name=class_name,
min_confidence=min_confidence, min_confidence=min_confidence,
@@ -160,14 +262,25 @@ def get_detection_run_geojson(
) )
def get_dataset_detection_geojson( def get_dataset_detection_geojson(
dataset_id: UUID, dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None, analysis_run_id: UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope( return envelope(
DetectionService.detections_to_geojson( DetectionService.detections_to_geojson(
db, db,
limit=limit,
analysis_run_id=analysis_run_id, analysis_run_id=analysis_run_id,
dataset_id=dataset_id, dataset_id=dataset_id,
class_name=class_name, class_name=class_name,
@@ -176,6 +289,27 @@ def get_dataset_detection_geojson(
) )
@router.post("/runs/compare", response_model=Envelope[DetectionComparisonResponse])
def compare_detection_runs(payload: DetectionComparisonRequest, db: Session = Depends(get_db)) -> dict:
"""Rank several runs against one reference on average precision.
The workbench ranks model variants by a stored F1 measured at each
variant's own confidence threshold, which orders the thresholds as much as
the models. Average precision describes the whole ranking a model produced.
Comparability is reported first: runs over different rasters, different
references or different inference coverage are not alternatives.
"""
return envelope(
DetectionComparisonService.compare_runs(
db,
analysis_run_ids=payload.analysis_run_ids,
reference_dataset_id=payload.reference_dataset_id,
iou_threshold=payload.iou_threshold,
)
)
@router.post( @router.post(
"/runs/{analysis_run_id}/qa/reference", "/runs/{analysis_run_id}/qa/reference",
response_model=Envelope[AnalysisQaResponse], response_model=Envelope[AnalysisQaResponse],
@@ -183,8 +317,12 @@ def get_dataset_detection_geojson(
def compare_detection_run_with_reference( def compare_detection_run_with_reference(
analysis_run_id: UUID, analysis_run_id: UUID,
payload: DetectionQaRequest, payload: DetectionQaRequest,
request: Request,
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
run = DetectionService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope( return envelope(
DetectionService.compare_detections_with_reference( DetectionService.compare_detections_with_reference(
db=db, db=db,
@@ -193,5 +331,6 @@ def compare_detection_run_with_reference(
iou_threshold=payload.iou_threshold, iou_threshold=payload.iou_threshold,
class_name=payload.class_name, class_name=payload.class_name,
min_confidence=payload.min_confidence, min_confidence=payload.min_confidence,
calibration_thresholds=payload.calibration_thresholds,
) )
) )
+64 -11
View File
@@ -2,10 +2,12 @@ from __future__ import annotations
from uuid import UUID from uuid import UUID
from fastapi import APIRouter, Depends, Query from fastapi import APIRouter, Depends, Query, Request
from fastapi.responses import FileResponse from fastapi.responses import FileResponse
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.api.guest_scope import assert_guest_project_scope, guest_project_scope
from app.core.errors import AppError
from app.db.session import get_db from app.db.session import get_db
from app.schemas import Envelope from app.schemas import Envelope
from app.schemas.export import ( from app.schemas.export import (
@@ -19,13 +21,30 @@ from app.schemas.export import (
ReportExportRequest, ReportExportRequest,
) )
from app.services.export_service import ExportService from app.services.export_service import ExportService
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService
from app.utils.response import envelope from app.utils.response import envelope
router = APIRouter(prefix="/exports", tags=["exports"]) router = APIRouter(prefix="/exports", tags=["exports"])
@router.post("/geojson", response_model=Envelope[ExportCreateResponse]) @router.post("/geojson", response_model=Envelope[ExportCreateResponse])
def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db)): def export_geojson(
payload: GeoJsonExportRequest,
request: Request,
db: Session = Depends(get_db),
):
if guest_project_scope(request) is not None:
if payload.export_kind in {"dataset", "vector_selection"} and payload.dataset_id is not None:
dataset = DatasetService.get_dataset(db, payload.dataset_id)
assert_guest_project_scope(request, dataset.project_id)
elif payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
run = DetectionService.get_run(db, payload.analysis_run_id)
assert_guest_project_scope(request, run.project_id)
elif payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
run = SegmentationService.get_run(db, payload.analysis_run_id)
assert_guest_project_scope(request, run.project_id)
if payload.export_kind == "vector_selection" and payload.dataset_id is not None and payload.bbox is not None: if payload.export_kind == "vector_selection" and payload.dataset_id is not None and payload.bbox is not None:
return envelope( return envelope(
ExportService.export_vector_selection_geojson( ExportService.export_vector_selection_geojson(
@@ -39,7 +58,12 @@ def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db))
) )
if payload.export_kind == "detection_run" and payload.analysis_run_id is not None: if payload.export_kind == "detection_run" and payload.analysis_run_id is not None:
return envelope( return envelope(
ExportService.export_detection_run_geojson(db, payload.analysis_run_id, payload.name).model_dump(mode="json") ExportService.export_detection_run_geojson(
db,
payload.analysis_run_id,
payload.name,
intended_use=payload.intended_use,
).model_dump(mode="json")
) )
if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None: if payload.export_kind == "segmentation_run" and payload.analysis_run_id is not None:
return envelope( return envelope(
@@ -47,21 +71,40 @@ def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db))
) )
if payload.dataset_id is not None: if payload.dataset_id is not None:
return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json")) return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json"))
return envelope({}) raise AppError(
code="INVALID_EXPORT_REQUEST",
message="GeoJSON export request does not match any supported export target",
status_code=422,
)
@router.post("/metadata", response_model=Envelope[ExportCreateResponse]) @router.post("/metadata", response_model=Envelope[ExportCreateResponse])
def export_project_metadata(payload: MetadataExportRequest, db: Session = Depends(get_db)): def export_project_metadata(
payload: MetadataExportRequest,
request: Request,
db: Session = Depends(get_db),
):
assert_guest_project_scope(request, payload.project_id)
return envelope(ExportService.export_project_metadata(db, payload.project_id, payload.name).model_dump(mode="json")) return envelope(ExportService.export_project_metadata(db, payload.project_id, payload.name).model_dump(mode="json"))
@router.post("/report", response_model=Envelope[ExportCreateResponse]) @router.post("/report", response_model=Envelope[ExportCreateResponse])
def export_project_report(payload: ReportExportRequest, db: Session = Depends(get_db)): def export_project_report(
payload: ReportExportRequest,
request: Request,
db: Session = Depends(get_db),
):
assert_guest_project_scope(request, payload.project_id)
return envelope(ExportService.export_project_report(db, payload.project_id, payload.name).model_dump(mode="json")) return envelope(ExportService.export_project_report(db, payload.project_id, payload.name).model_dump(mode="json"))
@router.post("/map-result", response_model=Envelope[ExportCreateResponse]) @router.post("/map-result", response_model=Envelope[ExportCreateResponse])
def export_map_result(payload: MapResultExportRequest, db: Session = Depends(get_db)): def export_map_result(
payload: MapResultExportRequest,
request: Request,
db: Session = Depends(get_db),
):
assert_guest_project_scope(request, payload.project_id)
return envelope(ExportService.export_map_result(db, payload).model_dump(mode="json")) return envelope(ExportService.export_map_result(db, payload).model_dump(mode="json"))
@@ -71,25 +114,35 @@ def export_map_result(payload: MapResultExportRequest, db: Session = Depends(get
) )
def list_project_exports( def list_project_exports(
project_id: UUID, project_id: UUID,
request: Request,
limit: int = Query(default=50, ge=1, le=100), limit: int = Query(default=50, ge=1, le=100),
offset: int = Query(default=0, ge=0), offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db), db: Session = Depends(get_db),
): ):
assert_guest_project_scope(request, project_id)
return envelope(ExportService.list_project_exports(db, project_id, limit=limit, offset=offset).model_dump(mode="json")) return envelope(ExportService.list_project_exports(db, project_id, limit=limit, offset=offset).model_dump(mode="json"))
@router.get("/{export_id}", response_model=Envelope[ExportRead]) @router.get("/{export_id}", response_model=Envelope[ExportRead])
def get_export(export_id: UUID, db: Session = Depends(get_db)): def get_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
return envelope(ExportService.get_export(db, export_id).model_dump(mode="json")) export = ExportService.get_export(db, export_id)
assert_guest_project_scope(request, export.project_id)
return envelope(export.model_dump(mode="json"))
@router.get("/{export_id}/download") @router.get("/{export_id}/download")
def download_export(export_id: UUID, db: Session = Depends(get_db)): def download_export(export_id: UUID, request: Request, db: Session = Depends(get_db)):
if guest_project_scope(request) is not None:
export = ExportService.get_export(db, export_id)
assert_guest_project_scope(request, export.project_id)
path = ExportService.get_export_download_path(db, export_id) path = ExportService.get_export_download_path(db, export_id)
media_type = "text/html" if path.suffix.lower() in {".html", ".htm"} else "application/json" media_type = "text/html" if path.suffix.lower() in {".html", ".htm"} else "application/json"
return FileResponse(path, filename=path.name, media_type=media_type) return FileResponse(path, filename=path.name, media_type=media_type)
@router.get("/{export_id}/content", response_model=Envelope[ExportContentResponse]) @router.get("/{export_id}/content", response_model=Envelope[ExportContentResponse])
def get_export_content(export_id: UUID, db: Session = Depends(get_db)): def get_export_content(export_id: UUID, request: Request, db: Session = Depends(get_db)):
if guest_project_scope(request) is not None:
export = ExportService.get_export(db, export_id)
assert_guest_project_scope(request, export.project_id)
return envelope(ExportService.get_export_content(db, export_id).model_dump(mode="json")) return envelope(ExportService.get_export_content(db, export_id).model_dump(mode="json"))
+20 -2
View File
@@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from fastapi import APIRouter, Depends from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.errors import AppError from app.core.errors import AppError
@@ -43,6 +43,19 @@ def _assert_project_exists(db: Session, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
def _assert_guest_project_scope(request: Request, project_id) -> None:
principal = getattr(request.state, "auth_principal", None)
if (
getattr(principal, "role", None) == "guest"
and getattr(principal, "project_id", None) != project_id
):
raise AppError(
code="GUEST_PROJECT_SCOPE_REQUIRED",
message="Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
status_code=403,
)
def _normalize_layer_input(layers: list[str] | None) -> list[str]: def _normalize_layer_input(layers: list[str] | None) -> list[str]:
return [layer.strip() for layer in (layers or []) if isinstance(layer, str) and layer.strip()] return [layer.strip() for layer in (layers or []) if isinstance(layer, str) and layer.strip()]
@@ -66,7 +79,12 @@ def get_coverage_catalog() -> dict:
@router.post("/coverage/resolve", response_model=Envelope[CoverageResolveResponse]) @router.post("/coverage/resolve", response_model=Envelope[CoverageResolveResponse])
def resolve_project_coverage(payload: CoverageResolveRequest, db: Session = Depends(get_db)) -> dict: def resolve_project_coverage(
payload: CoverageResolveRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
_assert_guest_project_scope(request, payload.project_id)
result = CoverageRegistryService.resolve( result = CoverageRegistryService.resolve(
db, db,
project_id=payload.project_id, project_id=payload.project_id,
+12 -15
View File
@@ -47,21 +47,18 @@ def _database_checks() -> dict[str, str]:
with get_engine().connect() as connection: with get_engine().connect() as connection:
connection.execute(text("SELECT 1")) connection.execute(text("SELECT 1"))
checks["database"] = "ok" checks["database"] = "ok"
postgis_version = connection.execute( connection.execute(
text("SELECT PostGIS_Version()") text("SELECT PostGIS_Version()")
).scalar_one() ).scalar_one()
checks["postgis"] = f"ok:{postgis_version}" checks["postgis"] = "ok"
database_head = connection.execute( database_head = connection.execute(
text("SELECT version_num FROM alembic_version") text("SELECT version_num FROM alembic_version")
).scalar_one() ).scalar_one()
expected_heads = _expected_migration_heads() expected_heads = _expected_migration_heads()
if len(expected_heads) == 1 and database_head == expected_heads[0]: if len(expected_heads) == 1 and database_head == expected_heads[0]:
checks["migration"] = f"ok:{database_head}" checks["migration"] = "ok"
else: else:
checks["migration"] = ( checks["migration"] = "degraded"
f"degraded:database={database_head};"
f"expected={','.join(expected_heads) or 'none'}"
)
except Exception: except Exception:
return checks return checks
return checks return checks
@@ -94,9 +91,7 @@ def _readiness_payload() -> HealthResponse:
return HealthResponse( return HealthResponse(
status="ok" if ready else "degraded", status="ok" if ready else "degraded",
service="geointel-backend", service="geointel-backend",
version=settings.app_version, version="public",
build_sha=settings.build_sha,
build_time=settings.build_time,
database=checks["database"], database=checks["database"],
postgis=checks["postgis"], postgis=checks["postgis"],
migration=checks["migration"], migration=checks["migration"],
@@ -107,13 +102,10 @@ def _readiness_payload() -> HealthResponse:
@router.get("/health/live", response_model=HealthResponse) @router.get("/health/live", response_model=HealthResponse)
def liveness() -> HealthResponse: def liveness() -> HealthResponse:
settings = get_settings()
return HealthResponse( return HealthResponse(
status="ok", status="ok",
service="geointel-backend", service="geointel-backend",
version=settings.app_version, version="public",
build_sha=settings.build_sha,
build_time=settings.build_time,
) )
@@ -147,6 +139,11 @@ def capabilities() -> SystemCapabilitiesEnvelope:
) )
yolo_configured = bool(configured_yolo and configured_yolo.configured) yolo_configured = bool(configured_yolo and configured_yolo.configured)
yolo_status = configured_yolo.status if configured_yolo else "not_configured" yolo_status = configured_yolo.status if configured_yolo else "not_configured"
configured_sam = ModelRegistryService.get_model_capability(
settings.sam_model_id,
settings=settings,
task_type="segmentation",
)
postgis_ready = _database_checks()["postgis"].startswith("ok:") postgis_ready = _database_checks()["postgis"].startswith("ok:")
return SystemCapabilitiesEnvelope( return SystemCapabilitiesEnvelope(
data=SystemCapabilities( data=SystemCapabilities(
@@ -155,7 +152,7 @@ def capabilities() -> SystemCapabilitiesEnvelope:
geopandas=_dependency_enabled("geopandas"), geopandas=_dependency_enabled("geopandas"),
yolo=yolo_configured, yolo=yolo_configured,
yolo_status=yolo_status, yolo_status=yolo_status,
sam=False, sam=bool(configured_sam and configured_sam.configured),
grb="bounded", grb="bounded",
sentinel="planned", sentinel="planned",
version=settings.app_version, version=settings.app_version,
+23 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
from typing import Literal from typing import Literal
from uuid import UUID from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, status from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.db.session import get_db from app.db.session import get_db
@@ -17,12 +17,34 @@ router = APIRouter(prefix="/projects", tags=["projects"])
@router.get("", response_model=Envelope[ProjectList]) @router.get("", response_model=Envelope[ProjectList])
def list_projects( def list_projects(
request: Request,
limit: int = Query(default=50, ge=1, le=200), limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0), offset: int = Query(default=0, ge=0),
name: str | None = Query(default=None, min_length=1, max_length=255), name: str | None = Query(default=None, min_length=1, max_length=255),
project_status: Literal["active", "archived", "all"] = Query(default="active", alias="status"), project_status: Literal["active", "archived", "all"] = Query(default="active", alias="status"),
db: Session = Depends(get_db), db: Session = Depends(get_db),
): ):
principal = getattr(request.state, "auth_principal", None)
if principal is not None and principal.role == "guest":
project = ProjectService.get_project(db, principal.project_id)
status_matches = bool(
project is not None
and (project_status == "all" or project.status == project_status)
)
name_matches = bool(
project is not None
and (name is None or name.casefold() in project.name.casefold())
)
matches = project is not None and status_matches and name_matches
visible = [project] if matches and offset == 0 else []
return envelope(
{
"items": [ProjectRead.model_validate(item).model_dump() for item in visible[:limit]],
"total": 1 if matches else 0,
"limit": limit,
"offset": offset,
}
)
projects, total = ProjectService.list_projects( projects, total = ProjectService.list_projects(
db, db,
limit=limit, limit=limit,
+14 -1
View File
@@ -40,9 +40,22 @@ def list_quality_checks(
def get_quality_check_evidence_geojson( def get_quality_check_evidence_geojson(
project_id: UUID, project_id: UUID,
quality_check_id: UUID, quality_check_id: UUID,
limit: int = Query(
default=QualityEvidenceService.DEFAULT_EVIDENCE_LIMIT,
ge=0,
le=100_000,
description="Maximum evidence features to draw; 0 returns everything. Misses and false positives first.",
),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
return envelope(QualityEvidenceService.evidence_geojson(db, project_id=project_id, quality_check_id=quality_check_id)) return envelope(
QualityEvidenceService.evidence_geojson(
db,
project_id=project_id,
quality_check_id=quality_check_id,
limit=limit,
)
)
@router.get( @router.get(
+123 -11
View File
@@ -2,14 +2,20 @@ from __future__ import annotations
from uuid import UUID from uuid import UUID
from fastapi import APIRouter, Depends from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.api.guest_scope import (
assert_guest_project_scope,
guest_project_scope,
guest_scoped_project_filter,
)
from app.db.session import get_db from app.db.session import get_db
from app.schemas import ( from app.schemas import (
AnalysisQaResponse, AnalysisQaResponse,
Envelope, Envelope,
GeoJsonFeatureCollection, GeoJsonFeatureCollection,
JobRead,
SegmentationListResponse, SegmentationListResponse,
SegmentationModelsResponse, SegmentationModelsResponse,
SegmentationQaRequest, SegmentationQaRequest,
@@ -20,6 +26,8 @@ from app.schemas import (
SegmentationRunResponse, SegmentationRunResponse,
) )
from app.services.model_registry_service import ModelRegistryService from app.services.model_registry_service import ModelRegistryService
from app.services.dataset_service import DatasetService
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService from app.services.segmentation_service import SegmentationService
from app.utils.response import envelope from app.utils.response import envelope
@@ -32,7 +40,12 @@ def list_segmentation_models() -> dict:
@router.post("/run", response_model=Envelope[SegmentationRunResponse]) @router.post("/run", response_model=Envelope[SegmentationRunResponse])
def run_segmentation(payload: SegmentationRunRequest, db: Session = Depends(get_db)) -> dict: def run_segmentation(
payload: SegmentationRunRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
assert_guest_project_scope(request, payload.project_id)
result = SegmentationService.run_segmentation( result = SegmentationService.run_segmentation(
db=db, db=db,
project_id=payload.project_id, project_id=payload.project_id,
@@ -46,18 +59,58 @@ def run_segmentation(payload: SegmentationRunRequest, db: Session = Depends(get_
return envelope(result.model_dump()) return envelope(result.model_dump())
@router.get("/runs", response_model=Envelope[SegmentationRunListResponse]) @router.post("/run-async", response_model=Envelope[JobRead])
def list_segmentation_runs( def queue_segmentation(
project_id: UUID | None = None, payload: SegmentationRunRequest,
dataset_id: UUID | None = None, request: Request,
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
return envelope(SegmentationService.list_runs(db, project_id=project_id, dataset_id=dataset_id).model_dump()) """Queue a segmentation run for the background worker.
Configured segmentation walks the same tile manifest as detection and is
just as unsuited to running inside the request. Poll ``GET /jobs/{id}``.
"""
assert_guest_project_scope(request, payload.project_id)
job = SegmentationService.enqueue_segmentation(
db=db,
project_id=payload.project_id,
dataset_id=payload.dataset_id,
model_id=payload.model_id,
confidence_threshold=payload.confidence_threshold,
class_filter=payload.class_filter,
tile_manifest_path=payload.tile_manifest_path,
parameters_json=payload.parameters_json,
)
return envelope(JobRead.model_validate(job).model_dump(mode="json"))
@router.get("/runs", response_model=Envelope[SegmentationRunListResponse])
def list_segmentation_runs(
request: Request,
project_id: UUID | None = None,
dataset_id: UUID | None = None,
limit: int = Query(default=DetectionService.DEFAULT_RUN_LIST_LIMIT, ge=0, le=5_000),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
) -> dict:
project_id = guest_scoped_project_filter(request, project_id)
return envelope(
SegmentationService.list_runs(
db, project_id=project_id, dataset_id=dataset_id, limit=limit, offset=offset
).model_dump()
)
@router.get("/runs/{analysis_run_id}", response_model=Envelope[SegmentationRunRead]) @router.get("/runs/{analysis_run_id}", response_model=Envelope[SegmentationRunRead])
def get_segmentation_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -> dict: def get_segmentation_run(
return envelope(SegmentationService.get_run(db, analysis_run_id).model_dump()) analysis_run_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope(run.model_dump())
@router.get( @router.get(
@@ -66,14 +119,27 @@ def get_segmentation_run(analysis_run_id: UUID, db: Session = Depends(get_db)) -
) )
def list_segmentation_run_outputs( def list_segmentation_run_outputs(
analysis_run_id: UUID, analysis_run_id: UUID,
request: Request,
dataset_id: UUID | None = None, dataset_id: UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope( return envelope(
SegmentationService.list_segmentations( SegmentationService.list_segmentations(
db, db,
limit=limit,
offset=offset,
analysis_run_id=analysis_run_id, analysis_run_id=analysis_run_id,
dataset_id=dataset_id, dataset_id=dataset_id,
class_name=class_name, class_name=class_name,
@@ -88,14 +154,27 @@ def list_segmentation_run_outputs(
) )
def list_dataset_segmentations( def list_dataset_segmentations(
dataset_id: UUID, dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None, analysis_run_id: UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope( return envelope(
SegmentationService.list_segmentations( SegmentationService.list_segmentations(
db, db,
limit=limit,
offset=offset,
analysis_run_id=analysis_run_id, analysis_run_id=analysis_run_id,
dataset_id=dataset_id, dataset_id=dataset_id,
class_name=class_name, class_name=class_name,
@@ -105,8 +184,14 @@ def list_dataset_segmentations(
@router.get("/segmentations/{segmentation_id}", response_model=Envelope[SegmentationRead]) @router.get("/segmentations/{segmentation_id}", response_model=Envelope[SegmentationRead])
def get_segmentation(segmentation_id: UUID, db: Session = Depends(get_db)) -> dict: def get_segmentation(
return envelope(SegmentationService.get_segmentation(db, segmentation_id).model_dump()) segmentation_id: UUID,
request: Request,
db: Session = Depends(get_db),
) -> dict:
segmentation = SegmentationService.get_segmentation(db, segmentation_id)
assert_guest_project_scope(request, segmentation.project_id)
return envelope(segmentation.model_dump())
@router.get( @router.get(
@@ -115,13 +200,24 @@ def get_segmentation(segmentation_id: UUID, db: Session = Depends(get_db)) -> di
) )
def get_segmentation_run_geojson( def get_segmentation_run_geojson(
analysis_run_id: UUID, analysis_run_id: UUID,
request: Request,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope( return envelope(
SegmentationService.segmentations_to_geojson( SegmentationService.segmentations_to_geojson(
db, db,
limit=limit,
analysis_run_id=analysis_run_id, analysis_run_id=analysis_run_id,
class_name=class_name, class_name=class_name,
min_confidence=min_confidence, min_confidence=min_confidence,
@@ -135,14 +231,25 @@ def get_segmentation_run_geojson(
) )
def get_dataset_segmentation_geojson( def get_dataset_segmentation_geojson(
dataset_id: UUID, dataset_id: UUID,
request: Request,
analysis_run_id: UUID | None = None, analysis_run_id: UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int = Query(
default=DetectionService.DEFAULT_RESULT_LIMIT,
ge=0,
le=50_000,
description="Maximum results to return; 0 returns everything. Highest confidence first.",
),
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
dataset = DatasetService.get_dataset(db, dataset_id)
assert_guest_project_scope(request, dataset.project_id)
return envelope( return envelope(
SegmentationService.segmentations_to_geojson( SegmentationService.segmentations_to_geojson(
db, db,
limit=limit,
analysis_run_id=analysis_run_id, analysis_run_id=analysis_run_id,
dataset_id=dataset_id, dataset_id=dataset_id,
class_name=class_name, class_name=class_name,
@@ -158,8 +265,12 @@ def get_dataset_segmentation_geojson(
def compare_segmentation_run_with_reference( def compare_segmentation_run_with_reference(
analysis_run_id: UUID, analysis_run_id: UUID,
payload: SegmentationQaRequest, payload: SegmentationQaRequest,
request: Request,
db: Session = Depends(get_db), db: Session = Depends(get_db),
) -> dict: ) -> dict:
if guest_project_scope(request) is not None:
run = SegmentationService.get_run(db, analysis_run_id)
assert_guest_project_scope(request, run.project_id)
return envelope( return envelope(
SegmentationService.compare_segmentations_with_reference( SegmentationService.compare_segmentations_with_reference(
db=db, db=db,
@@ -168,5 +279,6 @@ def compare_segmentation_run_with_reference(
iou_threshold=payload.iou_threshold, iou_threshold=payload.iou_threshold,
class_name=payload.class_name, class_name=payload.class_name,
min_confidence=payload.min_confidence, min_confidence=payload.min_confidence,
calibration_thresholds=payload.calibration_thresholds,
) )
) )
@@ -0,0 +1,87 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Area, Dataset
from app.schemas.common import Envelope
from app.schemas.operations import VectorSelectionResponse
from app.schemas.selection_partitions import VectorPartitionSelectionRequest
from app.services.vector_feature_service import VectorFeatureService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}", tags=["selection-partitions"])
def _product_identity(dataset: Dataset) -> str:
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
return str(metadata.get("product_key") or dataset.reference_layer_name or "")
@router.post(
"/datasets/vector/partitions/select",
response_model=Envelope[VectorSelectionResponse],
)
def select_vector_partitions(
project_id: UUID,
payload: VectorPartitionSelectionRequest,
db: Session = Depends(get_db),
):
datasets = db.query(Dataset).filter(Dataset.id.in_(payload.dataset_ids)).all()
by_id = {dataset.id: dataset for dataset in datasets}
ordered = [by_id.get(dataset_id) for dataset_id in payload.dataset_ids]
if any(dataset is None or dataset.project_id != project_id for dataset in ordered):
raise AppError(code="DATASET_NOT_FOUND", message="One or more selection partitions were not found", status_code=404)
typed_datasets = [dataset for dataset in ordered if dataset is not None]
if any(dataset.dataset_type not in {"vector", "geojson"} or dataset.status != "ready" for dataset in typed_datasets):
raise AppError(
code="INVALID_VECTOR_PARTITIONS",
message="Every selection partition must be a ready vector dataset",
status_code=409,
)
source_names = {dataset.source_name for dataset in typed_datasets}
product_keys = {_product_identity(dataset) for dataset in typed_datasets}
if len(source_names) != 1 or len(product_keys) != 1:
raise AppError(
code="VECTOR_PARTITION_SOURCE_MISMATCH",
message="Selection partitions must belong to one governed source product",
details={"source_names": sorted(str(value) for value in source_names), "product_keys": sorted(product_keys)},
status_code=409,
)
selection_geometry = None
selection_area_id = None
if payload.area_id is not None:
selection_area = db.get(Area, payload.area_id)
if selection_area is None or selection_area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
selection_geometry, _covers_full_area = VectorFeatureService.constrain_bbox_to_area(
payload.bbox.model_dump(),
selection_area.geometry,
)
selection_area_id = selection_area.id
representative = typed_datasets[0]
dataset_ids = [dataset.id for dataset in typed_datasets]
result = VectorFeatureService.select_features_by_bbox(
db,
dataset_id=representative.id,
dataset_ids=dataset_ids,
bbox=payload.bbox.model_dump(),
limit=payload.limit,
dataset=representative,
selection_geometry=selection_geometry,
selection_area_id=selection_area_id,
deduplicate_source_features=True,
)
result.update(
partition_count=len(dataset_ids),
source_name=representative.source_name,
dataset_ids=dataset_ids,
)
return envelope(VectorSelectionResponse(**result).model_dump(exclude_none=True))
+119
View File
@@ -0,0 +1,119 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
from app.schemas import (
DatasetLineageEdgeRead,
DatasetProvenanceRead,
DatasetQuarantineRead,
Envelope,
ItemList,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from app.utils.response import envelope
router = APIRouter(tags=["source-registry"])
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
def list_source_registry(
classification: str | None = None,
db: Session = Depends(get_db),
) -> dict:
query = (
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
)
if classification:
query = query.filter(SourceRegistry.classification == classification.strip().lower())
rows = (
query.group_by(SourceRegistry.id)
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
.all()
)
items = [_source_read(source, snapshot_count=count) for source, count in rows]
return envelope({"items": items, "total": len(items)})
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
normalized_key = source_key.strip().lower()
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
if source is None:
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
snapshots = (
db.query(SourceSnapshot)
.filter(SourceSnapshot.source_registry_id == source.id)
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
.all()
)
detail = SourceRegistryDetailRead(
source=_source_read(source, snapshot_count=len(snapshots)),
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
)
return envelope(detail)
@router.get(
"/projects/{project_id}/datasets/{dataset_id}/provenance",
response_model=Envelope[DatasetProvenanceRead],
)
def get_dataset_provenance(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
) -> dict:
if db.get(Project, project_id) is None:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if dataset is None or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
lineage = (
db.query(DatasetLineageEdge)
.filter(
or_(
DatasetLineageEdge.parent_dataset_id == dataset.id,
DatasetLineageEdge.child_dataset_id == dataset.id,
)
)
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
.all()
)
quarantines = (
db.query(DatasetQuarantine)
.filter(DatasetQuarantine.dataset_id == dataset.id)
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
.all()
)
result = DatasetProvenanceRead(
dataset_id=dataset.id,
source=_source_read(source) if source else None,
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
data_contract_key=dataset.data_contract_key,
data_contract_version=dataset.data_contract_version,
validation_status=dataset.validation_status,
validation_report_json=dataset.validation_report_json,
provenance_status=dataset.provenance_status,
lineage_status=dataset.lineage_status,
quarantine_status=dataset.quarantine_status,
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
)
return envelope(result)
+298 -3
View File
@@ -1,4 +1,6 @@
from pydantic import Field, field_validator from urllib.parse import urlsplit
from pydantic import AliasChoices, Field, field_validator, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -12,24 +14,119 @@ class Settings(BaseSettings):
app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV") app_env: str = Field(default="development", validation_alias="GEOINTEL_ENV")
app_version: str = Field( app_version: str = Field(
default="1.0.0-rc.1", default="1.0.0",
validation_alias="GEOINTEL_APP_VERSION", validation_alias="GEOINTEL_APP_VERSION",
) )
build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA") build_sha: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_SHA")
build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME") build_time: str | None = Field(default=None, validation_alias="GEOINTEL_BUILD_TIME")
api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX") api_prefix: str = Field(default="/api/v1", validation_alias="GEOINTEL_API_PREFIX")
auth_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_ENABLED")
auth_require_https: bool = Field(default=False, validation_alias="GEOINTEL_AUTH_REQUIRE_HTTPS")
auth_username: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_USERNAME")
auth_password_hash: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_PASSWORD_HASH")
auth_session_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTH_SESSION_SECRET")
authentik_issuer: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ISSUER")
authentik_client_id: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_ID")
authentik_client_secret: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_CLIENT_SECRET")
authentik_allowed_email: str | None = Field(default=None, validation_alias="GEOINTEL_AUTHENTIK_ALLOWED_EMAIL")
public_base_url: str = Field(
default="http://localhost:1202",
validation_alias="GEOINTEL_PUBLIC_BASE_URL",
)
auth_session_ttl_seconds: int = Field(
default=43_200,
ge=900,
le=604_800,
validation_alias="GEOINTEL_AUTH_SESSION_TTL_SECONDS",
)
guest_access_enabled: bool = Field(
default=False,
validation_alias="GEOINTEL_GUEST_ACCESS_ENABLED",
)
guest_display_name: str = Field(
default="Gast",
min_length=1,
max_length=64,
validation_alias="GEOINTEL_GUEST_DISPLAY_NAME",
)
guest_session_ttl_seconds: int = Field(
default=7_200,
ge=900,
le=86_400,
validation_alias="GEOINTEL_GUEST_SESSION_TTL_SECONDS",
)
guest_login_requests_per_minute: int = Field(
default=3,
ge=1,
le=60,
validation_alias="GEOINTEL_GUEST_LOGIN_REQUESTS_PER_MINUTE",
)
guest_compute_requests_per_minute: int = Field(
default=4,
ge=1,
le=120,
validation_alias="GEOINTEL_GUEST_COMPUTE_REQUESTS_PER_MINUTE",
)
guest_compute_max_concurrency: int = Field(
default=2,
ge=1,
le=16,
validation_alias="GEOINTEL_GUEST_COMPUTE_MAX_CONCURRENCY",
)
database_url: str = Field( database_url: str = Field(
default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1", default="postgresql+psycopg://geointel:geointel@localhost:5432/geointel?connect_timeout=1",
validation_alias="DATABASE_URL", validation_alias="DATABASE_URL",
) )
storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT") storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT")
max_upload_mb: int = Field(default=500, validation_alias="MAX_UPLOAD_MB") # Analysis consumes only artifacts under storage_root. Provisioning
# workflows that stage tiles elsewhere before ingest can opt out.
allow_external_artifact_paths: bool = Field(
default=False, validation_alias="GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS"
)
max_upload_mb: int = Field(
default=500,
ge=1,
le=2_048,
validation_alias=AliasChoices("GEOINTEL_MAX_UPLOAD_MB", "MAX_UPLOAD_MB"),
)
max_in_memory_vector_mb: int = Field(
default=64,
ge=1,
le=256,
validation_alias="GEOINTEL_MAX_IN_MEMORY_VECTOR_MB",
)
max_raster_pixels: int = Field(
default=40_000_000,
ge=1,
le=500_000_000,
validation_alias="GEOINTEL_MAX_RASTER_PIXELS",
)
max_raster_bands: int = Field(
default=16,
ge=1,
le=256,
validation_alias="GEOINTEL_MAX_RASTER_BANDS",
)
max_decoded_raster_mb: int = Field(
default=1024,
ge=16,
le=8192,
validation_alias="GEOINTEL_MAX_DECODED_RASTER_MB",
)
orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED") orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED")
orthophoto_wms_url: str = Field( orthophoto_wms_url: str = Field(
default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms", default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
validation_alias="ORTHOPHOTO_WMS_URL", validation_alias="ORTHOPHOTO_WMS_URL",
) )
orthophoto_wms_layer: str = Field(default="Ortho", validation_alias="ORTHOPHOTO_WMS_LAYER") orthophoto_wms_layer: str = Field(default="Ortho", validation_alias="ORTHOPHOTO_WMS_LAYER")
spw_orthophoto_wms_url: str = Field(
default="https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_LAST/MapServer/WMSServer",
validation_alias="SPW_ORTHOPHOTO_WMS_URL",
)
brussels_orthophoto_wms_url: str = Field(
default="https://geoservices-grid.irisnet.be/geoserver/urbisgrid/ows",
validation_alias="BRUSSELS_ORTHOPHOTO_WMS_URL",
)
orthophoto_resolution_m: float = Field(default=1.0, gt=0, validation_alias="ORTHOPHOTO_RESOLUTION_M") orthophoto_resolution_m: float = Field(default=1.0, gt=0, validation_alias="ORTHOPHOTO_RESOLUTION_M")
orthophoto_min_side_m: float = Field(default=128.0, gt=0, validation_alias="ORTHOPHOTO_MIN_SIDE_M") orthophoto_min_side_m: float = Field(default=128.0, gt=0, validation_alias="ORTHOPHOTO_MIN_SIDE_M")
orthophoto_max_side_m: float = Field(default=1024.0, gt=0, validation_alias="ORTHOPHOTO_MAX_SIDE_M") orthophoto_max_side_m: float = Field(default=1024.0, gt=0, validation_alias="ORTHOPHOTO_MAX_SIDE_M")
@@ -152,6 +249,27 @@ class Settings(BaseSettings):
le=8760, le=8760,
validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS", validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS",
) )
spw_picc_enabled: bool = Field(default=True, validation_alias="SPW_PICC_ENABLED")
spw_picc_mapserver_url: str = Field(
default=(
"https://geoservices.wallonie.be/arcgis/rest/services/"
"TOPOGRAPHIE/PICC_VDIFF/MapServer"
),
validation_alias="SPW_PICC_MAPSERVER_URL",
)
spw_flood_hazard_enabled: bool = Field(default=True, validation_alias="SPW_FLOOD_HAZARD_ENABLED")
spw_flood_hazard_mapserver_url: str = Field(
default=(
"https://geoservices.wallonie.be/arcgis/rest/services/"
"EAU/ALEA_INOND/MapServer"
),
validation_alias="SPW_FLOOD_HAZARD_MAPSERVER_URL",
)
urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED")
urbis_wfs_url: str = Field(
default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows",
validation_alias="URBIS_WFS_URL",
)
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED") dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
dhmv_wcs_url: str = Field( dhmv_wcs_url: str = Field(
default="https://geo.api.vlaanderen.be/DHMV/wcs", default="https://geo.api.vlaanderen.be/DHMV/wcs",
@@ -195,6 +313,12 @@ class Settings(BaseSettings):
le=250_000, le=250_000,
validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES", validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES",
) )
bathymetry_profiles_max_pages: int = Field(
default=200,
ge=1,
le=5_000,
validation_alias="BATHYMETRY_PROFILES_MAX_PAGES",
)
bathymetry_profiles_timeout_seconds: int = Field( bathymetry_profiles_timeout_seconds: int = Field(
default=120, default=120,
ge=1, ge=1,
@@ -207,6 +331,11 @@ class Settings(BaseSettings):
le=256, le=256,
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB", validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
) )
bathymetry_raster_max_pixels: int = Field(
default=30_000_000,
ge=1,
validation_alias="BATHYMETRY_RASTER_MAX_PIXELS",
)
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED") mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
mdk_bathymetry_wcs_url: str = Field( mdk_bathymetry_wcs_url: str = Field(
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs", default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
@@ -229,11 +358,58 @@ class Settings(BaseSettings):
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs", default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
validation_alias="THEMATIC_RASTER_WCS_URL", validation_alias="THEMATIC_RASTER_WCS_URL",
) )
mdk_bathymetry_acquisition_enabled: bool = Field(
default=False,
validation_alias="MDK_BATHYMETRY_ACQUISITION_ENABLED",
)
mdk_bathymetry_coverage_id: str | None = Field(default=None, validation_alias="MDK_BATHYMETRY_COVERAGE_ID")
mdk_bathymetry_request_crs: str = Field(default="EPSG:4326", validation_alias="MDK_BATHYMETRY_REQUEST_CRS")
mdk_bathymetry_max_bbox_deg2: float = Field(
default=0.25,
gt=0,
validation_alias="MDK_BATHYMETRY_MAX_BBOX_DEG2",
)
mdk_bathymetry_acquisition_timeout_seconds: int = Field(
default=120,
ge=1,
validation_alias="MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS",
)
mdk_bathymetry_acquisition_max_response_mb: int = Field(
default=160,
ge=1,
validation_alias="MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB",
)
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M") thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M") thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS") thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS") thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB") thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
walous_enabled: bool = Field(default=True, validation_alias="WALOUS_ENABLED")
walous_source_dir: str = Field(
default="/app/storage/source-cache/walous",
validation_alias="WALOUS_SOURCE_DIR",
)
walous_analysis_resolution_m: float = Field(
default=10.0,
ge=1.0,
le=100.0,
validation_alias="WALOUS_ANALYSIS_RESOLUTION_M",
)
walous_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="WALOUS_MAX_SIDE_M")
walous_max_pixels: int = Field(default=36_000_000, ge=1, validation_alias="WALOUS_MAX_PIXELS")
spw_terrain_enabled: bool = Field(default=True, validation_alias="SPW_TERRAIN_ENABLED")
spw_terrain_source_dir: str = Field(
default="/app/storage/source-cache/spw-terrain",
validation_alias="SPW_TERRAIN_SOURCE_DIR",
)
spw_terrain_analysis_resolution_m: float = Field(
default=5.0,
ge=1.0,
le=10.0,
validation_alias="SPW_TERRAIN_ANALYSIS_RESOLUTION_M",
)
spw_terrain_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="SPW_TERRAIN_MAX_SIDE_M")
spw_terrain_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="SPW_TERRAIN_MAX_PIXELS")
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL") redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL") log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL") sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
@@ -241,6 +417,14 @@ class Settings(BaseSettings):
default=False, default=False,
validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP", validation_alias="GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP",
) )
aoi_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_AOI_WORKER_ENABLED")
aoi_worker_poll_seconds: float = Field(default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_AOI_WORKER_POLL_SECONDS")
# Executes queued detection.run / segmentation.run jobs so tiled GPU
# inference never blocks an HTTP request.
analysis_worker_enabled: bool = Field(default=False, validation_alias="GEOINTEL_ANALYSIS_WORKER_ENABLED")
analysis_worker_poll_seconds: float = Field(
default=2.0, ge=0.5, le=60.0, validation_alias="GEOINTEL_ANALYSIS_WORKER_POLL_SECONDS"
)
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS") database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED") yolo_enabled: bool = Field(default=False, validation_alias="YOLO_ENABLED")
yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR") yolo_models_dir: str = Field(default="/app/models", validation_alias="YOLO_MODELS_DIR")
@@ -248,12 +432,67 @@ class Settings(BaseSettings):
yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID") yolo_model_id: str = Field(default="yolo-configured", validation_alias="YOLO_MODEL_ID")
yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME") yolo_model_display_name: str = Field(default="Configured YOLO detector", validation_alias="YOLO_MODEL_DISPLAY_NAME")
yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION") yolo_model_version: str | None = Field(default=None, validation_alias="YOLO_MODEL_VERSION")
yolo_model_classes: str = Field(default="building", validation_alias="YOLO_MODEL_CLASSES")
yolo_enforce_validation_scope: bool = Field(default=False, validation_alias="YOLO_ENFORCE_VALIDATION_SCOPE")
yolo_validation_scope_manifest_path: str | None = Field(
default=None,
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_PATH",
)
yolo_validation_scope_manifest_sha256: str | None = Field(
default=None,
validation_alias="YOLO_VALIDATION_SCOPE_MANIFEST_SHA256",
)
# Deprecated compatibility field. Mutable Area names are never an
# inference authorization boundary; deployments must use the immutable
# checksum-bound scope manifest above.
yolo_validated_area_names: str = Field(default="Mol,Kempen", validation_alias="YOLO_VALIDATED_AREA_NAMES")
yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE") yolo_device: str = Field(default="cpu", validation_alias="YOLO_DEVICE")
yolo_require_cuda: bool = Field(default=False, validation_alias="YOLO_REQUIRE_CUDA")
yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE") yolo_image_size: int = Field(default=640, validation_alias="YOLO_IMAGE_SIZE")
yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES") yolo_max_tiles: int = Field(default=100, validation_alias="YOLO_MAX_TILES")
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS") yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD") yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
yolo_suppress_tile_edge_detections: bool = Field(
default=True, validation_alias="YOLO_SUPPRESS_TILE_EDGE_DETECTIONS"
)
# Intersection over the smaller box. The candidate evaluation freezes this
# during calibration; serving a promoted model at a different value means
# the runtime suppresses detections the gate counted.
yolo_containment_nms_threshold: float = Field(
default=0.85, ge=0.0, le=1.0, validation_alias="YOLO_CONTAINMENT_NMS_THRESHOLD"
)
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE") yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
yolo_seg_enabled: bool = Field(default=False, validation_alias="YOLO_SEG_ENABLED")
yolo_seg_model_path: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_PATH")
yolo_seg_model_id: str = Field(default="yolo-seg-configured", validation_alias="YOLO_SEG_MODEL_ID")
yolo_seg_model_display_name: str = Field(
default="Configured YOLO segmentation",
validation_alias="YOLO_SEG_MODEL_DISPLAY_NAME",
)
yolo_seg_model_version: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_VERSION")
sam_enabled: bool = Field(default=False, validation_alias="SAM_ENABLED")
sam_model_path: str | None = Field(default=None, validation_alias="SAM_MODEL_PATH")
sam_model_id: str = Field(default="sam-configured", validation_alias="SAM_MODEL_ID")
sam_model_display_name: str = Field(
default="Configured SAM segmentation",
validation_alias="SAM_MODEL_DISPLAY_NAME",
)
sam_model_version: str | None = Field(default=None, validation_alias="SAM_MODEL_VERSION")
segmentation_max_masks_per_tile: int = Field(default=300, ge=1, validation_alias="SEGMENTATION_MAX_MASKS_PER_TILE")
# Masks and boxes overlap differently, so segmentation carries its own
# containment value rather than borrowing the detector's.
segmentation_containment_nms_threshold: float = Field(
default=0.85,
ge=0.0,
le=1.0,
validation_alias="SEGMENTATION_CONTAINMENT_NMS_THRESHOLD",
)
segmentation_duplicate_iou_threshold: float = Field(
default=0.5,
ge=0.0,
le=1.0,
validation_alias="SEGMENTATION_DUPLICATE_IOU_THRESHOLD",
)
ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED") ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED")
ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL") ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL")
ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL") ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL")
@@ -284,6 +523,62 @@ class Settings(BaseSettings):
raise ValueError("OLLAMA_BASE_URL must use http or https") raise ValueError("OLLAMA_BASE_URL must use http or https")
return normalized return normalized
@model_validator(mode="after")
def validate_operator_auth(self) -> "Settings":
self.guest_display_name = self.guest_display_name.strip()
if not self.guest_display_name:
raise ValueError("GEOINTEL_GUEST_DISPLAY_NAME must not be blank")
for field_name in (
"authentik_issuer",
"authentik_client_id",
"authentik_client_secret",
"authentik_allowed_email",
):
value = getattr(self, field_name)
setattr(self, field_name, value.strip() if value else None)
self.public_base_url = self.public_base_url.strip().rstrip("/")
authentik_values = (
self.authentik_issuer,
self.authentik_client_id,
self.authentik_client_secret,
self.authentik_allowed_email,
)
if any(authentik_values) and not all(authentik_values):
raise ValueError("All GEOINTEL_AUTHENTIK_* values must be configured together")
if all(authentik_values):
if not self.auth_enabled:
raise ValueError("GEOINTEL_AUTH_ENABLED must be true when Authentik is configured")
for label, value in (
("GEOINTEL_AUTHENTIK_ISSUER", self.authentik_issuer),
("GEOINTEL_PUBLIC_BASE_URL", self.public_base_url),
):
parsed = urlsplit(str(value))
if (
parsed.scheme != "https"
or not parsed.hostname
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
):
raise ValueError(f"{label} must be an absolute HTTPS URL without credentials, query or fragment")
public_url = urlsplit(self.public_base_url)
if public_url.path not in ("", "/"):
raise ValueError("GEOINTEL_PUBLIC_BASE_URL must not contain a path")
if "@" not in str(self.authentik_allowed_email) or any(
character.isspace() for character in str(self.authentik_allowed_email)
):
raise ValueError("GEOINTEL_AUTHENTIK_ALLOWED_EMAIL must be one valid e-mail address")
if not self.auth_enabled:
return self
if not (self.auth_username or "").strip():
raise ValueError("GEOINTEL_AUTH_USERNAME is required when authentication is enabled")
if not (self.auth_password_hash or "").startswith("pbkdf2_sha256$"):
raise ValueError("GEOINTEL_AUTH_PASSWORD_HASH must be a PBKDF2-SHA256 hash")
if len(self.auth_session_secret or "") < 32:
raise ValueError("GEOINTEL_AUTH_SESSION_SECRET must contain at least 32 characters")
return self
def get_settings() -> Settings: def get_settings() -> Settings:
return Settings() return Settings()
+14
View File
@@ -0,0 +1,14 @@
from __future__ import annotations
from uuid import UUID
# Stable server-owned identity: a public session must never attach itself to an
# operator project merely because the display names happen to match.
PUBLIC_DEMO_PROJECT_ID = UUID("6f7e6f12-9b62-4a3f-a5a0-4b3bb6b2c901")
PUBLIC_DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
PUBLIC_DEMO_PROJECT_MARKER = "geointel:public-demo:v1"
def is_public_demo_project(project_id: UUID) -> bool:
return project_id == PUBLIC_DEMO_PROJECT_ID
+250 -4
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import asyncio
import re import re
import time import time
import uuid import uuid
@@ -11,18 +12,21 @@ from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from app.api.routes import analysis, areas, assistant, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, temporal from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
from app.core.config import get_settings from app.core.config import get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.core.logging import configure_logging from app.core.logging import configure_logging
from app.core.request_context import reset_request_id, set_request_id from app.core.request_context import reset_request_id, set_request_id
from app.db.session import SessionLocal from app.db.session import SessionLocal
from app.services.runtime_reconciliation_service import RuntimeReconciliationService from app.services.runtime_reconciliation_service import RuntimeReconciliationService
from app.services.auth_service import AuthService
from app.services.analysis_job_worker import AnalysisJobWorker
from app.services.aoi_operation_worker import AoiOperationWorker
logger = logging.getLogger("geointel") logger = logging.getLogger("geointel")
SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
UNSAFE_HOST = re.compile(r"[/\\@\s\x00-\x1f\x7f]") UNSAFE_HOST = re.compile(r"[/\\@?#\s\x00-\x1f\x7f]")
def _to_error_payload( def _to_error_payload(
@@ -45,14 +49,19 @@ def create_app() -> FastAPI:
@asynccontextmanager @asynccontextmanager
async def lifespan(_: FastAPI): async def lifespan(_: FastAPI):
worker_stop = asyncio.Event()
worker_task = None
analysis_worker_task = None
if settings.reconcile_interrupted_runs_on_startup: if settings.reconcile_interrupted_runs_on_startup:
db = SessionLocal() db = SessionLocal()
try: try:
result = RuntimeReconciliationService.reconcile(db) result = RuntimeReconciliationService.reconcile(db)
logger.info( logger.info(
"Runtime reconciliation completed: jobs=%s analysis_runs=%s", "Runtime reconciliation completed: jobs=%s analysis_runs=%s resumed_aoi_partitions=%s exhausted_aoi_partitions=%s",
result.interrupted_jobs, result.interrupted_jobs,
result.interrupted_analysis_runs, result.interrupted_analysis_runs,
result.resumed_aoi_partitions,
result.exhausted_aoi_partitions,
) )
except Exception: except Exception:
db.rollback() db.rollback()
@@ -60,7 +69,19 @@ def create_app() -> FastAPI:
raise raise
finally: finally:
db.close() db.close()
yield if settings.aoi_worker_enabled:
worker_task = asyncio.create_task(AoiOperationWorker.run(worker_stop, settings.aoi_worker_poll_seconds))
if settings.analysis_worker_enabled:
analysis_worker_task = asyncio.create_task(
AnalysisJobWorker.run(worker_stop, settings.analysis_worker_poll_seconds)
)
try:
yield
finally:
worker_stop.set()
for task in (worker_task, analysis_worker_task):
if task is not None:
await task
app = FastAPI( app = FastAPI(
title="GeoIntel", title="GeoIntel",
@@ -79,7 +100,9 @@ def create_app() -> FastAPI:
) )
app.include_router(health.router) app.include_router(health.router)
app.include_router(auth.router, prefix=settings.api_prefix)
app.include_router(analysis.router, prefix=settings.api_prefix) app.include_router(analysis.router, prefix=settings.api_prefix)
app.include_router(aoi_operations.router, prefix=settings.api_prefix)
app.include_router(projects.router, prefix=settings.api_prefix) app.include_router(projects.router, prefix=settings.api_prefix)
app.include_router(areas.router, prefix=settings.api_prefix) app.include_router(areas.router, prefix=settings.api_prefix)
app.include_router(datasets.router, prefix=settings.api_prefix) app.include_router(datasets.router, prefix=settings.api_prefix)
@@ -87,10 +110,12 @@ def create_app() -> FastAPI:
app.include_router(quality_checks.router, prefix=settings.api_prefix) app.include_router(quality_checks.router, prefix=settings.api_prefix)
app.include_router(exports.router, prefix=settings.api_prefix) app.include_router(exports.router, prefix=settings.api_prefix)
app.include_router(external.router, prefix=settings.api_prefix) app.include_router(external.router, prefix=settings.api_prefix)
app.include_router(source_registry.router, prefix=settings.api_prefix)
app.include_router(demo.router, prefix=settings.api_prefix) app.include_router(demo.router, prefix=settings.api_prefix)
app.include_router(qa.router, prefix=settings.api_prefix) app.include_router(qa.router, prefix=settings.api_prefix)
app.include_router(detection.router, prefix=settings.api_prefix) app.include_router(detection.router, prefix=settings.api_prefix)
app.include_router(segmentation.router, prefix=settings.api_prefix) app.include_router(segmentation.router, prefix=settings.api_prefix)
app.include_router(selection_partitions.router, prefix=settings.api_prefix)
app.include_router(temporal.router, prefix=settings.api_prefix) app.include_router(temporal.router, prefix=settings.api_prefix)
app.include_router(assistant.router, prefix=settings.api_prefix) app.include_router(assistant.router, prefix=settings.api_prefix)
@@ -102,6 +127,7 @@ def create_app() -> FastAPI:
token = set_request_id(request_id) token = set_request_id(request_id)
started_at = time.perf_counter() started_at = time.perf_counter()
raw_path = str(request.scope.get("path") or "") raw_path = str(request.scope.get("path") or "")
guest_compute_acquired = False
try: try:
host = request.headers.get("host", "") host = request.headers.get("host", "")
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
@@ -127,6 +153,224 @@ def create_app() -> FastAPI:
) )
response.headers["x-request-id"] = request_id response.headers["x-request-id"] = request_id
return response return response
public_auth_paths = {
f"{settings.api_prefix}/auth/session",
f"{settings.api_prefix}/auth/login",
f"{settings.api_prefix}/auth/guest",
f"{settings.api_prefix}/auth/logout",
f"{settings.api_prefix}/auth/authentik/start",
f"{settings.api_prefix}/auth/authentik/callback",
}
direct_loopback_request = (
request.client is not None
and request.client.host in {"127.0.0.1", "::1"}
and not request.headers.get("x-real-ip")
and not request.headers.get("x-forwarded-for")
)
if (
settings.auth_enabled
and raw_path.startswith(f"{settings.api_prefix}/")
and raw_path not in public_auth_paths
and not direct_loopback_request
):
principal = AuthService.verify_session_token(
request.cookies.get(auth.COOKIE_NAME),
settings,
)
if principal is None:
response = JSONResponse(
status_code=401,
content=_to_error_payload(
"AUTHENTICATION_REQUIRED",
"Meld u aan om de GeoIntel API te gebruiken.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
request.state.auth_principal = principal
if principal.role == "guest":
project_path_prefix = f"{settings.api_prefix}/projects/"
guest_project_root = f"{project_path_prefix}{principal.project_id}"
if raw_path.startswith(project_path_prefix):
scoped_path = raw_path[len(project_path_prefix):]
requested_project_id = scoped_path.split("/", 1)[0]
if str(principal.project_id) != requested_project_id:
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_PROJECT_SCOPE_REQUIRED",
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
query_project_id = request.query_params.get("project_id")
if query_project_id and query_project_id != str(principal.project_id):
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_PROJECT_SCOPE_REQUIRED",
"Deze gastensessie heeft alleen toegang tot de GeoIntel-demowerkruimte.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
guest_safe_read_paths = {
f"{settings.api_prefix}/projects",
f"{settings.api_prefix}/external/providers",
f"{settings.api_prefix}/assistant/status",
f"{settings.api_prefix}/assistant/models",
f"{settings.api_prefix}/detection/models",
f"{settings.api_prefix}/detection/model-assets",
f"{settings.api_prefix}/detection/yolo/preflight",
f"{settings.api_prefix}/segmentation/models",
}
normalized_path = raw_path.rstrip("/") or "/"
guest_project_read = (
normalized_path == guest_project_root
or normalized_path.startswith(f"{guest_project_root}/")
)
is_read_request = request.method in {"GET", "HEAD", "OPTIONS"}
if is_read_request:
if (
normalized_path == f"{settings.api_prefix}/detection/yolo/preflight"
and request.query_params.get("check_model_load", "").lower() in {"1", "true", "yes", "on"}
):
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_MODEL_LOAD_FORBIDDEN",
"Model loading is available to authenticated operators only.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
guest_scoped_analysis_read = (
query_project_id == str(principal.project_id)
and normalized_path.startswith(
(
f"{settings.api_prefix}/detection/",
f"{settings.api_prefix}/segmentation/",
f"{settings.api_prefix}/exports/",
)
)
)
if (
normalized_path not in guest_safe_read_paths
and not guest_project_read
and not guest_scoped_analysis_read
):
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_ROUTE_NOT_AVAILABLE",
"Deze API-route maakt geen deel uit van de afgeschermde GeoIntel-demo.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
else:
guest_safe_post_paths = {
f"{settings.api_prefix}/demo/workflow",
f"{settings.api_prefix}/external/coverage/resolve",
f"{settings.api_prefix}/analysis/change-detection",
}
guest_scoped_analysis_post_paths = {
f"{settings.api_prefix}/detection/run",
f"{settings.api_prefix}/detection/run-async",
f"{settings.api_prefix}/segmentation/run",
f"{settings.api_prefix}/segmentation/run-async",
f"{settings.api_prefix}/qa/detections-vs-reference",
f"{settings.api_prefix}/exports/geojson",
f"{settings.api_prefix}/exports/metadata",
f"{settings.api_prefix}/exports/report",
f"{settings.api_prefix}/exports/map-result",
}
guest_safe_post_suffixes = (
"/acquire",
"/vector/select",
"/vector/select/derive",
"/raster/tile",
"/raster/bathymetry/select",
"/raster/terrain/select",
"/raster/flood-hazard/select",
"/raster/thematic/select",
"/raster/walous/select",
"/temporal/compare",
"/datasets/vector/partitions/select",
"/datasets/bathymetry/profiles/partitions/select",
)
is_guest_safe_post = request.method == "POST" and (
raw_path in guest_safe_post_paths
or (
raw_path in guest_scoped_analysis_post_paths
and query_project_id == str(principal.project_id)
)
or (
query_project_id == str(principal.project_id)
and raw_path.startswith(
(
f"{settings.api_prefix}/detection/runs/",
f"{settings.api_prefix}/segmentation/runs/",
)
)
and raw_path.endswith("/qa/reference")
)
or (
raw_path.startswith(project_path_prefix)
and (
raw_path.endswith(guest_safe_post_suffixes)
or raw_path.endswith("/assistant/query")
)
)
)
if not is_guest_safe_post:
response = JSONResponse(
status_code=403,
content=_to_error_payload(
"GUEST_READ_ONLY",
"Gasttoegang laat alleen projectgebonden demo-analyses toe. Meld u aan als operator voor beheerwijzigingen.",
request_id=request_id,
),
)
response.headers["x-request-id"] = request_id
return response
retry_after = AuthService.consume_guest_request(
f"guest-compute:{principal.session_id}",
max_requests=settings.guest_compute_requests_per_minute,
)
if retry_after:
response = JSONResponse(
status_code=429,
content=_to_error_payload(
"GUEST_COMPUTE_RATE_LIMITED",
"The public demo compute budget is temporarily exhausted.",
request_id=request_id,
),
)
response.headers["retry-after"] = str(retry_after)
response.headers["x-request-id"] = request_id
return response
guest_compute_acquired = AuthService.try_acquire_guest_compute(
max_concurrency=settings.guest_compute_max_concurrency,
)
if not guest_compute_acquired:
response = JSONResponse(
status_code=429,
content=_to_error_payload(
"GUEST_COMPUTE_BUSY",
"The public demo is already processing its maximum number of jobs.",
request_id=request_id,
),
)
response.headers["retry-after"] = "10"
response.headers["x-request-id"] = request_id
return response
response = await call_next(request) response = await call_next(request)
response.headers["x-request-id"] = request_id response.headers["x-request-id"] = request_id
logger.info( logger.info(
@@ -139,6 +383,8 @@ def create_app() -> FastAPI:
) )
return response return response
finally: finally:
if guest_compute_acquired:
AuthService.release_guest_compute()
reset_request_id(token) reset_request_id(token)
@app.exception_handler(AppError) @app.exception_handler(AppError)
+1 -1
View File
@@ -1 +1 @@
from app.models import * from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API
+27 -1
View File
@@ -1,9 +1,33 @@
from .entities import AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature from .entities import (
AoiOperation,
AoiOperationPartition,
AnalysisRun,
Area,
Dataset,
DatasetLineageEdge,
DatasetQuarantine,
DatasetVersion,
Detection,
DetectionReview,
Export,
Job,
Metric,
Project,
QualityCheck,
Segmentation,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
__all__ = [ __all__ = [
"AnalysisRun", "AnalysisRun",
"AoiOperation",
"AoiOperationPartition",
"Area", "Area",
"Dataset", "Dataset",
"DatasetLineageEdge",
"DatasetQuarantine",
"DatasetVersion", "DatasetVersion",
"Detection", "Detection",
"DetectionReview", "DetectionReview",
@@ -13,5 +37,7 @@ __all__ = [
"Project", "Project",
"QualityCheck", "QualityCheck",
"Segmentation", "Segmentation",
"SourceRegistry",
"SourceSnapshot",
"VectorFeature", "VectorFeature",
] ]
+426 -1
View File
@@ -12,13 +12,44 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.db.base import Base from app.db.base import Base
SOURCE_CLASSIFICATIONS = (
"authoritative",
"corroborative",
"contextual",
"derived",
"experimental",
)
SOURCE_FRESHNESS_STATUSES = (
"unknown",
"current",
"due",
"stale",
"not_applicable",
"review_required",
)
SOURCE_INGEST_STATUSES = (
"registered",
"configured",
"not_configured",
"available",
"ingested",
"failed",
"quarantined",
"legacy_unverified",
)
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
class Project(Base): class Project(Base):
__tablename__ = "projects" __tablename__ = "projects"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
name: Mapped[str] = mapped_column(String(255), nullable=False) name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True) description: Mapped[str | None] = mapped_column(Text, nullable=True)
region: Mapped[str] = mapped_column(String(120), default="Kempen") region: Mapped[str] = mapped_column(String(120), default="Belgium and Belgian North Sea")
status: Mapped[str] = mapped_column(String(32), default="active") status: Mapped[str] = mapped_column(String(32), default="active")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
@@ -42,6 +73,123 @@ class Area(Base):
project: Mapped[Project] = relationship("Project", back_populates="areas") project: Mapped[Project] = relationship("Project", back_populates="areas")
class SourceRegistry(Base):
"""Server-owned source identity and authority contract.
Dataset metadata remains descriptive until a governed importer binds a
dataset to both this registry entry and an immutable SourceSnapshot.
"""
__tablename__ = "source_registry"
__table_args__ = (
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
CheckConstraint(
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
name="ck_source_registry_classification",
),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_registry_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_registry_ingest_status",
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
classification: Mapped[str] = mapped_column(String(32), nullable=False)
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
)
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
class SourceSnapshot(Base):
"""Immutable source-version evidence recorded by governed ingestion."""
__tablename__ = "source_snapshots"
__table_args__ = (
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_snapshots_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_snapshots_ingest_status",
),
CheckConstraint(
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_source_snapshots_checksum_sha256",
),
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
Index("ix_source_snapshots_checksum", "checksum_sha256"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_registry_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
)
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
class Dataset(Base): class Dataset(Base):
__tablename__ = "datasets" __tablename__ = "datasets"
__table_args__ = ( __table_args__ = (
@@ -49,6 +197,27 @@ class Dataset(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from", "valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_datasets_temporal_valid_range", name="ck_datasets_temporal_valid_range",
), ),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_datasets_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_lineage_status",
),
CheckConstraint(
"quarantine_status IN ('not_quarantined', 'quarantined')",
name="ck_datasets_quarantine_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_datasets_ingest_key_not_blank",
),
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
Index( Index(
"ix_datasets_project_temporal_series_observed", "ix_datasets_project_temporal_series_observed",
"project_id", "project_id",
@@ -69,6 +238,7 @@ class Dataset(Base):
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True) content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column( derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), UUID(as_uuid=True),
ForeignKey("datasets.id", ondelete="SET NULL"), ForeignKey("datasets.id", ondelete="SET NULL"),
@@ -84,6 +254,43 @@ class Dataset(Base):
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True) reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
quarantine_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_quarantined",
server_default="not_quarantined",
comment="not_quarantined | quarantined",
)
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True) temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
@@ -106,6 +313,23 @@ class Dataset(Base):
back_populates="dataset", back_populates="dataset",
cascade="all, delete-orphan", cascade="all, delete-orphan",
) )
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.parent_dataset_id",
back_populates="parent_dataset",
cascade="all, delete-orphan",
)
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.child_dataset_id",
back_populates="child_dataset",
cascade="all, delete-orphan",
)
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
)
class DatasetVersion(Base): class DatasetVersion(Base):
@@ -115,7 +339,24 @@ class DatasetVersion(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from", "valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_dataset_versions_temporal_valid_range", name="ck_dataset_versions_temporal_valid_range",
), ),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_dataset_versions_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_lineage_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_dataset_versions_ingest_key_not_blank",
),
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True), Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
) )
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -127,11 +368,135 @@ class DatasetVersion(Base):
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions") dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
class DatasetLineageEdge(Base):
"""Immutable relationship between input/output datasets and transforms."""
__tablename__ = "dataset_lineage_edges"
__table_args__ = (
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
UniqueConstraint(
"parent_dataset_id",
"child_dataset_id",
"relation_type",
"transformation_name",
name="uq_dataset_lineage_edges_relation",
),
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
parent_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
)
child_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
)
class DatasetQuarantine(Base):
"""Durable fail-closed record for rejected or doubtful source artifacts."""
__tablename__ = "dataset_quarantines"
__table_args__ = (
CheckConstraint(
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
name="ck_dataset_quarantines_target_present",
),
CheckConstraint(
"status IN ('quarantined', 'released', 'rejected')",
name="ck_dataset_quarantines_status",
),
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
)
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
stage: Mapped[str] = mapped_column(String(64), nullable=False)
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
status: Mapped[str] = mapped_column(
String(32), nullable=False, default="quarantined", server_default="quarantined"
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
class VectorFeature(Base): class VectorFeature(Base):
@@ -361,3 +726,63 @@ class Job(Base):
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
class AoiOperation(Base):
__tablename__ = "aoi_operations"
__table_args__ = (
CheckConstraint(
"status IN ('queued', 'running', 'partial', 'success', 'failed', 'cancelled')",
name="ck_aoi_operations_status",
),
Index("ix_aoi_operations_project_status", "project_id", "status"),
Index("ix_aoi_operations_geometry", "geometry", postgresql_using="gist"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
project_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False)
area_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("areas.id", ondelete="SET NULL"), nullable=True)
parent_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
operation_type: Mapped[str] = mapped_column(String(128), nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
request_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
plan_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
class AoiOperationPartition(Base):
__tablename__ = "aoi_operation_partitions"
__table_args__ = (
CheckConstraint(
"status IN ('queued', 'running', 'success', 'failed', 'skipped')",
name="ck_aoi_operation_partitions_status",
),
UniqueConstraint("operation_id", "partition_key", name="uq_aoi_operation_partition_key"),
Index("ix_aoi_operation_partitions_operation_status", "operation_id", "status"),
Index("ix_aoi_operation_partitions_geometry", "geometry", postgresql_using="gist"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
operation_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("aoi_operations.id", ondelete="CASCADE"), nullable=False)
child_job_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), ForeignKey("jobs.id", ondelete="SET NULL"), nullable=True)
partition_key: Mapped[str] = mapped_column(String(255), nullable=False)
provider_key: Mapped[str] = mapped_column(String(120), nullable=False)
product_key: Mapped[str] = mapped_column(String(120), nullable=False)
ordinal: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
geometry: Mapped[str] = mapped_column(Geometry("MultiPolygon", srid=4326, spatial_index=False), nullable=False)
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)
checkpoint_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
result_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+38
View File
@@ -32,6 +32,14 @@ from .source_catalog import (
SourceCatalogProbeReport, SourceCatalogProbeReport,
SourceCatalogProbeSummary, SourceCatalogProbeSummary,
) )
from .source_registry import (
DatasetProvenanceRead,
DatasetLineageEdgeRead,
DatasetQuarantineRead,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from .official_vector import ( from .official_vector import (
@@ -48,6 +56,8 @@ from .detection import (
DetectionRunListResponse, DetectionRunListResponse,
DetectionRunRead, DetectionRunRead,
DetectionRunRequest, DetectionRunRequest,
DetectionComparisonRequest,
DetectionComparisonResponse,
DetectionRunResponse, DetectionRunResponse,
ModelAssetListResponse, ModelAssetListResponse,
ModelAssetRead, ModelAssetRead,
@@ -78,6 +88,11 @@ from .dhmv import (
TerrainSelectionResponse, TerrainSelectionResponse,
TerrainSelectionSummary, TerrainSelectionSummary,
) )
from .spw_terrain import (
SpwTerrainAcquireRequest,
SpwTerrainAcquisitionResult,
SpwTerrainProductRead,
)
from .flood_hazard import ( from .flood_hazard import (
FloodHazardAcquireRequest, FloodHazardAcquireRequest,
FloodHazardAcquisitionResult, FloodHazardAcquisitionResult,
@@ -93,8 +108,14 @@ from .bathymetry import (
BathymetryPartitionFinalizationResult, BathymetryPartitionFinalizationResult,
BathymetryProfileAcquireRequest, BathymetryProfileAcquireRequest,
BathymetryProfileAcquisitionResult, BathymetryProfileAcquisitionResult,
BathymetryRasterMetric,
BathymetryRasterSelectionRequest,
BathymetryRasterSelectionResponse,
BathymetryRasterSelectionSummary,
BathymetrySourceProbeRead, BathymetrySourceProbeRead,
BathymetrySourceRead, BathymetrySourceRead,
MdkBathymetryAcquireRequest,
MdkBathymetryAcquisitionResult,
) )
from .thematic_raster import ( from .thematic_raster import (
ThematicRasterAcquireRequest, ThematicRasterAcquireRequest,
@@ -196,6 +217,12 @@ __all__ = [
"SourceCatalogProbeItem", "SourceCatalogProbeItem",
"SourceCatalogProbeReport", "SourceCatalogProbeReport",
"SourceCatalogProbeSummary", "SourceCatalogProbeSummary",
"SourceRegistryRead",
"SourceRegistryDetailRead",
"SourceSnapshotRead",
"DatasetLineageEdgeRead",
"DatasetQuarantineRead",
"DatasetProvenanceRead",
"GrbRefreshLayerPlan", "GrbRefreshLayerPlan",
"GrbRefreshPlan", "GrbRefreshPlan",
"GrbRefreshPlanSummary", "GrbRefreshPlanSummary",
@@ -213,6 +240,8 @@ __all__ = [
"DetectionRunListResponse", "DetectionRunListResponse",
"DetectionRunRead", "DetectionRunRead",
"DetectionRunRequest", "DetectionRunRequest",
"DetectionComparisonRequest",
"DetectionComparisonResponse",
"DetectionRunResponse", "DetectionRunResponse",
"ModelAssetListResponse", "ModelAssetListResponse",
"ModelAssetRead", "ModelAssetRead",
@@ -242,6 +271,9 @@ __all__ = [
"DhmvAcquireRequest", "DhmvAcquireRequest",
"DhmvAcquisitionResult", "DhmvAcquisitionResult",
"DhmvProductRead", "DhmvProductRead",
"SpwTerrainAcquireRequest",
"SpwTerrainAcquisitionResult",
"SpwTerrainProductRead",
"TerrainMetric", "TerrainMetric",
"TerrainPartitionSelectionRequest", "TerrainPartitionSelectionRequest",
"TerrainSelectionRequest", "TerrainSelectionRequest",
@@ -257,10 +289,16 @@ __all__ = [
"FloodHazardSelectionSummary", "FloodHazardSelectionSummary",
"BathymetryProfileAcquireRequest", "BathymetryProfileAcquireRequest",
"BathymetryProfileAcquisitionResult", "BathymetryProfileAcquisitionResult",
"BathymetryRasterMetric",
"BathymetryRasterSelectionRequest",
"BathymetryRasterSelectionResponse",
"BathymetryRasterSelectionSummary",
"BathymetryPartitionFinalizeRequest", "BathymetryPartitionFinalizeRequest",
"BathymetryPartitionFinalizationResult", "BathymetryPartitionFinalizationResult",
"BathymetrySourceProbeRead", "BathymetrySourceProbeRead",
"BathymetrySourceRead", "BathymetrySourceRead",
"MdkBathymetryAcquireRequest",
"MdkBathymetryAcquisitionResult",
"ThematicRasterAcquireRequest", "ThematicRasterAcquireRequest",
"ThematicRasterAcquisitionResult", "ThematicRasterAcquisitionResult",
"ThematicRasterMetric", "ThematicRasterMetric",
+19
View File
@@ -5,12 +5,22 @@ from uuid import UUID
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from app.schemas.operations import VectorSelectionBBox
class ChangeDetectionRequest(BaseModel): class ChangeDetectionRequest(BaseModel):
source_dataset_id: UUID source_dataset_id: UUID
target_dataset_id: UUID target_dataset_id: UUID
iou_threshold: float = Field(default=0.8, ge=0.0, le=1.0) iou_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
# Below this the two footprints are separate objects rather than one that
# was redrawn; between the two thresholds the change class is "modified".
modified_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
include_unchanged: bool = True include_unchanged: bool = True
# Without a selection the comparison covers both datasets in full, which is
# rarely the question and never a response a map can draw.
bbox: VectorSelectionBBox | None = None
area_id: UUID | None = None
preview_limit: int = Field(default=2_000, ge=1, le=20_000)
class ChangeDetectionSummary(BaseModel): class ChangeDetectionSummary(BaseModel):
@@ -20,8 +30,17 @@ class ChangeDetectionSummary(BaseModel):
target_feature_count: int target_feature_count: int
added_count: int added_count: int
removed_count: int removed_count: int
# A footprint that was redrawn rather than demolished and rebuilt. Without
# this class it appeared as one removal plus one addition.
modified_count: int = 0
unchanged_count: int unchanged_count: int
iou_threshold: float iou_threshold: float
modified_iou_threshold: float | None = None
selection_area_id: UUID | None = None
# Counts describe the whole selection; the GeoJSON is capped so a regional
# comparison does not return both datasets in one response.
preview_limit: int | None = None
preview_truncated: bool = False
warnings: list[str] = Field(default_factory=list) warnings: list[str] = Field(default_factory=list)
generated_at: datetime generated_at: datetime
geojson: dict geojson: dict
+75
View File
@@ -0,0 +1,75 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.operations import VectorSelectionBBox
class AoiOperationCreate(BaseModel):
area_id: UUID | None = None
bbox: VectorSelectionBBox | None = None
operation_type: str = Field(min_length=1, max_length=128)
provider_key: str = Field(min_length=1, max_length=120)
product_key: str = Field(min_length=1, max_length=120)
coverage_zone: str | None = Field(default=None, max_length=64)
max_partition_side_m: float | None = Field(default=None, gt=0, le=60_000)
max_attempts: int = Field(default=3, ge=1, le=10)
parameters_json: dict = Field(default_factory=dict)
class AoiPartitionRead(BaseModel):
id: UUID
partition_key: str
provider_key: str
product_key: str
ordinal: int
status: str
attempt_count: int
max_attempts: int
checkpoint_json: dict | None = None
result_json: dict | None = None
error_message: str | None = None
model_config = {"from_attributes": True}
class AoiOperationRead(BaseModel):
id: UUID
project_id: UUID
area_id: UUID | None = None
parent_job_id: UUID | None = None
operation_type: str
status: str
request_json: dict
plan_json: dict
result_json: dict | None = None
error_message: str | None = None
progress: float
partition_counts: dict[str, int]
partitions: list[AoiPartitionRead] = Field(default_factory=list)
created_at: datetime | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
class AoiOperationList(BaseModel):
items: list[AoiOperationRead]
total: int
class AoiPartitionCheckpoint(BaseModel):
checkpoint_json: dict = Field(default_factory=dict)
class AoiPartitionComplete(BaseModel):
result_json: dict = Field(default_factory=dict)
skipped: bool = False
class AoiPartitionFail(BaseModel):
error_message: str = Field(min_length=1, max_length=4000)
retryable: bool = True
details: dict = Field(default_factory=dict)
+14
View File
@@ -14,6 +14,7 @@ class AreaCreate(BaseModel):
class AreaUpdate(BaseModel): class AreaUpdate(BaseModel):
name: str | None = None name: str | None = None
geometry: dict | None = None
crs: str | None = None crs: str | None = None
@@ -39,3 +40,16 @@ class AreaList(BaseModel):
total: int total: int
limit: int limit: int
offset: int offset: int
class MunicipalitySearchItem(BaseModel):
niscode: str
name: str
name_nl: str | None = None
name_fr: str | None = None
name_de: str | None = None
class MunicipalitySearchList(BaseModel):
items: list[MunicipalitySearchItem]
total: int
+16
View File
@@ -66,12 +66,28 @@ class AssistantTemporalSeries(BaseModel):
observation_count: int observation_count: int
class AssistantEstimateDisclosure(BaseModel):
"""A value in the answer that the source itself calls an estimate.
Derived from metric metadata rather than from the generated sentences, so
the disclosure is present whatever wording the model chose.
"""
theme: str
label: str
unit: str
source: str
dataset_id: UUID
reason: str
class AssistantQueryResponse(BaseModel): class AssistantQueryResponse(BaseModel):
answer: str answer: str
model: str model: str
scope_label: str scope_label: str
context_metrics: list[AssistantContextMetric] context_metrics: list[AssistantContextMetric]
temporal_series: list[AssistantTemporalSeries] temporal_series: list[AssistantTemporalSeries]
estimate_disclosures: list[AssistantEstimateDisclosure] = Field(default_factory=list)
source_dataset_ids: list[UUID] source_dataset_ids: list[UUID]
warnings: list[str] warnings: list[str]
generated_at: datetime generated_at: datetime
+29
View File
@@ -0,0 +1,29 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field
from app.schemas.common import Envelope
class AuthLoginRequest(BaseModel):
username: str = Field(min_length=1, max_length=128)
password: str = Field(min_length=1, max_length=1024)
class AuthSession(BaseModel):
authentication_required: bool
authenticated: bool
username: str | None = None
expires_at: datetime | None = None
role: Literal["operator", "guest"] | None = None
guest_access_enabled: bool = False
authentik_enabled: bool = False
guest_project_id: UUID | None = None
class AuthSessionEnvelope(Envelope[AuthSession]):
pass
+62
View File
@@ -109,3 +109,65 @@ class BathymetrySourceProbeRead(BaseModel):
checked_at: datetime checked_at: datetime
message: str message: str
limitation_message: str limitation_message: str
class MdkBathymetryAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
force_refresh: bool = False
class MdkBathymetryAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
coverage_id: str
bbox_epsg4326: list[float]
vertical_reference: str
resolution_m: float = Field(gt=0)
attribution: str
limitation_message: str
class BathymetryRasterSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
class BathymetryRasterMetric(BaseModel):
metric_key: str
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
is_estimate: bool = False
class BathymetryRasterSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
primary_metric_key: str
metrics: list[BathymetryRasterMetric]
class BathymetryRasterSelectionResponse(BaseModel):
dataset_id: UUID
product_key: str
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
selected_cell_count: int = Field(ge=1)
valid_cell_count: int = Field(ge=1)
coverage_ratio: float = Field(ge=0, le=1)
# Set when the drawn selection is smaller than one source cell and the
# analysis was widened to the cells it touches, so the value covers more
# ground than was requested.
cell_selection_warning: str | None = None
resolution_m: float = Field(gt=0)
vertical_reference: str
survey_period: str
summary: BathymetryRasterSelectionSummary
unsupported_metrics: list[str]
limitation_message: str
generated_at: str
+16
View File
@@ -65,9 +65,25 @@ class CoverageResolutionItem(BaseModel):
status: CoverageStatus status: CoverageStatus
source_names: list[str] source_names: list[str]
materialized_dataset_ids: list[UUID] materialized_dataset_ids: list[UUID]
evidence: list["CoverageEvidenceItem"] = Field(default_factory=list)
limitation_message: str limitation_message: str
class CoverageEvidenceItem(BaseModel):
dataset_id: UUID
source_name: str
authority_level: CoverageAuthority
source_version: str | None = None
observed_at: str | None = None
published_at: str | None = None
crs: str | None = None
resolution: dict | None = None
coverage_bbox_epsg4326: list[float] | None = None
attribution: str | None = None
license_note: str | None = None
checksum_sha256: str | None = None
class CoverageResolveResponse(BaseModel): class CoverageResolveResponse(BaseModel):
project_id: UUID project_id: UUID
bbox: CoverageBBox bbox: CoverageBBox
+19
View File
@@ -35,6 +35,16 @@ class DatasetCreateResponse(BaseModel):
reference_layer_name: str | None = None reference_layer_name: str | None = None
source_metadata: dict | None = None source_metadata: dict | None = None
provenance_metadata: dict | None = None provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
imported_at: datetime | None = None imported_at: datetime | None = None
temporal_series_key: str | None = None temporal_series_key: str | None = None
observed_at: datetime | None = None observed_at: datetime | None = None
@@ -97,6 +107,15 @@ class DatasetVersionRead(BaseModel):
checksum_sha256: str | None = None checksum_sha256: str | None = None
source_metadata: dict | None = None source_metadata: dict | None = None
provenance_metadata: dict | None = None provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
created_at: datetime | None = None created_at: datetime | None = None
model_config = {"from_attributes": True} model_config = {"from_attributes": True}
+44
View File
@@ -18,6 +18,11 @@ class DetectionModelCapability(BaseModel):
status: str status: str
limitation_message: str limitation_message: str
version: str | None = None version: str | None = None
training_scope: str | None = None
validation_scope: str | None = None
validated_regions: list[str] = Field(default_factory=list)
nationally_validated: bool = False
operator_review_required: bool = True
class DetectionModelsResponse(BaseModel): class DetectionModelsResponse(BaseModel):
@@ -37,6 +42,10 @@ class ModelAssetRead(BaseModel):
size_bytes: int size_bytes: int
sha256: str sha256: str
active: bool active: bool
runtime_available: bool
runtime_status: str
governed_validation_status: str
promotion_status: str
status: str status: str
limitation_message: str limitation_message: str
will_download_models: bool = False will_download_models: bool = False
@@ -68,6 +77,28 @@ class DetectionQaRequest(BaseModel):
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class_name: str | None = None class_name: str | None = None
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
# Confidence cuts to report alongside the run's own operating point. They
# are read off the one matching pass, so a sweep costs no extra inference.
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
class DetectionComparisonRequest(BaseModel):
"""Place several runs side by side against one reference."""
analysis_run_ids: list[UUID] = Field(min_length=2, max_length=12)
reference_dataset_id: UUID
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class DetectionComparisonResponse(BaseModel):
reference_dataset_id: UUID
iou_threshold: float
# Whether these runs answer the same question at all, and why not if they
# do not. Numbers from incomparable runs are reported but never ranked as
# if they were alternatives.
comparability: dict
ranking_metric: str
rows: list[dict]
class DetectionRunResponse(BaseModel): class DetectionRunResponse(BaseModel):
@@ -105,7 +136,11 @@ class DetectionRunRead(BaseModel):
class DetectionRunListResponse(BaseModel): class DetectionRunListResponse(BaseModel):
items: list[DetectionRunRead] items: list[DetectionRunRead]
# ``total`` counts every run; ``items`` is the most recent page of them.
total: int total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
class DetectionRead(BaseModel): class DetectionRead(BaseModel):
@@ -128,7 +163,11 @@ class DetectionRead(BaseModel):
class DetectionListResponse(BaseModel): class DetectionListResponse(BaseModel):
items: list[DetectionRead] items: list[DetectionRead]
# ``total`` is the complete population; ``items`` is one page of it.
total: int total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
class YoloPreflightChecks(BaseModel): class YoloPreflightChecks(BaseModel):
@@ -136,8 +175,11 @@ class YoloPreflightChecks(BaseModel):
enabled: bool enabled: bool
dependencies_available: bool | None = None dependencies_available: bool | None = None
accelerator_ready: bool | None = None
model_path_set: bool | None = None model_path_set: bool | None = None
model_file_exists: bool | None = None model_file_exists: bool | None = None
model_provenance_manifest_path: str | None = None
model_provenance_valid: bool | None = None
model_load_requested: bool model_load_requested: bool
model_load_ok: bool | None = None model_load_ok: bool | None = None
manifest_path_set: bool | None = None manifest_path_set: bool | None = None
@@ -155,6 +197,8 @@ class YoloRuntimeDetails(BaseModel):
torch_version: str | None = None torch_version: str | None = None
ultralytics_version: str | None = None ultralytics_version: str | None = None
cuda_available: bool | None = None cuda_available: bool | None = None
configured_device: str
cuda_required: bool
class YoloPreflightResponse(BaseModel): class YoloPreflightResponse(BaseModel):
+4
View File
@@ -53,6 +53,10 @@ class DetectionReviewSummary(BaseModel):
false_positive_total: int false_positive_total: int
false_negative_total: int false_negative_total: int
decision_counts: dict[str, int] decision_counts: dict[str, int]
# The score with the operator's verdicts applied, next to the raw one. A
# finding adjudicated as a reference gap is not the model's error, and an
# interval covers what the unreviewed remainder could still turn out to be.
reviewed_metrics: dict | None = None
class DetectionReviewList(BaseModel): class DetectionReviewList(BaseModel):
+5
View File
@@ -58,6 +58,7 @@ class TerrainSelectionRequest(BaseModel):
class TerrainPartitionSelectionRequest(TerrainSelectionRequest): class TerrainPartitionSelectionRequest(TerrainSelectionRequest):
product_key: str = "dtm_1m" product_key: str = "dtm_1m"
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
class TerrainMetric(BaseModel): class TerrainMetric(BaseModel):
@@ -89,6 +90,10 @@ class TerrainSelectionResponse(BaseModel):
sample_count: int sample_count: int
slope_sample_count: int slope_sample_count: int
coverage_ratio: float coverage_ratio: float
# Set when the drawn selection is smaller than one source cell and the
# analysis was widened to the cells it touches, so the value covers more
# ground than was requested.
cell_selection_warning: str | None = None
resolution_m: float resolution_m: float
vertical_reference: str vertical_reference: str
summary: TerrainSelectionSummary summary: TerrainSelectionSummary
+4
View File
@@ -10,6 +10,7 @@ from app.schemas.operations import VectorSelectionBBox
ExportKind = Literal["dataset", "detection_run", "segmentation_run", "vector_selection"] ExportKind = Literal["dataset", "detection_run", "segmentation_run", "vector_selection"]
DetectionExportIntendedUse = Literal["review", "operational"]
MapResultMode = Literal["current", "evolution"] MapResultMode = Literal["current", "evolution"]
@@ -21,6 +22,7 @@ class GeoJsonExportRequest(BaseModel):
name: str | None = None name: str | None = None
bbox: VectorSelectionBBox | None = None bbox: VectorSelectionBBox | None = None
limit: int = 250 limit: int = 250
intended_use: DetectionExportIntendedUse = "review"
@model_validator(mode="after") @model_validator(mode="after")
def validate_target(self) -> "GeoJsonExportRequest": def validate_target(self) -> "GeoJsonExportRequest":
@@ -33,6 +35,8 @@ class GeoJsonExportRequest(BaseModel):
raise ValueError("bbox is required for vector selection GeoJSON exports") raise ValueError("bbox is required for vector selection GeoJSON exports")
if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None: if self.export_kind in {"detection_run", "segmentation_run"} and self.analysis_run_id is None:
raise ValueError("analysis_run_id is required for run GeoJSON exports") raise ValueError("analysis_run_id is required for run GeoJSON exports")
if self.intended_use == "operational" and self.export_kind != "detection_run":
raise ValueError("operational intended_use is supported only for detection run exports")
return self return self
+10 -1
View File
@@ -61,6 +61,7 @@ class FloodHazardSelectionRequest(BaseModel):
class FloodHazardPartitionSelectionRequest(FloodHazardSelectionRequest): class FloodHazardPartitionSelectionRequest(FloodHazardSelectionRequest):
product_key: str = "pluviaal_current_t100" product_key: str = "pluviaal_current_t100"
dataset_ids: list[UUID] | None = Field(default=None, min_length=1, max_length=4096)
class FloodHazardMetric(BaseModel): class FloodHazardMetric(BaseModel):
@@ -92,9 +93,17 @@ class FloodHazardSelectionResponse(BaseModel):
return_period_years: int return_period_years: int
selection_bbox: VectorSelectionBBox selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None selection_area_id: UUID | None = None
# Three populations kept apart: cells drawn, cells the model covers, and
# cells with a positive modelled depth. ``inundated_fraction`` is a share
# of the modelled cells, and is null when nothing was modelled — absence
# of a model is not evidence of zero risk.
selected_cell_count: int selected_cell_count: int
valid_cell_count: int = 0
no_data_cell_count: int = 0
data_coverage_ratio: float = 1.0
inundated_cell_count: int inundated_cell_count: int
inundated_fraction: float inundated_fraction: float | None = None
coverage_warning: str | None = None
resolution_m: float resolution_m: float
summary: FloodHazardSelectionSummary summary: FloodHazardSelectionSummary
unsupported_metrics: list[str] unsupported_metrics: list[str]
+1
View File
@@ -32,6 +32,7 @@ class OfficialVectorProductRead(BaseModel):
attribution: str attribution: str
license_note: str license_note: str
limitation_message: str limitation_message: str
coverage_zones: list[str]
class OfficialVectorAcquisitionResult(BaseModel): class OfficialVectorAcquisitionResult(BaseModel):
+6
View File
@@ -236,7 +236,13 @@ class VectorSelectionSummary(BaseModel):
metric_unit: str metric_unit: str
aggregation_method: str aggregation_method: str
primary_metric_key: str | None = None primary_metric_key: str | None = None
# ``feature_count`` counts whole features that touch the selection, while
# area and length metrics clip to it. These fields say how far the two
# populations diverge, so the numbers on one panel can be read together.
feature_count: int feature_count: int
fully_covered_feature_count: int | None = None
partially_covered_feature_count: int | None = None
selection_edge_warning: str | None = None
is_estimate: bool = False is_estimate: bool = False
warning: str | None = None warning: str | None = None
metrics: list[VectorSelectionMetric] = Field(default_factory=list) metrics: list[VectorSelectionMetric] = Field(default_factory=list)
+6 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from uuid import UUID from uuid import UUID
from pydantic import BaseModel from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox from .operations import VectorSelectionBBox
@@ -12,6 +12,7 @@ class OrthophotoAcquireRequest(BaseModel):
area_id: UUID | None = None area_id: UUID | None = None
product_key: str = "most_recent" product_key: str = "most_recent"
force_refresh: bool = False force_refresh: bool = False
resolution_m: float | None = Field(default=None, ge=0.1, le=2.0)
class OrthophotoProductRead(BaseModel): class OrthophotoProductRead(BaseModel):
@@ -24,6 +25,10 @@ class OrthophotoProductRead(BaseModel):
color_mode: str color_mode: str
catalog_url: str catalog_url: str
limitation_message: str limitation_message: str
provider: str
coverage_zone: str
attribution: str
license_note: str
class OrthophotoAcquisitionResult(BaseModel): class OrthophotoAcquisitionResult(BaseModel):
+1 -1
View File
@@ -10,7 +10,7 @@ from pydantic import BaseModel
class ProjectCreate(BaseModel): class ProjectCreate(BaseModel):
name: str name: str
description: str | None = None description: str | None = None
region: str | None = "Kempen" region: str | None = "Belgium and Belgian North Sea"
class ProjectUpdate(BaseModel): class ProjectUpdate(BaseModel):
+14
View File
@@ -19,8 +19,14 @@ class QaProviderComparisonRequest(BaseModel):
class QaProviderComparisonResult(BaseModel): class QaProviderComparisonResult(BaseModel):
status: str status: str
warnings: list[str] = Field(default_factory=list) warnings: list[str] = Field(default_factory=list)
# Counts of the population that was actually matched, so that
# ``matches + false_positives == candidate_feature_count`` holds even when
# an area filter or an unparseable geometry removed features. The ``_raw``
# fields keep the untouched dataset totals visible next to them.
candidate_feature_count: int candidate_feature_count: int
reference_feature_count: int reference_feature_count: int
candidate_feature_count_raw: int | None = None
reference_feature_count_raw: int | None = None
matches: int matches: int
false_positives: int false_positives: int
false_negatives: int false_negatives: int
@@ -83,7 +89,13 @@ class QualityEvidenceResponse(BaseModel):
candidate_dataset_id: UUID | None = None candidate_dataset_id: UUID | None = None
reference_dataset_id: UUID reference_dataset_id: UUID
analysis_run_id: UUID | None = None analysis_run_id: UUID | None = None
# The overlay is capped so a regional check stays reviewable; the counts in
# the quality check itself are always complete.
feature_count: int feature_count: int
total_feature_count: int | None = None
role_counts: dict[str, int] = Field(default_factory=dict)
truncated: bool = False
limit: int | None = None
warnings: list[str] = Field(default_factory=list) warnings: list[str] = Field(default_factory=list)
geojson: GeoJsonFeatureCollection geojson: GeoJsonFeatureCollection
@@ -109,6 +121,8 @@ class AnalysisQaResponse(BaseModel):
coverage: dict[str, Any] | None = None coverage: dict[str, Any] | None = None
temporal_compatibility: dict[str, Any] | None = None temporal_compatibility: dict[str, Any] | None = None
box_to_footprint_diagnostics: dict[str, Any] | None = None box_to_footprint_diagnostics: dict[str, Any] | None = None
precision_recall_curve: dict[str, Any] | None = None
calibration_sweep: list[dict[str, Any]] = Field(default_factory=list)
match_evidence: list[dict[str, Any]] = Field(default_factory=list) match_evidence: list[dict[str, Any]] = Field(default_factory=list)
false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list) false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list) false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)
+11
View File
@@ -32,6 +32,8 @@ class SegmentationQaRequest(BaseModel):
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
class_name: str | None = None class_name: str | None = None
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
# Read off the one matching pass, exactly as for detection.
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
class SegmentationRunResponse(BaseModel): class SegmentationRunResponse(BaseModel):
@@ -69,7 +71,11 @@ class SegmentationRunRead(BaseModel):
class SegmentationRunListResponse(BaseModel): class SegmentationRunListResponse(BaseModel):
items: list[SegmentationRunRead] items: list[SegmentationRunRead]
# ``total`` counts every run; ``items`` is the most recent page of them.
total: int total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
class SegmentationRead(BaseModel): class SegmentationRead(BaseModel):
@@ -96,4 +102,9 @@ class SegmentationRead(BaseModel):
class SegmentationListResponse(BaseModel): class SegmentationListResponse(BaseModel):
items: list[SegmentationRead] items: list[SegmentationRead]
# ``total`` describes the complete filtered population; ``items`` is one
# stable confidence-ranked page of it.
total: int total: int
limit: int | None = None
offset: int = 0
truncated: bool = False
@@ -0,0 +1,14 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class VectorPartitionSelectionRequest(BaseModel):
dataset_ids: list[UUID] = Field(min_length=1, max_length=4096)
bbox: VectorSelectionBBox
area_id: UUID | None = None
limit: int = Field(default=1000, ge=1, le=1000)
+120
View File
@@ -0,0 +1,120 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
class SourceRegistryRead(BaseModel):
"""Read-only, server-owned source-authority definition."""
id: UUID
source_key: str
display_name: str
classification: str
authority_name: str
authority_scope_json: dict
provider_adapter_key: str | None = None
source_url: str | None = None
license_name: str
license_url: str | None = None
usage_restrictions: str
default_crs: str
default_units: str
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
expected_geometry_types_json: list
expected_attributes_json: dict
usage_policy_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
registry_metadata_json: dict
created_at: datetime | None = None
updated_at: datetime | None = None
snapshot_count: int = 0
model_config = {"from_attributes": True}
class SourceSnapshotRead(BaseModel):
"""Immutable version/snapshot evidence attached to an imported dataset."""
id: UUID
source_registry_id: UUID
snapshot_key: str
source_version: str | None = None
snapshot_at: datetime | None = None
fetched_at: datetime | None = None
source_url: str | None = None
checksum_sha256: str | None = None
crs: str | None = None
units: str | None = None
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
observed_schema_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
snapshot_metadata_json: dict
created_at: datetime | None = None
model_config = {"from_attributes": True}
class SourceRegistryDetailRead(BaseModel):
source: SourceRegistryRead
snapshots: list[SourceSnapshotRead]
class DatasetLineageEdgeRead(BaseModel):
id: UUID
parent_dataset_id: UUID
child_dataset_id: UUID
parent_dataset_version_id: UUID | None = None
child_dataset_version_id: UUID | None = None
relation_type: str
transformation_name: str
transformation_version: str | None = None
parameters_json: dict | None = None
input_checksum_sha256: str | None = None
output_checksum_sha256: str | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
class DatasetQuarantineRead(BaseModel):
id: UUID
dataset_id: UUID | None = None
dataset_version_id: UUID | None = None
source_snapshot_id: UUID | None = None
stage: str
reason_code: str
details_json: dict | None = None
artifact_path: str | None = None
artifact_checksum_sha256: str | None = None
status: str
created_at: datetime | None = None
resolved_at: datetime | None = None
resolved_by: str | None = None
model_config = {"from_attributes": True}
class DatasetProvenanceRead(BaseModel):
dataset_id: UUID
source: SourceRegistryRead | None = None
snapshot: SourceSnapshotRead | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
+55
View File
@@ -0,0 +1,55 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class SpwTerrainAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "spw_mnt_1m_2021_2022"
resolution_m: float | None = Field(default=None, ge=1.0, le=10.0)
force_refresh: bool = False
class SpwTerrainProductRead(BaseModel):
key: str
display_name: str
surface_model: str
source_filename: str
native_resolution_m: float
analysis_resolution_m: float
source_crs: str
vertical_reference: str
acquisition_period: str
catalog_url: str
attribution: str
license_note: str
limitation_message: str
coverage_zones: list[str]
configured: bool
status: str
class SpwTerrainAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
surface_model: str
native_resolution_m: float
resolution_m: float
width: int
height: int
valid_pixel_count: int
nodata_value: float
bbox_epsg4326: list[float]
bbox_epsg3812: list[float]
vertical_reference: str
acquisition_period: str
attribution: str
limitation_message: str
+29 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from uuid import UUID from uuid import UUID
from pydantic import BaseModel from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox from .operations import VectorSelectionBBox
@@ -32,6 +32,30 @@ class ThematicRasterProductRead(BaseModel):
legend_max_label: str legend_max_label: str
included_source_values: list[int] included_source_values: list[int]
limitation_message: str limitation_message: str
analysis_resolution_m: float | None = None
coverage_zones: list[str] = Field(default_factory=list)
configured: bool = True
status: str = "configured"
class WalousAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
theme: str
metric_kind: str
resolution_m: float
width: int
height: int
valid_pixel_count: int
bbox_epsg4326: list[float]
bbox_epsg3812: list[float]
observation_year: int
source_value_unit: str
attribution: str
limitation_message: str
class ThematicRasterAcquisitionResult(BaseModel): class ThematicRasterAcquisitionResult(BaseModel):
@@ -89,6 +113,10 @@ class ThematicRasterSelectionResponse(BaseModel):
selected_cell_count: int selected_cell_count: int
valid_cell_count: int valid_cell_count: int
coverage_ratio: float coverage_ratio: float
# Set when the drawn selection is smaller than one source cell and the
# analysis was widened to the cells it touches, so the value covers more
# ground than was requested.
cell_selection_warning: str | None = None
resolution_m: float resolution_m: float
observation_year: int observation_year: int
summary: ThematicRasterSelectionSummary summary: ThematicRasterSelectionSummary
+172
View File
@@ -0,0 +1,172 @@
"""Background execution for queued analysis runs.
Tiled GPU inference is minutes of work. Running it inside the HTTP request
holds a worker thread for the whole duration, times the client out and leaves
the operator without progress. Queued ``detection.run`` and
``segmentation.run`` jobs are picked up here instead, mirroring the polling
worker the AOI operations already use so the runtime keeps one job model.
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any
from uuid import UUID
from app.db.session import SessionLocal
from app.models import Job
logger = logging.getLogger("geointel.analysis_worker")
class AnalysisJobWorker:
HANDLED_JOB_TYPES = ("detection.run", "segmentation.run")
BATCH_SIZE = 4
@staticmethod
def _uuid(value: Any) -> UUID | None:
if isinstance(value, UUID):
return value
try:
return UUID(str(value))
except (TypeError, ValueError):
return None
@staticmethod
def _dispatch(db, job: Job) -> Any:
# Imported lazily: both services import each other's helpers, and the
# worker must not add a third edge to that cycle at module load.
from app.services.detection_service import DetectionService
from app.services.segmentation_service import SegmentationService
parameters = job.parameters_json if isinstance(job.parameters_json, dict) else {}
project_id = AnalysisJobWorker._uuid(parameters.get("project_id"))
dataset_id = AnalysisJobWorker._uuid(parameters.get("dataset_id"))
if project_id is None or dataset_id is None:
raise ValueError("Queued analysis job is missing project_id or dataset_id")
common = {
"db": db,
"project_id": project_id,
"dataset_id": dataset_id,
"model_id": parameters.get("model_id"),
"confidence_threshold": float(parameters.get("confidence_threshold") or 0.0),
"class_filter": parameters.get("class_filter") or [],
"tile_manifest_path": parameters.get("tile_manifest_path"),
"parameters_json": parameters.get("parameters_json") or {},
"existing_job": job,
}
if job.job_type == "detection.run":
return DetectionService.run_detection(
model_asset_id=parameters.get("model_asset_id"),
**common,
)
return SegmentationService.run_segmentation(**common)
@staticmethod
def claim(db, job: Job) -> bool:
"""Take the job out of the queue, atomically. Returns whether we won.
Selecting and then updating in a second statement lets two workers
a restarted process overlapping the previous one, or a second replica
both start tiled GPU inference on the same row. The conditional update
makes exactly one caller see a row count of 1; the AOI worker beside
this one already claims with FOR UPDATE SKIP LOCKED for the same reason.
"""
claimed = (
db.query(Job)
.filter(Job.id == job.id, Job.status == "queued")
.update({Job.status: "running"}, synchronize_session=False)
)
db.commit()
if not claimed:
return False
job.status = "running"
return True
@staticmethod
def _finalize(db, job: Job, result: Any) -> None:
"""Close a job the handler left open.
The analysis services normally set the terminal status themselves.
If one returns without doing so, recording the outcome here is what
keeps the job from sitting in "running" for ever.
"""
if job.status != "running":
return
status = getattr(result, "status", None)
if status == "success":
job.status = "success"
job.result_json = {
"detection_count": getattr(result, "detection_count", None),
"segmentation_count": getattr(result, "segmentation_count", None),
}
else:
job.status = "failed"
job.error_message = getattr(result, "message", None) or "Analysis run did not complete"
job.result_json = {"error_code": getattr(result, "error_code", None) or "ANALYSIS_JOB_INCOMPLETE"}
db.add(job)
db.commit()
@staticmethod
def _mark_failed(db, job: Job, *, code: str, message: str) -> None:
try:
db.rollback()
except Exception:
pass
job.status = "failed"
job.error_message = message
job.result_json = {"error_code": code, "message": message}
db.add(job)
db.commit()
@staticmethod
def run_once(db=None) -> int:
"""Execute one batch of queued analysis jobs. Returns the batch size."""
owns_session = db is None
session = db if db is not None else SessionLocal()
try:
rows = [
job
for job in (
session.query(Job)
.filter(Job.status == "queued")
.filter(Job.job_type.in_(AnalysisJobWorker.HANDLED_JOB_TYPES))
.order_by(Job.created_at)
.limit(AnalysisJobWorker.BATCH_SIZE)
.all()
)
if job.job_type in AnalysisJobWorker.HANDLED_JOB_TYPES and job.status == "queued"
]
claimed_count = 0
for job in rows:
if not AnalysisJobWorker.claim(session, job):
# Another worker took it between the select and the claim.
continue
claimed_count += 1
try:
result = AnalysisJobWorker._dispatch(session, job)
AnalysisJobWorker._finalize(session, job, result)
except Exception as exc:
code = getattr(exc, "code", None) or "ANALYSIS_JOB_INTERNAL_ERROR"
message = getattr(exc, "message", None) or str(exc) or "Unexpected analysis job failure"
AnalysisJobWorker._mark_failed(session, job, code=str(code), message=str(message))
logger.exception("Analysis job failed job_id=%s job_type=%s", job.id, job.job_type)
return claimed_count
finally:
if owns_session:
session.close()
@staticmethod
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
while not stop_event.is_set():
processed = await asyncio.to_thread(AnalysisJobWorker.run_once)
if processed == 0:
try:
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
except TimeoutError:
pass
@@ -0,0 +1,252 @@
from __future__ import annotations
from uuid import UUID
from geoalchemy2.shape import to_shape
from app.core.errors import AppError
from app.models import AoiOperation, AoiOperationPartition
from app.schemas.grb import GrbAcquireRequest
from app.schemas.dhmv import DhmvAcquireRequest
from app.schemas.spw_terrain import SpwTerrainAcquireRequest
from app.schemas.official_vector import OfficialVectorAcquireRequest
from app.schemas.flood_hazard import FloodHazardAcquireRequest
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
from app.schemas.bathymetry import (
BathymetryProfileAcquireRequest,
MdkBathymetryAcquireRequest,
)
from app.schemas.job import JobCreate
from app.schemas.operations import VectorSelectionBBox
from app.schemas.orthophoto import OrthophotoAcquireRequest
from app.services.aoi_operation_service import AoiOperationService
from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.spw_terrain_service import SpwTerrainService
from app.services.official_vector_acquisition_service import (
OfficialVectorAcquisitionService,
)
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.thematic_raster_acquisition_service import (
ThematicRasterAcquisitionService,
)
from app.services.walous_land_cover_service import WalousLandCoverService
from app.services.bathymetry_profile_acquisition_service import (
BathymetryProfileAcquisitionService,
)
from app.services.mdk_bathymetry_acquisition_service import (
MdkBathymetryAcquisitionService,
)
from app.services.job_service import JobService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
class AoiOperationExecutor:
"""Execute one bounded partition through an existing governed provider."""
@staticmethod
def execute_next(db, project_id: UUID, operation_id: UUID) -> dict:
partition = AoiOperationService.claim_next(db, project_id, operation_id)
if partition is None:
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
operation = db.get(AoiOperation, operation_id)
child = JobService.create_job(
db,
JobCreate(
job_type=f"aoi.{operation.operation_type}.partition",
project_id=project_id,
parameters_json={
"aoi_operation_id": str(operation_id),
"partition_id": str(partition.id),
"partition_key": partition.partition_key,
"provider_key": partition.provider_key,
"product_key": partition.product_key,
},
),
)
partition = db.get(AoiOperationPartition, partition.id)
partition.child_job_id = child.id
db.add(partition)
db.commit()
JobService.mark_running(db, child.id)
try:
result = AoiOperationExecutor._dispatch(
db, project_id, operation, partition
)
output_id = (
result.get("output_dataset_id") if isinstance(result, dict) else None
)
JobService.mark_success(
db,
child.id,
result=result,
output_dataset_id=UUID(str(output_id)) if output_id else None,
)
return AoiOperationService.complete(
db, project_id, operation_id, partition.id, result
)
except AppError as exc:
JobService.mark_failed(
db, child.id, exc.message, {"code": exc.code, "details": exc.details}
)
return AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
exc.message,
AoiOperationExecutor._retryable(exc),
{"code": exc.code, "details": exc.details},
)
except Exception:
try:
db.rollback()
JobService.mark_failed(
db,
child.id,
"Unexpected partition execution error",
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
finally:
AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
"Unexpected partition execution error",
True,
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
raise
@staticmethod
def _dispatch(
db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition
) -> dict:
geometry = to_shape(partition.geometry)
min_x, min_y, max_x, max_y = geometry.bounds
bbox = VectorSelectionBBox(
min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326"
)
force_refresh = bool(
(operation.request_json or {})
.get("parameters_json", {})
.get("force_refresh", False)
)
if partition.provider_key == "grb":
return GrbAcquisitionService.acquire(
db,
project_id,
GrbAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "orthophoto":
return OrthophotoAcquisitionService.acquire(
db,
project_id,
OrthophotoAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "dhmv":
return DhmvAcquisitionService.acquire(
db,
project_id,
DhmvAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "spw_terrain":
return SpwTerrainService.acquire(
db,
project_id,
SpwTerrainAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "official_vector":
return OfficialVectorAcquisitionService.acquire(
db,
project_id,
OfficialVectorAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "flood_hazard":
return FloodHazardAcquisitionService.acquire(
db,
project_id,
FloodHazardAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "thematic_raster":
return ThematicRasterAcquisitionService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "walous":
return WalousLandCoverService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "bathymetry_profiles":
return BathymetryProfileAcquisitionService.acquire(
db,
project_id,
BathymetryProfileAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
if partition.provider_key == "mdk_bathymetry":
return MdkBathymetryAcquisitionService.acquire(
db,
project_id,
MdkBathymetryAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed AOI executor is registered for this provider",
details={"provider_key": partition.provider_key},
status_code=422,
)
@staticmethod
def _retryable(error: AppError) -> bool:
return error.status_code >= 500 or error.code.endswith(
("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")
)
@@ -0,0 +1,481 @@
from __future__ import annotations
from collections import Counter
from datetime import datetime, timezone
from hashlib import sha256
import math
from uuid import UUID, uuid4
from geoalchemy2.shape import from_shape, to_shape
from pyproj import Transformer
from shapely.geometry import MultiPolygon, Polygon, box
from shapely.ops import transform
from app.core.errors import AppError
from app.core.config import get_settings
from app.models import AoiOperation, AoiOperationPartition, Area, Project
from app.schemas.aoi_operation import AoiOperationCreate
class AoiOperationService:
MAX_PARTITIONS = 4096
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
SCOPE_AREA_NAMES = {
"belgium": "Belgium land",
"flanders": "Flanders",
"wallonia": "Wallonia",
"brussels": "Brussels-Capital Region",
"belgian_north_sea": "Belgian part of the North Sea",
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
"continental_shelf": "Belgian continental shelf beyond territorial sea",
}
@staticmethod
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
if db.get(Project, project_id) is None:
raise AppError(
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
)
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
if payload.coverage_zone:
geometry = AoiOperationService._clip_to_zone(
db, project_id, geometry, payload.coverage_zone
)
geometry = AoiOperationService._as_multipolygon(geometry)
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
partition_side_m = AoiOperationService._partition_side(
payload.provider_key, payload.max_partition_side_m
)
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
operation_id = uuid4()
now = datetime.now(timezone.utc)
operation = AoiOperation(
id=operation_id,
project_id=project_id,
area_id=payload.area_id,
operation_type=payload.operation_type,
status="queued",
geometry=from_shape(geometry, srid=4326),
request_json=payload.model_dump(mode="json", exclude_none=True),
plan_json={
"partition_strategy": "epsg31370_square_grid_intersection_v1",
"max_partition_side_m": partition_side_m,
"budget_source": "governed_provider_registry"
if payload.max_partition_side_m is None
else "stricter_operator_override",
"partition_count": len(cells),
"provider_key": payload.provider_key,
"product_key": payload.product_key,
},
created_at=now,
)
db.add(operation)
for ordinal, cell in enumerate(cells):
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
wgs84 = AoiOperationService._as_multipolygon(wgs84)
digest = sha256(wgs84.wkb).hexdigest()[:20]
db.add(
AoiOperationPartition(
id=uuid4(),
operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key,
product_key=payload.product_key,
ordinal=ordinal,
status="queued",
geometry=from_shape(wgs84, srid=4326),
attempt_count=0,
max_attempts=payload.max_attempts,
created_at=now,
)
)
db.commit()
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
if area_name is None:
raise AppError(
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
message="Unknown governed coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
scope = (
db.query(Area)
.filter(Area.project_id == project_id, Area.name == area_name)
.first()
)
if scope is None:
raise AppError(
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
message="The governed coverage-zone geometry is not persisted in this project",
details={"coverage_zone": zone},
status_code=409,
)
clipped = geometry.intersection(to_shape(scope.geometry))
if clipped.is_empty:
raise AppError(
code="AOI_OUTSIDE_PROVIDER_ZONE",
message="The AOI does not intersect the provider coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
return clipped
@staticmethod
def _as_multipolygon(geometry) -> MultiPolygon:
if isinstance(geometry, Polygon):
return MultiPolygon([geometry])
if isinstance(geometry, MultiPolygon):
return geometry
polygons = [
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
]
if not polygons:
raise AppError(
code="AOI_GEOMETRY_EMPTY",
message="AOI contains no polygonal area after clipping",
status_code=422,
)
return MultiPolygon(polygons)
@staticmethod
def _partition_side(provider_key: str, requested: float | None) -> float:
settings = get_settings()
def raster_side(
max_side_m: float, max_pixels: int, resolution_m: float
) -> float:
# Keep every square grid cell within both the provider's spatial
# extent limit and its decoded-pixel budget. The small safety
# margin absorbs ceil/edge rounding in the acquisition services.
pixel_limited_side = (
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
)
return min(float(max_side_m), pixel_limited_side)
budgets = {
"orthophoto": float(settings.orthophoto_max_side_m),
"grb": float(settings.grb_max_side_m),
"dhmv": raster_side(
settings.dhmv_max_side_m,
settings.dhmv_max_pixels,
settings.dhmv_resolution_m,
),
"spw_terrain": raster_side(
settings.spw_terrain_max_side_m,
settings.spw_terrain_max_pixels,
settings.spw_terrain_analysis_resolution_m,
),
"official_vector": 20_000.0,
"flood_hazard": raster_side(
settings.flood_hazard_max_side_m,
settings.flood_hazard_max_pixels,
settings.flood_hazard_resolution_m,
),
"thematic_raster": raster_side(
settings.thematic_raster_max_side_m,
settings.thematic_raster_max_pixels,
10.0,
),
"walous": raster_side(
settings.walous_max_side_m,
settings.walous_max_pixels,
settings.walous_analysis_resolution_m,
),
"bathymetry_profiles": 20_000.0,
"mdk_bathymetry": 20_000.0,
}
if provider_key not in budgets:
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed partition budget is registered for this provider",
details={"provider_key": provider_key},
status_code=422,
)
governed = budgets[provider_key]
return min(governed, float(requested)) if requested is not None else governed
@staticmethod
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
if (payload.area_id is None) == (payload.bbox is None):
raise AppError(
code="AOI_SELECTION_REQUIRED",
message="Provide exactly one area_id or bbox",
status_code=422,
)
if payload.area_id is not None:
area = db.get(Area, payload.area_id)
if area is None or area.project_id != project_id:
raise AppError(
code="AREA_NOT_FOUND", message="Area not found", status_code=404
)
return to_shape(area.geometry)
bbox = payload.bbox
if bbox is None or bbox.crs != "EPSG:4326":
raise AppError(
code="INVALID_AOI_CRS",
message="AOI bbox must use EPSG:4326",
status_code=422,
)
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
@staticmethod
def _partition(geometry, side_m: float) -> list:
min_x, min_y, max_x, max_y = geometry.bounds
columns = max(1, math.ceil((max_x - min_x) / side_m))
rows = max(1, math.ceil((max_y - min_y) / side_m))
if columns * rows > AoiOperationService.MAX_PARTITIONS:
raise AppError(
code="AOI_PARTITION_LIMIT_EXCEEDED",
message="AOI requires too many bounded partitions",
details={
"candidate_count": columns * rows,
"max_partitions": AoiOperationService.MAX_PARTITIONS,
},
status_code=422,
)
partitions = []
for row in range(rows):
for column in range(columns):
clipped = geometry.intersection(
box(
min_x + column * side_m,
min_y + row * side_m,
min(min_x + (column + 1) * side_m, max_x),
min(min_y + (row + 1) * side_m, max_y),
)
)
if not clipped.is_empty and clipped.area > 0:
partitions.append(clipped)
return partitions
@staticmethod
def read(db, project_id: UUID, operation_id: UUID) -> dict:
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
counts = Counter(partition.status for partition in partitions)
complete = counts["success"] + counts["skipped"]
return {
"id": operation.id,
"project_id": operation.project_id,
"area_id": operation.area_id,
"parent_job_id": operation.parent_job_id,
"operation_type": operation.operation_type,
"status": operation.status,
"request_json": operation.request_json,
"plan_json": operation.plan_json,
"result_json": operation.result_json,
"error_message": operation.error_message,
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
"partition_counts": dict(counts),
"partitions": partitions,
"created_at": operation.created_at,
"started_at": operation.started_at,
"finished_at": operation.finished_at,
}
@staticmethod
def list(db, project_id: UUID, limit: int = 50) -> dict:
rows = (
db.query(AoiOperation)
.filter(AoiOperation.project_id == project_id)
.order_by(AoiOperation.created_at.desc())
.limit(limit)
.all()
)
return {
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
"total": len(rows),
}
@staticmethod
def claim_next(db, project_id: UUID, operation_id: UUID):
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partition = (
db.query(AoiOperationPartition)
.filter(
AoiOperationPartition.operation_id == operation_id,
AoiOperationPartition.status == "queued",
)
.order_by(AoiOperationPartition.ordinal)
.with_for_update(skip_locked=True)
.first()
)
if partition is None:
return None
now = datetime.now(timezone.utc)
partition.status = "running"
partition.started_at = now
partition.attempt_count += 1
partition.error_message = None
operation.status = "running"
operation.started_at = operation.started_at or now
db.add(partition)
db.add(operation)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def checkpoint(
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status != "running":
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can be checkpointed",
status_code=409,
)
partition.checkpoint_json = checkpoint
db.add(partition)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def complete(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
result: dict,
skipped: bool = False,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status == "success" or partition.status == "skipped":
return AoiOperationService.read(db, project_id, operation_id)
if partition.status != "running":
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can complete",
status_code=409,
)
partition.status = "skipped" if skipped else "success"
partition.result_json = result
partition.finished_at = datetime.now(timezone.utc)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def fail(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
message: str,
retryable: bool,
details: dict,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
partition.error_message = message
partition.result_json = {"details": details}
partition.status = (
"queued"
if retryable and partition.attempt_count < partition.max_attempts
else "failed"
)
partition.finished_at = (
None if partition.status == "queued" else datetime.now(timezone.utc)
)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def _partition_row(db, project_id, operation_id, partition_id):
operation = db.get(AoiOperation, operation_id)
partition = db.get(AoiOperationPartition, partition_id)
if (
operation is None
or operation.project_id != project_id
or partition is None
or partition.operation_id != operation_id
):
raise AppError(
code="AOI_PARTITION_NOT_FOUND",
message="AOI partition not found",
status_code=404,
)
return partition
@staticmethod
def _refresh_parent(db, operation_id):
operation = db.get(AoiOperation, operation_id)
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
statuses = [partition.status for partition in partitions]
output_dataset_ids = []
for partition in partitions:
output_id = (
(partition.result_json or {}).get("output_dataset_id")
if isinstance(partition.result_json, dict)
else None
)
if output_id and str(output_id) not in output_dataset_ids:
output_dataset_ids.append(str(output_id))
operation.result_json = {
"partition_count": len(partitions),
"completed_partition_count": sum(
status in {"success", "skipped"} for status in statuses
),
"failed_partition_count": statuses.count("failed"),
"output_dataset_ids": output_dataset_ids,
"merge_contract": "source_aware_spatial_union",
"vector_deduplication": "source_feature_id_then_geometry",
"raster_deduplication": "governed_mosaic_grid",
"complete_coverage": bool(statuses)
and all(status in {"success", "skipped"} for status in statuses),
}
now = datetime.now(timezone.utc)
if statuses and all(status in {"success", "skipped"} for status in statuses):
operation.status = "success"
operation.finished_at = now
operation.error_message = None
elif "failed" in statuses and not any(
status in {"queued", "running"} for status in statuses
):
operation.status = (
"partial"
if any(status in {"success", "skipped"} for status in statuses)
else "failed"
)
operation.finished_at = now
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
db.add(operation)
db.commit()
@@ -0,0 +1,38 @@
from __future__ import annotations
import asyncio
import logging
from app.db.session import SessionLocal
from app.models import AoiOperation
from app.services.aoi_operation_executor import AoiOperationExecutor
logger = logging.getLogger("geointel.aoi_worker")
class AoiOperationWorker:
@staticmethod
def run_once() -> int:
db = SessionLocal()
try:
rows = db.query(AoiOperation).filter(AoiOperation.status.in_(("queued", "running"))).order_by(AoiOperation.created_at).limit(10).all()
for operation in rows:
try:
AoiOperationExecutor.execute_next(db, operation.project_id, operation.id)
except Exception:
db.rollback()
logger.exception("AOI partition execution failed operation_id=%s", operation.id)
return len(rows)
finally:
db.close()
@staticmethod
async def run(stop_event: asyncio.Event, poll_seconds: float) -> None:
while not stop_event.is_set():
processed = await asyncio.to_thread(AoiOperationWorker.run_once)
if processed == 0:
try:
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
except TimeoutError:
pass
+107 -8
View File
@@ -7,12 +7,90 @@ from geoalchemy2.shape import from_shape, to_shape
from shapely.geometry import mapping from shapely.geometry import mapping
from app.core.errors import AppError from app.core.errors import AppError
from app.models import Area, Project from app.models import Area, Dataset, Project, VectorFeature
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326
class AreaService: class AreaService:
@staticmethod
def _municipality_dataset(db: Session, project_id: uuid.UUID) -> Dataset | None:
return (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.reference_layer_name == "belgium_municipalities",
Dataset.status == "ready",
)
.order_by(Dataset.created_at.desc())
.first()
)
@staticmethod
def _filter_municipality_properties(properties_items: list[dict], query: str, limit: int) -> tuple[list[dict], int]:
normalized = query.strip().casefold()
matches: list[dict] = []
for properties in properties_items:
names = [str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger")]
niscode = str(properties.get("niscode") or "").strip()
if normalized and normalized not in " ".join([niscode, *names]).casefold():
continue
display_name = next((name for name in names if name), niscode)
matches.append({
"niscode": niscode,
"name": display_name,
"name_nl": names[0] or None,
"name_fr": names[1] or None,
"name_de": names[2] or None,
})
matches.sort(key=lambda item: (item["name"].casefold(), item["niscode"]))
return matches[:limit], len(matches)
@staticmethod
def search_municipalities(db: Session, project_id: uuid.UUID, query: str, limit: int = 20) -> tuple[list[dict], int]:
dataset = AreaService._municipality_dataset(db, project_id)
if dataset is None:
return [], 0
property_rows = (
db.query(VectorFeature.properties_json)
.filter(VectorFeature.dataset_id == dataset.id)
.all()
)
properties_items = [row[0] for row in property_rows if isinstance(row[0], dict)]
return AreaService._filter_municipality_properties(properties_items, query, limit)
@staticmethod
def activate_municipality(db: Session, project_id: uuid.UUID, niscode: str) -> Area:
normalized_code = niscode.strip()
dataset = AreaService._municipality_dataset(db, project_id)
if dataset is None:
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
feature = (
db.query(VectorFeature)
.filter(
VectorFeature.dataset_id == dataset.id,
VectorFeature.properties_json["niscode"].as_string() == normalized_code,
)
.first()
)
if feature is not None:
properties = feature.properties_json if isinstance(feature.properties_json, dict) else {}
display_name = next(
(str(properties.get(key) or "").strip() for key in ("namedut", "namefre", "nameger") if str(properties.get(key) or "").strip()),
normalized_code,
)
area_name = f"Gemeente {display_name} - NIS {normalized_code}"
existing = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
if existing is not None:
return existing
geometry = to_shape(feature.geometry)
return AreaService.create_area(
db,
project_id,
AreaCreate(name=area_name, geometry=mapping(geometry), crs="EPSG:4326"),
)
raise AppError(code="MUNICIPALITY_NOT_FOUND", message="Municipality is not available in the official NGI administrative layer", status_code=404)
@staticmethod @staticmethod
def serialize_area(area: Area) -> dict: def serialize_area(area: Area) -> dict:
geometry = to_shape(area.geometry) if area.geometry else None geometry = to_shape(area.geometry) if area.geometry else None
@@ -48,7 +126,11 @@ class AreaService:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
try: try:
multipolygon = normalize_to_multipolygon(payload.geometry) multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc: except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
@@ -56,8 +138,8 @@ class AreaService:
project_id=project_id, project_id=project_id,
name=payload.name.strip() or "Unnamed area", name=payload.name.strip() or "Unnamed area",
geometry=from_shape(multipolygon, srid=4326), geometry=from_shape(multipolygon, srid=4326),
original_crs=payload.crs or "EPSG:4326", original_crs=original_crs,
area_m2=area_m2(multipolygon), area_m2=metric_area,
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326), bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
) )
db.add(area) db.add(area)
@@ -79,11 +161,28 @@ class AreaService:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
changed = False changed = False
if payload.name: if payload.name is not None and payload.name.strip():
area.name = payload.name.strip() or area.name area.name = payload.name.strip() or area.name
changed = True changed = True
if payload.crs: if payload.crs is not None and payload.geometry is None:
area.original_crs = payload.crs raise AppError(
code="INVALID_AREA_CRS_UPDATE",
message="crs can only be supplied together with replacement geometry",
status_code=422,
)
if payload.geometry is not None:
try:
multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
area.geometry = from_shape(multipolygon, srid=4326)
area.original_crs = original_crs
area.area_m2 = metric_area
area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
changed = True changed = True
if not changed: if not changed:
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422) raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
+260
View File
@@ -0,0 +1,260 @@
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import secrets
import threading
import time
from collections import deque
from dataclasses import dataclass, field
from typing import Literal, cast
from uuid import UUID
from app.core.config import Settings
from app.core.public_demo import PUBLIC_DEMO_PROJECT_ID
@dataclass(frozen=True)
class AuthPrincipal:
username: str
expires_at: int
session_id: str = field(default_factory=lambda: secrets.token_urlsafe(12))
role: Literal["operator", "guest"] = "operator"
project_id: UUID | None = None
class AuthService:
HASH_NAME = "pbkdf2_sha256"
HASH_ITERATIONS = 600_000
MAX_FAILURES = 5
FAILURE_WINDOW_SECONDS = 300
_failures: dict[str, deque[float]] = {}
_failure_lock = threading.Lock()
_guest_requests: dict[str, deque[float]] = {}
_guest_request_lock = threading.Lock()
_active_guest_compute = 0
_guest_compute_lock = threading.Lock()
@staticmethod
def _b64_encode(value: bytes) -> str:
return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=")
@staticmethod
def _b64_decode(value: str) -> bytes:
return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
@classmethod
def hash_password(
cls,
password: str,
*,
salt: bytes | None = None,
iterations: int | None = None,
) -> str:
resolved_salt = salt or secrets.token_bytes(18)
resolved_iterations = iterations or cls.HASH_ITERATIONS
digest = hashlib.pbkdf2_hmac(
"sha256",
password.encode("utf-8"),
resolved_salt,
resolved_iterations,
)
return "$".join(
(
cls.HASH_NAME,
str(resolved_iterations),
cls._b64_encode(resolved_salt),
cls._b64_encode(digest),
)
)
@classmethod
def verify_password(cls, password: str, encoded: str) -> bool:
try:
algorithm, iterations_raw, salt_raw, expected_raw = encoded.split("$", 3)
if algorithm != cls.HASH_NAME:
return False
iterations = int(iterations_raw)
if iterations < 100_000 or iterations > 2_000_000:
return False
salt = cls._b64_decode(salt_raw)
expected = cls._b64_decode(expected_raw)
actual = hashlib.pbkdf2_hmac(
"sha256",
password.encode("utf-8"),
salt,
iterations,
)
return hmac.compare_digest(actual, expected)
except (TypeError, ValueError):
return False
@classmethod
def credentials_match(cls, username: str, password: str, settings: Settings) -> bool:
expected_username = settings.auth_username or ""
expected_password_hash = settings.auth_password_hash or ""
username_matches = hmac.compare_digest(
username.encode("utf-8"),
expected_username.encode("utf-8"),
)
password_matches = cls.verify_password(password, expected_password_hash)
return username_matches and password_matches
@classmethod
def create_session_token(
cls,
username: str,
settings: Settings,
*,
role: Literal["operator", "guest"] = "operator",
project_id: UUID | None = None,
ttl_seconds: int | None = None,
now: int | None = None,
) -> str:
issued_at = int(time.time() if now is None else now)
if role == "guest" and project_id is None:
raise ValueError("Guest sessions must be scoped to a demo project")
resolved_ttl = ttl_seconds if ttl_seconds is not None else (
settings.guest_session_ttl_seconds if role == "guest" else settings.auth_session_ttl_seconds
)
payload = {
"exp": issued_at + resolved_ttl,
"iat": issued_at,
"jti": secrets.token_urlsafe(12),
"role": role,
"sub": username,
"v": 2,
}
if project_id is not None:
payload["project_id"] = str(project_id)
encoded_payload = cls._b64_encode(
json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")
)
signature = hmac.new(
(settings.auth_session_secret or "").encode("utf-8"),
encoded_payload.encode("ascii"),
hashlib.sha256,
).digest()
return f"{encoded_payload}.{cls._b64_encode(signature)}"
@classmethod
def verify_session_token(
cls,
token: str | None,
settings: Settings,
*,
now: int | None = None,
) -> AuthPrincipal | None:
if not token:
return None
try:
encoded_payload, encoded_signature = token.split(".", 1)
expected_signature = hmac.new(
(settings.auth_session_secret or "").encode("utf-8"),
encoded_payload.encode("ascii"),
hashlib.sha256,
).digest()
supplied_signature = cls._b64_decode(encoded_signature)
if not hmac.compare_digest(expected_signature, supplied_signature):
return None
payload = json.loads(cls._b64_decode(encoded_payload))
username = str(payload.get("sub") or "")
expires_at = int(payload.get("exp") or 0)
issued_at = int(payload.get("iat") or 0)
version = int(payload.get("v") or 0)
role_value = str(payload.get("role") or "operator")
session_id = str(payload.get("jti") or "")
current = int(time.time() if now is None else now)
if version not in {1, 2} or role_value not in {"operator", "guest"} or not session_id:
return None
role = cast(Literal["operator", "guest"], role_value)
if issued_at <= 0 or issued_at > current + 60 or expires_at <= current:
return None
if role == "operator":
if username != settings.auth_username:
return None
max_ttl = settings.auth_session_ttl_seconds
project_id = None
else:
if not settings.guest_access_enabled or username != settings.guest_display_name:
return None
max_ttl = settings.guest_session_ttl_seconds
raw_project_id = payload.get("project_id")
if not raw_project_id:
return None
project_id = UUID(str(raw_project_id))
if project_id != PUBLIC_DEMO_PROJECT_ID:
return None
if expires_at - issued_at > max_ttl:
return None
return AuthPrincipal(
username=username,
expires_at=expires_at,
session_id=session_id,
role=role,
project_id=project_id,
)
except (TypeError, ValueError, json.JSONDecodeError, UnicodeDecodeError):
return None
@classmethod
def retry_after_seconds(cls, key: str, *, now: float | None = None) -> int:
current = time.monotonic() if now is None else now
with cls._failure_lock:
attempts = cls._failures.setdefault(key, deque())
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
attempts.popleft()
if len(attempts) < cls.MAX_FAILURES:
if not attempts:
cls._failures.pop(key, None)
return 0
return max(1, int(cls.FAILURE_WINDOW_SECONDS - (current - attempts[0])))
@classmethod
def record_failure(cls, key: str, *, now: float | None = None) -> None:
current = time.monotonic() if now is None else now
with cls._failure_lock:
attempts = cls._failures.setdefault(key, deque())
while attempts and current - attempts[0] >= cls.FAILURE_WINDOW_SECONDS:
attempts.popleft()
attempts.append(current)
@classmethod
def clear_failures(cls, key: str) -> None:
with cls._failure_lock:
cls._failures.pop(key, None)
@classmethod
def consume_guest_request(
cls,
key: str,
*,
max_requests: int,
window_seconds: int = 60,
now: float | None = None,
) -> int:
"""Record a guest action and return Retry-After seconds when limited."""
current = time.monotonic() if now is None else now
with cls._guest_request_lock:
attempts = cls._guest_requests.setdefault(key, deque())
while attempts and current - attempts[0] >= window_seconds:
attempts.popleft()
if len(attempts) >= max_requests:
return max(1, int(window_seconds - (current - attempts[0])))
attempts.append(current)
return 0
@classmethod
def try_acquire_guest_compute(cls, *, max_concurrency: int) -> bool:
with cls._guest_compute_lock:
if cls._active_guest_compute >= max_concurrency:
return False
cls._active_guest_compute += 1
return True
@classmethod
def release_guest_compute(cls) -> None:
with cls._guest_compute_lock:
cls._active_guest_compute = max(0, cls._active_guest_compute - 1)
@@ -0,0 +1,207 @@
from __future__ import annotations
import base64
import hashlib
import json
import secrets
from typing import Any
from urllib.error import HTTPError
from urllib.parse import urlencode, urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener
import jwt
from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
from app.core.config import Settings
MAX_OIDC_JSON_BYTES = 1_048_576
class _RejectRedirects(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ANN201
return None
class AuthentikOidcService:
def __init__(self, settings: Settings):
self.settings = settings
self.issuer = (settings.authentik_issuer or "").rstrip("/")
self.serializer = URLSafeTimedSerializer(
settings.auth_session_secret or "",
salt="geointel-authentik-v1",
)
@property
def enabled(self) -> bool:
return bool(
self.issuer
and self.settings.authentik_client_id
and self.settings.authentik_client_secret
and self.settings.authentik_allowed_email
)
@property
def redirect_uri(self) -> str:
return (
f"{self.settings.public_base_url.rstrip('/')}"
f"{self.settings.api_prefix}/auth/authentik/callback"
)
@staticmethod
def _origin(url: str) -> tuple[str, str, int]:
parsed = urlsplit(url)
if parsed.scheme != "https" or not parsed.hostname:
raise ValueError("OIDC URLs must use absolute HTTPS URLs")
return parsed.scheme, parsed.hostname.casefold(), parsed.port or 443
def _validate_endpoint(self, url: str) -> str:
parsed = urlsplit(url)
if (
self._origin(url) != self._origin(self.issuer)
or parsed.username
or parsed.password
or parsed.fragment
):
raise ValueError("OIDC endpoint is outside the configured issuer origin")
return url
def _fetch_json(
self,
url: str,
data: dict[str, str] | None = None,
) -> dict[str, Any]:
self._validate_endpoint(url)
encoded = urlencode(data).encode("utf-8") if data is not None else None
headers = {"Accept": "application/json"}
if encoded is not None:
headers["Content-Type"] = "application/x-www-form-urlencoded"
request = Request(url, data=encoded, headers=headers)
try:
with build_opener(_RejectRedirects()).open(request, timeout=10) as response:
declared_length = response.headers.get("Content-Length")
if declared_length and int(declared_length) > MAX_OIDC_JSON_BYTES:
raise ValueError("OIDC response exceeds the configured size limit")
raw = response.read(MAX_OIDC_JSON_BYTES + 1)
except HTTPError as exc:
raise ValueError("OIDC endpoint returned an HTTP error or redirect") from exc
if len(raw) > MAX_OIDC_JSON_BYTES:
raise ValueError("OIDC response exceeds the configured size limit")
payload = json.loads(raw)
if not isinstance(payload, dict):
raise ValueError("OIDC endpoint did not return a JSON object")
return payload
def _discovery(self) -> dict[str, Any]:
document = self._fetch_json(
f"{self.issuer}/.well-known/openid-configuration"
)
if str(document.get("issuer", "")).rstrip("/") != self.issuer:
raise ValueError("OIDC issuer mismatch")
for key in ("authorization_endpoint", "token_endpoint", "jwks_uri"):
endpoint = document.get(key)
if not isinstance(endpoint, str):
raise ValueError(f"OIDC discovery is missing {key}")
self._validate_endpoint(endpoint)
return document
def start(self) -> tuple[str, str]:
if not self.enabled:
raise ValueError("Authentik is not configured")
state = secrets.token_urlsafe(32)
nonce = secrets.token_urlsafe(32)
verifier = secrets.token_urlsafe(48)
flow = self.serializer.dumps(
{"state": state, "nonce": nonce, "verifier": verifier}
)
challenge = (
base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
.rstrip(b"=")
.decode()
)
discovery = self._discovery()
query = urlencode(
{
"client_id": self.settings.authentik_client_id,
"redirect_uri": self.redirect_uri,
"response_type": "code",
"scope": "openid email profile",
"state": state,
"nonce": nonce,
"code_challenge": challenge,
"code_challenge_method": "S256",
}
)
return f"{discovery['authorization_endpoint']}?{query}", flow
def finish(self, *, code: str, state: str, flow_cookie: str) -> dict[str, Any]:
if not self.enabled or not code:
raise ValueError("OIDC flow is incomplete")
try:
flow = self.serializer.loads(flow_cookie, max_age=600)
except (BadSignature, SignatureExpired) as exc:
raise ValueError("Invalid OIDC flow") from exc
if not isinstance(flow, dict):
raise ValueError("Invalid OIDC flow payload")
if not state or not secrets.compare_digest(state, str(flow.get("state", ""))):
raise ValueError("OIDC state mismatch")
verifier = str(flow.get("verifier", ""))
nonce = str(flow.get("nonce", ""))
if not verifier or not nonce:
raise ValueError("OIDC flow payload is incomplete")
discovery = self._discovery()
token_response = self._fetch_json(
str(discovery["token_endpoint"]),
{
"grant_type": "authorization_code",
"code": code,
"redirect_uri": self.redirect_uri,
"client_id": self.settings.authentik_client_id or "",
"client_secret": self.settings.authentik_client_secret or "",
"code_verifier": verifier,
},
)
token = str(token_response.get("id_token", ""))
if not token:
raise ValueError("OIDC token response has no ID token")
header = jwt.get_unverified_header(token)
if header.get("alg") != "RS256" or not header.get("kid"):
raise ValueError("OIDC ID token uses an unsupported signing header")
jwks = self._fetch_json(str(discovery["jwks_uri"]))
matching_keys = [
key
for key in jwks.get("keys", [])
if isinstance(key, dict) and key.get("kid") == header["kid"]
]
if len(matching_keys) != 1:
raise ValueError("OIDC signing key is missing or ambiguous")
signing_key = jwt.PyJWK.from_dict(matching_keys[0]).key
claims = jwt.decode(
token,
signing_key,
algorithms=["RS256"],
audience=self.settings.authentik_client_id,
issuer=discovery["issuer"],
options={
"require": [
"exp",
"iat",
"iss",
"aud",
"sub",
"nonce",
"email",
"email_verified",
]
},
)
if not secrets.compare_digest(str(claims.get("nonce", "")), nonce):
raise ValueError("OIDC nonce mismatch")
email = str(claims.get("email", "")).strip().casefold()
allowed = str(self.settings.authentik_allowed_email or "").strip().casefold()
if claims.get("email_verified") is not True or not secrets.compare_digest(
email, allowed
):
raise ValueError("OIDC identity is not authorized")
return claims
@@ -7,7 +7,7 @@ import math
from typing import Any, Callable from typing import Any, Callable
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.parse import urlencode from urllib.parse import urlencode
from urllib.request import Request, urlopen from urllib.request import Request
from uuid import UUID from uuid import UUID
from geoalchemy2.shape import to_shape from geoalchemy2.shape import to_shape
@@ -15,6 +15,7 @@ from shapely.geometry import Point, box, mapping
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.services.outbound_request_guard import guarded_opener
from app.models import Area, Dataset, DatasetVersion, Project from app.models import Area, Dataset, DatasetVersion, Project
from app.schemas.bathymetry import ( from app.schemas.bathymetry import (
BathymetryPartitionFinalizeRequest, BathymetryPartitionFinalizeRequest,
@@ -91,20 +92,21 @@ class BathymetryProfileAcquisitionService:
"authority_level": "authoritative", "authority_level": "authoritative",
"geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen", "geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen",
"data_kind": "bodemhoogteraster en XYZ-puntenwolk", "data_kind": "bodemhoogteraster en XYZ-puntenwolk",
"query_modes": ["download", "arcgis_map_service"], "query_modes": ["operator_archive", "bounded_raster", "arcgis_map_service"],
"vertical_reference": "mDNG", "vertical_reference": "mDNG",
"horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden", "horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden",
"native_resolution": "0,5 m", "native_resolution": "0,5 m",
"integration_status": "available_not_integrated", "integration_status": "operational",
"acquisition_supported": False, "acquisition_supported": True,
"configured": False, "configured": True,
"service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer", "service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer",
"catalog_url": "https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html", "catalog_url": "https://geoportail.wallonie.be/catalogue/0a544b42-0b30-4c8e-85e7-38149b99eae0.html",
"attribution": "Service public de Wallonie", "attribution": "Service public de Wallonie",
"license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.", "license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.",
"limitation_message": ( "limitation_message": (
"Dekking en meetjaar verschillen per vaarweg of reservoir. Integratie vereist een beheerde " "De gepinde officiële release kan begrensd als raster worden geïmporteerd via de operator. "
"download- en mosaïekstroom plus expliciete omzetting van mDNG." "Dekking verschilt per vaarweg; de waarden zijn bodemhoogtes in mDNG uit 2019-2022, "
"zonder stilzwijgende datumconversie of afleiding van actuele waterdiepte."
), ),
}, },
{ {
@@ -133,8 +135,37 @@ class BathymetryProfileAcquisitionService:
) )
@staticmethod @staticmethod
def list_sources() -> list[dict[str, Any]]: def list_sources(settings=None) -> list[dict[str, Any]]:
return [BathymetrySourceRead(**item).model_dump() for item in BathymetryProfileAcquisitionService._SOURCES] from app.core.config import get_settings
resolved_settings = settings or get_settings()
items: list[dict[str, Any]] = []
for source in BathymetryProfileAcquisitionService._SOURCES:
item = dict(source)
if item["key"] == "mdk_bcp_bathymetry":
mdk_configured = bool(
resolved_settings.mdk_bathymetry_acquisition_enabled
and (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
)
item["acquisition_supported"] = True
item["configured"] = mdk_configured
if mdk_configured:
item["integration_status"] = "operational"
item["limitation_message"] = (
"Begrensde WCS-acquisitie is expliciet ingeschakeld en draait alleen wanneer de "
"live readiness-probe bereikbaar is en het geconfigureerde coverage-id door de "
"capabilities wordt geadverteerd. Dieptes blijven LAT-gerefereerd; watervolume "
"blijft zonder compatibel wateroppervlak niet ondersteund."
)
else:
item["limitation_message"] = (
"Begrensde WCS-acquisitie bestaat maar staat uit. Zet "
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true en configureer MDK_BATHYMETRY_COVERAGE_ID "
"pas nadat de readiness-probe live 'reachable' rapporteert. Er wordt nooit "
"onbeveiligd of ongevalideerd gedownload."
)
items.append(item)
return [BathymetrySourceRead(**item).model_dump() for item in items]
@staticmethod @staticmethod
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]: def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
@@ -194,7 +225,7 @@ class BathymetryProfileAcquisitionService:
}, },
) )
try: try:
with (opener or urlopen)(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response: with (opener or guarded_opener(url))(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response:
limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024 limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024
content = response.read(limit + 1) content = response.read(limit + 1)
except HTTPError as exc: except HTTPError as exc:
@@ -245,6 +276,45 @@ class BathymetryProfileAcquisitionService:
"spatialRel": "esriSpatialRelIntersects", "spatialRel": "esriSpatialRelIntersects",
} }
@staticmethod
def _unseen_records(
page_features: list[Any],
seen_object_ids: set[str],
) -> list[dict[str, Any]]:
"""Every page must bring records the earlier pages did not.
An ArcGIS layer without ``supportsPagination`` accepts ``resultOffset``
and ignores it, answering every page with the first one. Advancing the
offset by the page length still reaches the announced count, so the
completeness check below passed while the dataset held N copies of page
one a silent substitution of the source data, which is the one thing
bounded acquisition exists to prevent.
"""
fresh: list[dict[str, Any]] = []
for item in page_features:
if not isinstance(item, dict):
continue
attributes = item.get("attributes")
object_id = attributes.get("OBJECTID") if isinstance(attributes, dict) else None
if object_id is None:
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile record has no OBJECTID, so pagination cannot be verified",
status_code=502,
)
key = str(object_id)
if key in seen_object_ids:
raise AppError(
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
message="VHA profile pagination repeated a record; the layer is not honouring resultOffset",
details={"object_id": key},
status_code=502,
)
seen_object_ids.add(key)
fresh.append(item)
return fresh
@staticmethod @staticmethod
def _fetch_profiles( def _fetch_profiles(
bbox_values: tuple[float, float, float, float], bbox_values: tuple[float, float, float, float],
@@ -276,8 +346,16 @@ class BathymetryProfileAcquisitionService:
features: list[dict[str, Any]] = [] features: list[dict[str, Any]] = []
response_hashes: list[str] = [] response_hashes: list[str] = []
request_urls: list[str] = [count_url] request_urls: list[str] = [count_url]
seen_object_ids: set[str] = set()
offset = 0 offset = 0
while offset < candidate_count: while offset < candidate_count:
if len(request_urls) > settings.bathymetry_profiles_max_pages:
raise AppError(
code="BATHYMETRY_SCOPE_TOO_LARGE",
message="VHA profile pagination exceeded the configured page limit; acquire smaller area partitions",
details={"max_pages": settings.bathymetry_profiles_max_pages},
status_code=422,
)
page_url = BathymetryProfileAcquisitionService._query_url( page_url = BathymetryProfileAcquisitionService._query_url(
base, base,
{ {
@@ -297,11 +375,13 @@ class BathymetryProfileAcquisitionService:
message="VHA profile response does not contain a feature list", message="VHA profile response does not contain a feature list",
status_code=502, status_code=502,
) )
features.extend(item for item in page_features if isinstance(item, dict))
response_hashes.append(page_sha) response_hashes.append(page_sha)
request_urls.append(page_url) request_urls.append(page_url)
if not page_features: if not page_features:
break break
features.extend(
BathymetryProfileAcquisitionService._unseen_records(page_features, seen_object_ids)
)
offset += len(page_features) offset += len(page_features)
if len(features) != candidate_count: if len(features) != candidate_count:
raise AppError( raise AppError(
@@ -332,7 +412,15 @@ class BathymetryProfileAcquisitionService:
for start in range(0, len(ordered_codes), 100): for start in range(0, len(ordered_codes), 100):
chunk = ordered_codes[start : start + 100] chunk = ordered_codes[start : start + 100]
offset = 0 offset = 0
seen_page_hashes: set[str] = set()
while True: while True:
if len(seen_page_hashes) >= settings.bathymetry_profiles_max_pages:
raise AppError(
code="BATHYMETRY_SCOPE_TOO_LARGE",
message="VHA watercourse pagination exceeded the configured page limit",
details={"max_pages": settings.bathymetry_profiles_max_pages},
status_code=422,
)
url = BathymetryProfileAcquisitionService._query_url( url = BathymetryProfileAcquisitionService._query_url(
base, base,
{ {
@@ -355,6 +443,16 @@ class BathymetryProfileAcquisitionService:
message="VHA watercourse response does not contain a feature list", message="VHA watercourse response does not contain a feature list",
status_code=502, status_code=502,
) )
if response_sha in seen_page_hashes:
# The names themselves deduplicate by code, so a stuck
# provider produced no visible change while the loop, which
# ended only on exceededTransferLimit, kept requesting.
raise AppError(
code="BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION",
message="VHA watercourse pagination returned the same page again",
status_code=502,
)
seen_page_hashes.add(response_sha)
for feature in page_features: for feature in page_features:
attributes = feature.get("attributes") if isinstance(feature, dict) else None attributes = feature.get("attributes") if isinstance(feature, dict) else None
if not isinstance(attributes, dict): if not isinstance(attributes, dict):
@@ -0,0 +1,366 @@
from __future__ import annotations
import io
import math
from datetime import UTC, datetime
from pathlib import Path
from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import box, mapping
from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.raster_cell_selection import select_cells
from app.models import Area, Dataset
from app.schemas.bathymetry import (
BathymetryRasterMetric,
BathymetryRasterSelectionRequest,
BathymetryRasterSelectionResponse,
BathymetryRasterSelectionSummary,
)
class BathymetryRasterAnalysisService:
SOURCE_NAME = "spw_bathymetry"
PRODUCT_KEY = "spw_bathymetry_50cm_mdng"
UNSUPPORTED_METRICS = [
"current_water_depth_m",
"water_volume_m3",
"vertical_datum_conversion",
]
LIMITATION = (
"De rasterwaarden zijn waterbodemhoogtes in mDNG uit een samengestelde SPW-opmeting "
"(2019-2022). Zonder een gelijktijdig waterpeil zijn actuele waterdiepte en watervolume "
"niet berekenbaar. mDNG wordt niet stilzwijgend naar TAW, LAT of een ander verticaal datum omgezet."
)
@staticmethod
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster" or dataset.source_name != BathymetryRasterAnalysisService.SOURCE_NAME:
raise AppError(
code="INVALID_BATHYMETRY_RASTER_DATASET",
message="Bathymetry analysis requires a governed SPW bathymetry raster dataset",
status_code=400,
)
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
raise AppError(
code="DATASET_FILE_MISSING",
message="Persisted bathymetry raster file is unavailable",
status_code=404,
)
return dataset
@staticmethod
def _metadata(dataset: Dataset) -> dict:
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
if (
metadata.get("product_key") != BathymetryRasterAnalysisService.PRODUCT_KEY
or metadata.get("theme") != "bathymetry"
or metadata.get("value_semantics") != "bed_elevation"
or metadata.get("vertical_reference") != "mDNG"
or metadata.get("source_crs") != "EPSG:3812"
):
raise AppError(
code="INVALID_BATHYMETRY_RASTER_METADATA",
message="Bathymetry raster provenance or value semantics are incomplete",
status_code=409,
)
return metadata
@staticmethod
def _selection_geometry(db, project_id: UUID, payload: BathymetryRasterSelectionRequest):
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
if payload.area_id is None:
return selection
area = db.get(Area, payload.area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(
code="INVALID_DATASET_SCOPE",
message="Area does not belong to this project",
status_code=400,
)
selection = selection.intersection(to_shape(area.geometry))
if selection.is_empty or selection.area <= 0:
raise AppError(
code="BATHYMETRY_SELECTION_OUTSIDE_AREA",
message="Selection does not overlap the selected work area",
status_code=422,
)
return selection
@staticmethod
def analyze(
db,
project_id: UUID,
dataset_id: UUID,
payload: BathymetryRasterSelectionRequest,
*,
settings: Settings | None = None,
) -> dict:
resolved_settings = settings or get_settings()
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
source_metadata = BathymetryRasterAnalysisService._metadata(dataset)
selection_4326 = BathymetryRasterAnalysisService._selection_geometry(db, project_id, payload)
try:
import numpy as np
import rasterio
from rasterio.mask import mask
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Rasterio and numpy are required for bathymetry analysis",
status_code=503,
) from exc
try:
with rasterio.open(dataset.storage_path) as source:
if source.crs is None or source.crs.to_epsg() != 3812:
raise AppError(
code="INVALID_DATASET_CRS",
message="SPW bathymetry raster CRS must be EPSG:3812",
status_code=409,
)
if source.count != 1:
raise AppError(
code="INVALID_BATHYMETRY_RASTER_BANDS",
message="SPW bathymetry requires one bed-elevation band",
status_code=409,
)
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
selection_metric = shapely_transform(transformer.transform, selection_4326)
analysis_geometry = selection_metric.intersection(box(*source.bounds))
if analysis_geometry.is_empty or analysis_geometry.area <= 0:
raise AppError(
code="BATHYMETRY_SELECTION_OUTSIDE_DATASET",
message="Selection does not overlap the persisted bathymetry raster",
status_code=422,
)
min_x, min_y, max_x, max_y = analysis_geometry.bounds
expected_cells = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil(
(max_y - min_y) / abs(source.res[1])
)
if expected_cells > resolved_settings.bathymetry_raster_max_pixels:
raise AppError(
code="BATHYMETRY_SELECTION_TOO_LARGE",
message="Bathymetry analysis exceeds the configured raster cell limit",
details={
"pixel_count": expected_cells,
"max_pixels": resolved_settings.bathymetry_raster_max_pixels,
},
status_code=422,
)
# ``all_touched`` keeps the values of cells the selection only
# clips, so a selection finer than one cell still has data to
# read. Which of those cells actually count is decided by
# ``select_cells`` below, so the normal result is unchanged.
clipped, clipped_transform = mask(
source,
[mapping(analysis_geometry)],
crop=True,
filled=False,
indexes=[1],
all_touched=True,
)
band = np.ma.asarray(clipped[0], dtype="float64")
raw = band.filled(np.nan)
cell_selection = select_cells(
analysis_geometry,
out_shape=band.shape,
transform=clipped_transform,
cell_area_m2=abs(float(source.res[0])) * abs(float(source.res[1])),
)
selected_cells = cell_selection.mask
valid_cells = selected_cells & ~np.ma.getmaskarray(band) & np.isfinite(raw)
if source.nodata is not None:
valid_cells &= ~np.isclose(raw, float(source.nodata))
values = raw[valid_cells]
if values.size == 0:
raise AppError(
code="BATHYMETRY_NO_VALID_DATA",
message="No surveyed waterbed cells occur in this selection",
status_code=422,
)
resolution_x = abs(float(source.res[0]))
resolution_y = abs(float(source.res[1]))
cell_area_m2 = resolution_x * resolution_y
except AppError:
raise
except Exception as exc:
raise AppError(
code="BATHYMETRY_ANALYSIS_FAILED",
message="The persisted bathymetry raster could not be analysed",
details={"reason": str(exc)},
status_code=500,
) from exc
def metric(key: str, label: str, value: float, unit: str, method: str) -> BathymetryRasterMetric:
return BathymetryRasterMetric(
metric_key=key,
metric_label=label,
metric_value=round(float(value), 4),
metric_unit=unit,
aggregation_method=method,
)
selected_cell_count = int(selected_cells.sum())
valid_cell_count = int(values.size)
vertical_unit = str(source_metadata["vertical_reference"])
coverage_ratio = valid_cell_count / max(1, selected_cell_count)
metrics = [
metric(
"bed_elevation_mean_m",
"Gemiddelde waterbodemhoogte",
values.mean(),
f"m {vertical_unit}",
"mean_valid_source_cells",
),
metric(
"bed_elevation_min_m",
"Laagste waterbodemhoogte",
values.min(),
f"m {vertical_unit}",
"minimum_valid_source_cells",
),
metric(
"bed_elevation_max_m",
"Hoogste waterbodemhoogte",
values.max(),
f"m {vertical_unit}",
"maximum_valid_source_cells",
),
metric(
"bed_elevation_p10_m",
"10e percentiel waterbodemhoogte",
np.percentile(values, 10),
f"m {vertical_unit}",
"percentile_10_valid_source_cells",
),
metric(
"bed_elevation_p90_m",
"90e percentiel waterbodemhoogte",
np.percentile(values, 90),
f"m {vertical_unit}",
"percentile_90_valid_source_cells",
),
metric(
"surveyed_bed_surface_ha",
"Oppervlakte met gemeten waterbodem",
valid_cell_count * cell_area_m2 / 10_000.0,
"ha",
"valid_source_cells_times_cell_area",
),
metric(
"bathymetry_coverage_pct",
"Dekking waterbodemmeting",
coverage_ratio * 100.0,
"%",
"valid_source_cells_divided_by_selected_cells",
),
]
primary = metrics[0]
response = BathymetryRasterSelectionResponse(
dataset_id=dataset.id,
product_key=BathymetryRasterAnalysisService.PRODUCT_KEY,
selection_bbox=payload.bbox,
selection_area_id=payload.area_id,
selected_cell_count=selected_cell_count,
valid_cell_count=valid_cell_count,
coverage_ratio=round(coverage_ratio, 6),
cell_selection_warning=cell_selection.warning,
resolution_m=round(max(resolution_x, resolution_y), 4),
vertical_reference=vertical_unit,
survey_period=str(source_metadata.get("survey_period") or "2019-2022"),
summary=BathymetryRasterSelectionSummary(
metric_label=primary.metric_label,
metric_value=primary.metric_value,
metric_unit=primary.metric_unit,
aggregation_method=primary.aggregation_method,
primary_metric_key=primary.metric_key,
metrics=metrics,
),
unsupported_metrics=BathymetryRasterAnalysisService.UNSUPPORTED_METRICS,
limitation_message=BathymetryRasterAnalysisService.LIMITATION,
generated_at=datetime.now(UTC).isoformat(),
)
return response.model_dump(mode="json")
@staticmethod
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id)
BathymetryRasterAnalysisService._metadata(dataset)
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Rasterio, numpy and Pillow are required for bathymetry rendering",
status_code=503,
) from exc
try:
with rasterio.open(dataset.storage_path) as source:
scale = min(1.0, max_dimension / max(source.width, source.height))
width = max(1, round(source.width * scale))
height = max(1, round(source.height * scale))
data = source.read(
1,
out_shape=(height, width),
masked=True,
resampling=Resampling.bilinear,
)
values = np.asarray(data.filled(np.nan), dtype="float64")
valid = np.isfinite(values) & ~np.ma.getmaskarray(data)
if source.nodata is not None:
valid &= ~np.isclose(values, float(source.nodata))
if not valid.any():
raise AppError(
code="BATHYMETRY_NO_VALID_DATA",
message="Bathymetry raster contains no renderable cells",
status_code=422,
)
low, high = np.percentile(values[valid], [2, 98])
if high <= low:
high = low + 1.0
normalized = np.clip((values - low) / (high - low), 0.0, 1.0)
normalized = np.where(valid, normalized, 0.0)
stops = np.asarray([0.0, 0.35, 0.7, 1.0])
colors = np.asarray(
[
[8, 47, 73],
[15, 118, 140],
[103, 190, 170],
[236, 224, 163],
],
dtype="float64",
)
rgba = np.zeros((height, width, 4), dtype="uint8")
for channel in range(3):
rgba[:, :, channel] = np.interp(
normalized,
stops,
colors[:, channel],
).astype("uint8")
rgba[:, :, 3] = np.where(valid, 220, 0).astype("uint8")
output = io.BytesIO()
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
return output.getvalue()
except AppError:
raise
except Exception as exc:
raise AppError(
code="BATHYMETRY_PREVIEW_FAILED",
message="The persisted bathymetry raster could not be rendered",
details={"reason": str(exc)},
status_code=500,
) from exc
+257 -64
View File
@@ -4,10 +4,12 @@ from datetime import datetime, timezone
from typing import Any from typing import Any
from uuid import UUID from uuid import UUID
from geoalchemy2.shape import to_shape from geoalchemy2.shape import from_shape, to_shape
from shapely.geometry import mapping from shapely.geometry import mapping
from shapely.geometry.base import BaseGeometry from shapely.geometry.base import BaseGeometry
from shapely.strtree import STRtree
from shapely.validation import make_valid from shapely.validation import make_valid
from sqlalchemy import func
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.errors import AppError from app.core.errors import AppError
@@ -28,97 +30,271 @@ class ChangeDetectionService:
target_dataset_id: UUID, target_dataset_id: UUID,
iou_threshold: float = 0.8, iou_threshold: float = 0.8,
include_unchanged: bool = True, include_unchanged: bool = True,
modified_threshold: float = 0.3,
bbox: dict[str, Any] | None = None,
area_id: UUID | None = None,
preview_limit: int = 2_000,
) -> ChangeDetectionSummary: ) -> ChangeDetectionSummary:
if source_dataset_id == target_dataset_id: if source_dataset_id == target_dataset_id:
raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400) raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400)
if iou_threshold < 0 or iou_threshold > 1: if iou_threshold < 0 or iou_threshold > 1:
raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400) raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400)
if modified_threshold < 0 or modified_threshold > iou_threshold:
raise AppError(
code="INVALID_PARAMETERS",
message="modified_threshold must be between 0 and iou_threshold",
status_code=400,
)
source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source") source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source")
target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target") target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target")
source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset) selection_geometry = ChangeDetectionService._selection_geometry(db, project_id, bbox=bbox, area_id=area_id)
target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset)
source_features, source_warnings = ChangeDetectionService._load_features(db, source_dataset, selection_geometry)
target_features, target_warnings = ChangeDetectionService._load_features(db, target_dataset, selection_geometry)
if not source_features: if not source_features:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422) raise AppError(code="EMPTY_VECTOR_DATASET", message="Source dataset has no comparable vector features", status_code=422)
if not target_features: if not target_features:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422) raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422)
matched_target_indices: set[int] = set() source_features = ChangeDetectionService.restrict_to_selection(source_features, selection_geometry, label="Source")
unchanged: list[dict[str, Any]] = [] target_features = ChangeDetectionService.restrict_to_selection(target_features, selection_geometry, label="Target")
removed: list[dict[str, Any]] = []
for source_feature in source_features: classified = ChangeDetectionService._classify_features(
best_iou = 0.0 source_features,
best_index: int | None = None target_features,
for target_index, target_feature in enumerate(target_features): iou_threshold=iou_threshold,
if target_index in matched_target_indices: modified_threshold=modified_threshold,
continue )
candidate_iou = ChangeDetectionService._iou(source_feature["geometry"], target_feature["geometry"])
if candidate_iou > best_iou:
best_iou = candidate_iou
best_index = target_index
if best_index is not None and best_iou >= iou_threshold: buckets: dict[str, list[dict[str, Any]]] = {"added": [], "removed": [], "modified": [], "unchanged": []}
matched_target_indices.add(best_index) for item in classified:
if include_unchanged: buckets[item["change_type"]].append(
unchanged.append( ChangeDetectionService._feature(
ChangeDetectionService._feature( geometry=item["geometry"],
geometry=source_feature["geometry"], change_type=item["change_type"],
change_type="unchanged", source_dataset_id=source_dataset_id,
source_dataset_id=source_dataset_id, target_dataset_id=target_dataset_id,
target_dataset_id=target_dataset_id, source_feature_id=item["source_feature_id"],
source_feature_id=source_feature["feature_id"], target_feature_id=item["target_feature_id"],
target_feature_id=target_features[best_index]["feature_id"], iou=item["iou"],
iou=best_iou, properties=item["properties"],
properties=source_feature["properties"],
)
)
else:
removed.append(
ChangeDetectionService._feature(
geometry=source_feature["geometry"],
change_type="removed",
source_dataset_id=source_dataset_id,
target_dataset_id=target_dataset_id,
source_feature_id=source_feature["feature_id"],
target_feature_id=None,
iou=best_iou if best_iou > 0 else None,
properties=source_feature["properties"],
)
) )
added = [
ChangeDetectionService._feature(
geometry=target_feature["geometry"],
change_type="added",
source_dataset_id=source_dataset_id,
target_dataset_id=target_dataset_id,
source_feature_id=None,
target_feature_id=target_feature["feature_id"],
iou=None,
properties=target_feature["properties"],
) )
for target_index, target_feature in enumerate(target_features)
if target_index not in matched_target_indices
]
geojson_features = added + removed + unchanged unchanged_count = len(buckets["unchanged"])
if not include_unchanged:
buckets["unchanged"] = []
geojson_features, preview_truncated = ChangeDetectionService.limit_preview(
buckets["added"] + buckets["removed"] + buckets["modified"] + buckets["unchanged"],
limit=preview_limit,
)
warnings = source_warnings + target_warnings
edge_count = sum(
1 for feature in source_features + target_features if feature.get("partially_covered")
)
if edge_count:
warnings.append(
f"{edge_count} objecten liggen deels buiten de selectie. Ze zijn volledig vergeleken, zodat de "
"selectierand zelf geen wijziging veroorzaakt."
)
if preview_truncated:
warnings.append(
f"De tellingen gelden voor de volledige selectie; de kaart toont maximaal {preview_limit} objecten, "
"wijzigingen eerst."
)
return ChangeDetectionSummary( return ChangeDetectionSummary(
source_dataset_id=source_dataset_id, source_dataset_id=source_dataset_id,
target_dataset_id=target_dataset_id, target_dataset_id=target_dataset_id,
source_feature_count=len(source_features), source_feature_count=len(source_features),
target_feature_count=len(target_features), target_feature_count=len(target_features),
added_count=len(added), added_count=len(buckets["added"]),
removed_count=len(removed), removed_count=len(buckets["removed"]),
unchanged_count=len(unchanged) if include_unchanged else len(matched_target_indices), modified_count=len(buckets["modified"]),
unchanged_count=unchanged_count,
iou_threshold=iou_threshold, iou_threshold=iou_threshold,
warnings=source_warnings + target_warnings, modified_iou_threshold=modified_threshold,
selection_area_id=area_id,
preview_limit=preview_limit,
preview_truncated=preview_truncated,
warnings=warnings,
generated_at=datetime.now(timezone.utc), generated_at=datetime.now(timezone.utc),
geojson={"type": "FeatureCollection", "features": geojson_features}, geojson={"type": "FeatureCollection", "features": geojson_features},
) )
@staticmethod
def _selection_geometry(
db: Session,
project_id: UUID,
*,
bbox: dict[str, Any] | None,
area_id: UUID | None,
) -> BaseGeometry | None:
"""Resolve the drawn rectangle against the named work area, if any."""
from app.models import Area
from shapely.geometry import box as shapely_box
selection = None
if bbox:
selection = shapely_box(
float(bbox["min_x"]), float(bbox["min_y"]), float(bbox["max_x"]), float(bbox["max_y"])
)
if area_id is None:
return selection
area = db.get(Area, area_id)
if area is None or area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
area_geometry = to_shape(area.geometry)
if selection is None:
return area_geometry
intersection = selection.intersection(area_geometry)
if intersection.is_empty or intersection.area <= 0:
raise AppError(
code="CHANGE_DETECTION_SELECTION_OUTSIDE_AREA",
message="Selection does not overlap the selected work area",
status_code=422,
)
return intersection
# Order the preview spends its budget in. An operator asking what changed
# is not helped by a cap filled with unchanged footprints.
PREVIEW_PRIORITY = {"modified": 0, "added": 1, "removed": 2, "unchanged": 3}
@staticmethod
def restrict_to_selection(
features: list[dict[str, Any]],
selection_geometry: BaseGeometry | None,
*,
label: str = "Dataset",
) -> list[dict[str, Any]]:
"""Keep the features a drawn selection reaches, and say which it cuts.
Geometry is deliberately *not* clipped. A change class describes a whole
object: comparing a clipped 2020 footprint against an unclipped 2024 one
would manufacture "modified" along the selection edge. Clipping is right
for an area metric and wrong for an identity comparison.
"""
if selection_geometry is None:
return features
kept: list[dict[str, Any]] = []
for feature in features:
geometry = feature["geometry"]
if not geometry.intersects(selection_geometry):
continue
kept.append({**feature, "partially_covered": not selection_geometry.covers(geometry)})
if not kept:
raise AppError(
code="CHANGE_DETECTION_SELECTION_EMPTY",
message=f"{label} dataset has no features inside this selection",
status_code=422,
)
return kept
@staticmethod
def limit_preview(
features: list[dict[str, Any]],
*,
limit: int,
) -> tuple[list[dict[str, Any]], bool]:
"""Cap the returned geometry without capping the counts.
``include_unchanged`` defaulted to true and nothing bounded the result,
so a regional comparison returned a FeatureCollection holding both
datasets in full. The counts describe the whole selection; the preview
describes what a map can usefully draw.
"""
if limit <= 0 or len(features) <= limit:
return features, False
ordered = sorted(
features,
key=lambda item: ChangeDetectionService.PREVIEW_PRIORITY.get(item["change_type"], 9),
)
return ordered[:limit], True
@staticmethod
def _classify_features(
source_features: list[dict[str, Any]],
target_features: list[dict[str, Any]],
*,
iou_threshold: float,
modified_threshold: float,
) -> list[dict[str, Any]]:
"""Pair source with target footprints and label how each one changed.
Matching is indexed rather than a full cross product: comparing two
municipal building layers is otherwise hundreds of millions of geometry
intersections. Sources are considered largest first so a big footprint
is not left over after a small neighbour claimed its counterpart.
"""
target_geometries = [feature["geometry"] for feature in target_features]
tree = STRtree(target_geometries) if target_geometries else None
claimed: set[int] = set()
classified: list[dict[str, Any]] = []
order = sorted(
range(len(source_features)),
key=lambda index: (-source_features[index]["geometry"].area, str(source_features[index]["feature_id"])),
)
for source_index in order:
source_feature = source_features[source_index]
geometry = source_feature["geometry"]
best_iou = 0.0
best_index: int | None = None
candidates = [] if tree is None else sorted(int(value) for value in tree.query(geometry))
for target_index in candidates:
if target_index in claimed:
continue
candidate_iou = ChangeDetectionService._iou(geometry, target_geometries[target_index])
if candidate_iou > best_iou:
best_iou = candidate_iou
best_index = target_index
if best_index is not None and best_iou >= iou_threshold:
claimed.add(best_index)
change_type = "unchanged"
elif best_index is not None and best_iou >= modified_threshold:
# The same object, redrawn: an annexe, a demolition of one wing,
# or a resurvey. Reporting it as removed + added would hide it.
claimed.add(best_index)
change_type = "modified"
else:
change_type = "removed"
classified.append(
{
"change_type": change_type,
"geometry": geometry if change_type != "modified" else target_geometries[best_index],
"source_feature_id": source_feature["feature_id"],
"target_feature_id": target_features[best_index]["feature_id"] if change_type != "removed" else None,
"iou": best_iou if best_iou > 0 else None,
"properties": source_feature["properties"],
}
)
classified.extend(
{
"change_type": "added",
"geometry": target_feature["geometry"],
"source_feature_id": None,
"target_feature_id": target_feature["feature_id"],
"iou": None,
"properties": target_feature["properties"],
}
for target_index, target_feature in enumerate(target_features)
if target_index not in claimed
)
return classified
@staticmethod @staticmethod
def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset: def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset:
dataset = db.get(Dataset, dataset_id) dataset = db.get(Dataset, dataset_id)
@@ -130,8 +306,25 @@ class ChangeDetectionService:
return dataset return dataset
@staticmethod @staticmethod
def _load_features(db: Session, dataset: Dataset) -> tuple[list[dict[str, Any]], list[str]]: def _load_features(
rows = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id).all() db: Session,
dataset: Dataset,
selection_geometry: BaseGeometry | None = None,
) -> tuple[list[dict[str, Any]], list[str]]:
query = db.query(VectorFeature).filter(VectorFeature.dataset_id == dataset.id)
if selection_geometry is not None and hasattr(query, "filter"):
# Bound the load in the database. Pulling a regional building layer
# into Python to then discard most of it costs memory and time for
# nothing, and the fallback below has no such option.
try:
query = query.filter(
func.ST_Intersects(VectorFeature.geometry, from_shape(selection_geometry, srid=4326))
)
except Exception:
# Lightweight unit-test sessions do not implement every spatial
# predicate; restrict_to_selection still bounds the population.
pass
rows = query.all()
warnings: list[str] = [] warnings: list[str] = []
if rows: if rows:
return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings return [ChangeDetectionService._row_to_feature(row) for row in rows], warnings
+149 -17
View File
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
from app.core.errors import AppError from app.core.errors import AppError
from app.models import Area, Dataset, Project from app.models import Area, Dataset, Project
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.schemas.coverage import ( from app.schemas.coverage import (
CoverageBBox, CoverageBBox,
CoverageCatalogResponse, CoverageCatalogResponse,
@@ -79,6 +80,7 @@ class _SourceDefinition:
contract: CoverageSourceContract contract: CoverageSourceContract
materialized_layer_names: tuple[str, ...] = () materialized_layer_names: tuple[str, ...] = ()
materialized_source_names: tuple[str, ...] = () materialized_source_names: tuple[str, ...] = ()
operational_themes: tuple[str, ...] = ()
def _contract( def _contract(
@@ -98,6 +100,7 @@ def _contract(
limitation_message: str, limitation_message: str,
materialized_layer_names: tuple[str, ...] = (), materialized_layer_names: tuple[str, ...] = (),
materialized_source_names: tuple[str, ...] = (), materialized_source_names: tuple[str, ...] = (),
operational_themes: tuple[str, ...] = (),
) -> _SourceDefinition: ) -> _SourceDefinition:
return _SourceDefinition( return _SourceDefinition(
contract=CoverageSourceContract( contract=CoverageSourceContract(
@@ -117,6 +120,7 @@ def _contract(
), ),
materialized_layer_names=materialized_layer_names, materialized_layer_names=materialized_layer_names,
materialized_source_names=materialized_source_names or (source_name,), materialized_source_names=materialized_source_names or (source_name,),
operational_themes=operational_themes,
) )
@@ -156,12 +160,17 @@ SOURCE_DEFINITIONS = (
themes=("admin", "population"), themes=("admin", "population"),
native_layers=("statistical_sectors", "population_statistics"), native_layers=("statistical_sectors", "population_statistics"),
geometry_types=("Polygon", "MultiPolygon", "Tabular"), geometry_types=("Polygon", "MultiPolygon", "Tabular"),
acquisition_mode="catalog_only", acquisition_mode="operator_archive",
integration_status="not_configured", integration_status="operational",
source_url="https://statbel.fgov.be/en/open-data", source_url="https://statbel.fgov.be/en/open-data",
attribution="Statbel", attribution="Statbel",
license_note="Consult the license of the selected Statbel release.", license_note="Consult the license of the selected Statbel release.",
limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.", limitation_message=(
"National editions require the governed plan-stage-review-apply operator; "
"population in partially selected sectors is area-weighted."
),
materialized_layer_names=("population",),
operational_themes=("population",),
), ),
_contract( _contract(
source_name="digitaal_vlaanderen", source_name="digitaal_vlaanderen",
@@ -200,6 +209,25 @@ SOURCE_DEFINITIONS = (
"agentschap_landbouw_zeevisserij_agricultural_parcels", "agentschap_landbouw_zeevisserij_agricultural_parcels",
), ),
), ),
_contract(
source_name="vmm_vha_bathymetry_profiles",
display_name="VHA historische dwarsprofielen",
authority_level="authoritative",
coverage_zones=("flanders",),
themes=("bathymetry",),
native_layers=("digitale_atlas_profile_points",),
geometry_types=("Point",),
acquisition_mode="bounded_api",
integration_status="operational",
source_url="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
attribution="Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas",
license_note="Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata.",
limitation_message=(
"Historische puntmetingen met bronafhankelijke meetdatum en verticale referentie; "
"geen continue actuele bodemkaart en zonder gelijktijdig waterpeil geen watervolume."
),
materialized_source_names=("vmm_vha_bathymetry_profiles",),
),
_contract( _contract(
source_name="spw_geoportail", source_name="spw_geoportail",
display_name="SPW Geoportail Wallonie", display_name="SPW Geoportail Wallonie",
@@ -219,12 +247,17 @@ SOURCE_DEFINITIONS = (
), ),
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"), native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"), geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
acquisition_mode="catalog_only", acquisition_mode="bounded_api",
integration_status="not_configured", integration_status="operational",
source_url="https://geoportail.wallonie.be/catalogue", source_url="https://geoportail.wallonie.be/catalogue",
attribution="Service public de Wallonie", attribution="Service public de Wallonie",
license_note="Consult the license of each Geoportail Wallonie product.", license_note="Consult the license of each Geoportail Wallonie product.",
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.", limitation_message=(
"Bounded PICC buildings, road axes and hydrography, the legally current flood-hazard polygons, "
"operator-imported SPW bathymetry and bounded SPW MNT terrain are operational; other Walloon themes remain separately governed."
),
materialized_source_names=("spw_picc", "spw_flood_hazard", "spw_walous_land_cover", "spw_bathymetry", "spw_terrain"),
operational_themes=("buildings", "roads", "surface_water", "land_cover_use", "elevation", "flood_climate", "bathymetry"),
), ),
_contract( _contract(
source_name="urbis", source_name="urbis",
@@ -234,12 +267,17 @@ SOURCE_DEFINITIONS = (
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"), themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"), native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"), geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
acquisition_mode="catalog_only", acquisition_mode="bounded_api",
integration_status="not_configured", integration_status="operational",
source_url="https://datastore.brussels", source_url="https://datastore.brussels",
attribution="Brussels UrbIS", attribution="Brussels UrbIS",
license_note="Consult the license of the selected UrbIS dataset.", license_note="Consult the license of the selected UrbIS dataset.",
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.", limitation_message=(
"Bounded UrbIS buildings, cadastral parcels, street axes and Land Cover blocks are operational. "
"Permanent water uses the official WB block class; no separate hydrography network is inferred."
),
materialized_source_names=("urbis",),
operational_themes=("buildings", "parcels", "roads", "surface_water", "land_cover_use"),
), ),
_contract( _contract(
source_name="rbins_marine_reporting_units", source_name="rbins_marine_reporting_units",
@@ -304,7 +342,10 @@ SOURCE_DEFINITIONS = (
source_url="https://www.vlaanderen.be/datavindplaats", source_url="https://www.vlaanderen.be/datavindplaats",
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)", attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
license_note="Consult the official product license before acquisition.", license_note="Consult the official product license before acquisition.",
limitation_message="Strict-TLS acquisition and vertical datum evidence are not yet sufficient; no depths are synthesized.", limitation_message=(
"Bounded strict-TLS WCS acquisition is implemented but stays disabled until the operator enables it "
"with a live-validated coverage id; no depths are synthesized."
),
), ),
) )
@@ -330,6 +371,27 @@ FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
"flood_climate": {"vmm_flood_hazard": ()}, "flood_climate": {"vmm_flood_hazard": ()},
} }
REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = {
"spw_geoportail": {
"buildings": {"spw_picc": ("buildings",)},
"roads": {"spw_picc": ("roads",)},
"surface_water": {"spw_picc": ("water",)},
"land_cover_use": {"spw_walous_land_cover": ()},
"elevation": {"spw_terrain": ()},
"flood_climate": {"spw_flood_hazard": ("flood_hazard",)},
"bathymetry": {"spw_bathymetry": ()},
"orthophoto": {"spw_orthophoto": ()},
},
"urbis": {
"buildings": {"urbis": ("buildings",)},
"parcels": {"urbis": ("parcels",)},
"roads": {"urbis": ("roads",)},
"surface_water": {"urbis": ("water",)},
"land_cover_use": {"urbis": ("space_occupation", "forest")},
"orthophoto": {"urbis_orthophoto": ()},
},
}
class CoverageRegistryService: class CoverageRegistryService:
@staticmethod @staticmethod
@@ -392,26 +454,60 @@ class CoverageRegistryService:
definition: _SourceDefinition, definition: _SourceDefinition,
theme: str, theme: str,
zone: str, zone: str,
) -> list[Dataset]: selection: Any,
) -> tuple[list[Dataset], bool]:
if definition.operational_themes and theme not in definition.operational_themes:
return [], False
matches: list[Dataset] = [] matches: list[Dataset] = []
bounded_scopes: list[Any] = []
zone_scoped_materialization = False
for dataset in datasets: for dataset in datasets:
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names: if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
continue continue
# A source-name claim alone must not cause an unsafe artifact to
# appear as operational authoritative coverage.
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
continue
layer_names = definition.materialized_layer_names layer_names = definition.materialized_layer_names
if definition.contract.source_name == "digitaal_vlaanderen": if definition.contract.source_name == "digitaal_vlaanderen":
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {}) theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
if dataset.source_name not in theme_sources: if dataset.source_name not in theme_sources:
continue continue
layer_names = theme_sources[dataset.source_name] layer_names = theme_sources[dataset.source_name]
elif definition.contract.source_name in REGIONAL_THEME_DATASETS:
theme_sources = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme, {})
if dataset.source_name not in theme_sources:
continue
layer_names = theme_sources[dataset.source_name]
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or [] coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
if isinstance(coverage_zones, str): if isinstance(coverage_zones, str):
coverage_zones = [coverage_zones] coverage_zones = [coverage_zones]
acquired_bbox = metadata.get("bbox_epsg4326")
if (
definition.contract.acquisition_mode == "bounded_api"
and isinstance(acquired_bbox, list)
and len(acquired_bbox) == 4
):
try:
acquired_scope = box(*(float(value) for value in acquired_bbox))
except (TypeError, ValueError):
continue
if not acquired_scope.is_valid or not acquired_scope.intersects(selection):
continue
bounded_scopes.append(acquired_scope)
elif definition.contract.acquisition_mode == "bounded_api" and coverage_zones:
zone_scoped_materialization = zone in coverage_zones or "belgium" in coverage_zones
layer_matches = not layer_names or dataset.reference_layer_name in layer_names layer_matches = not layer_names or dataset.reference_layer_name in layer_names
zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones
if layer_matches and zone_matches: if layer_matches and zone_matches:
matches.append(dataset) matches.append(dataset)
return matches if not matches:
return [], False
fully_covered = True
if definition.contract.acquisition_mode == "bounded_api":
fully_covered = zone_scoped_materialization or (bool(bounded_scopes) and unary_union(bounded_scopes).covers(selection))
return matches, fully_covered
@staticmethod @staticmethod
def _resolve_item( def _resolve_item(
@@ -419,6 +515,7 @@ class CoverageRegistryService:
zone: str, zone: str,
theme: str, theme: str,
datasets: list[Dataset], datasets: list[Dataset],
selection: Any,
) -> CoverageResolutionItem: ) -> CoverageResolutionItem:
definitions = [ definitions = [
definition definition
@@ -436,22 +533,51 @@ class CoverageRegistryService:
) )
materialized: list[Dataset] = [] materialized: list[Dataset] = []
evidence: list[dict[str, Any]] = []
source_statuses: list[str] = [] source_statuses: list[str] = []
limitations: list[str] = [] limitations: list[str] = []
for definition in definitions: for definition in definitions:
matches = CoverageRegistryService._matching_datasets( matches, fully_covered = CoverageRegistryService._matching_datasets(
datasets, datasets,
definition, definition,
theme, theme,
zone, zone,
selection,
) )
materialized.extend(matches) materialized.extend(matches)
if matches: for dataset in matches:
metadata = dataset.source_metadata if isinstance(getattr(dataset, "source_metadata", None), dict) else {}
observed_at = getattr(dataset, "observed_at", None)
published_at = metadata.get("published_at") or metadata.get("publication_date") or metadata.get("published_on")
evidence.append({
"dataset_id": dataset.id,
"source_name": str(dataset.source_name or definition.contract.source_name),
"authority_level": definition.contract.authority_level,
"source_version": getattr(dataset, "source_version", None),
"observed_at": observed_at.isoformat() if hasattr(observed_at, "isoformat") else (str(observed_at) if observed_at else None),
"published_at": str(published_at) if published_at else None,
"crs": getattr(dataset, "crs", None) or metadata.get("source_crs"),
"resolution": getattr(dataset, "resolution_json", None),
"coverage_bbox_epsg4326": metadata.get("bbox_epsg4326"),
"attribution": metadata.get("attribution") or definition.contract.attribution,
"license_note": metadata.get("license_note") or definition.contract.license_note,
"checksum_sha256": getattr(dataset, "checksum_sha256", None),
})
if matches and fully_covered:
source_statuses.append("operational") source_statuses.append("operational")
elif definition.contract.integration_status == "operational": elif matches:
source_statuses.append("partial")
elif (
definition.contract.integration_status == "operational"
and (not definition.operational_themes or theme in definition.operational_themes)
):
source_statuses.append("partial") source_statuses.append("partial")
else: else:
source_statuses.append(definition.contract.integration_status) source_statuses.append(
"not_configured"
if definition.contract.integration_status == "operational"
else definition.contract.integration_status
)
limitations.append(definition.contract.limitation_message) limitations.append(definition.contract.limitation_message)
best_status = max(source_statuses, key=STATUS_RANK.__getitem__) best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
@@ -461,6 +587,7 @@ class CoverageRegistryService:
status=best_status, status=best_status,
source_names=[definition.contract.source_name for definition in definitions], source_names=[definition.contract.source_name for definition in definitions],
materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)), materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)),
evidence=list({str(item["dataset_id"]): item for item in evidence}.values()),
limitation_message=" ".join(dict.fromkeys(limitations)), limitation_message=" ".join(dict.fromkeys(limitations)),
) )
@@ -491,7 +618,12 @@ class CoverageRegistryService:
) )
items = [ items = [
CoverageRegistryService._resolve_item(zone=zone, theme=theme, datasets=datasets) CoverageRegistryService._resolve_item(
zone=zone,
theme=theme,
datasets=datasets,
selection=selection,
)
for zone in zones for zone in zones
for theme in requested_themes for theme in requested_themes
] ]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,160 @@
"""Fail-closed quarantine decisions for validated data assets.
Persistence is intentionally delegated to the caller's transaction. This
module derives stable decisions from immutable validation reports and blocks a
quarantined or failed asset from training, production inference and derived
processing until an explicit, separately persisted release action exists.
"""
from __future__ import annotations
from dataclasses import dataclass
from enum import StrEnum
from hashlib import sha256
from typing import Any
import json
from app.core.errors import AppError
from app.services.data_contract_validation import QuarantineStatus, ValidationReport, ValidationStatus
class AssetUse(StrEnum):
TRAINING = "training"
PRODUCTION_INFERENCE = "production_inference"
DERIVED_PROCESSING = "derived_processing"
EXPORT = "export"
@dataclass(frozen=True)
class QuarantineDecision:
"""A deterministic, auditable quarantine decision for one report."""
asset_id: str
data_contract_key: str
data_contract_version: str
validation_report_sha256: str
validation_status: ValidationStatus
quarantine_status: QuarantineStatus
reason_codes: tuple[str, ...]
idempotency_key: str
requires_explicit_release: bool = False
@property
def eligible_for_use(self) -> bool:
return self.validation_status == ValidationStatus.PASSED and self.quarantine_status == QuarantineStatus.NOT_QUARANTINED
def to_dict(self) -> dict[str, Any]:
return {
"asset_id": self.asset_id,
"data_contract_key": self.data_contract_key,
"data_contract_version": self.data_contract_version,
"validation_report_sha256": self.validation_report_sha256,
"validation_status": self.validation_status.value,
"quarantine_status": self.quarantine_status.value,
"reason_codes": list(self.reason_codes),
"idempotency_key": self.idempotency_key,
"requires_explicit_release": self.requires_explicit_release,
"eligible_for_use": self.eligible_for_use,
}
class DataQuarantineService:
"""Derive and enforce safe use decisions from validation results."""
@staticmethod
def decide(
report: ValidationReport,
*,
previous: QuarantineDecision | None = None,
) -> QuarantineDecision:
"""Create a stable decision without silently releasing old quarantines.
A fresh passing validation report can be persisted as a new validated
version by the import transaction. It cannot automatically release an
existing quarantined record: the caller must explicitly record that
reviewed state transition against the new report/version.
"""
is_quarantined = report.validation_status == ValidationStatus.FAILED or report.quarantine_status == QuarantineStatus.QUARANTINED
failure_codes = tuple(
sorted(
{
issue.code
for issue in report.issues
if issue.severity.value == "error" or is_quarantined
}
)
)
requires_explicit_release = False
reason_codes = failure_codes
status = QuarantineStatus.QUARANTINED if is_quarantined else QuarantineStatus.NOT_QUARANTINED
if previous is not None and previous.quarantine_status == QuarantineStatus.QUARANTINED and not is_quarantined:
status = QuarantineStatus.QUARANTINED
requires_explicit_release = True
reason_codes = ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
idempotency_key = _decision_key(
asset_id=report.asset_id,
contract_key=report.data_contract_key,
contract_version=report.data_contract_version,
report_sha256=report.report_sha256,
quarantine_status=status,
reason_codes=reason_codes,
requires_explicit_release=requires_explicit_release,
)
return QuarantineDecision(
asset_id=report.asset_id,
data_contract_key=report.data_contract_key,
data_contract_version=report.data_contract_version,
validation_report_sha256=report.report_sha256,
validation_status=report.validation_status,
quarantine_status=status,
reason_codes=reason_codes,
idempotency_key=idempotency_key,
requires_explicit_release=requires_explicit_release,
)
@staticmethod
def require_eligible(decision: QuarantineDecision, *, use: AssetUse) -> None:
"""Raise a typed error before a non-eligible artifact reaches a pipeline."""
if decision.eligible_for_use:
return
raise AppError(
code="DATASET_QUARANTINED",
message="Dataset is quarantined or failed validation and cannot enter this pipeline.",
status_code=409,
details={
"asset_id": decision.asset_id,
"use": use.value,
"quarantine_status": decision.quarantine_status.value,
"validation_status": decision.validation_status.value,
"reason_codes": list(decision.reason_codes),
"validation_report_sha256": decision.validation_report_sha256,
"idempotency_key": decision.idempotency_key,
"requires_explicit_release": decision.requires_explicit_release,
},
)
def _decision_key(
*,
asset_id: str,
contract_key: str,
contract_version: str,
report_sha256: str,
quarantine_status: QuarantineStatus,
reason_codes: tuple[str, ...],
requires_explicit_release: bool,
) -> str:
payload = {
"asset_id": asset_id,
"contract_key": contract_key,
"contract_version": contract_version,
"report_sha256": report_sha256,
"quarantine_status": quarantine_status.value,
"reason_codes": list(reason_codes),
"requires_explicit_release": requires_explicit_release,
}
return sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
@@ -0,0 +1,440 @@
"""Fail-closed provenance gates at data-consumption boundaries.
Import validation protects newly staged assets, but a persisted record can
subsequently become quarantined or have its provenance marked incomplete. The
callers of this service therefore re-check the durable Dataset state directly
before production inference, QA, derived processing, export, or authoritative
coverage reporting.
The only legacy relaxation is deliberately narrow: an *explicitly tagged*
fixture with no Phase-2 state can be used for fixture QA. A caller-provided
``fixture_mode`` flag alone never creates that trust claim. Fixture data can
never become a production inference, derived-processing, export or
authoritative-coverage input, and it never relaxes a recorded failed,
incomplete, or quarantined state.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
import re
from typing import Any, Literal
from sqlalchemy import inspect as sa_inspect
from app.core.errors import AppError
from app.models import Dataset
DatasetConsumptionPurpose = Literal[
"production_inference",
"quality_assessment",
"reference_validation",
"derived_processing",
"authoritative_coverage",
"export",
]
_FIXTURE_SOURCE_KEYS = {"fixture", "test", "test_fixture", "test-fixture", "unit-test-fixture"}
_UNTRUSTED_SOURCE_KEYS = {"manual", "fixture", "experimental", "legacy_unknown"}
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
_CONSUMABLE_SNAPSHOT_FRESHNESS = {"current", "not_applicable"}
_VALID_PURPOSES = {
"production_inference",
"quality_assessment",
"reference_validation",
"derived_processing",
"authoritative_coverage",
"export",
}
@dataclass(frozen=True)
class DatasetConsumptionDecision:
"""Auditable decision returned by the consumption gate."""
eligible: bool
purpose: DatasetConsumptionPurpose
fixture_legacy_exception: bool
reasons: tuple[str, ...]
evidence: dict[str, Any]
class DatasetConsumptionGate:
"""Evaluate durable provenance before a Dataset is consumed downstream."""
@staticmethod
def _value(dataset: Any, field: str, default: Any = None) -> Any:
if isinstance(dataset, Mapping):
return dataset.get(field, default)
return getattr(dataset, field, default)
@staticmethod
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
@classmethod
def _normalise(cls, value: Any) -> str:
return str(value or "").strip().lower()
@classmethod
def is_explicit_fixture(cls, dataset: Any) -> bool:
source = cls._normalise(cls._value(dataset, "source"))
source_name = cls._normalise(cls._value(dataset, "source_name"))
metadata = cls._mapping(cls._value(dataset, "metadata_json"))
source_metadata = cls._mapping(cls._value(dataset, "source_metadata"))
provenance = cls._mapping(cls._value(dataset, "provenance_metadata"))
return bool(
source in _FIXTURE_SOURCE_KEYS
or source_name in _FIXTURE_SOURCE_KEYS
or metadata.get("fixture") is True
or metadata.get("fixture_mode") is True
or source_metadata.get("fixture") is True
or source_metadata.get("fixture_mode") is True
or provenance.get("fixture") is True
or provenance.get("fixture_mode") is True
)
@classmethod
def _phase2_state_is_absent(cls, dataset: Any) -> bool:
fields = (
"data_contract_key",
"data_contract_version",
"validation_status",
"provenance_status",
"lineage_status",
"quarantine_status",
"source_registry_id",
"source_snapshot_id",
)
return all(cls._value(dataset, field) in {None, ""} for field in fields)
@staticmethod
def _is_transient_orm_dataset(dataset: Any) -> bool:
"""Recognize only unpersisted ORM fixtures, never database rows."""
if not isinstance(dataset, Dataset):
return False
try:
return bool(sa_inspect(dataset).transient)
except Exception: # pragma: no cover - defensive for unusual test doubles
return False
@classmethod
def _evidence(
cls,
dataset: Any,
purpose: DatasetConsumptionPurpose,
reference_task: str | None = None,
) -> dict[str, Any]:
source_registry = cls._value(dataset, "source_registry")
source_snapshot = cls._value(dataset, "source_snapshot")
source_policy = cls._mapping(cls._value(source_registry, "usage_policy_json"))
validation_authority = cls._mapping(source_policy.get("validation_authority"))
reference_approvals = cls._mapping(source_policy.get("reference_validation_approvals"))
return {
"dataset_id": str(cls._value(dataset, "id") or ""),
"purpose": purpose,
"dataset_status": cls._normalise(cls._value(dataset, "status")),
"source": cls._normalise(cls._value(dataset, "source")),
"source_name": cls._normalise(cls._value(dataset, "source_name")),
"data_contract_key": cls._value(dataset, "data_contract_key"),
"data_contract_version": cls._value(dataset, "data_contract_version"),
"validation_status": cls._normalise(cls._value(dataset, "validation_status")),
"provenance_status": cls._normalise(cls._value(dataset, "provenance_status")),
"lineage_status": cls._normalise(cls._value(dataset, "lineage_status")),
"quarantine_status": cls._normalise(cls._value(dataset, "quarantine_status")),
"checksum_sha256": cls._value(dataset, "checksum_sha256"),
"source_registry_id": str(cls._value(dataset, "source_registry_id") or ""),
"source_snapshot_id": str(cls._value(dataset, "source_snapshot_id") or ""),
"source_classification": cls._normalise(cls._value(source_registry, "classification")),
"source_key": cls._normalise(cls._value(source_registry, "source_key")),
"source_ground_truth_allowed": source_policy.get("ground_truth_allowed") is True,
"source_validation_authority": validation_authority,
"source_reference_validation_approvals": reference_approvals,
"source_authority_scope": cls._mapping(cls._value(source_registry, "authority_scope_json")),
"reference_task": cls._normalise(reference_task),
"snapshot_source_registry_id": str(cls._value(source_snapshot, "source_registry_id") or ""),
"snapshot_ingest_status": cls._normalise(cls._value(source_snapshot, "ingest_status")),
"snapshot_freshness_status": cls._normalise(cls._value(source_snapshot, "freshness_status")),
"snapshot_checksum_sha256": cls._value(source_snapshot, "checksum_sha256"),
}
@classmethod
def evaluate(
cls,
dataset: Any,
*,
purpose: DatasetConsumptionPurpose,
fixture_mode: bool = False,
reference_task: str | None = None,
) -> DatasetConsumptionDecision:
"""Return a stable decision without mutating the Dataset.
``fixture_mode`` can be supplied only by an explicitly fixture-only
caller. It is evidence for a QA fixture path, never a relaxation for
a production boundary or an explicit unsafe state.
"""
if purpose not in _VALID_PURPOSES:
raise ValueError(f"Unsupported dataset-consumption purpose: {purpose}")
evidence = cls._evidence(dataset, purpose, reference_task)
reasons: list[str] = []
explicit_fixture = cls.is_explicit_fixture(dataset)
phase2_absent = cls._phase2_state_is_absent(dataset)
# These are irrevocable safety states. They are checked before a
# fixture exception, so fixture rows cannot hide a bad recorded state.
if evidence["dataset_status"] in {"failed", "quarantined"}:
reasons.append("dataset_status_unsafe")
if evidence["quarantine_status"] == "quarantined":
reasons.append("dataset_quarantined")
if evidence["validation_status"] == "failed":
reasons.append("validation_failed")
if evidence["provenance_status"] == "incomplete":
reasons.append("provenance_incomplete")
if evidence["lineage_status"] == "incomplete":
reasons.append("lineage_incomplete")
if evidence["snapshot_ingest_status"] in {"failed", "quarantined"}:
reasons.append("source_snapshot_unsafe")
# A source family can be authoritative while an individual snapshot
# remains too old or insufficiently described to trust. Historical
# data that is intentionally valid needs an explicit
# ``not_applicable`` contract policy; an omitted, due or stale status
# cannot silently enter a production boundary.
if evidence["source_snapshot_id"] and evidence["snapshot_freshness_status"] not in _CONSUMABLE_SNAPSHOT_FRESHNESS:
reasons.append("source_snapshot_freshness_not_eligible")
if reasons:
return DatasetConsumptionDecision(
eligible=False,
purpose=purpose,
fixture_legacy_exception=False,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
# Fixtures are evidence for tests and QA only. They cannot become
# production inference, derived processing or export inputs merely by
# presenting a fixture flag at a public service boundary.
if phase2_absent and explicit_fixture:
if purpose == "quality_assessment":
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
if purpose == "authoritative_coverage":
reasons.append("fixture_not_authoritative_coverage")
else:
reasons.append("fixture_qa_only")
elif phase2_absent and fixture_mode:
# `fixture_mode` is a caller flag, not a source trust claim. A
# manual/unknown production dataset must never self-designate as a
# fixture merely by supplying this parameter.
reasons.append("fixture_source_required")
# Existing service tests construct transient SQLAlchemy Dataset objects
# directly rather than retrieving a persisted row. A production
# `db.get()` result is persistent and never enters this branch. This
# compatibility path is intentionally unavailable to coverage, where
# a fixture must never appear authoritative.
if (
phase2_absent
and not reasons
and cls._is_transient_orm_dataset(dataset)
and purpose == "quality_assessment"
):
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
# QA unit tests intentionally use projection objects
# instead of persisted ORM Datasets. Those projections cannot enter an
# application API boundary; keep the exception isolated to read-only
# candidate verification. Inference, reference validation, derived
# processing, export and coverage never accept a projection.
if phase2_absent and not reasons and not isinstance(dataset, Dataset) and purpose == "quality_assessment":
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
if phase2_absent:
reasons.append("phase2_provenance_missing")
if evidence["dataset_status"] != "ready":
reasons.append("dataset_not_ready")
if evidence["validation_status"] != "passed":
reasons.append("validation_not_passed")
if evidence["provenance_status"] != "complete":
reasons.append("provenance_not_complete")
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
reasons.append("lineage_not_complete")
if evidence["quarantine_status"] != "not_quarantined":
reasons.append("quarantine_status_not_clear")
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
reasons.append("data_contract_not_versioned")
if not _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")):
reasons.append("dataset_checksum_invalid")
if not evidence["source_registry_id"]:
reasons.append("source_registry_missing")
if not evidence["source_snapshot_id"]:
reasons.append("source_snapshot_missing")
if evidence["source_registry_id"] and not evidence["source_classification"]:
reasons.append("source_registry_unresolved")
if evidence["source_snapshot_id"] and evidence["snapshot_ingest_status"] != "ingested":
reasons.append("source_snapshot_not_ingested")
if evidence["source_snapshot_id"] and not _CHECKSUM_SHA256.fullmatch(
str(evidence["snapshot_checksum_sha256"] or "")
):
reasons.append("source_snapshot_checksum_invalid")
if (
evidence["source_registry_id"]
and evidence["source_snapshot_id"]
and evidence["snapshot_source_registry_id"]
and evidence["source_registry_id"] != evidence["snapshot_source_registry_id"]
):
reasons.append("source_snapshot_registry_mismatch")
if (
_CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or ""))
and _CHECKSUM_SHA256.fullmatch(str(evidence["snapshot_checksum_sha256"] or ""))
and str(evidence["checksum_sha256"]).lower() != str(evidence["snapshot_checksum_sha256"]).lower()
):
reasons.append("source_snapshot_checksum_mismatch")
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
reasons.append("source_registry_identity_mismatch")
# Contextual, corroborative, authoritative and properly derived
# sources can serve their declared non-ground-truth roles once the
# full governed contract passes. Experimental/manual sources cannot
# cross a production boundary; only an explicitly marked fixture may
# participate in candidate QA.
experimental_source = (
evidence["source_classification"] == "experimental"
or evidence["source_key"] in _UNTRUSTED_SOURCE_KEYS
)
if experimental_source:
if purpose == "quality_assessment" and explicit_fixture:
pass
elif purpose == "quality_assessment":
reasons.append("experimental_source_requires_fixture_qa")
else:
reasons.append("experimental_source_not_allowed_for_purpose")
if purpose == "authoritative_coverage":
if evidence["source_classification"] != "authoritative":
reasons.append("coverage_source_not_authoritative")
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
reasons.append("coverage_source_identity_mismatch")
if purpose == "reference_validation":
if cls._normalise(cls._value(dataset, "dataset_role")) != "reference":
reasons.append("reference_dataset_role_required")
if evidence["source_classification"] != "authoritative":
reasons.append("reference_source_not_authoritative")
if evidence["source_ground_truth_allowed"] is not True:
reasons.append("reference_source_not_ground_truth_allowed")
if not evidence["reference_task"]:
reasons.append("reference_validation_task_required")
elif not cls._reference_task_is_approved(evidence):
reasons.append("reference_task_authority_not_approved")
return DatasetConsumptionDecision(
eligible=not reasons,
purpose=purpose,
fixture_legacy_exception=False,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
@classmethod
def assert_eligible(
cls,
dataset: Any,
*,
purpose: DatasetConsumptionPurpose,
fixture_mode: bool = False,
reference_task: str | None = None,
) -> DatasetConsumptionDecision:
decision = cls.evaluate(
dataset,
purpose=purpose,
fixture_mode=fixture_mode,
reference_task=reference_task,
)
if decision.eligible:
return decision
code = "DATASET_QUARANTINED" if any(
reason in {"dataset_status_unsafe", "dataset_quarantined", "validation_failed", "source_snapshot_unsafe"}
for reason in decision.reasons
) else "DATASET_PROVENANCE_INCOMPLETE"
raise AppError(
code=code,
message="Dataset cannot be consumed until its provenance and validation gates are satisfied.",
status_code=409,
details={
"dataset_id": decision.evidence["dataset_id"],
"purpose": purpose,
"reasons": list(decision.reasons),
"fixture_legacy_exception": decision.fixture_legacy_exception,
},
)
@classmethod
def eligible_for_authoritative_coverage(cls, dataset: Any) -> bool:
"""Return false instead of raising so coverage can report a gap safely."""
return cls.evaluate(dataset, purpose="authoritative_coverage").eligible
@staticmethod
def _reference_task_is_approved(evidence: Mapping[str, Any]) -> bool:
"""Require task-specific primary authority or an explicit zone approval.
``classification=authoritative`` is deliberately not a blanket
permission to serve as building truth. A source may be authoritative
for an address lifecycle or elevation product while remaining only
corroborative for footprint QA. A regional product that is marked
``*_pending_contract`` similarly remains blocked until an operator
records a narrow product-and-zone approval in its server-owned policy.
"""
task = str(evidence.get("reference_task") or "").strip().lower()
authority = DatasetConsumptionGate._normalise(
DatasetConsumptionGate._mapping(evidence.get("source_validation_authority")).get(task)
)
if authority == "primary":
return True
if authority not in {"approved", "approved_product_zone"}:
return False
approvals = DatasetConsumptionGate._mapping(evidence.get("source_reference_validation_approvals"))
approval = DatasetConsumptionGate._mapping(approvals.get(task))
if approval.get("approved") is not True:
return False
source_key = str(evidence.get("source_key") or "").strip().lower()
source_scope = DatasetConsumptionGate._mapping(evidence.get("source_authority_scope"))
source_zone = str(source_scope.get("zone") or source_scope.get("scope") or "").strip()
approved_keys = approval.get("source_keys")
approved_zones = approval.get("zones")
if not isinstance(approved_keys, list) or source_key not in {
str(value).strip().lower() for value in approved_keys
}:
return False
if not isinstance(approved_zones, list) or source_zone not in {
str(value).strip() for value in approved_zones
}:
return False
return True
File diff suppressed because it is too large Load Diff
+64 -35
View File
@@ -10,8 +10,15 @@ from uuid import UUID, uuid4
from geoalchemy2.shape import from_shape from geoalchemy2.shape import from_shape
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.core.public_demo import (
PUBLIC_DEMO_PROJECT_ID,
PUBLIC_DEMO_PROJECT_MARKER,
PUBLIC_DEMO_PROJECT_NAME,
)
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
from app.schemas.demo import DemoWorkflowResponse from app.schemas.demo import DemoWorkflowResponse
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.geojson_service import parse_geojson_payload from app.services.geojson_service import parse_geojson_payload
from app.services.qa_service import QaService from app.services.qa_service import QaService
from app.services.quality_service import QualityService from app.services.quality_service import QualityService
@@ -22,7 +29,8 @@ from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_mult
class DemoWorkflowService: class DemoWorkflowService:
PROJECT_NAME = "GeoIntel Demo - Building QA" PROJECT_ID = PUBLIC_DEMO_PROJECT_ID
PROJECT_NAME = PUBLIC_DEMO_PROJECT_NAME
AREA_NAME = "Demo AOI - Geel buildings" AREA_NAME = "Demo AOI - Geel buildings"
REFERENCE_FILENAME = "demo_reference_buildings.geojson" REFERENCE_FILENAME = "demo_reference_buildings.geojson"
CANDIDATE_FILENAME = "demo_predicted_buildings.geojson" CANDIDATE_FILENAME = "demo_predicted_buildings.geojson"
@@ -30,17 +38,17 @@ class DemoWorkflowService:
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json" EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
@staticmethod @staticmethod
def _add_initial_version(db: Session, dataset: Dataset) -> None: def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
db.add( version = DatasetVersion(
DatasetVersion( dataset_id=dataset.id,
dataset_id=dataset.id, version=1,
version=1, storage_path=dataset.storage_path,
storage_path=dataset.storage_path, checksum_sha256=dataset.checksum_sha256,
checksum_sha256=dataset.checksum_sha256, source_metadata=dataset.source_metadata,
source_metadata=dataset.source_metadata, provenance_metadata=dataset.provenance_metadata,
provenance_metadata=dataset.provenance_metadata,
)
) )
db.add(version)
return version
@staticmethod @staticmethod
def _repo_root() -> Path: def _repo_root() -> Path:
@@ -90,17 +98,19 @@ class DemoWorkflowService:
@staticmethod @staticmethod
def _find_existing_project(db: Session) -> Project | None: def _find_existing_project(db: Session) -> Project | None:
projects = ( project = db.get(Project, DemoWorkflowService.PROJECT_ID)
db.query(Project) if project is None:
.filter(Project.name == DemoWorkflowService.PROJECT_NAME) return None
.filter(Project.status != "deleted") if (
.order_by(Project.created_at.asc()) project.name != DemoWorkflowService.PROJECT_NAME
.all() or project.description != PUBLIC_DEMO_PROJECT_MARKER
) ):
for project in projects: raise AppError(
if DemoWorkflowService._has_complete_demo_state(db, project.id): code="PUBLIC_DEMO_IDENTITY_CONFLICT",
return project message="The reserved public-demo project identity is already in use.",
return projects[0] if projects else None status_code=409,
)
return project
@staticmethod @staticmethod
def _activate_explicit_demo_project(db: Session, project: Project | None) -> Project | None: def _activate_explicit_demo_project(db: Session, project: Project | None) -> Project | None:
@@ -233,24 +243,34 @@ class DemoWorkflowService:
crs=metadata.get("crs"), crs=metadata.get("crs"),
bounds_json=metadata.get("bounds_json"), bounds_json=metadata.get("bounds_json"),
metadata_json=metadata, metadata_json=metadata,
status="ready", status="validating",
) )
db.add(dataset) db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset) version = DemoWorkflowService._add_initial_version(db, dataset)
is_ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection=payload,
source_key="fixture",
operation="demo.fixture_vector",
operation_parameters={"fixture_name": filename, "role": role},
)
if is_ready:
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
commit=False,
)
db.commit() db.commit()
db.refresh(dataset) db.refresh(dataset)
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
)
return dataset return dataset
@staticmethod @staticmethod
def _create_demo_raster_bytes() -> bytes: def _create_demo_raster_bytes() -> bytes:
numpy = importlib.import_module("numpy") numpy = importlib.import_module("numpy")
rasterio = importlib.import_module("rasterio")
rasterio_io = importlib.import_module("rasterio.io") rasterio_io = importlib.import_module("rasterio.io")
rasterio_transform = importlib.import_module("rasterio.transform") rasterio_transform = importlib.import_module("rasterio.transform")
@@ -318,10 +338,19 @@ class DemoWorkflowService:
crs=metadata.get("crs"), crs=metadata.get("crs"),
bounds_json=bounds_json, bounds_json=bounds_json,
metadata_json=metadata, metadata_json=metadata,
status="ready", status="validating",
) )
db.add(dataset) db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset) version = DemoWorkflowService._add_initial_version(db, dataset)
DerivedDatasetGovernanceService.govern_raster(
db,
dataset=dataset,
dataset_version=version,
raster_metadata=metadata,
source_key="fixture",
operation="demo.fixture_raster",
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
)
db.commit() db.commit()
db.refresh(dataset) db.refresh(dataset)
return dataset return dataset
@@ -467,9 +496,9 @@ class DemoWorkflowService:
created = True created = True
else: else:
project = Project( project = Project(
id=uuid4(), id=DemoWorkflowService.PROJECT_ID,
name=DemoWorkflowService.PROJECT_NAME, name=DemoWorkflowService.PROJECT_NAME,
description="Offline fixture workflow: reference buildings, predicted buildings and persisted QA metrics.", description=PUBLIC_DEMO_PROJECT_MARKER,
region="Kempen", region="Kempen",
status="active", status="active",
) )
@@ -0,0 +1,516 @@
"""Governance for persisted derived and fixture datasets.
Dataset importers own raw-source ingestion. This small service owns the
other persistence boundary: artifacts produced inside the workbench (vector
and raster operations) and the explicitly local demo fixtures. It is kept
separate from :mod:`dataset_service` so an operation can never create a ready
dataset without a source registry binding, immutable snapshot, validation
report and, for derived results, a durable lineage edge.
The ``Session.query`` capability check deliberately preserves lightweight
unit-test doubles used by pre-Phase-2 tests. Real SQLAlchemy sessions always
take the governed branch; the compatibility branch is not reachable in the
application runtime.
"""
from __future__ import annotations
from datetime import datetime, timezone
from hashlib import sha256
import json
import re
from typing import Any, Mapping
from sqlalchemy.orm import Session
from app.models import Dataset, DatasetVersion
from app.services.data_contract_validation import (
IssueSeverity,
LineageStatus,
LineageEvidence,
ProvenanceStatus,
QuarantineStatus,
TransformationEvidence,
ValidationIssue,
ValidationReport,
ValidationStatus,
build_raster_ingest_input,
build_vector_ingest_input,
validate_registered_asset,
)
from app.services.data_quarantine_service import DataQuarantineService
from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate
from app.services.source_registry_service import SourceRegistryService
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
_CANONICAL_VECTOR_CRS = "EPSG:4326"
class DerivedDatasetGovernanceService:
"""Apply Phase-2 provenance rules to non-importer Dataset creation.
Callers add the dataset and first immutable version, then call one of the
``govern_*`` methods before committing. A failed contract deliberately
leaves the artifact and its durable quarantine record in the transaction;
it is never silently promoted to ``ready``.
"""
@staticmethod
def persistence_available(db: Session) -> bool:
"""Return whether this is a real ORM persistence session.
Historical unit tests use minimal fakes with ``add``/``commit`` only.
Keeping that explicitly isolated avoids pretending a fake test store
has source-registry guarantees while production remains fail-closed.
"""
return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None))
@classmethod
def govern_vector(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
feature_collection: Mapping[str, Any],
source_key: str,
operation: str,
parent_dataset: Dataset | None = None,
operation_parameters: Mapping[str, Any] | None = None,
) -> bool:
"""Validate and bind a vector result, returning ``True`` when ready."""
if not cls.persistence_available(db):
# Explicit compatibility for historical minimal test fixtures.
# Production sessions always have query/flush and never take this
# branch.
dataset.status = "ready"
return True
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
metadata = dict(dataset.metadata_json or {})
output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS)
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
snapshot = cls._record_snapshot(
db,
source_key=source_key,
dataset=dataset,
operation=operation,
source_crs=output_crs,
spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"},
geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json},
observed_schema={
"dataset_type": "vector",
"geometry_types": metadata.get("geometry_types") or [],
"feature_count": metadata.get("feature_count"),
},
)
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
report = validate_registered_asset(
build_vector_ingest_input(
asset_id=str(dataset.id),
source_crs=output_crs,
storage_crs=output_crs,
feature_collection=feature_collection,
checksum_sha256=dataset.checksum_sha256,
computed_checksum_sha256=dataset.checksum_sha256,
source_registry_id=str(source.id),
source_snapshot_id=str(snapshot.id),
imported_at=dataset.imported_at or datetime.now(timezone.utc),
metadata=cls._contract_metadata(metadata, source.license_name),
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
source_version=dataset.source_version,
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
lineage=lineage,
)
)
if parent_gate is not None and not parent_gate.eligible:
report = cls._with_parent_gate_failure(report, parent_gate)
return cls._apply(
db,
dataset=dataset,
dataset_version=dataset_version,
source=source,
snapshot=snapshot,
report=report,
stage="derived_vector_validation",
parent_dataset=parent_dataset,
operation=operation,
operation_parameters=operation_parameters,
)
@classmethod
def govern_raster(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
raster_metadata: Mapping[str, Any],
source_key: str,
operation: str,
parent_dataset: Dataset | None = None,
operation_parameters: Mapping[str, Any] | None = None,
) -> bool:
"""Validate and bind a raster result, returning ``True`` when ready."""
if not cls.persistence_available(db):
dataset.status = "ready"
return True
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
metadata = dict(raster_metadata or {})
output_crs = str(metadata.get("crs") or dataset.crs or "") or None
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
snapshot = cls._record_snapshot(
db,
source_key=source_key,
dataset=dataset,
operation=operation,
source_crs=output_crs,
spatial_resolution=cls._raster_resolution(metadata, output_crs),
geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json},
observed_schema={
"dataset_type": "raster",
"width": metadata.get("width"),
"height": metadata.get("height"),
"band_count": metadata.get("band_count"),
"dtype": metadata.get("dtype"),
},
)
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
report = validate_registered_asset(
build_raster_ingest_input(
asset_id=str(dataset.id),
source_crs=output_crs,
storage_crs=output_crs,
raster_profile=metadata,
bounds=metadata.get("bounds") or dataset.bounds_json,
resolution=cls._raster_resolution(metadata, output_crs),
checksum_sha256=dataset.checksum_sha256,
computed_checksum_sha256=dataset.checksum_sha256,
source_registry_id=str(source.id),
source_snapshot_id=str(snapshot.id),
imported_at=dataset.imported_at or datetime.now(timezone.utc),
metadata=cls._contract_metadata(metadata, source.license_name),
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
source_version=dataset.source_version,
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
lineage=lineage,
)
)
if parent_gate is not None and not parent_gate.eligible:
report = cls._with_parent_gate_failure(report, parent_gate)
return cls._apply(
db,
dataset=dataset,
dataset_version=dataset_version,
source=source,
snapshot=snapshot,
report=report,
stage="derived_raster_validation",
parent_dataset=parent_dataset,
operation=operation,
operation_parameters=operation_parameters,
)
@classmethod
def _record_snapshot(
cls,
db: Session,
*,
source_key: str,
dataset: Dataset,
operation: str,
source_crs: str | None,
spatial_resolution: Mapping[str, Any] | None,
geographic_coverage: Mapping[str, Any] | None,
observed_schema: Mapping[str, Any] | None,
):
checksum = cls._checksum_or_placeholder(dataset.checksum_sha256)
# A derived artifact has a distinct creation event even when its bytes
# equal a prior output. Include the immutable dataset id so source
# snapshots never collide on a different fetched_at timestamp.
snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}"
return SourceRegistryService.record_snapshot(
db,
source_key=source_key,
snapshot_key=snapshot_key,
checksum_sha256=checksum,
source_version=dataset.source_version or f"{operation}:1.0.0",
snapshot_at=dataset.observed_at,
fetched_at=dataset.imported_at or datetime.now(timezone.utc),
crs=source_crs,
units=cls._units_for_crs(source_crs),
spatial_resolution=dict(spatial_resolution or {"status": "unknown"}),
temporal_coverage={
"observed_at": cls._datetime_value(dataset.observed_at),
"valid_from": cls._datetime_value(dataset.valid_from),
"valid_to": cls._datetime_value(dataset.valid_to),
},
geographic_coverage=dict(geographic_coverage or {"status": "unknown"}),
observed_schema=dict(observed_schema or {"status": "unknown"}),
# A transform cannot establish source freshness. With no source
# observation it is not applicable; with one it still needs an
# explicit policy review rather than a fabricated "current" flag.
freshness_status="not_applicable" if dataset.observed_at is None else "review_required",
ingest_status="ingested",
known_limitations=[
"Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.",
],
snapshot_metadata={
"operation": operation,
"dataset_id": str(dataset.id),
"storage_path": dataset.storage_path,
"artifact_checksum_sha256": dataset.checksum_sha256,
},
)
@classmethod
def _apply(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
source: Any,
snapshot: Any,
report: ValidationReport,
stage: str,
parent_dataset: Dataset | None,
operation: str,
operation_parameters: Mapping[str, Any] | None,
) -> bool:
fields = report.persistence_fields()
dataset.validation_report_json = fields["validation_report_json"]
dataset_version.validation_report_json = fields["validation_report_json"]
SourceRegistryService.bind_dataset_provenance(
dataset,
source=source,
snapshot=snapshot,
data_contract_key=fields["data_contract_key"],
data_contract_version=fields["data_contract_version"],
validation_status=fields["validation_status"],
provenance_status=fields["provenance_status"],
lineage_status=fields["lineage_status"],
)
SourceRegistryService.bind_dataset_version_provenance(
dataset_version,
source=source,
snapshot=snapshot,
data_contract_key=fields["data_contract_key"],
data_contract_version=fields["data_contract_version"],
validation_status=fields["validation_status"],
provenance_status=fields["provenance_status"],
lineage_status=fields["lineage_status"],
)
# IDs for DatasetVersion defaults exist only after the caller adds and
# flushes both rows. The operation services call this before commit.
db.flush()
if parent_dataset is not None:
SourceRegistryService.record_lineage_edge(
db,
parent_dataset_id=parent_dataset.id,
child_dataset_id=dataset.id,
parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset),
child_dataset_version_id=dataset_version.id,
relation_type="derived_from",
transformation_name=operation,
transformation_version="1.0.0",
parameters=dict(operation_parameters or {}),
input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256),
output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
)
decision = DataQuarantineService.decide(report)
if decision.eligible_for_use:
dataset.status = "ready"
dataset.quarantine_status = "not_quarantined"
return True
SourceRegistryService.quarantine_dataset(
db,
dataset=dataset,
dataset_version=dataset_version,
source_snapshot=snapshot,
stage=stage,
reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"),
details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()},
artifact_path=dataset.storage_path,
artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
)
return False
@staticmethod
def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]:
values = dict(metadata)
values.setdefault("license", license_name)
return values
@classmethod
def _lineage_evidence(
cls,
parent_dataset: Dataset | None,
operation: str,
operation_parameters: Mapping[str, Any] | None,
) -> LineageEvidence:
upstream_ids: tuple[str, ...] = ()
upstream_checksums: tuple[str, ...] = ()
if parent_dataset is not None:
upstream_ids = (str(parent_dataset.id),)
# An invalid/missing parent checksum intentionally fails the
# derived contract rather than inventing traceability. The same
# applies to a legacy/unvalidated parent: it may remain visible
# as evidence, but cannot create a new ready derived asset.
parent_is_governed = (
parent_dataset.status == "ready"
and parent_dataset.quarantine_status == "not_quarantined"
and parent_dataset.validation_status == "passed"
and parent_dataset.provenance_status == "complete"
and parent_dataset.source_registry_id is not None
and parent_dataset.source_snapshot_id is not None
)
upstream_checksums = (
parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed",
)
transform_checksum = cls._stable_hash(
{"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})}
)
return LineageEvidence(
upstream_asset_ids=upstream_ids,
upstream_checksums_sha256=upstream_checksums,
transformations=(
TransformationEvidence(
name=operation,
version="1.0.0",
checksum_sha256=transform_checksum,
),
),
)
@staticmethod
def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None:
"""Evaluate the durable parent boundary before creating a ready child.
We deliberately turn a rejected parent into a child validation failure
(instead of simply raising): the output is then persisted with its
source snapshot, validation evidence and durable quarantine record.
This makes an attempted derivation from a manual or experimental
dataset observable and prevents a caller from bypassing the boundary
by invoking the governance service directly.
"""
if parent_dataset is None:
return None
return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing")
@staticmethod
def _with_parent_gate_failure(
report: ValidationReport,
decision: DatasetConsumptionDecision,
) -> ValidationReport:
"""Attach an auditable, fail-closed lineage failure to a report."""
issue = ValidationIssue(
code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING",
category="lineage",
field="lineage.parent_dataset",
message="Parent dataset failed the governed derived-processing consumption gate.",
severity=IssueSeverity.ERROR,
expected="eligible governed parent dataset",
observed={
"dataset_id": decision.evidence.get("dataset_id"),
"reasons": list(decision.reasons),
"source_key": decision.evidence.get("source_key"),
"source_classification": decision.evidence.get("source_classification"),
},
)
return ValidationReport(
asset_id=report.asset_id,
data_contract_key=report.data_contract_key,
data_contract_version=report.data_contract_version,
contract_fingerprint_sha256=report.contract_fingerprint_sha256,
validation_status=ValidationStatus.FAILED,
provenance_status=(
ProvenanceStatus.INCOMPLETE
if report.provenance_status == ProvenanceStatus.COMPLETE
else report.provenance_status
),
lineage_status=LineageStatus.INCOMPLETE,
quarantine_status=QuarantineStatus.QUARANTINED,
validation_scope=report.validation_scope,
checked_at=report.checked_at,
issues=(*report.issues, issue),
)
@staticmethod
def _latest_parent_version_id(db: Session, dataset: Dataset):
version = (
db.query(DatasetVersion)
.filter(DatasetVersion.dataset_id == dataset.id)
.order_by(DatasetVersion.version.desc())
.first()
)
return version.id if version is not None else None
@staticmethod
def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None:
values = metadata.get("resolution")
if not isinstance(values, (list, tuple)) or len(values) < 2:
return None
try:
return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)}
except (TypeError, ValueError):
return None
@staticmethod
def _resolution_unit(crs: str | None) -> str:
return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m"
@staticmethod
def _units_for_crs(crs: str | None) -> str:
return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres"
@staticmethod
def _temporal_unknown_reason(dataset: Dataset) -> str | None:
if dataset.observed_at is not None:
return None
return "Derived or fixture artifact inherits no precise observation timestamp from its input."
@staticmethod
def _source_version_unknown_reason(dataset: Dataset) -> str | None:
if dataset.source_version:
return None
return "Derived or fixture artifact has no source edition; the transform version is recorded separately."
@staticmethod
def _datetime_value(value: datetime | None) -> str | None:
return value.astimezone(timezone.utc).isoformat() if value is not None else None
@staticmethod
def _stable_hash(value: Mapping[str, Any]) -> str:
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
@staticmethod
def _valid_checksum(value: str | None) -> str | None:
normalized = str(value or "").strip().lower()
return normalized if _SHA256.fullmatch(normalized) else None
@classmethod
def _checksum_or_placeholder(cls, value: str | None) -> str:
checksum = cls._valid_checksum(value)
if checksum is not None:
return checksum
# The contract receives the original invalid/missing checksum and
# quarantines it. A deterministic placeholder only permits storing
# the rejected snapshot without fabricating a valid artifact hash.
return cls._stable_hash({"invalid_dataset_checksum": value or "missing"})
@@ -0,0 +1,200 @@
"""Placing two detection runs side by side, honestly.
The workbench ranks model variants by a stored F1, each measured at that
variant's own confidence threshold. That number says as much about the
threshold as about the model: a conservatively calibrated detector looks worse
than a liberal one without detecting anything differently. Average precision
describes the whole ranking the model produced and is the comparable figure.
Comparability comes first, though. Two runs over different rasters, scored
against different references, or covering different ground are not two answers
to one question, and no metric makes them so.
"""
from __future__ import annotations
from typing import Any
from app.core.errors import AppError
class DetectionComparisonService:
# A run whose inference footprint was never established cannot be placed
# beside one that was: their recalls have different denominators.
PROVEN_COVERAGE_MODES = ("persisted_tile_manifest_union",)
@staticmethod
def assess_comparability(entries: list[dict[str, Any]]) -> dict[str, Any]:
"""Say whether these runs answer the same question, and why not if they don't."""
if len(entries) < 2:
raise AppError(
code="DETECTION_COMPARISON_NEEDS_TWO_RUNS",
message="Comparing detection models requires at least two runs",
status_code=400,
)
reasons: list[str] = []
source_rasters = {str(entry.get("dataset_id")) for entry in entries}
if len(source_rasters) > 1:
reasons.append("different_source_raster")
references = {str(entry.get("reference_dataset_id")) for entry in entries}
if len(references) > 1:
reasons.append("different_reference_dataset")
if any(
str(entry.get("coverage_mode")) not in DetectionComparisonService.PROVEN_COVERAGE_MODES
for entry in entries
):
reasons.append("coverage_not_proven")
populations = {int(entry.get("reference_evaluated_count") or 0) for entry in entries}
if len(populations) > 1:
reasons.append("different_evaluated_population")
# Two runs that suppressed duplicates differently produced different
# candidate sets from the same model output, so their scores describe
# different pipelines. Runs from before these values were recorded
# carry none; absence is not a difference.
post_processing = {
(
entry.get("containment_suppression_threshold"),
entry.get("duplicate_iou_threshold"),
)
for entry in entries
}
if len(post_processing) > 1:
reasons.append("different_post_processing")
return {
"comparable": not reasons,
"blocking_reasons": reasons,
"source_raster_count": len(source_rasters),
"reference_dataset_count": len(references),
"evaluated_population_counts": sorted(populations),
}
@staticmethod
def rank(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Order runs by average precision, stating the margin and any tie.
Ranking on the F1 each run happened to be read at would order the
thresholds, not the models.
"""
ordered = sorted(
rows,
key=lambda row: (-(row.get("average_precision") or 0.0), str(row.get("model_asset_id") or "")),
)
if not ordered:
return []
leader = ordered[0].get("average_precision") or 0.0
tied_count = sum(1 for row in ordered if (row.get("average_precision") or 0.0) == leader)
runner_up = (ordered[1].get("average_precision") or 0.0) if len(ordered) > 1 else leader
ranked: list[dict[str, Any]] = []
for row in ordered:
average_precision = row.get("average_precision") or 0.0
is_leader = average_precision == leader
ranked.append(
{
**row,
"rank": 1 if is_leader else 1 + sum(
1 for other in ordered if (other.get("average_precision") or 0.0) > average_precision
),
# Distance behind the best run; zero for the leader itself.
"average_precision_gap": leader - average_precision,
"tied": is_leader and tied_count > 1,
# Only the leader has a lead; stating it on every row would
# invite reading a follower's gap as an advantage.
"lead_over_next": (leader - runner_up) if is_leader and tied_count == 1 else None,
}
)
return ranked
@staticmethod
def compare_runs(
db,
*,
analysis_run_ids: list,
reference_dataset_id,
iou_threshold: float = 0.5,
) -> dict[str, Any]:
"""Score several runs against one reference and rank them on AP.
Each run is scored through the same QA path the workbench uses, so the
comparison and the persisted quality checks cannot drift apart.
"""
# Lazy: detection_service imports this module's siblings at load.
from app.models import AnalysisRun
from app.services.detection_service import DetectionService
if len(set(analysis_run_ids)) < 2:
raise AppError(
code="DETECTION_COMPARISON_NEEDS_TWO_RUNS",
message="Comparing detection models requires at least two distinct runs",
status_code=400,
)
entries: list[dict[str, Any]] = []
rows: list[dict[str, Any]] = []
for analysis_run_id in analysis_run_ids:
result = DetectionService.compare_detections_with_reference(
db,
analysis_run_id=analysis_run_id,
reference_dataset_id=reference_dataset_id,
iou_threshold=iou_threshold,
)
run = db.get(AnalysisRun, analysis_run_id)
parameters = (run.parameters_json if run and isinstance(run.parameters_json, dict) else {}) or {}
run_result = (run.result_json if run and isinstance(run.result_json, dict) else {}) or {}
coverage = result.get("coverage") if isinstance(result.get("coverage"), dict) else {}
curve = result.get("precision_recall_curve") or {}
entries.append(
{
"analysis_run_id": analysis_run_id,
"dataset_id": getattr(run, "dataset_id", None),
"model_id": getattr(run, "model_name", None),
"model_asset_id": parameters.get("model_asset_id"),
"reference_dataset_id": reference_dataset_id,
"coverage_mode": coverage.get("mode"),
"reference_evaluated_count": coverage.get("reference_evaluated_count"),
# Recorded on the run itself, so two runs that suppressed
# duplicates differently cannot be ranked against each other.
"containment_suppression_threshold": run_result.get("containment_suppression_threshold"),
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
}
)
rows.append(
{
"analysis_run_id": str(analysis_run_id),
"quality_check_id": result.get("quality_check_id"),
"model_id": getattr(run, "model_name", None),
"model_asset_id": parameters.get("model_asset_id"),
"run_confidence_threshold": parameters.get("confidence_threshold"),
"average_precision": curve.get("average_precision"),
"best_f1": curve.get("best_f1"),
"best_f1_threshold": curve.get("best_f1_threshold"),
# The figure the workbench used to rank on, kept visible so
# the difference between the two readings is auditable.
"f1_at_run_threshold": result.get("f1_score"),
"precision_at_run_threshold": result.get("precision"),
"recall_at_run_threshold": result.get("recall"),
"containment_suppression_threshold": run_result.get("containment_suppression_threshold"),
"duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"),
}
)
comparability = DetectionComparisonService.assess_comparability(entries)
return {
"reference_dataset_id": str(reference_dataset_id),
"iou_threshold": iou_threshold,
"comparability": comparability,
"ranking_metric": "average_precision",
"rows": DetectionComparisonService.rank(rows),
}
@@ -8,6 +8,24 @@ from shapely.geometry import Polygon
from app.core.errors import AppError from app.core.errors import AppError
def _require_source_crs(crs: str | None, tile: dict[str, Any]) -> str:
"""Resolve the CRS a pixel coordinate is measured in, or fail.
Falling back to EPSG:4326 turned a missing manifest field into geometry
that sits in the wrong place while still looking like a valid polygon on
the map. A georeferenced result without a known CRS is not a result.
"""
for candidate in (crs, tile.get("crs"), tile.get("source_crs")):
if isinstance(candidate, str) and candidate.strip():
return candidate.strip()
raise AppError(
code="DETECTION_TILE_CRS_REQUIRED",
message="Georeferencing a tile requires explicit CRS metadata",
status_code=422,
)
def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs: str | None = None) -> Polygon: def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs: str | None = None) -> Polygon:
if len(bbox) != 4: if len(bbox) != 4:
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must contain four pixel coordinates", status_code=422) raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO detection bbox must contain four pixel coordinates", status_code=422)
@@ -28,7 +46,7 @@ def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs:
else: else:
corners = _corners_from_bounds(bbox=[x_min, y_min, x_max, y_max], tile=tile) corners = _corners_from_bounds(bbox=[x_min, y_min, x_max, y_max], tile=tile)
source_crs = crs or tile.get("crs") or tile.get("source_crs") or "EPSG:4326" source_crs = _require_source_crs(crs, tile)
if str(source_crs).upper() not in {"EPSG:4326", "4326"}: if str(source_crs).upper() not in {"EPSG:4326", "4326"}:
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True) transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
corners = [transformer.transform(x, y) for x, y in corners] corners = [transformer.transform(x, y) for x, y in corners]
@@ -39,6 +57,75 @@ def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs:
return polygon return polygon
def pixel_points_to_epsg4326_polygon(points: list[list[float]], tile: dict[str, Any], crs: str | None = None) -> Polygon:
if not isinstance(points, list) or len(points) < 3:
raise AppError(
code="SEGMENTATION_INVALID_MASK",
message="Segmentation mask polygon must contain at least three pixel points",
status_code=422,
)
try:
pixel_points = [(float(point[0]), float(point[1])) for point in points]
except (TypeError, ValueError, IndexError) as exc:
raise AppError(
code="SEGMENTATION_INVALID_MASK",
message="Segmentation mask polygon points must be numeric [x, y] pairs",
status_code=422,
) from exc
transform = tile.get("transform")
if isinstance(transform, list) and len(transform) >= 6:
coordinates = [_apply_gdal_transform(transform, x, y) for x, y in pixel_points]
else:
coordinates = [_project_pixel_with_bounds(tile, x, y) for x, y in pixel_points]
source_crs = _require_source_crs(crs, tile)
if str(source_crs).upper() not in {"EPSG:4326", "4326"}:
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
coordinates = [transformer.transform(x, y) for x, y in coordinates]
if coordinates[0] != coordinates[-1]:
coordinates.append(coordinates[0])
polygon = Polygon(coordinates)
if not polygon.is_valid:
from shapely.validation import make_valid
repaired = make_valid(polygon)
polygon = _largest_polygon(repaired)
if polygon is None or polygon.is_empty or not polygon.is_valid or polygon.area <= 0:
raise AppError(
code="SEGMENTATION_INVALID_GEOMETRY",
message="Georeferenced segmentation geometry is invalid",
status_code=422,
)
return polygon
def _largest_polygon(geometry: Any) -> Polygon | None:
if isinstance(geometry, Polygon):
return geometry
candidates = [geom for geom in getattr(geometry, "geoms", []) if isinstance(geom, Polygon) and geom.area > 0]
if not candidates:
return None
return max(candidates, key=lambda geom: geom.area)
def _project_pixel_with_bounds(tile: dict[str, Any], px: float, py: float) -> tuple[float, float]:
bounds = tile.get("bounds")
pixel_window = tile.get("pixel_window")
if not (isinstance(bounds, list) and len(bounds) == 4 and isinstance(pixel_window, list) and len(pixel_window) == 4):
raise AppError(
code="DETECTION_TILE_MANIFEST_INVALID",
message="Tile manifest entries require transform or bounds plus pixel_window for georeferencing",
status_code=422,
)
left, bottom, right, top = [float(value) for value in bounds]
_, _, width, height = [float(value) for value in pixel_window]
if width <= 0 or height <= 0:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile pixel_window must have positive size", status_code=422)
return (left + (px / width) * (right - left), top - (py / height) * (top - bottom))
def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]: def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]:
c, a, b, f, d, e = [float(value) for value in transform[:6]] c, a, b, f, d, e = [float(value) for value in transform[:6]]
return (a * x + b * y + c, d * x + e * y + f) return (a * x + b * y + c, d * x + e * y + f)
@@ -0,0 +1,235 @@
"""Threshold-independent detection metrics.
A single precision/recall/F1 triple describes one operating point. Which point
that is depends on the confidence threshold the operator typed, so two models
cannot be compared from it: a conservatively calibrated model looks worse at a
low cut and better at a high one without detecting anything differently.
This service produces the standard alternative the full precision/recall
curve over every confidence value that occurs in the run, the average
precision derived from it, and the threshold where F1 actually peaks using
the same greedy IoU matching rule as the rest of QA so the numbers stay
comparable with the persisted quality checks.
"""
from __future__ import annotations
from typing import Any
from shapely.geometry.base import BaseGeometry
from shapely.strtree import STRtree
from app.services.qa_service import QaService
class DetectionMetricsService:
SUPPORTED_GEOMETRY_TYPES = QaService.SUPPORTED_GEOMETRY_TYPES
@staticmethod
def _rank_candidates(
candidates: list[tuple[dict[str, Any], BaseGeometry]],
) -> list[tuple[float, str, dict[str, Any], BaseGeometry]]:
"""Order candidates by confidence, highest first, deterministically."""
ranked: list[tuple[float, str, dict[str, Any], BaseGeometry]] = []
for index, (feature, geometry) in enumerate(candidates):
if geometry.is_empty or geometry.geom_type not in DetectionMetricsService.SUPPORTED_GEOMETRY_TYPES:
continue
if geometry.area <= 0:
continue
confidence = QaService._feature_confidence(feature)
identifier = QaService._feature_identifier(feature, "candidate", index)
ranked.append((float(confidence if confidence is not None else 0.0), identifier, feature, geometry))
ranked.sort(key=lambda item: (-item[0], item[1]))
return ranked
@staticmethod
def _greedy_hits(
ranked: list[tuple[float, str, dict[str, Any], BaseGeometry]],
references: list[tuple[dict[str, Any], BaseGeometry]],
iou_threshold: float,
) -> tuple[list[bool], int]:
"""Mark each ranked candidate as a hit or a miss, best score first.
Walking the ranking once and consuming references as they are claimed
is exactly the COCO/PASCAL rule, and it is what makes the result
independent of the order rows came out of the database.
"""
supported = [
(index, geometry)
for index, (_, geometry) in enumerate(references)
if geometry.geom_type in DetectionMetricsService.SUPPORTED_GEOMETRY_TYPES
and not geometry.is_empty
and geometry.area > 0
]
reference_count = len(supported)
if not supported:
return [False] * len(ranked), 0
tree = STRtree([geometry for _, geometry in supported])
claimed: set[int] = set()
hits: list[bool] = []
for _, _, _, geometry in ranked:
best_iou = 0.0
best_index: int | None = None
for position in sorted(int(value) for value in tree.query(geometry)):
if position in claimed:
continue
_, reference_geometry = supported[position]
intersection_area = geometry.intersection(reference_geometry).area
if intersection_area <= 0:
continue
union_area = geometry.area + reference_geometry.area - intersection_area
if union_area <= 0:
continue
iou = intersection_area / union_area
if iou > best_iou:
best_iou = iou
best_index = position
if best_index is not None and best_iou >= iou_threshold:
claimed.add(best_index)
hits.append(True)
else:
hits.append(False)
return hits, reference_count
@staticmethod
def _average_precision(points: list[dict[str, Any]]) -> float:
"""Area under the precision/recall curve, with precision made monotone.
Interpolating precision to its running maximum from the right is the
VOC/COCO convention; without it the sawtooth from individual false
positives shows up as noise in the score.
"""
if not points:
return 0.0
recalls = [0.0] + [point["recall"] for point in points]
precisions = [1.0] + [point["precision"] for point in points]
for index in range(len(precisions) - 2, -1, -1):
precisions[index] = max(precisions[index], precisions[index + 1])
area = 0.0
for index in range(1, len(recalls)):
area += (recalls[index] - recalls[index - 1]) * precisions[index]
return area
@staticmethod
def operating_point(curve: dict[str, Any], *, min_confidence: float) -> dict[str, Any]:
"""The metrics that hold when detections below ``min_confidence`` are dropped.
Read off the curve rather than recomputed: the curve already walked the
ranking once, and every threshold is a prefix of that walk. Running the
model again per threshold would spend N GPU passes to reproduce numbers
that are already here.
"""
points = curve.get("points") or []
reference_count = int(curve.get("reference_count") or 0)
admitted = [point for point in points if point["confidence_threshold"] >= min_confidence]
# Points are cumulative down the ranking, so the last admitted one is
# the complete tally at this cut.
tally = admitted[-1] if admitted else None
true_positives = int(tally["true_positives"]) if tally else 0
false_positives = int(tally["false_positives"]) if tally else 0
false_negatives = max(0, reference_count - true_positives)
candidate_count = true_positives + false_positives
precision = true_positives / candidate_count if candidate_count else None
recall = true_positives / reference_count if reference_count else None
f1_score = None
if precision is not None and recall is not None:
f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
return {
"min_confidence": min_confidence,
"confidence_threshold": tally["confidence_threshold"] if tally else None,
"candidate_count": candidate_count,
"true_positives": true_positives,
"false_positives": false_positives,
"false_negatives": false_negatives,
"precision": precision,
"recall": recall,
"f1_score": f1_score,
}
@staticmethod
def calibration_sweep(curve: dict[str, Any], *, thresholds: list[float]) -> list[dict[str, Any]]:
"""Every requested operating point, strictest first, from one curve.
Recall cannot fall as the cut loosens that monotonicity is exactly
why a single run answers the whole sweep.
"""
ordered = sorted({float(value) for value in thresholds}, reverse=True)
rows = [
DetectionMetricsService.operating_point(curve, min_confidence=value) for value in ordered
]
best_f1 = max((row["f1_score"] or 0.0) for row in rows) if rows else 0.0
marked = False
for row in rows:
is_best = not marked and (row["f1_score"] or 0.0) == best_f1
row["best_f1_in_sweep"] = is_best
marked = marked or is_best
return rows
@staticmethod
def precision_recall_curve(
candidates: list[tuple[dict[str, Any], BaseGeometry]],
references: list[tuple[dict[str, Any], BaseGeometry]],
*,
iou_threshold: float,
) -> dict[str, Any]:
"""Sweep every confidence value present and describe the whole curve."""
ranked = DetectionMetricsService._rank_candidates(candidates)
hits, reference_count = DetectionMetricsService._greedy_hits(ranked, references, iou_threshold)
points: list[dict[str, Any]] = []
true_positives = 0
for position, hit in enumerate(hits):
if hit:
true_positives += 1
false_positives = position + 1 - true_positives
precision = true_positives / (position + 1)
recall = true_positives / reference_count if reference_count else 0.0
f1 = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
points.append(
{
"confidence_threshold": ranked[position][0],
"candidate_count": position + 1,
"true_positives": true_positives,
"false_positives": false_positives,
"false_negatives": max(0, reference_count - true_positives),
"precision": precision,
"recall": recall,
"f1_score": f1,
}
)
# Keep one point per distinct threshold: the last one, which is the
# complete tally for everything at or above that confidence.
deduplicated: list[dict[str, Any]] = []
for point in points:
if deduplicated and deduplicated[-1]["confidence_threshold"] == point["confidence_threshold"]:
deduplicated[-1] = point
else:
deduplicated.append(point)
best = max(deduplicated, key=lambda point: (point["f1_score"], point["confidence_threshold"]), default=None)
return {
"iou_threshold": iou_threshold,
"reference_count": reference_count,
"candidate_count": len(ranked),
"average_precision": DetectionMetricsService._average_precision(points),
"best_f1": best["f1_score"] if best else 0.0,
"best_f1_threshold": best["confidence_threshold"] if best else None,
"best_f1_precision": best["precision"] if best else None,
"best_f1_recall": best["recall"] if best else None,
"points": deduplicated,
}
+38 -1
View File
@@ -168,24 +168,61 @@ class DetectionQaService:
clipped_boundary_count=clipped_boundary_count, clipped_boundary_count=clipped_boundary_count,
) )
# A polygon whose area is within this fraction of its own bounding box is
# an axis-aligned rectangle for practical purposes.
RECTANGULAR_AREA_RATIO = 0.99
@staticmethod
def candidate_geometry_mode(geometries: list[tuple[dict[str, Any], BaseGeometry]]) -> str:
"""Say whether the candidates are detector boxes or true footprints.
It matters for reading the score. An axis-aligned box can never reach
IoU 1 against a rotated or L-shaped building footprint, so a strict
footprint IoU understates a box detector by a fixed amount that has
nothing to do with whether it found the building.
"""
polygonal = [
geometry
for _, geometry in geometries
if geometry.geom_type in {"Polygon", "MultiPolygon"} and geometry.area > 0
]
if not polygonal:
return "unknown"
rectangular = sum(
1
for geometry in polygonal
if geometry.area / geometry.envelope.area >= DetectionQaService.RECTANGULAR_AREA_RATIO
)
return "axis_aligned_boxes" if rectangular == len(polygonal) else "footprint_polygons"
@staticmethod @staticmethod
def box_to_footprint_diagnostics( def box_to_footprint_diagnostics(
strict_evidence: QaMatchEvidence, strict_evidence: QaMatchEvidence,
envelope_evidence: QaMatchEvidence, envelope_evidence: QaMatchEvidence,
*, *,
iou_threshold: float, iou_threshold: float,
candidate_geometry_mode: str = "unknown",
) -> dict[str, Any]: ) -> dict[str, Any]:
envelope_metrics = DetectionQaService._metrics(envelope_evidence) envelope_metrics = DetectionQaService._metrics(envelope_evidence)
return { diagnostics = {
"diagnostic_only": True, "diagnostic_only": True,
"canonical_method": "candidate_polygon_vs_reference_footprint_iou", "canonical_method": "candidate_polygon_vs_reference_footprint_iou",
"diagnostic_method": "candidate_polygon_vs_reference_envelope_iou", "diagnostic_method": "candidate_polygon_vs_reference_envelope_iou",
"iou_threshold": iou_threshold, "iou_threshold": iou_threshold,
"candidate_geometry_mode": candidate_geometry_mode,
"strict_matches": strict_evidence.matches, "strict_matches": strict_evidence.matches,
"envelope_matches": envelope_evidence.matches, "envelope_matches": envelope_evidence.matches,
"possible_box_to_footprint_mismatch_count": max(0, envelope_evidence.matches - strict_evidence.matches), "possible_box_to_footprint_mismatch_count": max(0, envelope_evidence.matches - strict_evidence.matches),
**envelope_metrics, **envelope_metrics,
} }
if candidate_geometry_mode == "axis_aligned_boxes":
diagnostics["interpretation"] = (
"Candidates are axis-aligned detector boxes. The strict footprint IoU therefore has a "
"ceiling below 1 for rotated or non-rectangular buildings; the envelope figures isolate "
"detection quality from that shape mismatch."
)
return diagnostics
@staticmethod @staticmethod
def _metrics(evidence: QaMatchEvidence) -> dict[str, Any]: def _metrics(evidence: QaMatchEvidence) -> dict[str, Any]:
@@ -1,12 +1,12 @@
from __future__ import annotations from __future__ import annotations
from collections import Counter from collections import Counter
from typing import Any
from uuid import UUID from uuid import UUID
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app.core.errors import AppError from app.core.errors import AppError
from app.services.reviewed_metrics_service import ReviewedMetricsService
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
from app.schemas.detection_review import ( from app.schemas.detection_review import (
DetectionReviewList, DetectionReviewList,
@@ -80,7 +80,11 @@ class DetectionReviewService:
return {(row.evidence_role, row.evidence_feature_id): row for row in rows} return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
@staticmethod @staticmethod
def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary: def _summary(
evidence: list[dict[str, str]],
reviews: dict[tuple[str, str], DetectionReview],
quality_check: QualityCheck | None = None,
) -> DetectionReviewSummary:
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence} evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
decisions = Counter( decisions = Counter(
reviews[key].decision if key in reviews else "unreviewed" reviews[key].decision if key in reviews else "unreviewed"
@@ -96,6 +100,43 @@ class DetectionReviewService:
false_positive_total=false_positive_total, false_positive_total=false_positive_total,
false_negative_total=false_negative_total, false_negative_total=false_negative_total,
decision_counts=dict(sorted(decisions.items())), decision_counts=dict(sorted(decisions.items())),
reviewed_metrics=DetectionReviewService._reviewed_metrics(evidence_keys, reviews, quality_check),
)
@staticmethod
def _reviewed_metrics(
evidence_keys: set[tuple[str, str]],
reviews: dict[tuple[str, str], DetectionReview],
quality_check: QualityCheck | None,
) -> dict | None:
"""The score with the operator's verdicts applied.
Without this the panel shows a precision the operator has already
disproved: a false positive adjudicated as a reference gap is not the
model's error, and the raw number keeps counting it as one.
"""
if quality_check is None:
return None
findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
matches = findings.get("matches")
false_positives = findings.get("false_positives")
false_negatives = findings.get("false_negatives")
if not all(isinstance(value, int) for value in (matches, false_positives, false_negatives)):
return None
per_role: dict[str, Counter] = {"false_positive": Counter(), "false_negative": Counter()}
for role, feature_id in evidence_keys:
review = reviews.get((role, feature_id))
if review is not None and role in per_role:
per_role[role][review.decision] += 1
return ReviewedMetricsService.adjudicate(
matches=int(matches),
false_positives=int(false_positives),
false_negatives=int(false_negatives),
false_positive_decisions=dict(per_role["false_positive"]),
false_negative_decisions=dict(per_role["false_negative"]),
) )
@staticmethod @staticmethod
@@ -180,7 +221,7 @@ class DetectionReviewService:
total=len(filtered), total=len(filtered),
limit=limit, limit=limit,
offset=offset, offset=offset,
summary=DetectionReviewService._summary(evidence, reviews), summary=DetectionReviewService._summary(evidence, reviews, quality_check),
) )
@staticmethod @staticmethod
+586 -58
View File
@@ -9,21 +9,31 @@ from typing import Any
from typing import Type from typing import Type
from geoalchemy2.shape import from_shape, to_shape from geoalchemy2.shape import from_shape, to_shape
from pyproj import Transformer
from shapely.geometry import box as shapely_box
from shapely.geometry import mapping, shape from shapely.geometry import mapping, shape
from shapely.ops import transform as shapely_transform
from shapely.strtree import STRtree
from sqlalchemy import func from sqlalchemy import func
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.core.request_context import get_request_id from app.core.request_context import get_request_id
from app.models import AnalysisRun, Dataset, Detection, Job, Project, VectorFeature from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, VectorFeature
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.detection_metrics_service import DetectionMetricsService
from app.services.detection_qa_service import DetectionQaService from app.services.detection_qa_service import DetectionQaService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.model_asset_catalog_service import ModelAssetCatalogService from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.model_registry_service import ModelRegistryService from app.services.model_registry_service import ModelRegistryService
from app.services.model_validation_scope_service import ModelValidationScopeService
from app.services.qa_service import QaService from app.services.qa_service import QaService
from app.services.storage_service import StorageService
from app.services.quality_service import QualityService from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.temporal_compatibility_service import TemporalCompatibilityService from app.services.temporal_compatibility_service import TemporalCompatibilityService
from app.services.tile_manifest_service import TileManifestService
from app.services.yolo_adapter import YoloDetectionAdapter from app.services.yolo_adapter import YoloDetectionAdapter
@@ -48,22 +58,11 @@ class DetectionService:
parameters_json: dict[str, Any] | None = None, parameters_json: dict[str, Any] | None = None,
settings: Settings | None = None, settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
existing_job: Job | None = None,
) -> DetectionRunResponse: ) -> DetectionRunResponse:
parameters = dict(parameters_json or {}) parameters = dict(parameters_json or {})
resolved_settings = settings or get_settings() resolved_settings = settings or get_settings()
project = db.get(Project, project_id) dataset = DetectionService._validate_run_request(db, project_id=project_id, dataset_id=dataset_id)
if not project:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster":
raise AppError(
code="INVALID_DATASET_TYPE",
message="Detection requires a raster dataset",
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
TemporalCompatibilityService.ensure_detection_source_supported(dataset) TemporalCompatibilityService.ensure_detection_source_supported(dataset)
selected_model_asset = None selected_model_asset = None
@@ -90,6 +89,24 @@ class DetectionService:
message="Configured YOLO inference requires an existing raster tile manifest path", message="Configured YOLO inference requires an existing raster tile manifest path",
status_code=400, status_code=400,
) )
requested_classes = {DetectionService._canonical_class_name(value) for value in (class_filter or [])}
unsupported_classes = sorted(requested_classes - set(model.supported_classes))
if unsupported_classes:
raise AppError(code="DETECTION_CLASS_NOT_VALIDATED", message="The selected model is not validated for one or more requested classes", details={"unsupported_classes": unsupported_classes, "supported_classes": model.supported_classes}, status_code=422)
if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope:
DetectionService._validate_model_area_scope(db, dataset, resolved_settings)
# Never enter a production inference path with a persisted dataset
# that has failed validation, incomplete provenance, or an active
# quarantine. Fixture detection is a separate QA/test-only path.
if model.model_id == "manual-fixture-detector":
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="quality_assessment",
fixture_mode=True,
)
elif model.model_id == resolved_settings.yolo_model_id and model.configured:
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
run_parameters = { run_parameters = {
"model_id": model.model_id, "model_id": model.model_id,
@@ -101,7 +118,7 @@ class DetectionService:
"tile_manifest_path": tile_manifest_path, "tile_manifest_path": tile_manifest_path,
"parameters_json": parameters, "parameters_json": parameters,
} }
job = DetectionService._create_job(db, project_id, dataset_id, run_parameters) job = DetectionService._create_job(db, project_id, dataset_id, run_parameters, existing_job=existing_job)
analysis_run = DetectionService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters) analysis_run = DetectionService._create_analysis_run(db, project_id, dataset_id, job.id, model, run_parameters)
logger.info( logger.info(
"detection_started request_id=%s project_id=%s dataset_id=%s job_id=%s analysis_run_id=%s model_id=%s", "detection_started request_id=%s project_id=%s dataset_id=%s job_id=%s analysis_run_id=%s model_id=%s",
@@ -130,18 +147,23 @@ class DetectionService:
) )
if model.model_id == "manual-fixture-detector": if model.model_id == "manual-fixture-detector":
detections = DetectionService._persist_fixture_detections( try:
db=db, detections = DetectionService._persist_fixture_detections(
project_id=project_id, db=db,
dataset_id=dataset_id, project_id=project_id,
analysis_run=analysis_run, dataset_id=dataset_id,
job=job, analysis_run=analysis_run,
model_name=model.model_id, job=job,
model_version=model.version, model_name=model.model_id,
raw_detections=parameters.get("fixture_detections"), model_version=model.version,
confidence_threshold=confidence_threshold, raw_detections=parameters.get("fixture_detections"),
class_filter=class_filter or [], confidence_threshold=confidence_threshold,
) class_filter=class_filter or [],
)
except Exception as exc:
# A rejected fixture payload must never leave the run stuck in "running".
DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR")
raise
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections)) DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
return DetectionRunResponse( return DetectionRunResponse(
analysis_run_id=analysis_run.id, analysis_run_id=analysis_run.id,
@@ -183,6 +205,10 @@ class DetectionService:
error_code=exc.code, error_code=exc.code,
message=exc.message, message=exc.message,
) )
except Exception as exc:
# An unexpected inference error must never leave the run stuck in "running".
DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR")
raise
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary) DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary)
return DetectionRunResponse( return DetectionRunResponse(
analysis_run_id=analysis_run.id, analysis_run_id=analysis_run.id,
@@ -195,8 +221,54 @@ class DetectionService:
message="YOLO detections persisted.", message="YOLO detections persisted.",
) )
DetectionService._mark_failed(
db,
analysis_run,
job,
code="DETECTION_MODEL_UNAVAILABLE",
message="Detection model is unavailable",
)
raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503) raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503)
@staticmethod
def _validate_model_area_scope(db, dataset: Dataset, settings: Settings) -> None:
area = db.get(Area, dataset.area_id) if dataset.area_id else None
if area is None or area.geometry is None:
raise AppError(
code="DETECTION_VALIDATION_SCOPE_UNAVAILABLE",
message="Configured YOLO inference requires a persisted Dataset area geometry.",
details={"dataset_id": str(dataset.id)},
status_code=422,
)
try:
area_geometry = to_shape(area.geometry)
except Exception as exc:
raise AppError(
code="DETECTION_VALIDATION_SCOPE_UNAVAILABLE",
message="The persisted Dataset area geometry cannot be validated for model inference.",
details={"dataset_id": str(dataset.id), "error_type": type(exc).__name__},
status_code=422,
) from exc
ModelValidationScopeService.assert_area_covered(
area_geometry=area_geometry,
manifest_path=settings.yolo_validation_scope_manifest_path,
expected_manifest_sha256=settings.yolo_validation_scope_manifest_sha256,
model_id=settings.yolo_model_id,
model_path=settings.yolo_model_path,
)
@staticmethod
def _fail_run_after_exception(db, analysis_run: AnalysisRun, job: Job, exc: Exception, fallback_code: str) -> None:
try:
db.rollback()
except Exception:
pass
code = getattr(exc, "code", None) or fallback_code
message = getattr(exc, "message", None) or "Unexpected internal error during analysis run"
try:
DetectionService._mark_failed(db, analysis_run, job, code=str(code), message=str(message))
except Exception:
pass
@staticmethod @staticmethod
def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead: def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead:
run = db.get(AnalysisRun, analysis_run_id) run = db.get(AnalysisRun, analysis_run_id)
@@ -210,6 +282,8 @@ class DetectionService:
*, *,
project_id: uuid.UUID | None = None, project_id: uuid.UUID | None = None,
dataset_id: uuid.UUID | None = None, dataset_id: uuid.UUID | None = None,
limit: int | None = None,
offset: int = 0,
) -> DetectionRunListResponse: ) -> DetectionRunListResponse:
query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "detection") query = db.query(AnalysisRun).filter(AnalysisRun.analysis_type == "detection")
if project_id is not None: if project_id is not None:
@@ -217,7 +291,16 @@ class DetectionService:
if dataset_id is not None: if dataset_id is not None:
query = query.filter(AnalysisRun.dataset_id == dataset_id) query = query.filter(AnalysisRun.dataset_id == dataset_id)
rows = query.order_by(AnalysisRun.created_at.desc()).all() rows = query.order_by(AnalysisRun.created_at.desc()).all()
return DetectionRunListResponse(items=[DetectionRunRead.model_validate(row) for row in rows], total=len(rows)) # Runs accumulate with every analysis; the panel draws the recent ones.
resolved_limit = DetectionService.DEFAULT_RUN_LIST_LIMIT if limit is None else int(limit)
page, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=offset)
return DetectionRunListResponse(
items=[DetectionRunRead.model_validate(row) for row in page],
total=total,
limit=resolved_limit,
offset=max(0, int(offset)),
truncated=truncated,
)
@staticmethod @staticmethod
def list_detections( def list_detections(
@@ -227,6 +310,8 @@ class DetectionService:
dataset_id: uuid.UUID | None = None, dataset_id: uuid.UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int | None = None,
offset: int = 0,
) -> DetectionListResponse: ) -> DetectionListResponse:
if analysis_run_id is not None: if analysis_run_id is not None:
run = db.get(AnalysisRun, analysis_run_id) run = db.get(AnalysisRun, analysis_run_id)
@@ -239,8 +324,15 @@ class DetectionService:
class_name=class_name, class_name=class_name,
min_confidence=min_confidence, min_confidence=min_confidence,
) )
items = [DetectionRead.model_validate(row) for row in rows] resolved_limit = DetectionService.DEFAULT_RESULT_LIMIT if limit is None else int(limit)
return DetectionListResponse(items=items, total=len(items)) page, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=offset)
return DetectionListResponse(
items=[DetectionRead.model_validate(row) for row in page],
total=total,
limit=resolved_limit,
offset=max(0, int(offset)),
truncated=truncated,
)
@staticmethod @staticmethod
def get_detection(db, detection_id: uuid.UUID) -> DetectionRead: def get_detection(db, detection_id: uuid.UUID) -> DetectionRead:
@@ -257,16 +349,27 @@ class DetectionService:
dataset_id: uuid.UUID | None = None, dataset_id: uuid.UUID | None = None,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
limit: int | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
detections = DetectionService._query_detection_rows( rows = DetectionService._query_detection_rows(
db, db,
analysis_run_id=analysis_run_id, analysis_run_id=analysis_run_id,
dataset_id=dataset_id, dataset_id=dataset_id,
class_name=class_name, class_name=class_name,
min_confidence=min_confidence, min_confidence=min_confidence,
) )
resolved_limit = DetectionService.DEFAULT_RESULT_LIMIT if limit is None else int(limit)
# Rows arrive ranked by confidence, so a capped overlay draws the
# strongest detections rather than an arbitrary slice.
detections, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=0)
return { return {
"type": "FeatureCollection", "type": "FeatureCollection",
"geointel_result_window": {
"feature_count": len(detections),
"total_feature_count": total,
"limit": resolved_limit,
"truncated": truncated,
},
"features": [ "features": [
{ {
"type": "Feature", "type": "Feature",
@@ -286,6 +389,7 @@ class DetectionService:
iou_threshold: float = 0.5, iou_threshold: float = 0.5,
class_name: str | None = None, class_name: str | None = None,
min_confidence: float | None = None, min_confidence: float | None = None,
calibration_thresholds: list[float] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
run = db.get(AnalysisRun, analysis_run_id) run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection": if not run or run.analysis_type != "detection":
@@ -305,15 +409,6 @@ class DetectionService:
reference_dataset, reference_dataset,
) )
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
candidate_geometries = raw_candidate_geometries
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {} run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
manifest_path = DetectionQaService.tile_manifest_path(run_parameters) manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
resolved_settings = get_settings() resolved_settings = get_settings()
@@ -328,9 +423,48 @@ class DetectionService:
status_code=422, status_code=422,
) )
fixture_parameters = run_parameters.get("parameters_json")
fixture_mode = bool(
run.model_name == "manual-fixture-detector"
and isinstance(fixture_parameters, dict)
and fixture_parameters.get("fixture_mode") is True
)
DatasetConsumptionGate.assert_eligible(
candidate_dataset,
purpose="quality_assessment",
fixture_mode=fixture_mode,
)
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
raw_candidate_geometries = [
(
{
"id": str(row.id),
"class_name": row.class_name,
# Confidence lets the matcher rank candidates the way
# detection benchmarks do instead of by row order.
"confidence": row.confidence,
},
to_shape(row.geometry),
)
for row in detections
]
candidate_geometries = raw_candidate_geometries
coverage = None coverage = None
if manifest_path: if manifest_path:
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles) manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles, resolved_settings)
coverage = DetectionQaService.build_tile_coverage( coverage = DetectionQaService.build_tile_coverage(
manifest, manifest,
manifest_path=manifest_path, manifest_path=manifest_path,
@@ -416,10 +550,40 @@ class DetectionService:
reference_envelopes, reference_envelopes,
iou_threshold, iou_threshold,
) )
candidate_geometry_mode = DetectionQaService.candidate_geometry_mode(candidate_geometries)
box_to_footprint_diagnostics = DetectionQaService.box_to_footprint_diagnostics( box_to_footprint_diagnostics = DetectionQaService.box_to_footprint_diagnostics(
evidence, evidence,
envelope_evidence, envelope_evidence,
iou_threshold=iou_threshold, iou_threshold=iou_threshold,
candidate_geometry_mode=candidate_geometry_mode,
)
box_to_footprint_diagnostics["envelope_precision_recall_curve"] = (
DetectionMetricsService.precision_recall_curve(
candidate_geometries,
reference_envelopes,
iou_threshold=iou_threshold,
)
)
if candidate_geometry_mode == "axis_aligned_boxes":
coverage_warnings.append(
"Candidates are axis-aligned detector boxes; strict footprint IoU cannot reach 1 for "
"rotated or non-rectangular buildings. See box_to_footprint_diagnostics."
)
# Threshold-independent view of the same populations, so the run can be
# compared with another model instead of only with itself.
precision_recall_curve = DetectionMetricsService.precision_recall_curve(
candidate_geometries,
reference_geometries,
iou_threshold=iou_threshold,
)
# Every requested confidence cut, answered from that one matching pass.
# Re-running inference per threshold spends N GPU passes to reproduce
# numbers already present here: suppression walks candidates in
# descending confidence, so the kept set above a cut does not depend on
# the threshold the run itself used.
calibration_sweep = DetectionMetricsService.calibration_sweep(
precision_recall_curve,
thresholds=list(calibration_thresholds or []),
) )
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
@@ -455,6 +619,8 @@ class DetectionService:
"coverage": coverage_summary, "coverage": coverage_summary,
"temporal_compatibility": temporal_compatibility, "temporal_compatibility": temporal_compatibility,
"box_to_footprint_diagnostics": box_to_footprint_diagnostics, "box_to_footprint_diagnostics": box_to_footprint_diagnostics,
"precision_recall_curve": precision_recall_curve,
"calibration_sweep": calibration_sweep,
"match_evidence": evidence.match_evidence, "match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence, "false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence, "false_negative_evidence": evidence.false_negative_evidence,
@@ -466,6 +632,9 @@ class DetectionService:
"mean_iou": mean_iou, "mean_iou": mean_iou,
"false_positive_count": evidence.false_positives, "false_positive_count": evidence.false_positives,
"false_negative_count": evidence.false_negatives, "false_negative_count": evidence.false_negatives,
"average_precision": precision_recall_curve["average_precision"],
"best_f1": precision_recall_curve["best_f1"],
"best_f1_threshold": precision_recall_curve["best_f1_threshold"],
}, },
) )
logger.info( logger.info(
@@ -500,13 +669,33 @@ class DetectionService:
"coverage": coverage_summary, "coverage": coverage_summary,
"temporal_compatibility": temporal_compatibility, "temporal_compatibility": temporal_compatibility,
"box_to_footprint_diagnostics": box_to_footprint_diagnostics, "box_to_footprint_diagnostics": box_to_footprint_diagnostics,
"precision_recall_curve": precision_recall_curve,
"calibration_sweep": calibration_sweep,
"match_evidence": evidence.match_evidence, "match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence, "false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence, "false_negative_evidence": evidence.false_negative_evidence,
} }
@staticmethod @staticmethod
def _create_job(db, project_id: uuid.UUID, dataset_id: uuid.UUID, parameters: dict[str, Any]) -> Job: def _create_job(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
parameters: dict[str, Any],
existing_job: Job | None = None,
) -> Job:
if existing_job is not None:
# A queued job already represents this run; reuse it so the client
# keeps polling one identifier from request to result.
existing_job.status = "running"
existing_job.dataset_id = dataset_id
existing_job.input_dataset_id = dataset_id
existing_job.parameters_json = {**(existing_job.parameters_json or {}), **parameters}
existing_job.started_at = DetectionService._now()
db.add(existing_job)
db.commit()
db.refresh(existing_job)
return existing_job
job = Job( job = Job(
id=uuid.uuid4(), id=uuid.uuid4(),
job_type="detection.run", job_type="detection.run",
@@ -522,6 +711,99 @@ class DetectionService:
db.refresh(job) db.refresh(job)
return job return job
@staticmethod
def enqueue_detection(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
model_id: str,
confidence_threshold: float,
model_asset_id: str | None = None,
class_filter: list[str] | None = None,
tile_manifest_path: str | None = None,
parameters_json: dict[str, Any] | None = None,
) -> Job:
"""Accept a detection run for background execution.
Everything cheap enough to answer inside the request is checked here,
so an operator learns about a missing dataset or an unvalidated class
immediately rather than from a job that fails minutes later.
"""
DetectionService._validate_run_request(
db,
project_id=project_id,
dataset_id=dataset_id,
)
job = Job(
id=uuid.uuid4(),
job_type="detection.run",
status="queued",
project_id=project_id,
dataset_id=dataset_id,
input_dataset_id=dataset_id,
parameters_json={
"project_id": str(project_id),
"dataset_id": str(dataset_id),
"model_id": model_id,
"model_asset_id": model_asset_id,
"confidence_threshold": confidence_threshold,
"class_filter": class_filter or [],
"tile_manifest_path": tile_manifest_path,
"parameters_json": dict(parameters_json or {}),
},
)
db.add(job)
db.commit()
db.refresh(job)
logger.info(
"detection_queued request_id=%s project_id=%s dataset_id=%s job_id=%s model_id=%s",
get_request_id(),
project_id,
dataset_id,
job.id,
model_id,
)
return job
@staticmethod
def _validate_run_request(db, *, project_id: uuid.UUID, dataset_id: uuid.UUID) -> Dataset:
project = db.get(Project, project_id)
if not project:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster":
raise AppError(
code="INVALID_DATASET_TYPE",
message="Detection requires a raster dataset",
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
return dataset
# A regional run holds tens of thousands of detections; the results table
# and the map overlay both read them after every run.
DEFAULT_RESULT_LIMIT = 2_000
DEFAULT_RUN_LIST_LIMIT = 200
@staticmethod
def paginate(rows: list[Any], *, limit: int, offset: int) -> tuple[list[Any], int, bool]:
"""Slice a result population, keeping the total intact.
``limit <= 0`` means "everything", for callers that genuinely need the
whole population and know what they are asking for.
"""
total = len(rows)
start = max(0, int(offset))
if limit <= 0:
return rows[start:], total, False
page = rows[start : start + int(limit)]
# Truncated means: this page is not the whole population.
return page, total, len(page) < total
@staticmethod @staticmethod
def _query_detection_rows( def _query_detection_rows(
db, db,
@@ -540,7 +822,14 @@ class DetectionService:
query = query.filter(Detection.class_name == class_name) query = query.filter(Detection.class_name == class_name)
if min_confidence is not None: if min_confidence is not None:
query = query.filter(Detection.confidence >= min_confidence) query = query.filter(Detection.confidence >= min_confidence)
return query.order_by(Detection.created_at.desc()).all() # ``created_at`` defaults to the transaction timestamp, so every
# detection in a run shares one value and ordering by it alone leaves
# the row order undefined. Confidence first, id as a stable tiebreak.
return query.order_by(
Detection.confidence.desc(),
Detection.created_at.desc(),
Detection.id.asc(),
).all()
@staticmethod @staticmethod
def _detection_properties(detection: Detection) -> dict[str, Any]: def _detection_properties(detection: Detection) -> dict[str, Any]:
@@ -679,16 +968,51 @@ class DetectionService:
settings: Settings, settings: Settings,
yolo_adapter_class: Type[YoloDetectionAdapter], yolo_adapter_class: Type[YoloDetectionAdapter],
) -> tuple[list[Detection], dict[str, Any]]: ) -> tuple[list[Detection], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles) manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
dataset = db.get(Dataset, dataset_id)
if dataset is None:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
manifest_binding = TileManifestService.validate_for_inference(
db,
dataset,
manifest,
manifest_path=tile_manifest_path or "",
settings=settings,
error_prefix="DETECTION",
)
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
model_path = Path(settings.yolo_model_path or "").expanduser() model_path = Path(settings.yolo_model_path or "").expanduser()
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id=model_name,
task_type="object_detection",
expected_model_version=model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
DetectionService._attach_runtime_model_provenance(
analysis_run,
job,
runtime_model_provenance,
)
adapter = yolo_adapter_class(settings) adapter = yolo_adapter_class(settings)
model = adapter.load_model(model_path) model = adapter.load_model(model_path)
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)} allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
candidates: list[dict[str, Any]] = [] candidates: list[dict[str, Any]] = []
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326" manifest_crs = DetectionService._require_manifest_crs(manifest)
for tile in manifest["tiles"]: raster_bounds = DetectionService._bounds_to_epsg4326(manifest.get("bounds"), manifest_crs)
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser()) tiles = list(manifest["tiles"])
for raw in adapter.predict_tile(model, tile_path, confidence_threshold): tile_paths = [
DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser(), settings) for tile in tiles
]
# Batched so the GPU is not idle between tiles; each tile keeps its own
# transform for georeferencing, so results stay per tile and in order.
detections_per_tile = adapter.predict_tiles(model, tile_paths, confidence_threshold)
for tile, tile_path, raw_detections in zip(tiles, tile_paths, detections_per_tile):
tile_crs = tile.get("crs") or manifest_crs
tile_bounds_4326 = DetectionService._bounds_to_epsg4326(tile.get("bounds"), tile_crs)
tile_edge_tolerance = DetectionService._tile_edge_tolerance(tile, tile_bounds_4326)
for raw in raw_detections:
model_class_name = str(raw.get("class_name") or "").strip() model_class_name = str(raw.get("class_name") or "").strip()
class_name = DetectionService._canonical_class_name(model_class_name) class_name = DetectionService._canonical_class_name(model_class_name)
confidence = float(raw.get("confidence", 0.0)) confidence = float(raw.get("confidence", 0.0))
@@ -699,7 +1023,7 @@ class DetectionService:
bbox = raw.get("bbox") bbox = raw.get("bbox")
if not isinstance(bbox, list): if not isinstance(bbox, list):
raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO adapter returned a detection without bbox", status_code=422) raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO adapter returned a detection without bbox", status_code=422)
geometry = pixel_bbox_to_epsg4326_polygon(bbox=bbox, tile=tile, crs=tile.get("crs") or manifest_crs) geometry = pixel_bbox_to_epsg4326_polygon(bbox=bbox, tile=tile, crs=tile_crs)
properties = dict(raw.get("properties") or {}) properties = dict(raw.get("properties") or {})
if model_class_name and model_class_name != class_name: if model_class_name and model_class_name != class_name:
properties.setdefault("model_class_name", model_class_name) properties.setdefault("model_class_name", model_class_name)
@@ -711,11 +1035,21 @@ class DetectionService:
"bbox": bbox, "bbox": bbox,
"source_tile_path": str(tile_path), "source_tile_path": str(tile_path),
"properties": {**properties, "tile_index": tile.get("index")}, "properties": {**properties, "tile_index": tile.get("index")},
"tile_bounds": tile_bounds_4326,
"tile_edge_tolerance": tile_edge_tolerance,
} }
) )
edge_filtered_candidates = candidates
if settings.yolo_suppress_tile_edge_detections:
edge_filtered_candidates = DetectionService._drop_tile_edge_truncations(
candidates,
raster_bounds=raster_bounds,
tolerance=0.0,
)
filtered_candidates = DetectionService._suppress_duplicate_candidates( filtered_candidates = DetectionService._suppress_duplicate_candidates(
candidates, edge_filtered_candidates,
iou_threshold=float(settings.yolo_duplicate_iou_threshold), iou_threshold=float(settings.yolo_duplicate_iou_threshold),
containment_threshold=float(settings.yolo_containment_nms_threshold),
) )
persisted: list[Detection] = [] persisted: list[Detection] = []
for candidate in filtered_candidates: for candidate in filtered_candidates:
@@ -738,7 +1072,10 @@ class DetectionService:
"y_max": float(bbox[3]), "y_max": float(bbox[3]),
}, },
source_tile_path=candidate["source_tile_path"], source_tile_path=candidate["source_tile_path"],
properties_json=candidate["properties"], properties_json={
**candidate["properties"],
"runtime_model_provenance": runtime_model_provenance.as_dict(),
},
) )
db.add(detection) db.add(detection)
persisted.append(detection) persisted.append(detection)
@@ -748,28 +1085,140 @@ class DetectionService:
return persisted, { return persisted, {
"raw_detection_count": len(candidates), "raw_detection_count": len(candidates),
"suppressed_detection_count": len(candidates) - len(filtered_candidates), "suppressed_detection_count": len(candidates) - len(filtered_candidates),
"tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates),
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold), "duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
"containment_suppression_threshold": float(settings.yolo_containment_nms_threshold),
"tile_manifest_binding": manifest_binding,
"runtime_model_provenance": runtime_model_provenance.as_dict(),
} }
@staticmethod
def _attach_tile_manifest_binding(
analysis_run: AnalysisRun,
job: Job,
binding: dict[str, Any],
) -> None:
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["tile_manifest_binding"] = dict(binding)
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["tile_manifest_binding"] = dict(binding)
job.parameters_json = job_parameters
@staticmethod
def _attach_runtime_model_provenance(
analysis_run: AnalysisRun,
job: Job,
provenance: RuntimeModelProvenance,
) -> None:
"""Persist byte-bound model evidence with the run before adapter loading.
Individual detections retain the same evidence in ``properties_json``;
this run-level copy is the compact audit root for a complete inference.
Assigning fresh dictionaries matters for SQLAlchemy JSON change tracking.
"""
evidence = provenance.as_dict()
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["runtime_model_provenance"] = evidence
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["runtime_model_provenance"] = evidence
job.parameters_json = job_parameters
@staticmethod @staticmethod
def _canonical_class_name(value: Any) -> str: def _canonical_class_name(value: Any) -> str:
return str(value or "").strip().casefold() return str(value or "").strip().casefold()
# An object wider than the tile overlap is truncated by both tiles, so the
# two halves barely intersect and IoU alone never suppresses them. Overlap
# measured against the smaller box catches that case; the threshold is
# deliberately strict so that terraced houses stay separate detections.
# Fallback only. The served value is configuration, so a promoted model can
# be run at the threshold its evaluation froze.
CONTAINMENT_SUPPRESSION_THRESHOLD = 0.85
@staticmethod @staticmethod
def _suppress_duplicate_candidates(candidates: list[dict[str, Any]], iou_threshold: float) -> list[dict[str, Any]]: def _suppress_duplicate_candidates(
candidates: list[dict[str, Any]],
iou_threshold: float,
containment_threshold: float | None = None,
) -> list[dict[str, Any]]:
if iou_threshold <= 0 or len(candidates) < 2: if iou_threshold <= 0 or len(candidates) < 2:
return candidates return candidates
if containment_threshold is None:
containment_threshold = DetectionService.CONTAINMENT_SUPPRESSION_THRESHOLD
ordered = sorted(
candidates,
key=lambda item: (-float(item["confidence"]), str(item.get("source_tile_path") or "")),
)
kept: list[dict[str, Any]] = [] kept: list[dict[str, Any]] = []
for candidate in sorted(candidates, key=lambda item: float(item["confidence"]), reverse=True): kept_geometries: list[Any] = []
tree = None
for candidate in ordered:
geometry = candidate["geometry"]
duplicate = False duplicate = False
for kept_candidate in kept: # Only geometries that actually touch this candidate can suppress
# it, so an index keeps a dense AOI from turning into an O(n^2) scan.
neighbour_indexes = range(len(kept)) if tree is None else (int(index) for index in tree.query(geometry))
for index in neighbour_indexes:
kept_candidate = kept[index]
if candidate["class_name"] != kept_candidate["class_name"]: if candidate["class_name"] != kept_candidate["class_name"]:
continue continue
if DetectionService._geometry_iou(candidate["geometry"], kept_candidate["geometry"]) >= iou_threshold: other = kept_geometries[index]
if DetectionService._geometry_iou(geometry, other) >= iou_threshold:
duplicate = True
break
if DetectionService._geometry_containment(geometry, other) >= containment_threshold:
duplicate = True duplicate = True
break break
if not duplicate: if not duplicate:
kept.append(candidate) kept.append(candidate)
kept_geometries.append(geometry)
tree = STRtree(kept_geometries)
return kept
@staticmethod
def _drop_tile_edge_truncations(
candidates: list[dict[str, Any]],
*,
raster_bounds: tuple[float, float, float, float] | None,
tolerance: float,
) -> list[dict[str, Any]]:
"""Discard boxes cut off by an interior tile edge.
Such a box describes only the part of the object that fell inside its
tile. Because tiles overlap, the neighbouring tile saw the object whole
and contributed the box worth keeping. A box against the outer raster
edge has no such neighbour and is kept.
"""
if raster_bounds is None or tolerance <= 0:
return candidates
raster_left, raster_bottom, raster_right, raster_top = raster_bounds
kept: list[dict[str, Any]] = []
for candidate in candidates:
tile_bounds = candidate.get("tile_bounds")
if not tile_bounds or len(tuple(tile_bounds)) != 4:
kept.append(candidate)
continue
tile_left, tile_bottom, tile_right, tile_top = (float(value) for value in tile_bounds)
left, bottom, right, top = candidate["geometry"].bounds
# A pixel-sized tolerance per tile: a fixed degree value would be
# wrong for both a 10 cm orthophoto and a coarse thematic raster.
tolerance = float(candidate.get("tile_edge_tolerance") or 0.0) or tolerance
touches_interior_edge = (
(abs(left - tile_left) <= tolerance and abs(tile_left - raster_left) > tolerance)
or (abs(right - tile_right) <= tolerance and abs(tile_right - raster_right) > tolerance)
or (abs(bottom - tile_bottom) <= tolerance and abs(tile_bottom - raster_bottom) > tolerance)
or (abs(top - tile_top) <= tolerance and abs(tile_top - raster_top) > tolerance)
)
if not touches_interior_edge:
kept.append(candidate)
return kept return kept
@staticmethod @staticmethod
@@ -785,14 +1234,90 @@ class DetectionService:
return intersection_area / union_area return intersection_area / union_area
@staticmethod @staticmethod
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]: def _geometry_containment(left, right) -> float:
"""Intersection over the smaller of the two areas."""
if left.is_empty or right.is_empty:
return 0.0
smaller_area = min(left.area, right.area)
if smaller_area <= 0:
return 0.0
intersection_area = left.intersection(right).area
if intersection_area <= 0:
return 0.0
return intersection_area / smaller_area
@staticmethod
def _require_manifest_crs(manifest: dict[str, Any]) -> str:
"""Refuse to georeference inference output against a guessed CRS.
Detection QA already rejects a tile without explicit CRS metadata.
Silently assuming EPSG:4326 on the inference side produced geometry
that looks plausible on a map but sits in the wrong place.
"""
raw_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
if not isinstance(raw_crs, str) or not raw_crs.strip():
raise AppError(
code="DETECTION_TILE_MANIFEST_INVALID",
message="Raster tile manifest requires explicit CRS metadata for georeferencing",
status_code=422,
)
return raw_crs.strip()
@staticmethod
def _bounds_to_epsg4326(bounds: Any, crs: str | None) -> tuple[float, float, float, float] | None:
if not isinstance(bounds, (list, tuple)) or len(bounds) != 4:
return None
try:
left, bottom, right, top = (float(value) for value in bounds)
except (TypeError, ValueError):
return None
if left >= right or bottom >= top:
return None
if not crs or str(crs).strip().upper() in {"EPSG:4326", "4326"}:
return (left, bottom, right, top)
try:
transformer = Transformer.from_crs(crs, "EPSG:4326", always_xy=True)
# Transform the whole rectangle, not just two corners: a projected
# box does not stay axis-aligned after reprojection.
projected = shapely_transform(transformer.transform, shapely_box(left, bottom, right, top))
return projected.bounds
except Exception:
return None
@staticmethod
def _tile_edge_tolerance(tile: dict[str, Any], tile_bounds_4326: tuple[float, float, float, float] | None) -> float:
"""One and a half pixels, expressed in the degrees the boxes live in."""
if tile_bounds_4326 is None:
return 0.0
pixel_window = tile.get("pixel_window")
if not (isinstance(pixel_window, (list, tuple)) and len(pixel_window) == 4):
return 0.0
try:
width = float(pixel_window[2])
height = float(pixel_window[3])
except (TypeError, ValueError):
return 0.0
if width <= 0 or height <= 0:
return 0.0
left, bottom, right, top = tile_bounds_4326
return 1.5 * max((right - left) / width, (top - bottom) / height)
@staticmethod
def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int, settings: Settings | None = None) -> dict[str, Any]:
if not tile_manifest_path: if not tile_manifest_path:
raise AppError( raise AppError(
code="DETECTION_TILE_MANIFEST_REQUIRED", code="DETECTION_TILE_MANIFEST_REQUIRED",
message="Configured YOLO inference requires an existing raster tile manifest path", message="Configured YOLO inference requires an existing raster tile manifest path",
status_code=400, status_code=400,
) )
manifest_path = Path(tile_manifest_path).expanduser() # The path arrives in the request, so it must name a governed artifact
# rather than an arbitrary file on the host.
manifest_path = StorageService.assert_within_storage_root(
tile_manifest_path, label="tile manifest", settings=settings
)
if not manifest_path.exists() or not manifest_path.is_file(): if not manifest_path.exists() or not manifest_path.is_file():
raise AppError( raise AppError(
code="DETECTION_TILE_MANIFEST_NOT_FOUND", code="DETECTION_TILE_MANIFEST_NOT_FOUND",
@@ -817,13 +1342,16 @@ class DetectionService:
return manifest return manifest
@staticmethod @staticmethod
def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path) -> Path: def _resolve_tile_path(tile: dict[str, Any], manifest_path: Path, settings: Settings | None = None) -> Path:
raw_path = tile.get("path") raw_path = tile.get("path")
if not isinstance(raw_path, str) or not raw_path: if not isinstance(raw_path, str) or not raw_path:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile manifest entries require a path", status_code=422) raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile manifest entries require a path", status_code=422)
tile_path = Path(raw_path).expanduser() tile_path = Path(raw_path).expanduser()
if not tile_path.is_absolute(): if not tile_path.is_absolute():
tile_path = manifest_path.parent / tile_path tile_path = manifest_path.parent / tile_path
# A manifest entry may name an absolute path; it is still only allowed
# to point at a tile the runtime itself produced.
tile_path = StorageService.assert_within_storage_root(tile_path, label="raster tile", settings=settings)
if not tile_path.exists() or not tile_path.is_file(): if not tile_path.exists() or not tile_path.is_file():
raise AppError( raise AppError(
code="DETECTION_TILE_NOT_FOUND", code="DETECTION_TILE_NOT_FOUND",
@@ -12,7 +12,7 @@ from pathlib import Path
from typing import Any, Callable from typing import Any, Callable
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.parse import urlencode from urllib.parse import urlencode
from urllib.request import Request, urlopen from urllib.request import Request
from uuid import UUID from uuid import UUID
from geoalchemy2.shape import to_shape from geoalchemy2.shape import to_shape
@@ -22,6 +22,7 @@ from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.services.outbound_request_guard import guarded_opener
from app.models import Area, Dataset, Project from app.models import Area, Dataset, Project
from app.schemas.dhmv import DhmvAcquireRequest, DhmvAcquisitionResult, DhmvProductRead from app.schemas.dhmv import DhmvAcquireRequest, DhmvAcquisitionResult, DhmvProductRead
from app.services.dataset_service import DatasetService from app.services.dataset_service import DatasetService
@@ -294,7 +295,7 @@ class DhmvAcquisitionService:
) )
max_bytes = settings.dhmv_max_response_mb * 1024 * 1024 max_bytes = settings.dhmv_max_response_mb * 1024 * 1024
try: try:
with (opener or urlopen)(request, timeout=settings.dhmv_timeout_seconds) as response: with (opener or guarded_opener(request_url))(request, timeout=settings.dhmv_timeout_seconds) as response:
content_type = str(response.headers.get("Content-Type", "")) content_type = str(response.headers.get("Content-Type", ""))
content_length = response.headers.get("Content-Length") content_length = response.headers.get("Content-Length")
if content_length and int(content_length) > max_bytes: if content_length and int(content_length) > max_bytes:
+267 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import json import json
import re import re
import uuid import uuid
from datetime import datetime, timezone
from html import escape from html import escape
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -23,6 +24,7 @@ from app.schemas.flood_hazard import FloodHazardPartitionSelectionRequest, Flood
from app.schemas.temporal import TemporalComparisonRequest from app.schemas.temporal import TemporalComparisonRequest
from app.schemas.thematic_raster import ThematicRasterSelectionRequest from app.schemas.thematic_raster import ThematicRasterSelectionRequest
from app.services.dataset_service import DatasetService from app.services.dataset_service import DatasetService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.detection_service import DetectionService from app.services.detection_service import DetectionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.segmentation_service import SegmentationService from app.services.segmentation_service import SegmentationService
@@ -34,12 +36,194 @@ from app.services.vector_feature_service import VectorFeatureService
class ExportService: class ExportService:
# RFC 7946 allows foreign members on a FeatureCollection and requires
# parsers to ignore ones they do not know, so the provenance travels with
# the file without breaking QGIS, ogr2ogr or any other reader.
PROVENANCE_MEMBER = "geointel_provenance"
@staticmethod
def provenance_member(
*,
source: str,
project_id: uuid.UUID,
dataset_id: uuid.UUID | None = None,
analysis_run_id: uuid.UUID | None = None,
source_name: str | None = None,
source_version: str | None = None,
observed_at: Any = None,
selection_bbox: dict[str, Any] | None = None,
selection_area_id: uuid.UUID | None = None,
feature_count: int | None = None,
total_feature_count: int | None = None,
truncated: bool = False,
warnings: list[str] | None = None,
) -> dict[str, Any]:
"""Describe an exported FeatureCollection inside the file itself.
A capped export previously recorded ``truncated`` on the export record
only, so the downloaded file looked complete. Completeness is derived
from the counts as well as the flag: a caller that forgets to pass the
flag cannot produce a file that claims to hold everything.
"""
complete = not truncated
if feature_count is not None and total_feature_count is not None:
complete = complete and feature_count >= total_feature_count
member: dict[str, Any] = {
"source": source,
"project_id": str(project_id),
"exported_at": datetime.now(timezone.utc).isoformat(),
"complete": complete,
"completeness_note": None,
}
if dataset_id is not None:
member["dataset_id"] = str(dataset_id)
if analysis_run_id is not None:
member["analysis_run_id"] = str(analysis_run_id)
if source_name:
member["source_name"] = source_name
if source_version:
member["source_version"] = source_version
if observed_at is not None:
member["observed_at"] = observed_at.isoformat() if hasattr(observed_at, "isoformat") else str(observed_at)
if selection_bbox is not None:
member["selection_bbox"] = selection_bbox
if selection_area_id is not None:
member["selection_area_id"] = str(selection_area_id)
if feature_count is not None:
member["feature_count"] = feature_count
if total_feature_count is not None:
member["total_feature_count"] = total_feature_count
if warnings:
member["warnings"] = list(warnings)
if not complete:
written = feature_count if feature_count is not None else "?"
available = total_feature_count if total_feature_count is not None else "?"
member["completeness_note"] = (
f"Dit bestand bevat {written} van {available} objecten uit de selectie. Het is een "
"begrensde uitsnede, geen volledige export."
)
return member
@staticmethod
def attach_provenance(feature_collection: dict[str, Any], member: dict[str, Any]) -> dict[str, Any]:
feature_collection[ExportService.PROVENANCE_MEMBER] = member
return feature_collection
@staticmethod
def _detection_export_trust(db: Session, run: AnalysisRun) -> dict[str, Any]:
"""Classify persisted AI output without turning confidence into truth."""
checks = (
db.query(QualityCheck)
.filter(
QualityCheck.analysis_run_id == run.id,
QualityCheck.check_type == "detections_vs_reference",
)
.order_by(QualityCheck.created_at.desc())
.all()
)
quality_check = checks[0] if checks else None
reasons: list[str] = []
reference_dataset = None
if quality_check is None:
reasons.append("authoritative_qa_missing")
else:
findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
coverage = findings.get("coverage") if isinstance(findings.get("coverage"), dict) else {}
temporal = findings.get("temporal_compatibility") if isinstance(findings.get("temporal_compatibility"), dict) else {}
warnings = findings.get("warnings") if isinstance(findings.get("warnings"), list) else []
if quality_check.status != "ok":
reasons.append("quality_check_not_ok")
if findings.get("unsupported_geometry") is True:
reasons.append("unsupported_geometry")
false_positives = findings.get("false_positives")
false_negatives = findings.get("false_negatives")
if (
isinstance(false_positives, bool)
or not isinstance(false_positives, (int, float))
or false_positives != 0
):
reasons.append("false_positives_present")
if (
isinstance(false_negatives, bool)
or not isinstance(false_negatives, (int, float))
or false_negatives != 0
):
reasons.append("false_negatives_present")
if warnings:
reasons.append("quality_warnings_present")
if coverage.get("applied") is not True:
reasons.append("inference_coverage_not_proven")
if temporal.get("status") != "compatible":
reasons.append("temporal_compatibility_not_proven")
reference_dataset = db.get(Dataset, quality_check.reference_dataset_id)
if reference_dataset is None:
reasons.append("reference_dataset_missing")
else:
try:
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
except AppError:
reasons.append("reference_not_authoritative_for_buildings")
operational_use_allowed = not reasons
return {
"schema_version": "geointel.result-trust/v1",
"classification": (
"authoritative_reference_checked_ai_output"
if operational_use_allowed
else "unverified_ai_review_output"
),
"authoritative": False,
"operational_use_allowed": operational_use_allowed,
"operator_review_required": True,
"quality_check_id": str(quality_check.id) if quality_check else None,
"reference_dataset_id": str(reference_dataset.id) if reference_dataset else None,
"blocking_reasons": sorted(set(reasons)),
"limitation": (
"AI output is not ground truth. Operational use is bounded to the exact source, AOI, model and reference QA evidence."
),
}
@staticmethod
def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset:
"""Block an output export when its persisted source dataset is unsafe."""
if not run.dataset_id:
raise AppError(
code="DATASET_PROVENANCE_INCOMPLETE",
message="Analysis output cannot be exported without a persisted source dataset.",
status_code=409,
)
dataset = db.get(Dataset, run.dataset_id)
if not dataset or dataset.project_id != run.project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Analysis source dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
return dataset
@staticmethod @staticmethod
def export_map_result( def export_map_result(
db: Session, db: Session,
payload: MapResultExportRequest, payload: MapResultExportRequest,
) -> ExportCreateResponse: ) -> ExportCreateResponse:
if payload.mode == "evolution": if payload.mode == "evolution":
earlier_dataset = db.get(Dataset, payload.earlier_dataset_id)
later_dataset = db.get(Dataset, payload.later_dataset_id)
if (
not earlier_dataset
or not later_dataset
or earlier_dataset.project_id != payload.project_id
or later_dataset.project_id != payload.project_id
):
raise AppError(code="DATASET_NOT_FOUND", message="Temporal export dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(earlier_dataset, purpose="export")
DatasetConsumptionGate.assert_eligible(later_dataset, purpose="export")
comparison = TemporalAnalysisService.compare( comparison = TemporalAnalysisService.compare(
db, db,
project_id=payload.project_id, project_id=payload.project_id,
@@ -86,6 +270,7 @@ class ExportService:
dataset = db.get(Dataset, payload.dataset_id) dataset = db.get(Dataset, payload.dataset_id)
if not dataset or dataset.project_id != payload.project_id: if not dataset or dataset.project_id != payload.project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
if dataset.dataset_type in DatasetService.VECTOR_TYPES: if dataset.dataset_type in DatasetService.VECTOR_TYPES:
if payload.partitioned: if payload.partitioned:
return ExportService.export_partitioned_vector_selection_geojson( return ExportService.export_partitioned_vector_selection_geojson(
@@ -219,6 +404,7 @@ class ExportService:
limit: int = 1000, limit: int = 1000,
name: str | None = None, name: str | None = None,
) -> ExportCreateResponse: ) -> ExportCreateResponse:
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders": if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
raise AppError( raise AppError(
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED", code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
@@ -308,6 +494,7 @@ class ExportService:
details={"dataset_type": dataset.dataset_type}, details={"dataset_type": dataset.dataset_type},
status_code=400, status_code=400,
) )
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
selection_kwargs: dict[str, Any] = { selection_kwargs: dict[str, Any] = {
"dataset_id": dataset_id, "dataset_id": dataset_id,
@@ -337,6 +524,28 @@ class ExportService:
preclipped_partition_filter=preclipped_partition_filter, preclipped_partition_filter=preclipped_partition_filter,
) )
selection = VectorFeatureService.select_features_by_bbox(db, **selection_kwargs) selection = VectorFeatureService.select_features_by_bbox(db, **selection_kwargs)
summary = selection.get("summary") if isinstance(selection.get("summary"), dict) else {}
ExportService.attach_provenance(
selection["geojson"],
ExportService.provenance_member(
source="vector_selection",
project_id=dataset.project_id,
dataset_id=dataset.id,
source_name=dataset.source_name,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
selection_bbox=selection["selection_bbox"],
selection_area_id=area_id,
feature_count=selection["feature_count"],
total_feature_count=selection.get("total_feature_count"),
truncated=selection["truncated"],
warnings=[
warning
for warning in (summary.get("selection_edge_warning"), summary.get("warning"))
if warning
],
),
)
filename = ExportService._filename(name, f"{dataset.id}-selection.geojson", ".geojson") filename = ExportService._filename(name, f"{dataset.id}-selection.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename) export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
metadata = { metadata = {
@@ -374,8 +583,21 @@ class ExportService:
details={"dataset_type": dataset.dataset_type}, details={"dataset_type": dataset.dataset_type},
status_code=400, status_code=400,
) )
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id) feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
ExportService.attach_provenance(
feature_collection,
ExportService.provenance_member(
source="dataset",
project_id=dataset.project_id,
dataset_id=dataset.id,
source_name=dataset.source_name,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
feature_count=len(feature_collection.get("features", [])),
),
)
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson") filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename) export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
metadata = { metadata = {
@@ -397,12 +619,43 @@ class ExportService:
return ExportService._create_response(export) return ExportService._create_response(export)
@staticmethod @staticmethod
def export_detection_run_geojson(db: Session, analysis_run_id: uuid.UUID, name: str | None = None) -> ExportCreateResponse: def export_detection_run_geojson(
db: Session,
analysis_run_id: uuid.UUID,
name: str | None = None,
*,
intended_use: str = "review",
) -> ExportCreateResponse:
run = db.get(AnalysisRun, analysis_run_id) run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection": if not run or run.analysis_type != "detection":
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404) raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
ExportService._assert_run_source_dataset_exportable(db, run)
trust = ExportService._detection_export_trust(db, run)
if intended_use == "operational" and not trust["operational_use_allowed"]:
raise AppError(
code="DETECTION_OPERATIONAL_EXPORT_BLOCKED",
message="Operational detection export requires complete authoritative QA with no remaining errors or warnings.",
details=trust,
status_code=409,
)
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id) feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
feature_collection["geointel_result"] = trust
ExportService.attach_provenance(
feature_collection,
ExportService.provenance_member(
source="detection_run",
project_id=run.project_id,
dataset_id=run.dataset_id,
analysis_run_id=run.id,
feature_count=len(feature_collection.get("features", [])),
),
)
for feature in feature_collection.get("features", []):
properties = feature.get("properties") if isinstance(feature, dict) else None
if isinstance(properties, dict):
properties["result_classification"] = trust["classification"]
properties["authoritative"] = False
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson") filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename) export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
metadata = { metadata = {
@@ -411,6 +664,8 @@ class ExportService:
"project_id": str(run.project_id), "project_id": str(run.project_id),
"dataset_id": str(run.dataset_id) if run.dataset_id else None, "dataset_id": str(run.dataset_id) if run.dataset_id else None,
"feature_count": len(feature_collection.get("features", [])), "feature_count": len(feature_collection.get("features", [])),
"intended_use": intended_use,
"result_trust": trust,
} }
export = ExportService._write_json_export( export = ExportService._write_json_export(
db, db,
@@ -428,8 +683,19 @@ class ExportService:
run = db.get(AnalysisRun, analysis_run_id) run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "segmentation": if not run or run.analysis_type != "segmentation":
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404) raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
ExportService._assert_run_source_dataset_exportable(db, run)
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id) feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
ExportService.attach_provenance(
feature_collection,
ExportService.provenance_member(
source="segmentation_run",
project_id=run.project_id,
dataset_id=run.dataset_id,
analysis_run_id=run.id,
feature_count=len(feature_collection.get("features", [])),
),
)
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson") filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename) export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
metadata = { metadata = {
@@ -12,7 +12,7 @@ from pathlib import Path
from typing import Any, Callable from typing import Any, Callable
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.parse import urlencode from urllib.parse import urlencode
from urllib.request import Request, urlopen from urllib.request import Request
from uuid import UUID from uuid import UUID
from xml.etree import ElementTree from xml.etree import ElementTree
@@ -23,6 +23,7 @@ from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.services.outbound_request_guard import guarded_opener
from app.models import Area, Dataset, Project from app.models import Area, Dataset, Project
from app.schemas.flood_hazard import FloodHazardAcquireRequest, FloodHazardAcquisitionResult, FloodHazardProductRead from app.schemas.flood_hazard import FloodHazardAcquireRequest, FloodHazardAcquisitionResult, FloodHazardProductRead
from app.services.dataset_service import DatasetService from app.services.dataset_service import DatasetService
@@ -256,7 +257,7 @@ class FloodHazardAcquisitionService:
request = Request(request_url, headers={"Accept": "*/*", "User-Agent": "GeoIntel/0.1 bounded-vmm-flood-hazard-acquisition"}) request = Request(request_url, headers={"Accept": "*/*", "User-Agent": "GeoIntel/0.1 bounded-vmm-flood-hazard-acquisition"})
max_bytes = settings.flood_hazard_max_response_mb * 1024 * 1024 max_bytes = settings.flood_hazard_max_response_mb * 1024 * 1024
try: try:
with (opener or urlopen)(request, timeout=settings.flood_hazard_timeout_seconds) as response: with (opener or guarded_opener(request_url))(request, timeout=settings.flood_hazard_timeout_seconds) as response:
content_type = str(response.headers.get("Content-Type", "")) content_type = str(response.headers.get("Content-Type", ""))
content_length = response.headers.get("Content-Length") content_length = response.headers.get("Content-Length")
if content_length and int(content_length) > max_bytes: if content_length and int(content_length) > max_bytes:
@@ -2,8 +2,10 @@ from __future__ import annotations
import io import io
import math import math
from dataclasses import dataclass
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any
from uuid import UUID from uuid import UUID
from geoalchemy2.shape import to_shape from geoalchemy2.shape import to_shape
@@ -13,6 +15,7 @@ from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.services.raster_cell_selection import select_cells
from app.models import Area, Dataset from app.models import Area, Dataset
from app.schemas.flood_hazard import ( from app.schemas.flood_hazard import (
FloodHazardMetric, FloodHazardMetric,
@@ -25,6 +28,83 @@ from app.services.flood_hazard_acquisition_service import FloodHazardAcquisition
from app.services.raster_partition_analysis_service import RasterPartitionAnalysisService from app.services.raster_partition_analysis_service import RasterPartitionAnalysisService
@dataclass(frozen=True)
class FloodHazardCellStatistics:
"""Cell populations behind one flood-hazard selection.
Three populations, deliberately kept apart:
``selected``
every cell whose centre falls inside the drawn selection;
``valid``
the subset the VMM raster actually models finite, not nodata;
``inundated``
the subset of valid cells with a positive modelled depth.
Risk is a share of what was modelled. Dividing by the selected cells
instead silently reports "no data" as "no risk", which for a selection
reaching past the modelled extent understates the hazard by whatever
fraction of the rectangle the model never covered.
"""
selected_cell_count: int
valid_cell_count: int
inundated_cell_count: int
depth_values: Any
@property
def no_data_cell_count(self) -> int:
return max(0, self.selected_cell_count - self.valid_cell_count)
@property
def data_coverage_ratio(self) -> float:
if self.selected_cell_count <= 0:
return 0.0
return self.valid_cell_count / self.selected_cell_count
@property
def inundated_fraction(self) -> float | None:
"""``None`` when nothing was modelled: absence of data is not a zero."""
if self.valid_cell_count <= 0:
return None
return self.inundated_cell_count / self.valid_cell_count
def inundated_area_ha(self, cell_area_m2: float) -> float:
return self.inundated_cell_count * cell_area_m2 / 10_000.0
def analysed_area_ha(self, cell_area_m2: float) -> float:
"""Area the model actually covers inside the selection."""
return self.valid_cell_count * cell_area_m2 / 10_000.0
def selected_area_ha(self, cell_area_m2: float) -> float:
"""Area of the selection as rasterised, model coverage aside."""
return self.selected_cell_count * cell_area_m2 / 10_000.0
@classmethod
def from_cells(cls, values: Any, selected: Any, *, nodata: float | None) -> "FloodHazardCellStatistics":
import numpy as np
raw = np.asarray(values, dtype="float64")
selected_mask = np.asarray(selected, dtype=bool)
has_data = selected_mask & np.isfinite(raw)
if nodata is not None:
has_data &= ~np.isclose(raw, float(nodata))
# A modelled zero or negative depth is data: it says "dry here", which
# is a different statement from "not modelled here".
inundated = has_data & (raw > 0.0)
return cls(
selected_cell_count=int(selected_mask.sum()),
valid_cell_count=int(has_data.sum()),
inundated_cell_count=int(inundated.sum()),
depth_values=raw[inundated],
)
class FloodHazardAnalysisService: class FloodHazardAnalysisService:
UNSUPPORTED_METRICS = [ UNSUPPORTED_METRICS = [
"bathymetry_depth_m", "bathymetry_depth_m",
@@ -36,6 +116,77 @@ class FloodHazardAnalysisService:
"gemodelleerde maxima op en is geen gelijktijdig opgeslagen watervolume, actuele waterstand of bathymetrie." "gemodelleerde maxima op en is geen gelijktijdig opgeslagen watervolume, actuele waterstand of bathymetrie."
) )
@staticmethod
def _coverage_metrics(stats: "FloodHazardCellStatistics", cell_area_m2: float, metric) -> list[FloodHazardMetric]:
"""Headline metrics, each stating which population it is a share of.
The analysed area is reported next to the drawn area so an operator can
see immediately how much of the rectangle the flood model covers. A
selection with no model data reports 0% coverage rather than 0% risk.
"""
metrics = [
metric(
"modelled_inundated_area_ha",
"Gemodelleerd overstroomd oppervlak",
stats.inundated_area_ha(cell_area_m2),
"ha",
"positive_depth_cells_times_cell_area",
),
metric(
"modelled_inundated_share_pct",
"Aandeel gemodelleerd gebied met diepte",
0.0 if stats.inundated_fraction is None else stats.inundated_fraction * 100.0,
"%",
"positive_depth_cells_divided_by_modelled_cells",
),
metric(
"modelled_area_ha",
"Oppervlak met overstromingsmodel",
stats.analysed_area_ha(cell_area_m2),
"ha",
"modelled_cells_times_cell_area",
),
metric(
"selection_area_ha",
"Oppervlak van de selectie",
stats.selected_area_ha(cell_area_m2),
"ha",
"selected_cells_times_cell_area",
),
metric(
"model_coverage_pct",
"Deel van de selectie met een model",
stats.data_coverage_ratio * 100.0,
"%",
"modelled_cells_divided_by_selected_cells",
),
]
return metrics
@staticmethod
def _combined_warning(stats: "FloodHazardCellStatistics", cell_selection_warning: str | None) -> str | None:
parts = [
part
for part in (cell_selection_warning, FloodHazardAnalysisService._coverage_warning(stats))
if part
]
return " ".join(parts) if parts else None
@staticmethod
def _coverage_warning(stats: "FloodHazardCellStatistics") -> str | None:
if stats.valid_cell_count <= 0:
return (
"Voor deze selectie bestaat geen VMM-overstromingsmodel. Er is dus geen overstromingsrisico "
"gemeten; dit is geen bevestiging dat het risico nul is."
)
if stats.data_coverage_ratio < 0.999:
return (
f"Het VMM-model dekt {stats.data_coverage_ratio * 100:.1f}% van deze selectie. Percentages gelden "
"voor het gemodelleerde deel, niet voor de volledige selectie."
)
return None
@staticmethod @staticmethod
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset: def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
dataset = db.get(Dataset, dataset_id) dataset = db.get(Dataset, dataset_id)
@@ -81,7 +232,6 @@ class FloodHazardAnalysisService:
try: try:
import numpy as np import numpy as np
import rasterio import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask from rasterio.mask import mask
except ImportError as exc: except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc
@@ -110,17 +260,32 @@ class FloodHazardAnalysisService:
details={"pixel_count": expected_cells, "max_pixels": resolved_settings.flood_hazard_max_pixels}, details={"pixel_count": expected_cells, "max_pixels": resolved_settings.flood_hazard_max_pixels},
status_code=422, status_code=422,
) )
clipped, clipped_transform = mask(source, [mapping(analysis_geometry)], crop=True, filled=False, indexes=[1]) # ``all_touched`` keeps the values of cells the selection only
# clips, so a selection finer than one cell still has data to
# read. Which of those cells actually count is decided by
# ``select_cells`` below, so the normal result is unchanged.
clipped, clipped_transform = mask(
source,
[mapping(analysis_geometry)],
crop=True,
filled=False,
indexes=[1],
all_touched=True,
)
depth = np.ma.asarray(clipped[0], dtype="float64") depth = np.ma.asarray(clipped[0], dtype="float64")
raw = depth.filled(np.nan) raw = depth.filled(np.nan)
selected_cells = geometry_mask([mapping(analysis_geometry)], out_shape=depth.shape, transform=clipped_transform, invert=True) cell_selection = select_cells(
nodata = source.nodata analysis_geometry,
valid = selected_cells & ~np.ma.getmaskarray(depth) & np.isfinite(raw) & (raw > 0.0) out_shape=depth.shape,
if nodata is not None: transform=clipped_transform,
valid &= raw != float(nodata) cell_area_m2=abs(float(source.res[0])) * abs(float(source.res[1])),
values = raw[valid] )
selected_cell_count = int(selected_cells.sum()) selected_cells = cell_selection.mask
inundated_cell_count = int(values.size) # A masked cell carries no model value, so fold the mask into
# the raw array before the populations are separated.
raw = np.where(np.ma.getmaskarray(depth), np.nan, raw)
stats = FloodHazardCellStatistics.from_cells(raw, selected_cells, nodata=source.nodata)
values = stats.depth_values
resolution_x = abs(float(source.res[0])) resolution_x = abs(float(source.res[0]))
resolution_y = abs(float(source.res[1])) resolution_y = abs(float(source.res[1]))
cell_area_m2 = resolution_x * resolution_y cell_area_m2 = resolution_x * resolution_y
@@ -143,18 +308,8 @@ class FloodHazardAnalysisService:
aggregation_method=method, aggregation_method=method,
) )
inundated_area_ha = inundated_cell_count * cell_area_m2 / 10_000.0 metrics = FloodHazardAnalysisService._coverage_metrics(stats, cell_area_m2, metric)
metrics = [ if stats.inundated_cell_count:
metric("modelled_inundated_area_ha", "Gemodelleerd overstroomd oppervlak", inundated_area_ha, "ha", "positive_depth_cells_times_cell_area"),
metric(
"modelled_inundated_share_pct",
"Aandeel selectie met gemodelleerde diepte",
inundated_cell_count / max(1, selected_cell_count) * 100.0,
"%",
"positive_depth_cells_divided_by_selected_cells",
),
]
if inundated_cell_count:
metrics.extend( metrics.extend(
[ [
metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"), metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"),
@@ -181,9 +336,14 @@ class FloodHazardAnalysisService:
return_period_years=product.return_period_years, return_period_years=product.return_period_years,
selection_bbox=payload.bbox, selection_bbox=payload.bbox,
selection_area_id=payload.area_id, selection_area_id=payload.area_id,
selected_cell_count=selected_cell_count, selected_cell_count=stats.selected_cell_count,
inundated_cell_count=inundated_cell_count, valid_cell_count=stats.valid_cell_count,
inundated_fraction=round(inundated_cell_count / max(1, selected_cell_count), 6), no_data_cell_count=stats.no_data_cell_count,
data_coverage_ratio=round(stats.data_coverage_ratio, 6),
inundated_cell_count=stats.inundated_cell_count,
inundated_fraction=(
None if stats.inundated_fraction is None else round(stats.inundated_fraction, 6)
),
resolution_m=round(max(resolution_x, resolution_y), 4), resolution_m=round(max(resolution_x, resolution_y), 4),
summary=FloodHazardSelectionSummary( summary=FloodHazardSelectionSummary(
metric_label=primary.metric_label, metric_label=primary.metric_label,
@@ -193,6 +353,7 @@ class FloodHazardAnalysisService:
primary_metric_key=primary.metric_key, primary_metric_key=primary.metric_key,
metrics=metrics, metrics=metrics,
), ),
coverage_warning=FloodHazardAnalysisService._combined_warning(stats, cell_selection.warning),
unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS, unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS,
limitation_message=FloodHazardAnalysisService.LIMITATION, limitation_message=FloodHazardAnalysisService.LIMITATION,
generated_at=datetime.now(UTC).isoformat(), generated_at=datetime.now(UTC).isoformat(),
@@ -225,6 +386,7 @@ class FloodHazardAnalysisService:
selection_geometry_4326=selection_4326, selection_geometry_4326=selection_4326,
nodata=FloodHazardAcquisitionService.NODATA, nodata=FloodHazardAcquisitionService.NODATA,
max_pixels=resolved_settings.flood_hazard_max_pixels, max_pixels=resolved_settings.flood_hazard_max_pixels,
dataset_ids=payload.dataset_ids,
) )
try: try:
import numpy as np import numpy as np
@@ -235,16 +397,12 @@ class FloodHazardAnalysisService:
status_code=503, status_code=503,
) from exc ) from exc
raw = partition.values stats = FloodHazardCellStatistics.from_cells(
valid = ( partition.values,
partition.selected_cells partition.selected_cells,
& np.isfinite(raw) nodata=FloodHazardAcquisitionService.NODATA,
& (raw != FloodHazardAcquisitionService.NODATA)
& (raw > 0.0)
) )
values = raw[valid] values = stats.depth_values
selected_cell_count = int(partition.selected_cells.sum())
inundated_cell_count = int(values.size)
cell_area_m2 = partition.resolution_x * partition.resolution_y cell_area_m2 = partition.resolution_x * partition.resolution_y
def metric(key: str, label: str, value: float, unit: str, method: str) -> FloodHazardMetric: def metric(key: str, label: str, value: float, unit: str, method: str) -> FloodHazardMetric:
@@ -256,24 +414,8 @@ class FloodHazardAnalysisService:
aggregation_method=method, aggregation_method=method,
) )
inundated_area_ha = inundated_cell_count * cell_area_m2 / 10_000.0 metrics = FloodHazardAnalysisService._coverage_metrics(stats, cell_area_m2, metric)
metrics = [ if stats.inundated_cell_count:
metric(
"modelled_inundated_area_ha",
"Gemodelleerd overstroomd oppervlak",
inundated_area_ha,
"ha",
"positive_depth_cells_times_cell_area",
),
metric(
"modelled_inundated_share_pct",
"Aandeel selectie met gemodelleerde diepte",
inundated_cell_count / max(1, selected_cell_count) * 100.0,
"%",
"positive_depth_cells_divided_by_selected_cells",
),
]
if inundated_cell_count:
metrics.extend( metrics.extend(
[ [
metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"), metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"),
@@ -301,9 +443,14 @@ class FloodHazardAnalysisService:
return_period_years=product.return_period_years, return_period_years=product.return_period_years,
selection_bbox=payload.bbox, selection_bbox=payload.bbox,
selection_area_id=payload.area_id, selection_area_id=payload.area_id,
selected_cell_count=selected_cell_count, selected_cell_count=stats.selected_cell_count,
inundated_cell_count=inundated_cell_count, valid_cell_count=stats.valid_cell_count,
inundated_fraction=round(inundated_cell_count / max(1, selected_cell_count), 6), no_data_cell_count=stats.no_data_cell_count,
data_coverage_ratio=round(stats.data_coverage_ratio, 6),
inundated_cell_count=stats.inundated_cell_count,
inundated_fraction=(
None if stats.inundated_fraction is None else round(stats.inundated_fraction, 6)
),
resolution_m=round(max(partition.resolution_x, partition.resolution_y), 4), resolution_m=round(max(partition.resolution_x, partition.resolution_y), 4),
summary=FloodHazardSelectionSummary( summary=FloodHazardSelectionSummary(
metric_label=primary.metric_label, metric_label=primary.metric_label,
@@ -313,6 +460,7 @@ class FloodHazardAnalysisService:
primary_metric_key=primary.metric_key, primary_metric_key=primary.metric_key,
metrics=metrics, metrics=metrics,
), ),
coverage_warning=FloodHazardAnalysisService._combined_warning(stats, partition.cell_selection_warning),
unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS, unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS,
limitation_message=( limitation_message=(
f"{FloodHazardAnalysisService.LIMITATION} De selectie werd exact berekend over " f"{FloodHazardAnalysisService.LIMITATION} De selectie werd exact berekend over "
@@ -16,6 +16,7 @@ from app.core.errors import AppError
from app.models import Area, Dataset, Project from app.models import Area, Dataset, Project
from app.schemas.assistant import ( from app.schemas.assistant import (
AssistantContextMetric, AssistantContextMetric,
AssistantEstimateDisclosure,
AssistantModelRead, AssistantModelRead,
AssistantQueryRequest, AssistantQueryRequest,
AssistantQueryResponse, AssistantQueryResponse,
@@ -112,6 +113,44 @@ class GeoAssistantService:
} }
return themes or None return themes or None
@classmethod
def estimate_disclosures(
cls,
metrics: list[AssistantContextMetric],
) -> list[AssistantEstimateDisclosure]:
"""List every estimated value behind the answer, straight from metadata.
``ensure_estimate_disclosure`` can only add a caveat when it recognises
the phrasing the model produced, which makes the guarantee dependent on
generated text. This derives the same statement from the source
metadata, so it holds regardless of how the answer was written.
"""
seen: set[tuple[str, UUID]] = set()
disclosures: list[AssistantEstimateDisclosure] = []
for metric in sorted(metrics, key=lambda item: (item.theme, item.label)):
if not metric.is_estimate:
continue
key = (metric.theme, metric.dataset_id)
if key in seen:
continue
seen.add(key)
topic = cls.ESTIMATE_TOPIC_LABELS.get(metric.theme, metric.label)
disclosures.append(
AssistantEstimateDisclosure(
theme=metric.theme,
label=metric.label,
unit=metric.unit,
source=metric.source,
dataset_id=metric.dataset_id,
reason=(
f"De bronmetadata van {metric.source} markeert {topic} als schatting, "
"geen exacte telling."
),
)
)
return disclosures
@classmethod @classmethod
def ensure_estimate_disclosure( def ensure_estimate_disclosure(
cls, cls,
@@ -702,6 +741,7 @@ class GeoAssistantService:
scope_label=scope_label, scope_label=scope_label,
context_metrics=metrics, context_metrics=metrics,
temporal_series=series, temporal_series=series,
estimate_disclosures=self.estimate_disclosures(metrics),
source_dataset_ids=dataset_ids, source_dataset_ids=dataset_ids,
warnings=warnings, warnings=warnings,
generated_at=datetime.now(timezone.utc), generated_at=datetime.now(timezone.utc),
@@ -9,7 +9,7 @@ from pathlib import Path
from typing import Any, Callable from typing import Any, Callable
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
from urllib.request import HTTPRedirectHandler, Request, build_opener from urllib.request import Request
from uuid import UUID from uuid import UUID
from geoalchemy2.shape import to_shape from geoalchemy2.shape import to_shape
@@ -20,20 +20,12 @@ from shapely.validation import make_valid
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.services.outbound_request_guard import guarded_opener
from app.models import Area, Dataset, Project from app.models import Area, Dataset, Project
from app.schemas.grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead from app.schemas.grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from app.services.dataset_service import DatasetService from app.services.dataset_service import DatasetService
class _RejectRedirects(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
del req, fp, code, msg, headers, newurl
return None
_NO_REDIRECT_OPENER = build_opener(_RejectRedirects())
@dataclass(frozen=True) @dataclass(frozen=True)
class GrbCollection: class GrbCollection:
name: str name: str
@@ -368,7 +360,7 @@ class GrbAcquisitionService:
}, },
) )
try: try:
with (opener or _NO_REDIRECT_OPENER.open)( with (opener or guarded_opener(url, allow_redirect=False))(
request, request,
timeout=settings.grb_timeout_seconds, timeout=settings.grb_timeout_seconds,
) as response: ) as response:
+16
View File
@@ -76,6 +76,22 @@ class JobService:
result_json["output_dataset_id"] = str(result_json["output_dataset_id"]) result_json["output_dataset_id"] = str(result_json["output_dataset_id"])
payload["result_json"] = result_json payload["result_json"] = result_json
raise raise
except Exception:
# An unexpected error must never leave the job stuck in "running".
try:
db.rollback()
except Exception:
pass
try:
JobService.mark_failed(
db,
created.id,
error_message="Unexpected internal error during synchronous job execution",
details={"code": "JOB_INTERNAL_ERROR"},
)
except Exception:
pass
raise
@staticmethod @staticmethod
def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]: def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]:
@@ -0,0 +1,335 @@
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request
from uuid import UUID
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.outbound_request_guard import guarded_opener
from app.models import Dataset
from app.schemas.bathymetry import MdkBathymetryAcquireRequest, MdkBathymetryAcquisitionResult
from app.services.dataset_service import DatasetService
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
class MdkBathymetryAcquisitionService:
"""Bounded, fail-closed GetCoverage acquisition for the MDK Belgian North Sea depth model.
Acquisition only runs when:
- the operator explicitly enabled acquisition and configured a coverage id,
- the live strict-TLS readiness probe reports ``reachable``,
- the configured coverage id is advertised by the live capabilities document,
- the requested EPSG:4326 bbox stays within the configured size bound.
No depth values are ever synthesized, no insecure TLS fallback exists and the
LAT vertical reference is persisted with every artifact so it can never be
silently compared with TAW or mDNG data.
"""
PROVIDER = "mdk_bcp_bathymetry"
VERTICAL_REFERENCE = "LAT"
NATIVE_RESOLUTION_M = 20.0
MAX_PIXELS_PER_SIDE = 4096
LIMITATION = (
"Dieptewaarden zijn LAT-gerefereerd en gelden voor de bemonsterde survey-periode van het officiële "
"MDK-model. LAT mag nooit zonder gedocumenteerde datumtransformatie met TAW- of mDNG-gegevens worden "
"vergeleken; watervolume blijft zonder compatibel wateroppervlak niet ondersteund."
)
ATTRIBUTION = "Agentschap Maritieme Dienstverlening en Kust (MDK)"
LICENSE_NOTE = "Consult the official MDK product license before redistribution."
@staticmethod
def acquire(
db,
project_id: UUID,
payload: MdkBathymetryAcquireRequest,
*,
settings: Settings | None = None,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
if not resolved_settings.mdk_bathymetry_acquisition_enabled:
raise AppError(
code="MDK_BATHYMETRY_ACQUISITION_DISABLED",
message=(
"MDK bathymetry acquisition is disabled. Enable it explicitly with "
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true after the readiness probe reports reachable."
),
status_code=409,
)
coverage_id = (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
if not coverage_id:
raise AppError(
code="MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED",
message="MDK_BATHYMETRY_COVERAGE_ID is not configured; GeoIntel will not guess coverage identifiers.",
status_code=409,
)
bbox = MdkBathymetryAcquisitionService._validated_bbox(payload, resolved_settings)
probe = MdkBathymetryProbeService.probe(settings=resolved_settings, opener=opener)
if probe.get("status") != "reachable":
raise AppError(
code="MDK_BATHYMETRY_ENDPOINT_NOT_READY",
message="The live MDK readiness probe does not report a reachable, TLS-verified WCS endpoint.",
details={"probe_status": probe.get("status"), "probe_message": probe.get("message")},
status_code=502,
)
if coverage_id not in (probe.get("coverage_identifiers") or []):
raise AppError(
code="MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED",
message="The configured coverage id is not advertised by the live MDK capabilities document.",
details={
"configured_coverage_id": coverage_id,
"advertised_coverage_identifiers": probe.get("coverage_identifiers") or [],
},
status_code=502,
)
request_url = MdkBathymetryAcquisitionService._get_coverage_url(resolved_settings, coverage_id, bbox)
request_hash = hashlib.sha256(request_url.encode("utf-8")).hexdigest()
filename = f"mdk_bathymetry_{request_hash[:12]}.tif"
if not payload.force_refresh:
cached = MdkBathymetryAcquisitionService._cached_dataset(db, project_id, filename)
if cached is not None:
return MdkBathymetryAcquisitionResult(
output_dataset_id=cached.id,
reused=True,
provider=MdkBathymetryAcquisitionService.PROVIDER,
coverage_id=coverage_id,
bbox_epsg4326=bbox,
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
).model_dump(mode="json")
content, content_type = MdkBathymetryAcquisitionService._fetch(request_url, resolved_settings, opener)
validation = MdkBathymetryAcquisitionService._validate_geotiff(content)
acquired_at = datetime.now(UTC)
dataset = DatasetService.import_raster_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=content,
source=f"MDK Belgian Continental Shelf WCS {coverage_id}",
source_name=MdkBathymetryAcquisitionService.PROVIDER,
source_metadata={
"provider": MdkBathymetryAcquisitionService.PROVIDER,
"service": "WCS",
"service_version": "1.0.0",
"coverage_id": coverage_id,
"vertical_reference": MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
"native_resolution_m": MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
"bbox_epsg4326": bbox,
"attribution": MdkBathymetryAcquisitionService.ATTRIBUTION,
"license_note": MdkBathymetryAcquisitionService.LICENSE_NOTE,
"raster_validation": validation,
},
provenance_metadata={
"acquisition": "explicit_bounded_wcs_get_coverage",
"acquired_at": acquired_at.isoformat(),
"request_url": request_url,
"request_hash": request_hash,
"response_content_type": content_type,
"coverage_sha256": hashlib.sha256(content).hexdigest(),
"probe_status": probe.get("status"),
"probe_response_sha256": probe.get("response_sha256"),
"probe_checked_at": probe.get("checked_at"),
"limitation_message": MdkBathymetryAcquisitionService.LIMITATION,
},
)
return MdkBathymetryAcquisitionResult(
output_dataset_id=dataset.id,
reused=False,
provider=MdkBathymetryAcquisitionService.PROVIDER,
coverage_id=coverage_id,
bbox_epsg4326=bbox,
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
).model_dump(mode="json")
@staticmethod
def _validated_bbox(payload: MdkBathymetryAcquireRequest, settings: Settings) -> list[float]:
bbox = payload.bbox
min_x, min_y, max_x, max_y = (
float(bbox.min_x),
float(bbox.min_y),
float(bbox.max_x),
float(bbox.max_y),
)
if max_x <= min_x or max_y <= min_y:
raise AppError(
code="MDK_BATHYMETRY_INVALID_BBOX",
message="The requested bbox must have positive width and height in EPSG:4326.",
status_code=422,
)
area_deg2 = (max_x - min_x) * (max_y - min_y)
if area_deg2 > float(settings.mdk_bathymetry_max_bbox_deg2):
raise AppError(
code="MDK_BATHYMETRY_BBOX_TOO_LARGE",
message="The requested bbox exceeds the configured bounded acquisition size.",
details={
"bbox_area_deg2": area_deg2,
"max_bbox_deg2": float(settings.mdk_bathymetry_max_bbox_deg2),
},
status_code=422,
)
return [min_x, min_y, max_x, max_y]
@staticmethod
def _get_coverage_url(settings: Settings, coverage_id: str, bbox: list[float]) -> str:
parsed = urlsplit(settings.mdk_bathymetry_wcs_url.strip())
if parsed.scheme.lower() != "https" or not parsed.hostname:
raise AppError(
code="MDK_BATHYMETRY_INVALID_CONFIGURATION",
message="MDK bathymetry acquisition requires an absolute HTTPS WCS URL.",
status_code=409,
)
width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox)
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
parameters.update(
{
"service": "WCS",
"request": "GetCoverage",
"version": "1.0.0",
"coverage": coverage_id,
"crs": settings.mdk_bathymetry_request_crs,
"bbox": ",".join(f"{value:.8f}" for value in bbox),
"width": str(width),
"height": str(height),
"format": "GeoTIFF",
}
)
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
@staticmethod
def _pixel_dimensions(bbox: list[float]) -> tuple[int, int]:
min_x, min_y, max_x, max_y = bbox
# Approximate meters per degree near the Belgian North Sea (~51.5N).
meters_per_deg_lat = 111_320.0
meters_per_deg_lon = 69_400.0
width = int((max_x - min_x) * meters_per_deg_lon / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
height = int((max_y - min_y) * meters_per_deg_lat / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
width = max(1, min(width, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
height = max(1, min(height, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
return width, height
@staticmethod
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
request = Request(
request_url,
headers={
"Accept": "image/tiff,*/*;q=0.1",
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-bounded-acquisition",
},
)
max_bytes = settings.mdk_bathymetry_acquisition_max_response_mb * 1024 * 1024
try:
with (opener or guarded_opener(request_url))(request, timeout=settings.mdk_bathymetry_acquisition_timeout_seconds) as response:
content_type = str(response.headers.get("Content-Type", "")) if hasattr(response, "headers") else ""
content = response.read(max_bytes + 1)
except HTTPError as exc:
preview = exc.read(300).decode("utf-8", errors="replace")
raise AppError(
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
message="The MDK WCS could not complete the bounded GetCoverage request.",
details={"provider_status_code": int(exc.code), "response_preview": preview},
status_code=502,
) from exc
except (URLError, TimeoutError, OSError) as exc:
raise AppError(
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
message="The MDK WCS could not be reached for the bounded GetCoverage request.",
details={"reason": str(exc)},
status_code=502,
) from exc
if len(content) > max_bytes:
raise AppError(
code="MDK_BATHYMETRY_RESPONSE_TOO_LARGE",
message="The MDK coverage response exceeds the configured size limit.",
status_code=502,
)
if not content.startswith((b"II*\x00", b"MM\x00*")):
preview = content[:300].decode("utf-8", errors="replace")
raise AppError(
code="MDK_BATHYMETRY_INVALID_RESPONSE",
message="The MDK WCS did not return a GeoTIFF coverage.",
details={"content_type": content_type, "response_preview": preview},
status_code=502,
)
return content, content_type
@staticmethod
def _validate_geotiff(content: bytes) -> dict[str, Any]:
try:
import numpy as np
from rasterio.io import MemoryFile
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Rasterio is required to validate the MDK bathymetry coverage before persistence.",
status_code=503,
) from exc
try:
with MemoryFile(content) as memory, memory.open() as source:
if source.count < 1:
raise AppError(
code="MDK_BATHYMETRY_INVALID_RESPONSE",
message="The MDK coverage contains no raster bands.",
status_code=502,
)
band = source.read(1, masked=True)
valid = band.compressed()
if valid.size == 0:
raise AppError(
code="MDK_BATHYMETRY_NO_VALID_DATA",
message="The MDK coverage contains no valid depth cells in this selection.",
status_code=422,
)
return {
"crs": str(source.crs) if source.crs else None,
"width": int(source.width),
"height": int(source.height),
"nodata": None if source.nodata is None else float(source.nodata),
"valid_cell_count": int(valid.size),
"minimum_value": float(np.min(valid)),
"maximum_value": float(np.max(valid)),
}
except AppError:
raise
except Exception as exc: # rasterio raises many distinct errors for corrupt input
raise AppError(
code="MDK_BATHYMETRY_INVALID_RESPONSE",
message="The MDK coverage could not be opened as a valid GeoTIFF.",
details={"reason": str(exc)},
status_code=502,
) from exc
@staticmethod
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
from pathlib import Path
candidate = (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.name == filename,
Dataset.source_name == MdkBathymetryAcquisitionService.PROVIDER,
Dataset.status == "ready",
)
.order_by(Dataset.imported_at.desc())
.first()
)
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
@@ -6,11 +6,12 @@ import ssl
from typing import Any, Callable from typing import Any, Callable
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen from urllib.request import Request
from xml.etree import ElementTree from xml.etree import ElementTree
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.schemas.bathymetry import BathymetrySourceProbeRead from app.schemas.bathymetry import BathymetrySourceProbeRead
from app.services.outbound_request_guard import guarded_opener
class MdkBathymetryProbeService: class MdkBathymetryProbeService:
@@ -43,7 +44,7 @@ class MdkBathymetryProbeService:
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe", "User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe",
}, },
) )
with (opener or urlopen)(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response: with (opener or guarded_opener(capabilities_url))(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response:
limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024 limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024
content = response.read(limit + 1) content = response.read(limit + 1)
if len(content) > limit: if len(content) > limit:
@@ -24,11 +24,18 @@ class ModelAssetCatalogService:
if not model_directory.exists() or not model_directory.is_dir(): if not model_directory.exists() or not model_directory.is_dir():
return ModelAssetListResponse(items=[], total=0, model_directory=str(model_directory)) return ModelAssetListResponse(items=[], total=0, model_directory=str(model_directory))
items = [ candidate_paths = [
ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path) path
for path in sorted(model_directory.iterdir(), key=lambda item: item.name.lower()) for path in sorted(model_directory.iterdir(), key=lambda item: item.name.lower())
if path.is_file() and path.suffix.lower() in ModelAssetCatalogService.SUPPORTED_SUFFIXES if path.is_file() and path.suffix.lower() in ModelAssetCatalogService.SUPPORTED_SUFFIXES
] ]
if active_model_path is not None:
candidate_paths = [path for path in candidate_paths if path.resolve() == active_model_path]
items = [
ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path)
for path in candidate_paths
]
return ModelAssetListResponse(items=items, total=len(items), model_directory=str(model_directory)) return ModelAssetListResponse(items=items, total=len(items), model_directory=str(model_directory))
@staticmethod @staticmethod
@@ -72,8 +79,16 @@ class ModelAssetCatalogService:
size_bytes=path.stat().st_size, size_bytes=path.stat().st_size,
sha256=ModelAssetCatalogService._sha256(path), sha256=ModelAssetCatalogService._sha256(path),
active=active_model_path == resolved_path, active=active_model_path == resolved_path,
status="available", runtime_available=True,
limitation_message="Local runtime model asset. GeoIntel will not download or mutate model weights.", runtime_status="active" if active_model_path == resolved_path else "available",
governed_validation_status="not_verified_by_catalog",
promotion_status="not_verified_by_catalog",
status="runtime_active" if active_model_path == resolved_path else "runtime_available",
limitation_message=(
"Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion."
if active_model_path == resolved_path
else "Local runtime model asset. Governed validation and promotion are not established by this catalog."
),
will_download_models=False, will_download_models=False,
) )
+189 -30
View File
@@ -5,7 +5,13 @@ from typing import Type
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.schemas.detection import DetectionModelCapability from app.schemas.detection import DetectionModelCapability
from app.services.segmentation_adapter import (
SamSegmentationAdapter,
YoloSegmentationAdapter,
)
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_adapter import YoloDetectionAdapter from app.services.yolo_adapter import YoloDetectionAdapter
from app.core.errors import AppError
class ModelRegistryService: class ModelRegistryService:
@@ -14,10 +20,16 @@ class ModelRegistryService:
settings: Settings | None = None, settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
task_type: str = "object_detection", task_type: str = "object_detection",
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> list[DetectionModelCapability]: ) -> list[DetectionModelCapability]:
resolved_settings = settings or get_settings() resolved_settings = settings or get_settings()
if task_type == "segmentation": if task_type == "segmentation":
return ModelRegistryService.list_segmentation_model_capabilities() return ModelRegistryService.list_segmentation_model_capabilities(
settings=resolved_settings,
yolo_seg_adapter_class=yolo_seg_adapter_class,
sam_adapter_class=sam_adapter_class,
)
if task_type != "object_detection": if task_type != "object_detection":
return [] return []
return [ return [
@@ -32,7 +44,9 @@ class ModelRegistryService:
limitation_message="YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed.", limitation_message="YOLO/PyTorch inference is not configured in Sprint 8; no model is downloaded or executed.",
version=None, version=None,
), ),
ModelRegistryService._configured_yolo_capability(resolved_settings, yolo_adapter_class), ModelRegistryService._configured_yolo_capability(
resolved_settings, yolo_adapter_class
),
DetectionModelCapability( DetectionModelCapability(
model_id="manual-fixture-detector", model_id="manual-fixture-detector",
display_name="Manual fixture detector", display_name="Manual fixture detector",
@@ -52,15 +66,28 @@ class ModelRegistryService:
settings: Settings | None = None, settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
task_type: str = "object_detection", task_type: str = "object_detection",
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> DetectionModelCapability | None: ) -> DetectionModelCapability | None:
normalized = model_id.strip() normalized = model_id.strip()
for model in ModelRegistryService.list_model_capabilities(settings=settings, yolo_adapter_class=yolo_adapter_class, task_type=task_type): for model in ModelRegistryService.list_model_capabilities(
settings=settings,
yolo_adapter_class=yolo_adapter_class,
task_type=task_type,
yolo_seg_adapter_class=yolo_seg_adapter_class,
sam_adapter_class=sam_adapter_class,
):
if model.model_id == normalized: if model.model_id == normalized:
return model return model
return None return None
@staticmethod @staticmethod
def list_segmentation_model_capabilities() -> list[DetectionModelCapability]: def list_segmentation_model_capabilities(
settings: Settings | None = None,
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> list[DetectionModelCapability]:
resolved_settings = settings or get_settings()
return [ return [
DetectionModelCapability( DetectionModelCapability(
model_id="segmentation-placeholder", model_id="segmentation-placeholder",
@@ -70,7 +97,7 @@ class ModelRegistryService:
supported_classes=["building", "vegetation", "water", "landuse"], supported_classes=["building", "vegetation", "water", "landuse"],
configured=False, configured=False,
status="not_configured", status="not_configured",
limitation_message="Segmentation inference is not configured in Sprint 9; no SAM/YOLO-seg model is downloaded or executed.", limitation_message="Segmentation inference is not configured for this placeholder; no model is downloaded or executed.",
version=None, version=None,
), ),
DetectionModelCapability( DetectionModelCapability(
@@ -84,30 +111,128 @@ class ModelRegistryService:
limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.", limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.",
version="fixture-v1", version="fixture-v1",
), ),
DetectionModelCapability( ModelRegistryService._configured_yolo_seg_capability(
model_id="yolo-seg-configured", resolved_settings, yolo_seg_adapter_class
display_name="Configured YOLO segmentation",
framework="ultralytics/pytorch",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="YOLO-seg is not configured in Sprint 9. GeoIntel will not download segmentation model weights automatically.",
version=None,
), ),
DetectionModelCapability( ModelRegistryService._configured_sam_capability(
model_id="sam-configured", resolved_settings, sam_adapter_class
display_name="Configured SAM segmentation",
framework="sam",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="SAM is not configured in Sprint 9 and is not installed as a backend dependency.",
version=None,
), ),
] ]
@staticmethod
def _configured_yolo_seg_capability(
settings: Settings,
adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
) -> DetectionModelCapability:
configured = False
status = "not_configured"
limitation = (
"YOLO segmentation is disabled. Set YOLO_SEG_ENABLED=true and YOLO_SEG_MODEL_PATH to a local "
"segmentation model file to enable inference. GeoIntel never downloads model weights automatically."
)
model_path = (
Path(settings.yolo_seg_model_path).expanduser()
if settings.yolo_seg_model_path
else None
)
if settings.yolo_seg_enabled:
if not adapter_class.dependencies_available():
status = "dependency_unavailable"
limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
elif model_path is None:
limitation = "YOLO_SEG_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically."
elif not model_path.exists() or not model_path.is_file():
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.yolo_seg_model_id,
task_type="segmentation",
expected_model_version=settings.yolo_seg_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured YOLO segmentation weights are not runnable until their immutable "
f"runtime provenance sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.yolo_seg_model_id,
display_name=settings.yolo_seg_model_display_name,
framework="ultralytics/pytorch",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=configured,
status=status,
limitation_message=limitation,
version=settings.yolo_seg_model_version,
)
@staticmethod
def _configured_sam_capability(
settings: Settings,
adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> DetectionModelCapability:
configured = False
status = "not_configured"
limitation = (
"SAM is disabled. Set SAM_ENABLED=true and SAM_MODEL_PATH to a local SAM-compatible model file to "
"enable class-agnostic segmentation. GeoIntel never downloads model weights automatically."
)
model_path = (
Path(settings.sam_model_path).expanduser()
if settings.sam_model_path
else None
)
if settings.sam_enabled:
if not adapter_class.dependencies_available():
status = "dependency_unavailable"
limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
elif model_path is None:
limitation = "SAM_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically."
elif not model_path.exists() or not model_path.is_file():
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.sam_model_id,
task_type="segmentation",
expected_model_version=settings.sam_model_version,
allowed_frameworks=("ultralytics/sam", "sam", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured SAM weights are not runnable until their immutable runtime provenance "
f"sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.sam_model_id,
display_name=settings.sam_model_display_name,
framework="ultralytics/sam",
task_type="segmentation",
supported_classes=["segment"],
configured=configured,
status=status,
limitation_message=limitation,
version=settings.sam_model_version,
)
@staticmethod @staticmethod
def _configured_yolo_capability( def _configured_yolo_capability(
settings: Settings, settings: Settings,
@@ -116,7 +241,11 @@ class ModelRegistryService:
configured = False configured = False
status = "not_configured" status = "not_configured"
limitation = "YOLO is disabled. Set YOLO_ENABLED=true and YOLO_MODEL_PATH to a local model file to enable inference." limitation = "YOLO is disabled. Set YOLO_ENABLED=true and YOLO_MODEL_PATH to a local model file to enable inference."
model_path = Path(settings.yolo_model_path).expanduser() if settings.yolo_model_path else None model_path = (
Path(settings.yolo_model_path).expanduser()
if settings.yolo_model_path
else None
)
if settings.yolo_enabled: if settings.yolo_enabled:
if not yolo_adapter_class.dependencies_available(): if not yolo_adapter_class.dependencies_available():
@@ -127,18 +256,48 @@ class ModelRegistryService:
elif not model_path.exists() or not model_path.is_file(): elif not model_path.exists() or not model_path.is_file():
limitation = "YOLO_MODEL_PATH does not point to an existing local model file. GeoIntel will not download model weights automatically." limitation = "YOLO_MODEL_PATH does not point to an existing local model file. GeoIntel will not download model weights automatically."
else: else:
configured = True try:
status = "configured" validate_runtime = getattr(yolo_adapter_class, "validate_runtime", None)
limitation = "Configured for local YOLO inference over an existing raster tile manifest." if validate_runtime is not None:
yolo_adapter_class(settings).validate_runtime()
except AppError as exc:
status = "accelerator_unavailable"
limitation = exc.message
else:
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.yolo_model_id,
task_type="object_detection",
expected_model_version=settings.yolo_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured YOLO weights are not runnable until their immutable runtime provenance "
f"sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
return DetectionModelCapability( return DetectionModelCapability(
model_id=settings.yolo_model_id, model_id=settings.yolo_model_id,
display_name=settings.yolo_model_display_name, display_name=settings.yolo_model_display_name,
framework="ultralytics/pytorch", framework="ultralytics/pytorch",
task_type="object_detection", task_type="object_detection",
supported_classes=["building", "road", "water", "landuse"], supported_classes=[value.strip().lower() for value in settings.yolo_model_classes.split(",") if value.strip()],
configured=configured, configured=configured,
status=status, status=status,
limitation_message=limitation, limitation_message=limitation,
version=settings.yolo_model_version, version=settings.yolo_model_version,
training_scope=(
"Operator-managed local weights; the runtime has no nationally governed training-corpus evidence."
),
validation_scope="Mol and the Kempen operator evidence; no Belgian national validation matrix is bound.",
validated_regions=["flanders_mol_kempen"],
nationally_validated=False,
operator_review_required=True,
) )
@@ -0,0 +1,142 @@
"""Checksum-bound geographic validation scope for production model inference."""
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
import re
from typing import Any
from shapely.geometry import shape
from shapely.geometry.base import BaseGeometry
from app.core.errors import AppError
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
class ModelValidationScopeService:
"""Load an immutable, model-bound AOI and prove that an input is covered."""
SCHEMA_VERSION = "geointel.model-validation-scope/v1"
@classmethod
def assert_area_covered(
cls,
*,
area_geometry: BaseGeometry,
manifest_path: str | None,
expected_manifest_sha256: str | None,
model_id: str,
model_path: str | None,
) -> dict[str, str]:
path = Path(manifest_path).expanduser() if manifest_path else None
expected_checksum = (expected_manifest_sha256 or "").strip().lower()
if path is None or not expected_checksum:
cls._raise(
"DETECTION_VALIDATION_SCOPE_NOT_CONFIGURED",
"Configured YOLO inference requires a checksum-bound geographic validation-scope manifest.",
)
if not _SHA256.fullmatch(expected_checksum):
cls._raise(
"DETECTION_VALIDATION_SCOPE_INVALID",
"The configured validation-scope checksum must be a lowercase SHA-256 digest.",
manifest_path=str(path),
)
try:
raw_manifest = path.read_bytes()
payload = json.loads(raw_manifest.decode("utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
cls._raise(
"DETECTION_VALIDATION_SCOPE_INVALID",
"The configured validation-scope manifest is missing or unreadable.",
manifest_path=str(path),
error_type=type(exc).__name__,
)
observed_manifest_sha256 = sha256(raw_manifest).hexdigest()
if observed_manifest_sha256 != expected_checksum:
cls._raise(
"DETECTION_VALIDATION_SCOPE_CHECKSUM_MISMATCH",
"The validation-scope manifest does not match its configured checksum.",
manifest_path=str(path),
expected=expected_checksum,
observed=observed_manifest_sha256,
)
if not isinstance(payload, dict) or payload.get("schema_version") != cls.SCHEMA_VERSION:
cls._raise(
"DETECTION_VALIDATION_SCOPE_INVALID",
"The validation-scope manifest has an unsupported schema.",
manifest_path=str(path),
)
if payload.get("model_id") != model_id:
cls._raise(
"DETECTION_VALIDATION_SCOPE_MODEL_MISMATCH",
"The validation scope is not bound to the selected model identity.",
expected_model_id=model_id,
observed_model_id=payload.get("model_id"),
)
configured_model_path = Path(model_path).expanduser() if model_path else None
if configured_model_path is None or not configured_model_path.is_file():
cls._raise(
"DETECTION_VALIDATION_SCOPE_MODEL_UNAVAILABLE",
"The model bytes bound by the validation scope are unavailable.",
)
declared_model_sha256 = str(payload.get("model_sha256") or "").strip().lower()
observed_model_sha256 = cls._file_sha256(configured_model_path)
if not _SHA256.fullmatch(declared_model_sha256) or declared_model_sha256 != observed_model_sha256:
cls._raise(
"DETECTION_VALIDATION_SCOPE_MODEL_MISMATCH",
"The validation scope is not bound to the exact selected model bytes.",
expected=declared_model_sha256 or None,
observed=observed_model_sha256,
)
if payload.get("crs") != "EPSG:4326":
cls._raise(
"DETECTION_VALIDATION_SCOPE_INVALID",
"The validation-scope geometry must explicitly use EPSG:4326.",
observed_crs=payload.get("crs"),
)
try:
scope_geometry = shape(payload.get("geometry"))
except Exception as exc:
cls._raise(
"DETECTION_VALIDATION_SCOPE_INVALID",
"The validation-scope geometry is not valid GeoJSON.",
error_type=type(exc).__name__,
)
if (
scope_geometry.is_empty
or not scope_geometry.is_valid
or scope_geometry.geom_type not in {"Polygon", "MultiPolygon"}
):
cls._raise(
"DETECTION_VALIDATION_SCOPE_INVALID",
"The validation scope must be a non-empty valid Polygon or MultiPolygon.",
geometry_type=scope_geometry.geom_type,
)
if area_geometry.is_empty or not area_geometry.is_valid or not scope_geometry.covers(area_geometry):
cls._raise(
"DETECTION_VALIDATION_SCOPE_UNAVAILABLE",
"Configured YOLO inference is not validated for the complete Dataset area.",
scope_key=payload.get("scope_key"),
)
return {
"scope_key": str(payload.get("scope_key") or "unspecified"),
"manifest_path": str(path.resolve()),
"manifest_sha256": observed_manifest_sha256,
"model_sha256": observed_model_sha256,
}
@staticmethod
def _file_sha256(path: Path) -> str:
digest = sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
@staticmethod
def _raise(code: str, message: str, **details: Any) -> None:
raise AppError(code=code, message=message, details=details, status_code=422)

Some files were not shown because too many files have changed in this diff Show More