feat: add governed nationwide AOI orchestration and CUDA enforcement
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-26 05:23:33 +02:00
parent 25b6f1ab39
commit 2be72fac58
50 changed files with 1596 additions and 51 deletions
+10 -1
View File
@@ -398,7 +398,7 @@ def test_all_in_one_dockerfile_caches_dependencies_and_uses_cpu_torch_for_ai_run
smoke_index = dockerfile.index("RUN python scripts/gis_import_smoke.py")
assert metadata_copy_index < placeholder_readme_index < dependency_install_index < backend_copy_index < smoke_index
assert "GEOINTEL_TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu" in dockerfile
assert "GEOINTEL_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu130" in dockerfile
assert "GEOINTEL_TORCH_VERSION=2.13.0" in dockerfile
assert "GEOINTEL_TORCHVISION_VERSION=0.28.0" in dockerfile
assert '--index-url "$GEOINTEL_TORCH_INDEX_URL"' in dockerfile
@@ -424,3 +424,12 @@ def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None:
assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script
assert '-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD"' in run_script
assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script
def test_unraid_ai_runtime_requests_nvidia_and_fails_closed() -> None:
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
assert "--gpus all" in run_script
assert 'YOLO_DEVICE="${YOLO_DEVICE:-cuda:0}"' in run_script
assert 'YOLO_REQUIRE_CUDA="${YOLO_REQUIRE_CUDA:-true}"' in run_script
assert '-e YOLO_REQUIRE_CUDA="$YOLO_REQUIRE_CUDA"' in run_script
assert "https://download.pytorch.org/whl/cu130" in dockerfile
@@ -225,6 +225,26 @@ def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
assert outside.items[0].materialized_dataset_ids == []
def test_bounded_partition_union_can_be_operational() -> None:
project_id = uuid4()
left_id = uuid4()
right_id = uuid4()
datasets = [
SimpleNamespace(id=left_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.50, 50.50, 4.60, 50.60]}),
SimpleNamespace(id=right_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.60, 50.50, 4.70, 50.60]}),
]
session = FakeSession(
project=SimpleNamespace(id=project_id),
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)), scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8))],
datasets=datasets,
)
result = CoverageRegistryService.resolve(session, project_id, CoverageBBox(minx=4.51, miny=50.51, maxx=4.69, maxy=50.59), ["buildings"])
assert result.items[0].status == "operational"
assert result.items[0].materialized_dataset_ids == [left_id, right_id]
def test_spw_bathymetry_materialization_is_source_specific() -> None:
project_id = uuid4()
scope = [
@@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import datetime, timezone
from unittest.mock import MagicMock
from app.models import AnalysisRun, Job
from app.models import AoiOperation, AoiOperationPartition, AnalysisRun, Job
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
@@ -11,9 +11,15 @@ def test_reconciliation_terminalizes_only_running_work() -> None:
db = MagicMock()
jobs = MagicMock()
runs = MagicMock()
resumable_partitions = MagicMock()
exhausted_partitions = MagicMock()
operations = MagicMock()
jobs.filter.return_value.update.return_value = 5
runs.filter.return_value.update.return_value = 2
db.query.side_effect = [jobs, runs]
resumable_partitions.filter.return_value.update.return_value = 3
exhausted_partitions.filter.return_value.update.return_value = 1
operations.filter.return_value.update.return_value = 2
db.query.side_effect = [jobs, runs, resumable_partitions, exhausted_partitions, operations]
finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc)
result = RuntimeReconciliationService.reconcile(
@@ -23,8 +29,13 @@ def test_reconciliation_terminalizes_only_running_work() -> None:
assert result.interrupted_jobs == 5
assert result.interrupted_analysis_runs == 2
assert result.resumed_aoi_partitions == 3
assert result.exhausted_aoi_partitions == 1
jobs.filter.assert_called_once()
runs.filter.assert_called_once()
resumable_partitions.filter.assert_called_once()
exhausted_partitions.filter.assert_called_once()
operations.filter.assert_called_once()
job_values = jobs.filter.return_value.update.call_args.args[0]
run_values = runs.filter.return_value.update.call_args.args[0]
assert job_values[Job.status] == "failed"
@@ -33,6 +44,12 @@ def test_reconciliation_terminalizes_only_running_work() -> None:
assert run_values[AnalysisRun.status] == "failed"
assert run_values[AnalysisRun.finished_at] == finished_at
assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message]
resumed_values = resumable_partitions.filter.return_value.update.call_args.args[0]
exhausted_values = exhausted_partitions.filter.return_value.update.call_args.args[0]
operation_values = operations.filter.return_value.update.call_args.args[0]
assert resumed_values[AoiOperationPartition.status] == "queued"
assert exhausted_values[AoiOperationPartition.status] == "failed"
assert operation_values[AoiOperation.status] == "queued"
db.commit.assert_called_once_with()
@@ -83,7 +83,7 @@ def test_vector_partition_route_combines_one_governed_product(monkeypatch) -> No
def test_vector_partition_request_has_a_bounded_fan_out() -> None:
with pytest.raises(ValidationError):
VectorPartitionSelectionRequest(
dataset_ids=[uuid4() for _ in range(17)],
dataset_ids=[uuid4() for _ in range(4097)],
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
)
@@ -0,0 +1,86 @@
from __future__ import annotations
from pathlib import Path
import pytest
from shapely.geometry import box
from app.core.errors import AppError
from app.services.aoi_operation_executor import AoiOperationExecutor
from app.services.aoi_operation_service import AoiOperationService
ROOT = Path(__file__).resolve().parents[2]
def test_partition_plan_covers_aoi_without_overlapping_area() -> None:
aoi = box(0, 0, 25_000, 18_000)
partitions = AoiOperationService._partition(aoi, 10_000)
assert len(partitions) == 6
assert sum(partition.area for partition in partitions) == pytest.approx(aoi.area)
assert all(partition.within(aoi) for partition in partitions)
for index, partition in enumerate(partitions):
for other in partitions[index + 1 :]:
assert partition.intersection(other).area == pytest.approx(0.0)
def test_partition_plan_intersects_irregular_aoi_exactly() -> None:
aoi = box(0, 0, 20_000, 20_000).difference(box(5_000, 5_000, 15_000, 15_000))
partitions = AoiOperationService._partition(aoi, 8_000)
assert sum(partition.area for partition in partitions) == pytest.approx(aoi.area)
assert all(not partition.intersects(box(5_001, 5_001, 14_999, 14_999)) for partition in partitions)
def test_partition_plan_fails_before_unbounded_fanout(monkeypatch) -> None:
monkeypatch.setattr(AoiOperationService, "MAX_PARTITIONS", 4)
with pytest.raises(AppError) as exc_info:
AoiOperationService._partition(box(0, 0, 30_000, 30_000), 10_000)
assert exc_info.value.code == "AOI_PARTITION_LIMIT_EXCEEDED"
assert exc_info.value.details["candidate_count"] == 9
def test_executor_retries_only_transient_provider_failures() -> None:
assert AoiOperationExecutor._retryable(AppError(code="UPSTREAM_UNAVAILABLE", message="down", status_code=503)) is True
assert AoiOperationExecutor._retryable(AppError(code="INVALID_SCOPE", message="bad", status_code=422)) is False
def test_provider_budget_is_automatic_and_override_can_only_be_stricter() -> None:
governed = AoiOperationService._partition_side("grb", None)
assert governed > 0
assert AoiOperationService._partition_side("grb", governed * 2) == governed
assert AoiOperationService._partition_side("grb", governed / 2) == governed / 2
@pytest.mark.parametrize(
("provider_key", "max_pixels", "resolution_m"),
[
("dhmv", 12_000_000, 5.0),
("flood_hazard", 12_000_000, 5.0),
("spw_terrain", 12_000_000, 5.0),
("thematic_raster", 30_000_000, 10.0),
("walous", 36_000_000, 10.0),
],
)
def test_raster_provider_budget_never_exceeds_decoded_pixel_limit(
provider_key: str, max_pixels: int, resolution_m: float
) -> None:
side_m = AoiOperationService._partition_side(provider_key, None)
assert (side_m / resolution_m) ** 2 < max_pixels
def test_migration_and_api_are_registered() -> None:
migration = (ROOT / "backend/alembic/versions/202607260001_aoi_operations.py").read_text(encoding="utf-8")
main = (ROOT / "backend/app/main.py").read_text(encoding="utf-8")
route = (ROOT / "backend/app/api/routes/aoi_operations.py").read_text(encoding="utf-8")
assert 'down_revision = "202607160001"' in migration
assert '"aoi_operations"' in migration
assert '"aoi_operation_partitions"' in migration
assert "app.include_router(aoi_operations.router" in main
assert '"/{operation_id}/execute-next"' in route
assert '"/{operation_id}/partitions/{partition_id}/checkpoint"' in route
@@ -167,7 +167,7 @@ def test_powershell_tower_deploy_streams_remote_script_to_bash() -> None:
powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
assert "[System.Text.UTF8Encoding]::new($false)" in powershell
assert "[System.IO.File]::WriteAllText($localScriptPath, $remoteScript, $utf8NoBom)" in powershell
assert "[System.IO.File]::WriteAllText($localScriptPath, $remoteScriptLf, $utf8NoBom)" in powershell
assert "& scp @scpArgs" in powershell
assert "& ssh @sshRunArgs" in powershell
assert "bash '$remoteScriptPath'" in powershell
@@ -192,8 +192,10 @@ def test_frontend_and_unraid_icon_assets_are_present() -> None:
index = (ROOT / "frontend" / "index.html").read_text(encoding="utf-8")
assert "<svg" in deploy_icon
assert "GeoIntel Kempen" in deploy_icon
assert deploy_icon == frontend_icon
assert "<title id=\"title\">GeoIntel</title>" in deploy_icon
assert "Een geometrische G als geografische lens" in deploy_icon
assert "Een geometrische G als geografische lens" in frontend_icon
assert deploy_png.read_bytes() == frontend_png.read_bytes()
assert deploy_png.read_bytes() == frontend_png.read_bytes()
assert deploy_png.stat().st_size > 1000
assert '<link rel="icon" type="image/svg+xml" href="/geointel-icon.svg" />' in index
+43 -1
View File
@@ -2,13 +2,15 @@ from __future__ import annotations
import json
from pathlib import Path
import sys
from types import SimpleNamespace
from uuid import uuid4
import pytest
from app.core.config import Settings
from app.core.errors import AppError
from app.models import AnalysisRun, Dataset, Detection, Job, Project
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.detection_service import DetectionService
from app.services.model_registry_service import ModelRegistryService
@@ -235,6 +237,7 @@ def test_yolo_configured_model_reports_configured_with_local_model_and_dependenc
assert model.nationally_validated is False
assert model.operator_review_required is True
assert model.validated_regions == ["flanders_mol_kempen"]
assert model.supported_classes == ["building"]
assert "Mol and the Kempen" in (model.validation_scope or "")
@@ -246,6 +249,45 @@ def test_yolo_dependency_check_uses_real_imports_not_find_spec() -> None:
assert "import torch" in source
def test_yolo_runtime_fails_closed_when_cuda_is_required_but_unavailable(tmp_path: Path, monkeypatch) -> None:
settings = _settings(tmp_path, yolo_device="cuda:0", yolo_require_cuda=True)
monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)))
with pytest.raises(AppError) as exc_info:
YoloDetectionAdapter(settings).validate_runtime()
assert exc_info.value.code == "DETECTION_ACCELERATOR_UNAVAILABLE"
def test_yolo_runtime_rejects_cpu_device_when_cuda_is_required(tmp_path: Path, monkeypatch) -> None:
settings = _settings(tmp_path, yolo_device="cpu", yolo_require_cuda=True)
monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True)))
with pytest.raises(AppError) as exc_info:
YoloDetectionAdapter(settings).validate_runtime()
assert exc_info.value.code == "DETECTION_ACCELERATOR_MISCONFIGURED"
def test_yolo_validation_scope_requires_persisted_validated_area(tmp_path: Path) -> None:
dataset = Dataset(id=uuid4(), project_id=uuid4(), name="image.tif", dataset_type="raster", source="test", area_id=uuid4())
wrong_area = Area(id=dataset.area_id, project_id=dataset.project_id, name="Brussels", geometry="MULTIPOLYGON EMPTY")
db = FakeSession(objects={(Area, dataset.area_id): wrong_area})
with pytest.raises(AppError) as exc_info:
DetectionService._validate_model_area_scope(db, dataset, _settings(tmp_path, yolo_validated_area_names="Mol,Kempen"))
assert exc_info.value.code == "DETECTION_VALIDATION_SCOPE_UNAVAILABLE"
def test_yolo_validation_scope_accepts_bound_mol_area(tmp_path: Path) -> None:
dataset = Dataset(id=uuid4(), project_id=uuid4(), name="image.tif", dataset_type="raster", source="test", area_id=uuid4())
area = Area(id=dataset.area_id, project_id=dataset.project_id, name="Gemeente Mol", geometry="MULTIPOLYGON EMPTY")
db = FakeSession(objects={(Area, dataset.area_id): area})
DetectionService._validate_model_area_scope(db, dataset, _settings(tmp_path, yolo_validated_area_names="Mol,Kempen"))
def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)