feat: add governed nationwide AOI orchestration and CUDA enforcement
This commit is contained in:
@@ -398,7 +398,7 @@ def test_all_in_one_dockerfile_caches_dependencies_and_uses_cpu_torch_for_ai_run
|
||||
smoke_index = dockerfile.index("RUN python scripts/gis_import_smoke.py")
|
||||
|
||||
assert metadata_copy_index < placeholder_readme_index < dependency_install_index < backend_copy_index < smoke_index
|
||||
assert "GEOINTEL_TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu" in dockerfile
|
||||
assert "GEOINTEL_TORCH_INDEX_URL=https://download.pytorch.org/whl/cu130" in dockerfile
|
||||
assert "GEOINTEL_TORCH_VERSION=2.13.0" in dockerfile
|
||||
assert "GEOINTEL_TORCHVISION_VERSION=0.28.0" in dockerfile
|
||||
assert '--index-url "$GEOINTEL_TORCH_INDEX_URL"' in dockerfile
|
||||
@@ -424,3 +424,12 @@ def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None:
|
||||
assert '-e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS"' in run_script
|
||||
assert '-e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD"' in run_script
|
||||
assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script
|
||||
def test_unraid_ai_runtime_requests_nvidia_and_fails_closed() -> None:
|
||||
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
|
||||
assert "--gpus all" in run_script
|
||||
assert 'YOLO_DEVICE="${YOLO_DEVICE:-cuda:0}"' in run_script
|
||||
assert 'YOLO_REQUIRE_CUDA="${YOLO_REQUIRE_CUDA:-true}"' in run_script
|
||||
assert '-e YOLO_REQUIRE_CUDA="$YOLO_REQUIRE_CUDA"' in run_script
|
||||
assert "https://download.pytorch.org/whl/cu130" in dockerfile
|
||||
|
||||
@@ -225,6 +225,26 @@ def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
|
||||
assert outside.items[0].materialized_dataset_ids == []
|
||||
|
||||
|
||||
def test_bounded_partition_union_can_be_operational() -> None:
|
||||
project_id = uuid4()
|
||||
left_id = uuid4()
|
||||
right_id = uuid4()
|
||||
datasets = [
|
||||
SimpleNamespace(id=left_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.50, 50.50, 4.60, 50.60]}),
|
||||
SimpleNamespace(id=right_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.60, 50.50, 4.70, 50.60]}),
|
||||
]
|
||||
session = FakeSession(
|
||||
project=SimpleNamespace(id=project_id),
|
||||
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)), scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8))],
|
||||
datasets=datasets,
|
||||
)
|
||||
|
||||
result = CoverageRegistryService.resolve(session, project_id, CoverageBBox(minx=4.51, miny=50.51, maxx=4.69, maxy=50.59), ["buildings"])
|
||||
|
||||
assert result.items[0].status == "operational"
|
||||
assert result.items[0].materialized_dataset_ids == [left_id, right_id]
|
||||
|
||||
|
||||
def test_spw_bathymetry_materialization_is_source_specific() -> None:
|
||||
project_id = uuid4()
|
||||
scope = [
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.models import AnalysisRun, Job
|
||||
from app.models import AoiOperation, AoiOperationPartition, AnalysisRun, Job
|
||||
from app.services.runtime_reconciliation_service import RuntimeReconciliationService
|
||||
|
||||
|
||||
@@ -11,9 +11,15 @@ def test_reconciliation_terminalizes_only_running_work() -> None:
|
||||
db = MagicMock()
|
||||
jobs = MagicMock()
|
||||
runs = MagicMock()
|
||||
resumable_partitions = MagicMock()
|
||||
exhausted_partitions = MagicMock()
|
||||
operations = MagicMock()
|
||||
jobs.filter.return_value.update.return_value = 5
|
||||
runs.filter.return_value.update.return_value = 2
|
||||
db.query.side_effect = [jobs, runs]
|
||||
resumable_partitions.filter.return_value.update.return_value = 3
|
||||
exhausted_partitions.filter.return_value.update.return_value = 1
|
||||
operations.filter.return_value.update.return_value = 2
|
||||
db.query.side_effect = [jobs, runs, resumable_partitions, exhausted_partitions, operations]
|
||||
finished_at = datetime(2026, 7, 17, 20, 0, tzinfo=timezone.utc)
|
||||
|
||||
result = RuntimeReconciliationService.reconcile(
|
||||
@@ -23,8 +29,13 @@ def test_reconciliation_terminalizes_only_running_work() -> None:
|
||||
|
||||
assert result.interrupted_jobs == 5
|
||||
assert result.interrupted_analysis_runs == 2
|
||||
assert result.resumed_aoi_partitions == 3
|
||||
assert result.exhausted_aoi_partitions == 1
|
||||
jobs.filter.assert_called_once()
|
||||
runs.filter.assert_called_once()
|
||||
resumable_partitions.filter.assert_called_once()
|
||||
exhausted_partitions.filter.assert_called_once()
|
||||
operations.filter.assert_called_once()
|
||||
job_values = jobs.filter.return_value.update.call_args.args[0]
|
||||
run_values = runs.filter.return_value.update.call_args.args[0]
|
||||
assert job_values[Job.status] == "failed"
|
||||
@@ -33,6 +44,12 @@ def test_reconciliation_terminalizes_only_running_work() -> None:
|
||||
assert run_values[AnalysisRun.status] == "failed"
|
||||
assert run_values[AnalysisRun.finished_at] == finished_at
|
||||
assert "PROCESS_INTERRUPTED" in run_values[AnalysisRun.error_message]
|
||||
resumed_values = resumable_partitions.filter.return_value.update.call_args.args[0]
|
||||
exhausted_values = exhausted_partitions.filter.return_value.update.call_args.args[0]
|
||||
operation_values = operations.filter.return_value.update.call_args.args[0]
|
||||
assert resumed_values[AoiOperationPartition.status] == "queued"
|
||||
assert exhausted_values[AoiOperationPartition.status] == "failed"
|
||||
assert operation_values[AoiOperation.status] == "queued"
|
||||
db.commit.assert_called_once_with()
|
||||
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ def test_vector_partition_route_combines_one_governed_product(monkeypatch) -> No
|
||||
def test_vector_partition_request_has_a_bounded_fan_out() -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
VectorPartitionSelectionRequest(
|
||||
dataset_ids=[uuid4() for _ in range(17)],
|
||||
dataset_ids=[uuid4() for _ in range(4097)],
|
||||
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.2},
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.aoi_operation_executor import AoiOperationExecutor
|
||||
from app.services.aoi_operation_service import AoiOperationService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_partition_plan_covers_aoi_without_overlapping_area() -> None:
|
||||
aoi = box(0, 0, 25_000, 18_000)
|
||||
partitions = AoiOperationService._partition(aoi, 10_000)
|
||||
|
||||
assert len(partitions) == 6
|
||||
assert sum(partition.area for partition in partitions) == pytest.approx(aoi.area)
|
||||
assert all(partition.within(aoi) for partition in partitions)
|
||||
for index, partition in enumerate(partitions):
|
||||
for other in partitions[index + 1 :]:
|
||||
assert partition.intersection(other).area == pytest.approx(0.0)
|
||||
|
||||
|
||||
def test_partition_plan_intersects_irregular_aoi_exactly() -> None:
|
||||
aoi = box(0, 0, 20_000, 20_000).difference(box(5_000, 5_000, 15_000, 15_000))
|
||||
partitions = AoiOperationService._partition(aoi, 8_000)
|
||||
|
||||
assert sum(partition.area for partition in partitions) == pytest.approx(aoi.area)
|
||||
assert all(not partition.intersects(box(5_001, 5_001, 14_999, 14_999)) for partition in partitions)
|
||||
|
||||
|
||||
def test_partition_plan_fails_before_unbounded_fanout(monkeypatch) -> None:
|
||||
monkeypatch.setattr(AoiOperationService, "MAX_PARTITIONS", 4)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
AoiOperationService._partition(box(0, 0, 30_000, 30_000), 10_000)
|
||||
|
||||
assert exc_info.value.code == "AOI_PARTITION_LIMIT_EXCEEDED"
|
||||
assert exc_info.value.details["candidate_count"] == 9
|
||||
|
||||
|
||||
def test_executor_retries_only_transient_provider_failures() -> None:
|
||||
assert AoiOperationExecutor._retryable(AppError(code="UPSTREAM_UNAVAILABLE", message="down", status_code=503)) is True
|
||||
assert AoiOperationExecutor._retryable(AppError(code="INVALID_SCOPE", message="bad", status_code=422)) is False
|
||||
|
||||
|
||||
def test_provider_budget_is_automatic_and_override_can_only_be_stricter() -> None:
|
||||
governed = AoiOperationService._partition_side("grb", None)
|
||||
assert governed > 0
|
||||
assert AoiOperationService._partition_side("grb", governed * 2) == governed
|
||||
assert AoiOperationService._partition_side("grb", governed / 2) == governed / 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider_key", "max_pixels", "resolution_m"),
|
||||
[
|
||||
("dhmv", 12_000_000, 5.0),
|
||||
("flood_hazard", 12_000_000, 5.0),
|
||||
("spw_terrain", 12_000_000, 5.0),
|
||||
("thematic_raster", 30_000_000, 10.0),
|
||||
("walous", 36_000_000, 10.0),
|
||||
],
|
||||
)
|
||||
def test_raster_provider_budget_never_exceeds_decoded_pixel_limit(
|
||||
provider_key: str, max_pixels: int, resolution_m: float
|
||||
) -> None:
|
||||
side_m = AoiOperationService._partition_side(provider_key, None)
|
||||
|
||||
assert (side_m / resolution_m) ** 2 < max_pixels
|
||||
|
||||
|
||||
def test_migration_and_api_are_registered() -> None:
|
||||
migration = (ROOT / "backend/alembic/versions/202607260001_aoi_operations.py").read_text(encoding="utf-8")
|
||||
main = (ROOT / "backend/app/main.py").read_text(encoding="utf-8")
|
||||
route = (ROOT / "backend/app/api/routes/aoi_operations.py").read_text(encoding="utf-8")
|
||||
|
||||
assert 'down_revision = "202607160001"' in migration
|
||||
assert '"aoi_operations"' in migration
|
||||
assert '"aoi_operation_partitions"' in migration
|
||||
assert "app.include_router(aoi_operations.router" in main
|
||||
assert '"/{operation_id}/execute-next"' in route
|
||||
assert '"/{operation_id}/partitions/{partition_id}/checkpoint"' in route
|
||||
@@ -167,7 +167,7 @@ def test_powershell_tower_deploy_streams_remote_script_to_bash() -> None:
|
||||
powershell = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8")
|
||||
|
||||
assert "[System.Text.UTF8Encoding]::new($false)" in powershell
|
||||
assert "[System.IO.File]::WriteAllText($localScriptPath, $remoteScript, $utf8NoBom)" in powershell
|
||||
assert "[System.IO.File]::WriteAllText($localScriptPath, $remoteScriptLf, $utf8NoBom)" in powershell
|
||||
assert "& scp @scpArgs" in powershell
|
||||
assert "& ssh @sshRunArgs" in powershell
|
||||
assert "bash '$remoteScriptPath'" in powershell
|
||||
@@ -192,8 +192,10 @@ def test_frontend_and_unraid_icon_assets_are_present() -> None:
|
||||
index = (ROOT / "frontend" / "index.html").read_text(encoding="utf-8")
|
||||
|
||||
assert "<svg" in deploy_icon
|
||||
assert "GeoIntel Kempen" in deploy_icon
|
||||
assert deploy_icon == frontend_icon
|
||||
assert "<title id=\"title\">GeoIntel</title>" in deploy_icon
|
||||
assert "Een geometrische G als geografische lens" in deploy_icon
|
||||
assert "Een geometrische G als geografische lens" in frontend_icon
|
||||
assert deploy_png.read_bytes() == frontend_png.read_bytes()
|
||||
assert deploy_png.read_bytes() == frontend_png.read_bytes()
|
||||
assert deploy_png.stat().st_size > 1000
|
||||
assert '<link rel="icon" type="image/svg+xml" href="/geointel-icon.svg" />' in index
|
||||
|
||||
@@ -2,13 +2,15 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project
|
||||
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project
|
||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
@@ -235,6 +237,7 @@ def test_yolo_configured_model_reports_configured_with_local_model_and_dependenc
|
||||
assert model.nationally_validated is False
|
||||
assert model.operator_review_required is True
|
||||
assert model.validated_regions == ["flanders_mol_kempen"]
|
||||
assert model.supported_classes == ["building"]
|
||||
assert "Mol and the Kempen" in (model.validation_scope or "")
|
||||
|
||||
|
||||
@@ -246,6 +249,45 @@ def test_yolo_dependency_check_uses_real_imports_not_find_spec() -> None:
|
||||
assert "import torch" in source
|
||||
|
||||
|
||||
def test_yolo_runtime_fails_closed_when_cuda_is_required_but_unavailable(tmp_path: Path, monkeypatch) -> None:
|
||||
settings = _settings(tmp_path, yolo_device="cuda:0", yolo_require_cuda=True)
|
||||
monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: False)))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
YoloDetectionAdapter(settings).validate_runtime()
|
||||
|
||||
assert exc_info.value.code == "DETECTION_ACCELERATOR_UNAVAILABLE"
|
||||
|
||||
|
||||
def test_yolo_runtime_rejects_cpu_device_when_cuda_is_required(tmp_path: Path, monkeypatch) -> None:
|
||||
settings = _settings(tmp_path, yolo_device="cpu", yolo_require_cuda=True)
|
||||
monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True)))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
YoloDetectionAdapter(settings).validate_runtime()
|
||||
|
||||
assert exc_info.value.code == "DETECTION_ACCELERATOR_MISCONFIGURED"
|
||||
|
||||
|
||||
def test_yolo_validation_scope_requires_persisted_validated_area(tmp_path: Path) -> None:
|
||||
dataset = Dataset(id=uuid4(), project_id=uuid4(), name="image.tif", dataset_type="raster", source="test", area_id=uuid4())
|
||||
wrong_area = Area(id=dataset.area_id, project_id=dataset.project_id, name="Brussels", geometry="MULTIPOLYGON EMPTY")
|
||||
db = FakeSession(objects={(Area, dataset.area_id): wrong_area})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DetectionService._validate_model_area_scope(db, dataset, _settings(tmp_path, yolo_validated_area_names="Mol,Kempen"))
|
||||
|
||||
assert exc_info.value.code == "DETECTION_VALIDATION_SCOPE_UNAVAILABLE"
|
||||
|
||||
|
||||
def test_yolo_validation_scope_accepts_bound_mol_area(tmp_path: Path) -> None:
|
||||
dataset = Dataset(id=uuid4(), project_id=uuid4(), name="image.tif", dataset_type="raster", source="test", area_id=uuid4())
|
||||
area = Area(id=dataset.area_id, project_id=dataset.project_id, name="Gemeente Mol", geometry="MULTIPOLYGON EMPTY")
|
||||
db = FakeSession(objects={(Area, dataset.area_id): area})
|
||||
|
||||
DetectionService._validate_model_area_scope(db, dataset, _settings(tmp_path, yolo_validated_area_names="Mol,Kempen"))
|
||||
|
||||
|
||||
def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
Reference in New Issue
Block a user