diff --git a/.env.example b/.env.example index e910f0a5..562a7aa7 100644 --- a/.env.example +++ b/.env.example @@ -74,6 +74,14 @@ MDK_BATHYMETRY_PROBE_ENABLED=true MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20 MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4 +# Bounded MDK acquisition stays fail-closed until the readiness probe reports +# "reachable" and an advertised coverage id is configured explicitly. +MDK_BATHYMETRY_ACQUISITION_ENABLED=false +MDK_BATHYMETRY_COVERAGE_ID= +MDK_BATHYMETRY_REQUEST_CRS=EPSG:4326 +MDK_BATHYMETRY_MAX_BBOX_DEG2=0.25 +MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS=120 +MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB=160 THEMATIC_RASTER_ENABLED=true THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs THEMATIC_RASTER_MIN_SIDE_M=100 @@ -94,6 +102,21 @@ YOLO_MAX_TILES=100 YOLO_MAX_DETECTIONS=1000 YOLO_DUPLICATE_IOU_THRESHOLD=0.5 YOLO_BATCH_SIZE=1 + +# Local segmentation models. GeoIntel never downloads model weights +# automatically; point these to existing local files to enable inference. +YOLO_SEG_ENABLED=false +YOLO_SEG_MODEL_PATH= +YOLO_SEG_MODEL_ID=yolo-seg-configured +YOLO_SEG_MODEL_DISPLAY_NAME=Configured YOLO segmentation +YOLO_SEG_MODEL_VERSION= +SAM_ENABLED=false +SAM_MODEL_PATH= +SAM_MODEL_ID=sam-configured +SAM_MODEL_DISPLAY_NAME=Configured SAM segmentation +SAM_MODEL_VERSION= +SEGMENTATION_MAX_MASKS_PER_TILE=300 +SEGMENTATION_DUPLICATE_IOU_THRESHOLD=0.5 ENABLE_GRB_WFS=false GRB_WFS_URL= OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d98da35..eabc7a83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,72 @@ # Changelog +## Unreleased - Post-V1 capability completion (2026-07-19) + +- Made `Belgium and North Sea Workbench` the unconditional frontend startup + context, moved the initial MapLibre viewport to national extent and removed + Mol/Kempen defaults from project/area forms and end-user source copy. Mol and + Kempen remain golden regression data only. +- Added bounded official UrbIS Land Cover products for Brussels using the + live-validated `urbisvector:Blocks` WFS layer: total land-cover blocks, + FO/GB forest and park blocks, and WB permanent-water blocks. Persisted + geometries retain source class codes and expose real hectare metrics. +- Restricted the production model-asset catalog to the explicit + `YOLO_MODEL_PATH` file so training/smoke checkpoints no longer pollute the + end-user selector. The Detection Lab now states the local Mol/Kempen + validation scope and explicitly warns that the model is not nationally + validated. + +- Implemented real local segmentation inference: `YoloSegmentationAdapter` and + `SamSegmentationAdapter` (ultralytics interface) run over existing raster + tile manifests, georeference mask polygons to EPSG:4326, suppress duplicate + masks by IoU, compute geodesic areas and persist `Segmentation` rows with + local-inference provenance. The segmentation model registry now reports + `yolo-seg-configured` and `sam-configured` dynamically from + `YOLO_SEG_ENABLED`/`YOLO_SEG_MODEL_PATH` and `SAM_ENABLED`/`SAM_MODEL_PATH`. + Everything stays fail-closed: no weights are downloaded automatically and a + missing file or dependency reports an explicit unavailable status. +- Implemented bounded MDK Belgian North Sea bathymetry acquisition + (`POST /datasets/bathymetry/mdk/acquire`): WCS 1.0.0 GetCoverage behind the + existing strict-TLS readiness probe. Acquisition requires explicit + `MDK_BATHYMETRY_ACQUISITION_ENABLED=true`, a coverage id advertised by the + live capabilities document, a bounded EPSG:4326 bbox, GeoTIFF validation via + rasterio and persists LAT vertical-reference provenance. The bathymetry + source registry now reports `acquisition_supported=true` with + `configured=false` until the operator opts in. +- Added the live-validated `urbis_street_axes` product + (`urbisvector:StreetAxes`, INSPIRE_ID identity, LineString geometry) so the + Brussels roads theme becomes operational through the existing bounded UrbIS + WFS engine. The live capabilities advertise no hydrography feature type, so + Brussels surface water intentionally remains `not_configured`. +- Extended `.env.example` with the new segmentation and MDK acquisition + variables and updated `docs/KNOWN_LIMITATIONS.md` and `docs/TODO.md`. + +### Functional audit fixes (beyond documented scope) + +- Runs can no longer be orphaned in `running`: rejected fixture payloads, + unexpected inference errors and the unreachable model fall-through in both + `DetectionService.run_detection` and `SegmentationService.run_segmentation` + now mark the analysis run and job `failed` before propagating the error, and + `JobService.run_sync_job` marks the job failed on unexpected non-AppError + exceptions as well (`tests/test_run_state_consistency.py`). +- `/api/v1/system/capabilities` no longer hardcodes `sam=false`; it reports + the real configured state of the SAM segmentation capability. +- `POST /exports/geojson` fails closed with `INVALID_EXPORT_REQUEST` instead of + silently returning an empty envelope when no export target matches. +- The segmentation workbench now auto-selects a configured non-fixture + segmentation model when one exists, mirroring the detection workbench. +- Runtime parity: `YOLO_SEG_*`, `SAM_*`, `SEGMENTATION_*` and + `MDK_BATHYMETRY_ACQUISITION_*` are now wired through `docker-compose.yml`, + `docker-compose.unraid.yml`, `deploy/unraid/run-dockerman-container.sh`, + `deploy/unraid/geointel.env.example` and the DockerMan template. Without + this the new segmentation and bathymetry features could never be enabled in + the deployed runtimes. Compose deployments now also reconcile interrupted + runs after a restart (`GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP=true`), + matching the Unraid runtime. Guarded by + `test_segmentation_and_mdk_acquisition_are_configurable_in_every_runtime` + and `test_compose_reconciles_interrupted_runs_after_restart_like_unraid_runtime`. + ## 1.0.0 - Final Belgium and Belgian North Sea release (2026-07-19) - Fixed rectangle analysis so it materializes and reads all applicable diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index 483fc67c..cf51dce6 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -50,6 +50,7 @@ from app.schemas import ( BathymetryProfileAcquireRequest, BathymetryRasterSelectionRequest, BathymetryRasterSelectionResponse, + MdkBathymetryAcquireRequest, ThematicRasterAcquireRequest, ThematicRasterProductRead, ThematicRasterSelectionResponse, @@ -92,6 +93,7 @@ from app.services.flood_hazard_acquisition_service import FloodHazardAcquisition from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService from app.services.bathymetry_raster_analysis_service import BathymetryRasterAnalysisService +from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService @@ -342,6 +344,25 @@ def probe_mdk_bathymetry_readiness(project_id: UUID, db: Session = Depends(get_d return envelope(MdkBathymetryProbeService.probe()) +@router.post( + "/datasets/bathymetry/mdk/acquire", + response_model=Envelope[JobRead], +) +def acquire_bounded_mdk_bathymetry( + project_id: UUID, + payload: MdkBathymetryAcquireRequest, + db: Session = Depends(get_db), +): + job = JobService.run_sync_job( + db=db, + project_id=project_id, + job_type="raster.mdk_bathymetry.acquire", + parameters=payload.model_dump(mode="json"), + operation=lambda: MdkBathymetryAcquisitionService.acquire(db, project_id, payload), + ) + return envelope(job) + + @router.post( "/datasets/bathymetry/profiles/acquire", response_model=Envelope[JobRead], diff --git a/backend/app/api/routes/exports.py b/backend/app/api/routes/exports.py index f14a253d..d1c7ac48 100644 --- a/backend/app/api/routes/exports.py +++ b/backend/app/api/routes/exports.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends, Query from fastapi.responses import FileResponse from sqlalchemy.orm import Session +from app.core.errors import AppError from app.db.session import get_db from app.schemas import Envelope from app.schemas.export import ( @@ -47,7 +48,11 @@ def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db)) ) if payload.dataset_id is not None: return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json")) - return envelope({}) + raise AppError( + code="INVALID_EXPORT_REQUEST", + message="GeoJSON export request does not match any supported export target", + status_code=422, + ) @router.post("/metadata", response_model=Envelope[ExportCreateResponse]) diff --git a/backend/app/api/routes/health.py b/backend/app/api/routes/health.py index 093f96bd..7d479f60 100644 --- a/backend/app/api/routes/health.py +++ b/backend/app/api/routes/health.py @@ -147,6 +147,11 @@ def capabilities() -> SystemCapabilitiesEnvelope: ) yolo_configured = bool(configured_yolo and configured_yolo.configured) yolo_status = configured_yolo.status if configured_yolo else "not_configured" + configured_sam = ModelRegistryService.get_model_capability( + settings.sam_model_id, + settings=settings, + task_type="segmentation", + ) postgis_ready = _database_checks()["postgis"].startswith("ok:") return SystemCapabilitiesEnvelope( data=SystemCapabilities( @@ -155,7 +160,7 @@ def capabilities() -> SystemCapabilitiesEnvelope: geopandas=_dependency_enabled("geopandas"), yolo=yolo_configured, yolo_status=yolo_status, - sam=False, + sam=bool(configured_sam and configured_sam.configured), grb="bounded", sentinel="planned", version=settings.app_version, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 1bbc5dca..ad8cec9e 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -247,6 +247,27 @@ class Settings(BaseSettings): default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs", validation_alias="THEMATIC_RASTER_WCS_URL", ) + mdk_bathymetry_acquisition_enabled: bool = Field( + default=False, + validation_alias="MDK_BATHYMETRY_ACQUISITION_ENABLED", + ) + mdk_bathymetry_coverage_id: str | None = Field(default=None, validation_alias="MDK_BATHYMETRY_COVERAGE_ID") + mdk_bathymetry_request_crs: str = Field(default="EPSG:4326", validation_alias="MDK_BATHYMETRY_REQUEST_CRS") + mdk_bathymetry_max_bbox_deg2: float = Field( + default=0.25, + gt=0, + validation_alias="MDK_BATHYMETRY_MAX_BBOX_DEG2", + ) + mdk_bathymetry_acquisition_timeout_seconds: int = Field( + default=120, + ge=1, + validation_alias="MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS", + ) + mdk_bathymetry_acquisition_max_response_mb: int = Field( + default=160, + ge=1, + validation_alias="MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB", + ) thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M") thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M") thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS") @@ -272,6 +293,29 @@ class Settings(BaseSettings): yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS") yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD") yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE") + yolo_seg_enabled: bool = Field(default=False, validation_alias="YOLO_SEG_ENABLED") + yolo_seg_model_path: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_PATH") + yolo_seg_model_id: str = Field(default="yolo-seg-configured", validation_alias="YOLO_SEG_MODEL_ID") + yolo_seg_model_display_name: str = Field( + default="Configured YOLO segmentation", + validation_alias="YOLO_SEG_MODEL_DISPLAY_NAME", + ) + yolo_seg_model_version: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_VERSION") + sam_enabled: bool = Field(default=False, validation_alias="SAM_ENABLED") + sam_model_path: str | None = Field(default=None, validation_alias="SAM_MODEL_PATH") + sam_model_id: str = Field(default="sam-configured", validation_alias="SAM_MODEL_ID") + sam_model_display_name: str = Field( + default="Configured SAM segmentation", + validation_alias="SAM_MODEL_DISPLAY_NAME", + ) + sam_model_version: str | None = Field(default=None, validation_alias="SAM_MODEL_VERSION") + segmentation_max_masks_per_tile: int = Field(default=300, ge=1, validation_alias="SEGMENTATION_MAX_MASKS_PER_TILE") + segmentation_duplicate_iou_threshold: float = Field( + default=0.5, + ge=0.0, + le=1.0, + validation_alias="SEGMENTATION_DUPLICATE_IOU_THRESHOLD", + ) ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED") ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL") ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL") diff --git a/backend/app/models/entities.py b/backend/app/models/entities.py index 9ec05743..f8b3b28e 100644 --- a/backend/app/models/entities.py +++ b/backend/app/models/entities.py @@ -18,7 +18,7 @@ class Project(Base): id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) name: Mapped[str] = mapped_column(String(255), nullable=False) description: Mapped[str | None] = mapped_column(Text, nullable=True) - region: Mapped[str] = mapped_column(String(120), default="Kempen") + region: Mapped[str] = mapped_column(String(120), default="Belgium and Belgian North Sea") status: Mapped[str] = mapped_column(String(32), default="active") created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 7f8ee75f..f98e568a 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -99,6 +99,8 @@ from .bathymetry import ( BathymetryRasterSelectionSummary, BathymetrySourceProbeRead, BathymetrySourceRead, + MdkBathymetryAcquireRequest, + MdkBathymetryAcquisitionResult, ) from .thematic_raster import ( ThematicRasterAcquireRequest, @@ -265,6 +267,8 @@ __all__ = [ "BathymetryPartitionFinalizationResult", "BathymetrySourceProbeRead", "BathymetrySourceRead", + "MdkBathymetryAcquireRequest", + "MdkBathymetryAcquisitionResult", "ThematicRasterAcquireRequest", "ThematicRasterAcquisitionResult", "ThematicRasterMetric", diff --git a/backend/app/schemas/bathymetry.py b/backend/app/schemas/bathymetry.py index b1b51ac2..d2bbceda 100644 --- a/backend/app/schemas/bathymetry.py +++ b/backend/app/schemas/bathymetry.py @@ -111,6 +111,24 @@ class BathymetrySourceProbeRead(BaseModel): limitation_message: str +class MdkBathymetryAcquireRequest(BaseModel): + bbox: VectorSelectionBBox + area_id: UUID | None = None + force_refresh: bool = False + + +class MdkBathymetryAcquisitionResult(BaseModel): + output_dataset_id: UUID + reused: bool + provider: str + coverage_id: str + bbox_epsg4326: list[float] + vertical_reference: str + resolution_m: float = Field(gt=0) + attribution: str + limitation_message: str + + class BathymetryRasterSelectionRequest(BaseModel): bbox: VectorSelectionBBox area_id: UUID | None = None diff --git a/backend/app/schemas/project.py b/backend/app/schemas/project.py index 7964a3ac..5aa7f85a 100644 --- a/backend/app/schemas/project.py +++ b/backend/app/schemas/project.py @@ -10,7 +10,7 @@ from pydantic import BaseModel class ProjectCreate(BaseModel): name: str description: str | None = None - region: str | None = "Kempen" + region: str | None = "Belgium and Belgian North Sea" class ProjectUpdate(BaseModel): diff --git a/backend/app/services/bathymetry_profile_acquisition_service.py b/backend/app/services/bathymetry_profile_acquisition_service.py index d4ba7b79..bf3a19c8 100644 --- a/backend/app/services/bathymetry_profile_acquisition_service.py +++ b/backend/app/services/bathymetry_profile_acquisition_service.py @@ -134,8 +134,37 @@ class BathymetryProfileAcquisitionService: ) @staticmethod - def list_sources() -> list[dict[str, Any]]: - return [BathymetrySourceRead(**item).model_dump() for item in BathymetryProfileAcquisitionService._SOURCES] + def list_sources(settings=None) -> list[dict[str, Any]]: + from app.core.config import get_settings + + resolved_settings = settings or get_settings() + items: list[dict[str, Any]] = [] + for source in BathymetryProfileAcquisitionService._SOURCES: + item = dict(source) + if item["key"] == "mdk_bcp_bathymetry": + mdk_configured = bool( + resolved_settings.mdk_bathymetry_acquisition_enabled + and (resolved_settings.mdk_bathymetry_coverage_id or "").strip() + ) + item["acquisition_supported"] = True + item["configured"] = mdk_configured + if mdk_configured: + item["integration_status"] = "operational" + item["limitation_message"] = ( + "Begrensde WCS-acquisitie is expliciet ingeschakeld en draait alleen wanneer de " + "live readiness-probe bereikbaar is en het geconfigureerde coverage-id door de " + "capabilities wordt geadverteerd. Dieptes blijven LAT-gerefereerd; watervolume " + "blijft zonder compatibel wateroppervlak niet ondersteund." + ) + else: + item["limitation_message"] = ( + "Begrensde WCS-acquisitie bestaat maar staat uit. Zet " + "MDK_BATHYMETRY_ACQUISITION_ENABLED=true en configureer MDK_BATHYMETRY_COVERAGE_ID " + "pas nadat de readiness-probe live 'reachable' rapporteert. Er wordt nooit " + "onbeveiligd of ongevalideerd gedownload." + ) + items.append(item) + return [BathymetrySourceRead(**item).model_dump() for item in items] @staticmethod def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]: diff --git a/backend/app/services/coverage_registry_service.py b/backend/app/services/coverage_registry_service.py index d9e2c198..9ae42407 100644 --- a/backend/app/services/coverage_registry_service.py +++ b/backend/app/services/coverage_registry_service.py @@ -272,11 +272,11 @@ SOURCE_DEFINITIONS = ( attribution="Brussels UrbIS", license_note="Consult the license of the selected UrbIS dataset.", limitation_message=( - "Bounded UrbIS buildings and cadastral parcels are operational; " - "other Brussels themes remain unavailable until separately governed." + "Bounded UrbIS buildings, cadastral parcels, street axes and Land Cover blocks are operational. " + "Permanent water uses the official WB block class; no separate hydrography network is inferred." ), materialized_source_names=("urbis",), - operational_themes=("buildings", "parcels"), + operational_themes=("buildings", "parcels", "roads", "surface_water", "land_cover_use"), ), _contract( source_name="rbins_marine_reporting_units", @@ -341,7 +341,10 @@ SOURCE_DEFINITIONS = ( source_url="https://www.vlaanderen.be/datavindplaats", attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)", license_note="Consult the official product license before acquisition.", - limitation_message="Strict-TLS acquisition and vertical datum evidence are not yet sufficient; no depths are synthesized.", + limitation_message=( + "Bounded strict-TLS WCS acquisition is implemented but stays disabled until the operator enables it " + "with a live-validated coverage id; no depths are synthesized." + ), ), ) @@ -377,6 +380,9 @@ REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = { "urbis": { "buildings": {"urbis": ("buildings",)}, "parcels": {"urbis": ("parcels",)}, + "roads": {"urbis": ("roads",)}, + "surface_water": {"urbis": ("water",)}, + "land_cover_use": {"urbis": ("space_occupation", "forest")}, }, } diff --git a/backend/app/services/detection_georeferencing.py b/backend/app/services/detection_georeferencing.py index b80a627c..ee79a557 100644 --- a/backend/app/services/detection_georeferencing.py +++ b/backend/app/services/detection_georeferencing.py @@ -39,6 +39,75 @@ def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs: return polygon +def pixel_points_to_epsg4326_polygon(points: list[list[float]], tile: dict[str, Any], crs: str | None = None) -> Polygon: + if not isinstance(points, list) or len(points) < 3: + raise AppError( + code="SEGMENTATION_INVALID_MASK", + message="Segmentation mask polygon must contain at least three pixel points", + status_code=422, + ) + try: + pixel_points = [(float(point[0]), float(point[1])) for point in points] + except (TypeError, ValueError, IndexError) as exc: + raise AppError( + code="SEGMENTATION_INVALID_MASK", + message="Segmentation mask polygon points must be numeric [x, y] pairs", + status_code=422, + ) from exc + + transform = tile.get("transform") + if isinstance(transform, list) and len(transform) >= 6: + coordinates = [_apply_gdal_transform(transform, x, y) for x, y in pixel_points] + else: + coordinates = [_project_pixel_with_bounds(tile, x, y) for x, y in pixel_points] + + source_crs = crs or tile.get("crs") or tile.get("source_crs") or "EPSG:4326" + if str(source_crs).upper() not in {"EPSG:4326", "4326"}: + transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True) + coordinates = [transformer.transform(x, y) for x, y in coordinates] + + if coordinates[0] != coordinates[-1]: + coordinates.append(coordinates[0]) + polygon = Polygon(coordinates) + if not polygon.is_valid: + from shapely.validation import make_valid + + repaired = make_valid(polygon) + polygon = _largest_polygon(repaired) + if polygon is None or polygon.is_empty or not polygon.is_valid or polygon.area <= 0: + raise AppError( + code="SEGMENTATION_INVALID_GEOMETRY", + message="Georeferenced segmentation geometry is invalid", + status_code=422, + ) + return polygon + + +def _largest_polygon(geometry: Any) -> Polygon | None: + if isinstance(geometry, Polygon): + return geometry + candidates = [geom for geom in getattr(geometry, "geoms", []) if isinstance(geom, Polygon) and geom.area > 0] + if not candidates: + return None + return max(candidates, key=lambda geom: geom.area) + + +def _project_pixel_with_bounds(tile: dict[str, Any], px: float, py: float) -> tuple[float, float]: + bounds = tile.get("bounds") + pixel_window = tile.get("pixel_window") + if not (isinstance(bounds, list) and len(bounds) == 4 and isinstance(pixel_window, list) and len(pixel_window) == 4): + raise AppError( + code="DETECTION_TILE_MANIFEST_INVALID", + message="Tile manifest entries require transform or bounds plus pixel_window for georeferencing", + status_code=422, + ) + left, bottom, right, top = [float(value) for value in bounds] + _, _, width, height = [float(value) for value in pixel_window] + if width <= 0 or height <= 0: + raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile pixel_window must have positive size", status_code=422) + return (left + (px / width) * (right - left), top - (py / height) * (top - bottom)) + + def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]: c, a, b, f, d, e = [float(value) for value in transform[:6]] return (a * x + b * y + c, d * x + e * y + f) diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 5ebc8fd9..6d915011 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -130,18 +130,23 @@ class DetectionService: ) if model.model_id == "manual-fixture-detector": - detections = DetectionService._persist_fixture_detections( - db=db, - project_id=project_id, - dataset_id=dataset_id, - analysis_run=analysis_run, - job=job, - model_name=model.model_id, - model_version=model.version, - raw_detections=parameters.get("fixture_detections"), - confidence_threshold=confidence_threshold, - class_filter=class_filter or [], - ) + try: + detections = DetectionService._persist_fixture_detections( + db=db, + project_id=project_id, + dataset_id=dataset_id, + analysis_run=analysis_run, + job=job, + model_name=model.model_id, + model_version=model.version, + raw_detections=parameters.get("fixture_detections"), + confidence_threshold=confidence_threshold, + class_filter=class_filter or [], + ) + except Exception as exc: + # A rejected fixture payload must never leave the run stuck in "running". + DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR") + raise DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections)) return DetectionRunResponse( analysis_run_id=analysis_run.id, @@ -183,6 +188,10 @@ class DetectionService: error_code=exc.code, message=exc.message, ) + except Exception as exc: + # An unexpected inference error must never leave the run stuck in "running". + DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR") + raise DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary) return DetectionRunResponse( analysis_run_id=analysis_run.id, @@ -195,8 +204,28 @@ class DetectionService: message="YOLO detections persisted.", ) + DetectionService._mark_failed( + db, + analysis_run, + job, + code="DETECTION_MODEL_UNAVAILABLE", + message="Detection model is unavailable", + ) raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503) + @staticmethod + def _fail_run_after_exception(db, analysis_run: AnalysisRun, job: Job, exc: Exception, fallback_code: str) -> None: + try: + db.rollback() + except Exception: + pass + code = getattr(exc, "code", None) or fallback_code + message = getattr(exc, "message", None) or "Unexpected internal error during analysis run" + try: + DetectionService._mark_failed(db, analysis_run, job, code=str(code), message=str(message)) + except Exception: + pass + @staticmethod def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead: run = db.get(AnalysisRun, analysis_run_id) diff --git a/backend/app/services/job_service.py b/backend/app/services/job_service.py index c7cf9d2f..9a7a29cf 100644 --- a/backend/app/services/job_service.py +++ b/backend/app/services/job_service.py @@ -76,6 +76,22 @@ class JobService: result_json["output_dataset_id"] = str(result_json["output_dataset_id"]) payload["result_json"] = result_json raise + except Exception: + # An unexpected error must never leave the job stuck in "running". + try: + db.rollback() + except Exception: + pass + try: + JobService.mark_failed( + db, + created.id, + error_message="Unexpected internal error during synchronous job execution", + details={"code": "JOB_INTERNAL_ERROR"}, + ) + except Exception: + pass + raise @staticmethod def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]: diff --git a/backend/app/services/mdk_bathymetry_acquisition_service.py b/backend/app/services/mdk_bathymetry_acquisition_service.py new file mode 100644 index 00000000..2510c0d9 --- /dev/null +++ b/backend/app/services/mdk_bathymetry_acquisition_service.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from urllib.request import Request, urlopen +from uuid import UUID + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import Dataset +from app.schemas.bathymetry import MdkBathymetryAcquireRequest, MdkBathymetryAcquisitionResult +from app.services.dataset_service import DatasetService +from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService + + +class MdkBathymetryAcquisitionService: + """Bounded, fail-closed GetCoverage acquisition for the MDK Belgian North Sea depth model. + + Acquisition only runs when: + + - the operator explicitly enabled acquisition and configured a coverage id, + - the live strict-TLS readiness probe reports ``reachable``, + - the configured coverage id is advertised by the live capabilities document, + - the requested EPSG:4326 bbox stays within the configured size bound. + + No depth values are ever synthesized, no insecure TLS fallback exists and the + LAT vertical reference is persisted with every artifact so it can never be + silently compared with TAW or mDNG data. + """ + + PROVIDER = "mdk_bcp_bathymetry" + VERTICAL_REFERENCE = "LAT" + NATIVE_RESOLUTION_M = 20.0 + MAX_PIXELS_PER_SIDE = 4096 + LIMITATION = ( + "Dieptewaarden zijn LAT-gerefereerd en gelden voor de bemonsterde survey-periode van het officiële " + "MDK-model. LAT mag nooit zonder gedocumenteerde datumtransformatie met TAW- of mDNG-gegevens worden " + "vergeleken; watervolume blijft zonder compatibel wateroppervlak niet ondersteund." + ) + ATTRIBUTION = "Agentschap Maritieme Dienstverlening en Kust (MDK)" + LICENSE_NOTE = "Consult the official MDK product license before redistribution." + + @staticmethod + def acquire( + db, + project_id: UUID, + payload: MdkBathymetryAcquireRequest, + *, + settings: Settings | None = None, + opener: Callable[..., Any] | None = None, + ) -> dict[str, Any]: + resolved_settings = settings or get_settings() + if not resolved_settings.mdk_bathymetry_acquisition_enabled: + raise AppError( + code="MDK_BATHYMETRY_ACQUISITION_DISABLED", + message=( + "MDK bathymetry acquisition is disabled. Enable it explicitly with " + "MDK_BATHYMETRY_ACQUISITION_ENABLED=true after the readiness probe reports reachable." + ), + status_code=409, + ) + coverage_id = (resolved_settings.mdk_bathymetry_coverage_id or "").strip() + if not coverage_id: + raise AppError( + code="MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED", + message="MDK_BATHYMETRY_COVERAGE_ID is not configured; GeoIntel will not guess coverage identifiers.", + status_code=409, + ) + + bbox = MdkBathymetryAcquisitionService._validated_bbox(payload, resolved_settings) + + probe = MdkBathymetryProbeService.probe(settings=resolved_settings, opener=opener) + if probe.get("status") != "reachable": + raise AppError( + code="MDK_BATHYMETRY_ENDPOINT_NOT_READY", + message="The live MDK readiness probe does not report a reachable, TLS-verified WCS endpoint.", + details={"probe_status": probe.get("status"), "probe_message": probe.get("message")}, + status_code=502, + ) + if coverage_id not in (probe.get("coverage_identifiers") or []): + raise AppError( + code="MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED", + message="The configured coverage id is not advertised by the live MDK capabilities document.", + details={ + "configured_coverage_id": coverage_id, + "advertised_coverage_identifiers": probe.get("coverage_identifiers") or [], + }, + status_code=502, + ) + + request_url = MdkBathymetryAcquisitionService._get_coverage_url(resolved_settings, coverage_id, bbox) + request_hash = hashlib.sha256(request_url.encode("utf-8")).hexdigest() + filename = f"mdk_bathymetry_{request_hash[:12]}.tif" + + if not payload.force_refresh: + cached = MdkBathymetryAcquisitionService._cached_dataset(db, project_id, filename) + if cached is not None: + return MdkBathymetryAcquisitionResult( + output_dataset_id=cached.id, + reused=True, + provider=MdkBathymetryAcquisitionService.PROVIDER, + coverage_id=coverage_id, + bbox_epsg4326=bbox, + vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE, + resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M, + attribution=MdkBathymetryAcquisitionService.ATTRIBUTION, + limitation_message=MdkBathymetryAcquisitionService.LIMITATION, + ).model_dump(mode="json") + + content, content_type = MdkBathymetryAcquisitionService._fetch(request_url, resolved_settings, opener) + validation = MdkBathymetryAcquisitionService._validate_geotiff(content) + acquired_at = datetime.now(UTC) + + dataset = DatasetService.import_raster_bytes( + db, + project_id=project_id, + area_id=payload.area_id, + filename=filename, + content=content, + source=f"MDK Belgian Continental Shelf WCS {coverage_id}", + source_name=MdkBathymetryAcquisitionService.PROVIDER, + source_metadata={ + "provider": MdkBathymetryAcquisitionService.PROVIDER, + "service": "WCS", + "service_version": "1.0.0", + "coverage_id": coverage_id, + "vertical_reference": MdkBathymetryAcquisitionService.VERTICAL_REFERENCE, + "native_resolution_m": MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M, + "bbox_epsg4326": bbox, + "attribution": MdkBathymetryAcquisitionService.ATTRIBUTION, + "license_note": MdkBathymetryAcquisitionService.LICENSE_NOTE, + "raster_validation": validation, + }, + provenance_metadata={ + "acquisition": "explicit_bounded_wcs_get_coverage", + "acquired_at": acquired_at.isoformat(), + "request_url": request_url, + "request_hash": request_hash, + "response_content_type": content_type, + "coverage_sha256": hashlib.sha256(content).hexdigest(), + "probe_status": probe.get("status"), + "probe_response_sha256": probe.get("response_sha256"), + "probe_checked_at": probe.get("checked_at"), + "limitation_message": MdkBathymetryAcquisitionService.LIMITATION, + }, + ) + return MdkBathymetryAcquisitionResult( + output_dataset_id=dataset.id, + reused=False, + provider=MdkBathymetryAcquisitionService.PROVIDER, + coverage_id=coverage_id, + bbox_epsg4326=bbox, + vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE, + resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M, + attribution=MdkBathymetryAcquisitionService.ATTRIBUTION, + limitation_message=MdkBathymetryAcquisitionService.LIMITATION, + ).model_dump(mode="json") + + @staticmethod + def _validated_bbox(payload: MdkBathymetryAcquireRequest, settings: Settings) -> list[float]: + bbox = payload.bbox + min_x, min_y, max_x, max_y = ( + float(bbox.min_x), + float(bbox.min_y), + float(bbox.max_x), + float(bbox.max_y), + ) + if max_x <= min_x or max_y <= min_y: + raise AppError( + code="MDK_BATHYMETRY_INVALID_BBOX", + message="The requested bbox must have positive width and height in EPSG:4326.", + status_code=422, + ) + area_deg2 = (max_x - min_x) * (max_y - min_y) + if area_deg2 > float(settings.mdk_bathymetry_max_bbox_deg2): + raise AppError( + code="MDK_BATHYMETRY_BBOX_TOO_LARGE", + message="The requested bbox exceeds the configured bounded acquisition size.", + details={ + "bbox_area_deg2": area_deg2, + "max_bbox_deg2": float(settings.mdk_bathymetry_max_bbox_deg2), + }, + status_code=422, + ) + return [min_x, min_y, max_x, max_y] + + @staticmethod + def _get_coverage_url(settings: Settings, coverage_id: str, bbox: list[float]) -> str: + parsed = urlsplit(settings.mdk_bathymetry_wcs_url.strip()) + if parsed.scheme.lower() != "https" or not parsed.hostname: + raise AppError( + code="MDK_BATHYMETRY_INVALID_CONFIGURATION", + message="MDK bathymetry acquisition requires an absolute HTTPS WCS URL.", + status_code=409, + ) + width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox) + parameters = dict(parse_qsl(parsed.query, keep_blank_values=True)) + parameters.update( + { + "service": "WCS", + "request": "GetCoverage", + "version": "1.0.0", + "coverage": coverage_id, + "crs": settings.mdk_bathymetry_request_crs, + "bbox": ",".join(f"{value:.8f}" for value in bbox), + "width": str(width), + "height": str(height), + "format": "GeoTIFF", + } + ) + return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), "")) + + @staticmethod + def _pixel_dimensions(bbox: list[float]) -> tuple[int, int]: + min_x, min_y, max_x, max_y = bbox + # Approximate meters per degree near the Belgian North Sea (~51.5N). + meters_per_deg_lat = 111_320.0 + meters_per_deg_lon = 69_400.0 + width = int((max_x - min_x) * meters_per_deg_lon / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M) + height = int((max_y - min_y) * meters_per_deg_lat / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M) + width = max(1, min(width, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE)) + height = max(1, min(height, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE)) + return width, height + + @staticmethod + def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]: + request = Request( + request_url, + headers={ + "Accept": "image/tiff,*/*;q=0.1", + "User-Agent": "GeoIntel/1.0 MDK-bathymetry-bounded-acquisition", + }, + ) + max_bytes = settings.mdk_bathymetry_acquisition_max_response_mb * 1024 * 1024 + try: + with (opener or urlopen)(request, timeout=settings.mdk_bathymetry_acquisition_timeout_seconds) as response: + content_type = str(response.headers.get("Content-Type", "")) if hasattr(response, "headers") else "" + content = response.read(max_bytes + 1) + except HTTPError as exc: + preview = exc.read(300).decode("utf-8", errors="replace") + raise AppError( + code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE", + message="The MDK WCS could not complete the bounded GetCoverage request.", + details={"provider_status_code": int(exc.code), "response_preview": preview}, + status_code=502, + ) from exc + except (URLError, TimeoutError, OSError) as exc: + raise AppError( + code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE", + message="The MDK WCS could not be reached for the bounded GetCoverage request.", + details={"reason": str(exc)}, + status_code=502, + ) from exc + if len(content) > max_bytes: + raise AppError( + code="MDK_BATHYMETRY_RESPONSE_TOO_LARGE", + message="The MDK coverage response exceeds the configured size limit.", + status_code=502, + ) + if not content.startswith((b"II*\x00", b"MM\x00*")): + preview = content[:300].decode("utf-8", errors="replace") + raise AppError( + code="MDK_BATHYMETRY_INVALID_RESPONSE", + message="The MDK WCS did not return a GeoTIFF coverage.", + details={"content_type": content_type, "response_preview": preview}, + status_code=502, + ) + return content, content_type + + @staticmethod + def _validate_geotiff(content: bytes) -> dict[str, Any]: + try: + import numpy as np + from rasterio.io import MemoryFile + except ImportError as exc: + raise AppError( + code="RASTER_PROCESSING_UNAVAILABLE", + message="Rasterio is required to validate the MDK bathymetry coverage before persistence.", + status_code=503, + ) from exc + try: + with MemoryFile(content) as memory, memory.open() as source: + if source.count < 1: + raise AppError( + code="MDK_BATHYMETRY_INVALID_RESPONSE", + message="The MDK coverage contains no raster bands.", + status_code=502, + ) + band = source.read(1, masked=True) + valid = band.compressed() + if valid.size == 0: + raise AppError( + code="MDK_BATHYMETRY_NO_VALID_DATA", + message="The MDK coverage contains no valid depth cells in this selection.", + status_code=422, + ) + return { + "crs": str(source.crs) if source.crs else None, + "width": int(source.width), + "height": int(source.height), + "nodata": None if source.nodata is None else float(source.nodata), + "valid_cell_count": int(valid.size), + "minimum_value": float(np.min(valid)), + "maximum_value": float(np.max(valid)), + } + except AppError: + raise + except Exception as exc: # rasterio raises many distinct errors for corrupt input + raise AppError( + code="MDK_BATHYMETRY_INVALID_RESPONSE", + message="The MDK coverage could not be opened as a valid GeoTIFF.", + details={"reason": str(exc)}, + status_code=502, + ) from exc + + @staticmethod + def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None: + from pathlib import Path + + candidate = ( + db.query(Dataset) + .filter( + Dataset.project_id == project_id, + Dataset.name == filename, + Dataset.source_name == MdkBathymetryAcquisitionService.PROVIDER, + Dataset.status == "ready", + ) + .order_by(Dataset.imported_at.desc()) + .first() + ) + return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None diff --git a/backend/app/services/model_asset_catalog_service.py b/backend/app/services/model_asset_catalog_service.py index 9d75eb49..bb73011e 100644 --- a/backend/app/services/model_asset_catalog_service.py +++ b/backend/app/services/model_asset_catalog_service.py @@ -24,11 +24,18 @@ class ModelAssetCatalogService: if not model_directory.exists() or not model_directory.is_dir(): return ModelAssetListResponse(items=[], total=0, model_directory=str(model_directory)) - items = [ - ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path) + candidate_paths = [ + path for path in sorted(model_directory.iterdir(), key=lambda item: item.name.lower()) if path.is_file() and path.suffix.lower() in ModelAssetCatalogService.SUPPORTED_SUFFIXES ] + if active_model_path is not None: + candidate_paths = [path for path in candidate_paths if path.resolve() == active_model_path] + + items = [ + ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path) + for path in candidate_paths + ] return ModelAssetListResponse(items=items, total=len(items), model_directory=str(model_directory)) @staticmethod @@ -72,8 +79,12 @@ class ModelAssetCatalogService: size_bytes=path.stat().st_size, sha256=ModelAssetCatalogService._sha256(path), active=active_model_path == resolved_path, - status="available", - limitation_message="Local runtime model asset. GeoIntel will not download or mutate model weights.", + status="approved" if active_model_path == resolved_path else "available", + limitation_message=( + "Approved local runtime model asset. GeoIntel will not download or mutate model weights." + if active_model_path == resolved_path + else "Local development model asset. Configure it explicitly before production use." + ), will_download_models=False, ) diff --git a/backend/app/services/model_registry_service.py b/backend/app/services/model_registry_service.py index 3334a65b..3a980cae 100644 --- a/backend/app/services/model_registry_service.py +++ b/backend/app/services/model_registry_service.py @@ -5,6 +5,7 @@ from typing import Type from app.core.config import Settings, get_settings from app.schemas.detection import DetectionModelCapability +from app.services.segmentation_adapter import SamSegmentationAdapter, YoloSegmentationAdapter from app.services.yolo_adapter import YoloDetectionAdapter @@ -14,10 +15,16 @@ class ModelRegistryService: settings: Settings | None = None, yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, task_type: str = "object_detection", + yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter, + sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter, ) -> list[DetectionModelCapability]: resolved_settings = settings or get_settings() if task_type == "segmentation": - return ModelRegistryService.list_segmentation_model_capabilities() + return ModelRegistryService.list_segmentation_model_capabilities( + settings=resolved_settings, + yolo_seg_adapter_class=yolo_seg_adapter_class, + sam_adapter_class=sam_adapter_class, + ) if task_type != "object_detection": return [] return [ @@ -52,15 +59,28 @@ class ModelRegistryService: settings: Settings | None = None, yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter, task_type: str = "object_detection", + yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter, + sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter, ) -> DetectionModelCapability | None: normalized = model_id.strip() - for model in ModelRegistryService.list_model_capabilities(settings=settings, yolo_adapter_class=yolo_adapter_class, task_type=task_type): + for model in ModelRegistryService.list_model_capabilities( + settings=settings, + yolo_adapter_class=yolo_adapter_class, + task_type=task_type, + yolo_seg_adapter_class=yolo_seg_adapter_class, + sam_adapter_class=sam_adapter_class, + ): if model.model_id == normalized: return model return None @staticmethod - def list_segmentation_model_capabilities() -> list[DetectionModelCapability]: + def list_segmentation_model_capabilities( + settings: Settings | None = None, + yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter, + sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter, + ) -> list[DetectionModelCapability]: + resolved_settings = settings or get_settings() return [ DetectionModelCapability( model_id="segmentation-placeholder", @@ -70,7 +90,7 @@ class ModelRegistryService: supported_classes=["building", "vegetation", "water", "landuse"], configured=False, status="not_configured", - limitation_message="Segmentation inference is not configured in Sprint 9; no SAM/YOLO-seg model is downloaded or executed.", + limitation_message="Segmentation inference is not configured for this placeholder; no model is downloaded or executed.", version=None, ), DetectionModelCapability( @@ -84,30 +104,86 @@ class ModelRegistryService: limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.", version="fixture-v1", ), - DetectionModelCapability( - model_id="yolo-seg-configured", - display_name="Configured YOLO segmentation", - framework="ultralytics/pytorch", - task_type="segmentation", - supported_classes=["building", "vegetation", "water", "landuse"], - configured=False, - status="not_configured", - limitation_message="YOLO-seg is not configured in Sprint 9. GeoIntel will not download segmentation model weights automatically.", - version=None, - ), - DetectionModelCapability( - model_id="sam-configured", - display_name="Configured SAM segmentation", - framework="sam", - task_type="segmentation", - supported_classes=["building", "vegetation", "water", "landuse"], - configured=False, - status="not_configured", - limitation_message="SAM is not configured in Sprint 9 and is not installed as a backend dependency.", - version=None, - ), + ModelRegistryService._configured_yolo_seg_capability(resolved_settings, yolo_seg_adapter_class), + ModelRegistryService._configured_sam_capability(resolved_settings, sam_adapter_class), ] + @staticmethod + def _configured_yolo_seg_capability( + settings: Settings, + adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter, + ) -> DetectionModelCapability: + configured = False + status = "not_configured" + limitation = ( + "YOLO segmentation is disabled. Set YOLO_SEG_ENABLED=true and YOLO_SEG_MODEL_PATH to a local " + "segmentation model file to enable inference. GeoIntel never downloads model weights automatically." + ) + model_path = Path(settings.yolo_seg_model_path).expanduser() if settings.yolo_seg_model_path else None + + if settings.yolo_seg_enabled: + if not adapter_class.dependencies_available(): + status = "dependency_unavailable" + limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]." + elif model_path is None: + limitation = "YOLO_SEG_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically." + elif not model_path.exists() or not model_path.is_file(): + limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically." + else: + configured = True + status = "configured" + limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest." + + return DetectionModelCapability( + model_id=settings.yolo_seg_model_id, + display_name=settings.yolo_seg_model_display_name, + framework="ultralytics/pytorch", + task_type="segmentation", + supported_classes=["building", "vegetation", "water", "landuse"], + configured=configured, + status=status, + limitation_message=limitation, + version=settings.yolo_seg_model_version, + ) + + @staticmethod + def _configured_sam_capability( + settings: Settings, + adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter, + ) -> DetectionModelCapability: + configured = False + status = "not_configured" + limitation = ( + "SAM is disabled. Set SAM_ENABLED=true and SAM_MODEL_PATH to a local SAM-compatible model file to " + "enable class-agnostic segmentation. GeoIntel never downloads model weights automatically." + ) + model_path = Path(settings.sam_model_path).expanduser() if settings.sam_model_path else None + + if settings.sam_enabled: + if not adapter_class.dependencies_available(): + status = "dependency_unavailable" + limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]." + elif model_path is None: + limitation = "SAM_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically." + elif not model_path.exists() or not model_path.is_file(): + limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically." + else: + configured = True + status = "configured" + limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest." + + return DetectionModelCapability( + model_id=settings.sam_model_id, + display_name=settings.sam_model_display_name, + framework="ultralytics/sam", + task_type="segmentation", + supported_classes=["segment"], + configured=configured, + status=status, + limitation_message=limitation, + version=settings.sam_model_version, + ) + @staticmethod def _configured_yolo_capability( settings: Settings, diff --git a/backend/app/services/official_vector_acquisition_service.py b/backend/app/services/official_vector_acquisition_service.py index ab7f93c5..8c8e8553 100644 --- a/backend/app/services/official_vector_acquisition_service.py +++ b/backend/app/services/official_vector_acquisition_service.py @@ -71,6 +71,7 @@ class OfficialVectorProduct: response_crs: str = "EPSG:4326" identity_field: str | None = None requires_coverage_area: bool = False + property_filter: dict[str, tuple[str, ...]] | None = None class OfficialVectorAcquisitionService: @@ -509,6 +510,57 @@ class OfficialVectorAcquisitionService: identity_field="INSPIRE_ID", requires_coverage_area=True, ), + OfficialVectorProduct( + key="urbis_street_axes", + display_name="UrbIS street axes", + theme="roads", + provider="Paradigm Brussels", + source_name="urbis", + reference_layer_name="roads", + service_type="WFS 2.0", + collection="urbisvector:StreetAxes", + source_crs="EPSG:31370", + source_version="2026-06-06", + observation_label="UrbIS revision 6 June 2026", + authority_level="authoritative", + catalog_url=( + "https://datastore.brussels/web/data/dataset/" + "2cf42541-1813-11ef-8a81-00090ffe0001" + ), + attribution="Paradigm Brussels - UrbIS", + license_note="UrbIS topographic layers are published under CC0.", + limitation_message=( + "UrbIS street axes describe topographic road geometry for the Brussels-Capital " + "Region and are not a routing network or a traffic measurement." + ), + source="UrbIS WFS", + observed_at=datetime(2026, 6, 6, tzinfo=UTC), + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "road_length", + "method": "intersection_length", + "label": "Wegaslengte", + "unit": "km", + "geometry_dimension": 1, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "road_segment_count", + "method": "feature_count", + "label": "Wegsegmenten", + "unit": "objecten", + "geometry_dimension": 1, + }, + ), + geometry_types=("LineString", "MultiLineString"), + coverage_zones=("brussels",), + endpoint_kind="urbis_wfs", + response_crs="EPSG:31370", + identity_field="INSPIRE_ID", + requires_coverage_area=True, + ), OfficialVectorProduct( key="urbis_cadastral_parcels", display_name="UrbIS cadastral parcels", @@ -561,6 +613,143 @@ class OfficialVectorAcquisitionService: identity_field="INSPIRE_ID", requires_coverage_area=True, ), + OfficialVectorProduct( + key="urbis_land_cover_blocks", + display_name="UrbIS land cover blocks", + theme="space_occupation", + provider="Paradigm Brussels", + source_name="urbis", + reference_layer_name="space_occupation", + service_type="WFS 2.0", + collection="urbisvector:Blocks", + source_crs="EPSG:31370", + source_version="UrbIS Land Cover 1.0; live WFS checked 2026-07-22", + observation_label="Current UrbIS land-cover WFS", + authority_level="authoritative", + catalog_url="https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_NL20240401.pdf", + attribution="Paradigm Brussels - UrbIS Land Cover", + license_note="UrbIS Land Cover is available through the official download and WFS service; retain source attribution.", + limitation_message=( + "UrbIS blocks describe physical and biological land cover. They are not zoning, ownership or legal land use. " + "The WFS does not expose a separate observation date per feature." + ), + source="UrbIS WFS", + observed_at=None, + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "land_cover_area", + "method": "intersection_area", + "label": "Landbedekking", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "land_cover_block_count", + "method": "feature_count", + "label": "Landbedekkingsblokken", + "unit": "objecten", + "geometry_dimension": 2, + }, + ), + coverage_zones=("brussels",), + endpoint_kind="urbis_wfs", + response_crs="EPSG:31370", + identity_field="INSPIRE_ID", + requires_coverage_area=True, + ), + OfficialVectorProduct( + key="urbis_forest_parks", + display_name="UrbIS forests and parks", + theme="forest", + provider="Paradigm Brussels", + source_name="urbis", + reference_layer_name="forest", + service_type="WFS 2.0", + collection="urbisvector:Blocks", + source_crs="EPSG:31370", + source_version="UrbIS Land Cover 1.0; live WFS checked 2026-07-22", + observation_label="Current UrbIS land-cover WFS", + authority_level="authoritative", + catalog_url="https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_NL20240401.pdf", + attribution="Paradigm Brussels - UrbIS Land Cover", + license_note="UrbIS Land Cover is available through the official download and WFS service; retain source attribution.", + limitation_message="Includes only UrbIS block types FO (forest/woodland) and GB (parks); street trees and smaller green elements are not inferred.", + source="UrbIS WFS", + observed_at=None, + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "forest_park_area", + "method": "intersection_area", + "label": "Bos- en parkoppervlakte", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "forest_park_count", + "method": "feature_count", + "label": "Bos- en parkzones", + "unit": "objecten", + "geometry_dimension": 2, + }, + ), + coverage_zones=("brussels",), + endpoint_kind="urbis_wfs", + response_crs="EPSG:31370", + identity_field="INSPIRE_ID", + requires_coverage_area=True, + property_filter={"TYPE": ("FO", "GB")}, + ), + OfficialVectorProduct( + key="urbis_water_surfaces", + display_name="UrbIS permanent water surfaces", + theme="water", + provider="Paradigm Brussels", + source_name="urbis", + reference_layer_name="water", + service_type="WFS 2.0", + collection="urbisvector:Blocks", + source_crs="EPSG:31370", + source_version="UrbIS Land Cover 1.0; live WFS checked 2026-07-22", + observation_label="Current UrbIS land-cover WFS", + authority_level="authoritative", + catalog_url="https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_NL20240401.pdf", + attribution="Paradigm Brussels - UrbIS Land Cover", + license_note="UrbIS Land Cover is available through the official download and WFS service; retain source attribution.", + limitation_message="Includes only UrbIS block type WB: canals, lakes and watercourses with predominantly permanent water.", + source="UrbIS WFS", + observed_at=None, + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "water_surface_area", + "method": "intersection_area", + "label": "Permanent wateroppervlak", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "water_surface_count", + "method": "feature_count", + "label": "Waterzones", + "unit": "objecten", + "geometry_dimension": 2, + }, + ), + coverage_zones=("brussels",), + endpoint_kind="urbis_wfs", + response_crs="EPSG:31370", + identity_field="INSPIRE_ID", + requires_coverage_area=True, + property_filter={"TYPE": ("WB",)}, + ), ) return {product.key: product for product in products} @@ -1081,6 +1270,12 @@ class OfficialVectorAcquisitionService: scope_metric: Any, coverage_scope: str, ) -> dict[str, Any] | None: + raw = dict(feature.get("properties") or {}) + if product.property_filter and any( + str(raw.get(property_name) or "") not in allowed_values + for property_name, allowed_values in product.property_filter.items() + ): + return None dimension = 2 if any("Polygon" in item for item in product.geometry_types) else 1 try: source_geometry = shape(feature.get("geometry")) @@ -1122,7 +1317,6 @@ class OfficialVectorAcquisitionService: ) if clipped_wgs84 is None: return None - raw = dict(feature.get("properties") or {}) identity = ( raw.get(product.identity_field or "") or feature.get("id") diff --git a/backend/app/services/project_service.py b/backend/app/services/project_service.py index 57829541..a3e63015 100644 --- a/backend/app/services/project_service.py +++ b/backend/app/services/project_service.py @@ -31,7 +31,11 @@ class ProjectService: @staticmethod def create_project(db: Session, payload: ProjectCreate) -> ProjectRead: - project = Project(name=payload.name.strip(), description=(payload.description or "").strip() or None, region=payload.region or "Kempen") + project = Project( + name=payload.name.strip(), + description=(payload.description or "").strip() or None, + region=payload.region or "Belgium and Belgian North Sea", + ) db.add(project) db.commit() db.refresh(project) diff --git a/backend/app/services/segmentation_adapter.py b/backend/app/services/segmentation_adapter.py index 8fd47676..73b4b1bc 100644 --- a/backend/app/services/segmentation_adapter.py +++ b/backend/app/services/segmentation_adapter.py @@ -1,8 +1,13 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path from typing import Any, Protocol +from app.core.config import Settings +from app.core.errors import AppError +from app.services.yolo_adapter import _prediction_source, _to_list + @dataclass(frozen=True) class SegmentationAdapterResult: @@ -23,6 +28,159 @@ class SegmentationAdapter(Protocol): """Future segmentation adapters must local-import model dependencies inside execution paths.""" +class _UltralyticsSegmentationAdapterBase: + """Shared local-inference plumbing for ultralytics-backed segmentation models. + + Model weights are never downloaded automatically; a missing local file or + missing dependency fails closed with an explicit error. + """ + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + @staticmethod + def dependencies_available() -> bool: + try: + import torch # noqa: F401 + import ultralytics # noqa: F401 + except Exception: + return False + return True + + def _require_model_file(self, model_path: Path) -> None: + if not model_path.exists() or not model_path.is_file(): + raise AppError( + code="SEGMENTATION_MODEL_UNAVAILABLE", + message="Configured segmentation model file does not exist", + details={"model_path": str(model_path)}, + status_code=503, + ) + if not self.dependencies_available(): + raise AppError( + code="SEGMENTATION_DEPENDENCY_UNAVAILABLE", + message="Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai].", + status_code=503, + ) + + def _predict(self, model, tile_path: Path, confidence_threshold: float) -> list[Any]: + if not tile_path.exists() or not tile_path.is_file(): + raise AppError( + code="SEGMENTATION_TILE_NOT_FOUND", + message="Tile referenced by manifest does not exist", + details={"tile_path": str(tile_path)}, + status_code=422, + ) + try: + with _prediction_source(tile_path) as prediction_source: + return model.predict( + source=prediction_source, + conf=float(confidence_threshold), + imgsz=int(self.settings.yolo_image_size), + device=self.settings.yolo_device, + verbose=False, + ) + except AppError: + raise + except Exception as exc: + raise AppError( + code="SEGMENTATION_INFERENCE_FAILED", + message="Configured segmentation inference failed for a raster tile", + details={"tile_path": str(tile_path), "error": str(exc)}, + status_code=503, + ) from exc + + def _extract_masks(self, results: list[Any], default_class_name: str | None = None) -> list[dict[str, Any]]: + segmentations: list[dict[str, Any]] = [] + max_masks = int(self.settings.segmentation_max_masks_per_tile) + for result in results: + names = getattr(result, "names", {}) or {} + masks = getattr(result, "masks", None) + if masks is None: + continue + polygons = getattr(masks, "xy", None) or [] + boxes = getattr(result, "boxes", None) + confidence_values = _to_list(getattr(boxes, "conf", [])) if boxes is not None else [] + class_values = _to_list(getattr(boxes, "cls", [])) if boxes is not None else [] + bbox_values = _to_list(getattr(boxes, "xyxy", [])) if boxes is not None else [] + for index, polygon in enumerate(polygons): + if len(segmentations) >= max_masks: + return segmentations + points = _to_list(polygon) + if not isinstance(points, list) or len(points) < 3: + continue + class_id = int(class_values[index]) if index < len(class_values) else -1 + if default_class_name is not None: + class_name = default_class_name + else: + class_name = str(names.get(class_id, class_id)) + confidence = float(confidence_values[index]) if index < len(confidence_values) else None + bbox = [float(value) for value in bbox_values[index]] if index < len(bbox_values) else None + segmentations.append( + { + "class_name": class_name, + "confidence": confidence, + "points": [[float(point[0]), float(point[1])] for point in points], + "bbox": bbox, + "properties": {"class_id": class_id}, + } + ) + return segmentations + + +class YoloSegmentationAdapter(_UltralyticsSegmentationAdapterBase): + def load_model(self, model_path: Path): + self._require_model_file(model_path) + try: + from ultralytics import YOLO + except ImportError as exc: + raise AppError( + code="SEGMENTATION_DEPENDENCY_UNAVAILABLE", + message="YOLO segmentation dependencies are not importable. Install backend optional extras with geointel-backend[ai].", + status_code=503, + ) from exc + try: + return YOLO(str(model_path)) + except Exception as exc: + raise AppError( + code="SEGMENTATION_MODEL_LOAD_FAILED", + message="Configured YOLO segmentation model could not be loaded", + details={"model_path": str(model_path)}, + status_code=503, + ) from exc + + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict[str, Any]]: + results = self._predict(model, tile_path, confidence_threshold) + return self._extract_masks(results) + + +class SamSegmentationAdapter(_UltralyticsSegmentationAdapterBase): + """Class-agnostic SAM segmentation through the ultralytics SAM interface.""" + + def load_model(self, model_path: Path): + self._require_model_file(model_path) + try: + from ultralytics import SAM + except ImportError as exc: + raise AppError( + code="SEGMENTATION_DEPENDENCY_UNAVAILABLE", + message="SAM segmentation requires the ultralytics SAM interface. Install backend optional extras with geointel-backend[ai].", + status_code=503, + ) from exc + try: + return SAM(str(model_path)) + except Exception as exc: + raise AppError( + code="SEGMENTATION_MODEL_LOAD_FAILED", + message="Configured SAM model could not be loaded", + details={"model_path": str(model_path)}, + status_code=503, + ) from exc + + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict[str, Any]]: + results = self._predict(model, tile_path, confidence_threshold) + return self._extract_masks(results, default_class_name="segment") + + class FixtureSegmentationAdapter: def segment(self, raw_segmentations: Any) -> list[SegmentationAdapterResult]: if not isinstance(raw_segmentations, list): diff --git a/backend/app/services/segmentation_service.py b/backend/app/services/segmentation_service.py index f55d0a19..62dc63fb 100644 --- a/backend/app/services/segmentation_service.py +++ b/backend/app/services/segmentation_service.py @@ -19,10 +19,16 @@ from app.schemas.segmentation import ( SegmentationRunRead, SegmentationRunResponse, ) +from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon +from app.services.detection_service import DetectionService from app.services.model_registry_service import ModelRegistryService from app.services.qa_service import QaService from app.services.quality_service import QualityService -from app.services.segmentation_adapter import FixtureSegmentationAdapter +from app.services.segmentation_adapter import ( + FixtureSegmentationAdapter, + SamSegmentationAdapter, + YoloSegmentationAdapter, +) class SegmentationService: @@ -41,6 +47,8 @@ class SegmentationService: tile_manifest_path: str | None = None, parameters_json: dict[str, Any] | None = None, settings: Settings | None = None, + yolo_seg_adapter_class: type[YoloSegmentationAdapter] = YoloSegmentationAdapter, + sam_adapter_class: type[SamSegmentationAdapter] = SamSegmentationAdapter, ) -> SegmentationRunResponse: parameters = dict(parameters_json or {}) resolved_settings = settings or get_settings() @@ -58,7 +66,13 @@ class SegmentationService: status_code=400, ) - model = ModelRegistryService.get_model_capability(model_id, task_type="segmentation") + model = ModelRegistryService.get_model_capability( + model_id, + settings=resolved_settings, + task_type="segmentation", + yolo_seg_adapter_class=yolo_seg_adapter_class, + sam_adapter_class=sam_adapter_class, + ) if model is None: raise AppError(code="SEGMENTATION_MODEL_NOT_FOUND", message="Segmentation model not found", status_code=404) if model.model_id == "fixture-segmenter" and parameters.get("fixture_mode") is not True: @@ -67,6 +81,13 @@ class SegmentationService: message="Fixture segmenter requires explicit fixture_mode=true", status_code=400, ) + configured_model_ids = {resolved_settings.yolo_seg_model_id, resolved_settings.sam_model_id} + if model.model_id in configured_model_ids and model.configured and not tile_manifest_path: + raise AppError( + code="SEGMENTATION_TILE_MANIFEST_REQUIRED", + message="Configured segmentation inference requires an existing raster tile manifest path", + status_code=400, + ) run_parameters = { "model_id": model.model_id, @@ -100,19 +121,24 @@ class SegmentationService: ) if model.model_id == "fixture-segmenter": - segmentations = SegmentationService._persist_fixture_segmentations( - db=db, - project_id=project_id, - dataset_id=dataset_id, - analysis_run=analysis_run, - job=job, - model_name=model.model_id, - model_version=model.version, - raw_segmentations=parameters.get("fixture_segmentations"), - confidence_threshold=confidence_threshold, - class_filter=class_filter or [], - settings=resolved_settings, - ) + try: + segmentations = SegmentationService._persist_fixture_segmentations( + db=db, + project_id=project_id, + dataset_id=dataset_id, + analysis_run=analysis_run, + job=job, + model_name=model.model_id, + model_version=model.version, + raw_segmentations=parameters.get("fixture_segmentations"), + confidence_threshold=confidence_threshold, + class_filter=class_filter or [], + settings=resolved_settings, + ) + except Exception as exc: + # A rejected fixture payload must never leave the run stuck in "running". + SegmentationService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="SEGMENTATION_INTERNAL_ERROR") + raise SegmentationService._mark_success(db, analysis_run, job, segmentation_count=len(segmentations)) return SegmentationRunResponse( analysis_run_id=analysis_run.id, @@ -125,8 +151,80 @@ class SegmentationService: message="Fixture segmentations persisted.", ) + if model.model_id in configured_model_ids: + try: + segmentations, postprocess_summary = SegmentationService._run_configured_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + analysis_run=analysis_run, + job=job, + model_name=model.model_id, + model_version=model.version, + tile_manifest_path=tile_manifest_path, + confidence_threshold=confidence_threshold, + class_filter=class_filter or [], + settings=resolved_settings, + yolo_seg_adapter_class=yolo_seg_adapter_class, + sam_adapter_class=sam_adapter_class, + ) + except AppError as exc: + SegmentationService._mark_failed(db, analysis_run, job, code=exc.code, message=exc.message) + return SegmentationRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="failed", + segmentation_count=0, + error_code=exc.code, + message=exc.message, + ) + except Exception as exc: + # An unexpected inference error must never leave the run stuck in "running". + SegmentationService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="SEGMENTATION_INTERNAL_ERROR") + raise + SegmentationService._mark_success( + db, + analysis_run, + job, + segmentation_count=len(segmentations), + extra_result=postprocess_summary, + ) + return SegmentationRunResponse( + analysis_run_id=analysis_run.id, + job_id=job.id, + project_id=project_id, + dataset_id=dataset_id, + model_id=model.model_id, + status="success", + segmentation_count=len(segmentations), + message="Configured segmentation inference persisted georeferenced masks.", + ) + + SegmentationService._mark_failed( + db, + analysis_run, + job, + code="SEGMENTATION_MODEL_UNAVAILABLE", + message="Segmentation model is unavailable", + ) raise AppError(code="SEGMENTATION_MODEL_UNAVAILABLE", message="Segmentation model is unavailable", status_code=503) + @staticmethod + def _fail_run_after_exception(db, analysis_run: AnalysisRun, job: Job, exc: Exception, fallback_code: str) -> None: + try: + db.rollback() + except Exception: + pass + code = getattr(exc, "code", None) or fallback_code + message = getattr(exc, "message", None) or "Unexpected internal error during analysis run" + try: + SegmentationService._mark_failed(db, analysis_run, job, code=str(code), message=str(message)) + except Exception: + pass + @staticmethod def get_run(db, analysis_run_id: uuid.UUID) -> SegmentationRunRead: run = db.get(AnalysisRun, analysis_run_id) @@ -378,8 +476,10 @@ class SegmentationService: db.refresh(job) @staticmethod - def _mark_success(db, analysis_run: AnalysisRun, job: Job, segmentation_count: int) -> None: + def _mark_success(db, analysis_run: AnalysisRun, job: Job, segmentation_count: int, extra_result: dict[str, Any] | None = None) -> None: result = {"segmentation_count": segmentation_count} + if extra_result: + result.update(extra_result) analysis_run.status = "success" analysis_run.finished_at = SegmentationService._now() analysis_run.result_json = result @@ -392,6 +492,129 @@ class SegmentationService: db.refresh(analysis_run) db.refresh(job) + @staticmethod + def _run_configured_segmentation( + db, + project_id: uuid.UUID, + dataset_id: uuid.UUID, + analysis_run: AnalysisRun, + job: Job, + model_name: str, + model_version: str | None, + tile_manifest_path: str | None, + confidence_threshold: float, + class_filter: list[str], + settings: Settings, + yolo_seg_adapter_class: type[YoloSegmentationAdapter], + sam_adapter_class: type[SamSegmentationAdapter], + ) -> tuple[list[Segmentation], dict[str, Any]]: + manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles) + if model_name == settings.sam_model_id: + adapter = sam_adapter_class(settings) + model_path = Path(settings.sam_model_path or "").expanduser() + else: + adapter = yolo_seg_adapter_class(settings) + model_path = Path(settings.yolo_seg_model_path or "").expanduser() + model = adapter.load_model(model_path) + + allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)} + manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326" + candidates: list[dict[str, Any]] = [] + for tile in manifest["tiles"]: + tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser()) + for raw in adapter.predict_tile(model, tile_path, confidence_threshold): + model_class_name = str(raw.get("class_name") or "").strip() + class_name = DetectionService._canonical_class_name(model_class_name) + confidence = raw.get("confidence") + confidence = float(confidence) if confidence is not None else None + if allowed_classes and class_name not in allowed_classes: + continue + if confidence is not None and confidence < confidence_threshold: + continue + points = raw.get("points") + if not isinstance(points, list) or len(points) < 3: + continue + geometry = pixel_points_to_epsg4326_polygon(points=points, tile=tile, crs=tile.get("crs") or manifest_crs) + properties = dict(raw.get("properties") or {}) + if model_class_name and model_class_name != class_name: + properties.setdefault("model_class_name", model_class_name) + candidates.append( + { + "class_name": class_name, + "confidence": confidence if confidence is not None else 0.0, + "reported_confidence": confidence, + "geometry": geometry, + "bbox": raw.get("bbox"), + "source_tile_path": str(tile_path), + "tile_index": tile.get("index"), + "properties": {**properties, "tile_index": tile.get("index")}, + } + ) + filtered_candidates = DetectionService._suppress_duplicate_candidates( + candidates, + iou_threshold=float(settings.segmentation_duplicate_iou_threshold), + ) + persisted: list[Segmentation] = [] + for candidate in filtered_candidates: + geometry = candidate["geometry"] + if isinstance(geometry, Polygon): + geometry = MultiPolygon([geometry]) + bbox = candidate.get("bbox") + bbox_json = None + if isinstance(bbox, list) and len(bbox) == 4: + bbox_json = { + "x_min": float(bbox[0]), + "y_min": float(bbox[1]), + "x_max": float(bbox[2]), + "y_max": float(bbox[3]), + } + segmentation = Segmentation( + id=uuid.uuid4(), + project_id=project_id, + dataset_id=dataset_id, + analysis_run_id=analysis_run.id, + job_id=job.id, + model_name=model_name, + model_version=model_version, + class_name=candidate["class_name"], + confidence=candidate["reported_confidence"], + geometry=from_shape(geometry, srid=4326), + bbox_json=bbox_json, + area_m2=SegmentationService._geodesic_area_m2(geometry), + mask_path=None, + source_tile_path=candidate["source_tile_path"], + tile_index=candidate["tile_index"] if isinstance(candidate["tile_index"], int) else None, + properties_json=candidate["properties"], + provenance_json={ + "inference": "local", + "model_id": model_name, + "tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()), + "tile_index": candidate["tile_index"], + "device": settings.yolo_device, + }, + ) + db.add(segmentation) + persisted.append(segmentation) + db.commit() + for segmentation in persisted: + db.refresh(segmentation) + return persisted, { + "raw_segmentation_count": len(candidates), + "suppressed_segmentation_count": len(candidates) - len(filtered_candidates), + "duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold), + "tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()), + } + + @staticmethod + def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None: + try: + from pyproj import Geod + + area, _ = Geod(ellps="WGS84").geometry_area_perimeter(geometry) + return abs(float(area)) + except Exception: + return None + @staticmethod def _persist_fixture_segmentations( db, diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index a827950c..fd015306 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -277,6 +277,48 @@ def test_regional_official_vector_sources_are_configurable_in_every_runtime() -> assert f'Target="{key}"' in template +def test_segmentation_and_mdk_acquisition_are_configurable_in_every_runtime() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8") + run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") + env_example = (ROOT / ".env.example").read_text(encoding="utf-8") + unraid_env = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8") + template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8") + + for key in ( + "YOLO_SEG_ENABLED", + "YOLO_SEG_MODEL_PATH", + "SAM_ENABLED", + "SAM_MODEL_PATH", + "SEGMENTATION_MAX_MASKS_PER_TILE", + "SEGMENTATION_DUPLICATE_IOU_THRESHOLD", + "MDK_BATHYMETRY_ACQUISITION_ENABLED", + "MDK_BATHYMETRY_COVERAGE_ID", + "MDK_BATHYMETRY_MAX_BBOX_DEG2", + ): + assert key in compose, key + assert key in unraid_compose, key + assert f'{key}="${{{key}:-' in run_script, key + assert f'-e {key}="${key}"' in run_script, key + assert f"{key}=" in env_example, key + assert f"{key}=" in unraid_env, key + assert f'Target="{key}"' in template, key + + +def test_compose_reconciles_interrupted_runs_after_restart_like_unraid_runtime() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8") + + assert ( + "GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP: " + "${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}" + ) in compose + assert ( + 'GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP=' + '"${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true}"' + ) in start_script + + def test_docker_build_contexts_exclude_vendor_build_and_cache_outputs() -> None: required_patterns = { "node_modules", diff --git a/backend/tests/test_mdk_bathymetry_acquisition.py b/backend/tests/test_mdk_bathymetry_acquisition.py new file mode 100644 index 00000000..ea0a3e4f --- /dev/null +++ b/backend/tests/test_mdk_bathymetry_acquisition.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import io +from pathlib import Path +from uuid import uuid4 + +import pytest + +from app.core.config import Settings +from app.schemas.bathymetry import MdkBathymetryAcquireRequest +from app.schemas.operations import VectorSelectionBBox +from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService + +CAPABILITIES_XML = b""" + + + + depth_model_20m_lat + + + + +""" + + +class FakeResponse: + def __init__(self, content: bytes, content_type: str = "application/xml") -> None: + self._stream = io.BytesIO(content) + self.headers = {"Content-Type": content_type} + + def read(self, limit: int = -1) -> bytes: + return self._stream.read(limit) + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + +def _payload(**overrides) -> MdkBathymetryAcquireRequest: + values = { + "bbox": VectorSelectionBBox(min_x=2.5, min_y=51.3, max_x=2.6, max_y=51.4), + "force_refresh": True, + } + values.update(overrides) + return MdkBathymetryAcquireRequest(**values) + + +def _settings(**overrides) -> Settings: + values = { + "mdk_bathymetry_acquisition_enabled": True, + "mdk_bathymetry_coverage_id": "depth_model_20m_lat", + } + values.update(overrides) + return Settings(**values) + + +def test_acquisition_fails_closed_when_disabled() -> None: + settings = _settings(mdk_bathymetry_acquisition_enabled=False) + + with pytest.raises(Exception) as exc_info: + MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings) + + assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_ACQUISITION_DISABLED" + + +def test_acquisition_fails_closed_without_coverage_id() -> None: + settings = _settings(mdk_bathymetry_coverage_id=None) + + with pytest.raises(Exception) as exc_info: + MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings) + + assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED" + + +def test_acquisition_rejects_oversized_bbox() -> None: + settings = _settings(mdk_bathymetry_max_bbox_deg2=0.001) + + with pytest.raises(Exception) as exc_info: + MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings) + + assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_BBOX_TOO_LARGE" + + +def test_acquisition_requires_reachable_probe() -> None: + settings = _settings() + + def failing_opener(request, timeout=None): + raise OSError("connection refused") + + with pytest.raises(Exception) as exc_info: + MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=failing_opener) + + assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_ENDPOINT_NOT_READY" + + +def test_acquisition_requires_advertised_coverage_id() -> None: + settings = _settings(mdk_bathymetry_coverage_id="not_advertised_coverage") + + def opener(request, timeout=None): + return FakeResponse(CAPABILITIES_XML) + + with pytest.raises(Exception) as exc_info: + MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=opener) + + assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED" + + +def test_acquisition_rejects_non_geotiff_coverage_response() -> None: + settings = _settings() + responses = [] + + def opener(request, timeout=None): + url = request.full_url if hasattr(request, "full_url") else str(request) + responses.append(url) + if "GetCapabilities" in url: + return FakeResponse(CAPABILITIES_XML) + return FakeResponse(b"boom", "application/xml") + + with pytest.raises(Exception) as exc_info: + MdkBathymetryAcquisitionService.acquire(None, uuid4(), _payload(), settings=settings, opener=opener) + + assert getattr(exc_info.value, "code", None) == "MDK_BATHYMETRY_INVALID_RESPONSE" + assert any("GetCoverage" in url for url in responses) + coverage_urls = [url for url in responses if "GetCoverage" in url] + assert "coverage=depth_model_20m_lat" in coverage_urls[0] + assert "format=GeoTIFF" in coverage_urls[0] + + +def test_get_coverage_url_is_bounded_and_pinned() -> None: + settings = _settings() + bbox = [2.5, 51.3, 2.6, 51.4] + + url = MdkBathymetryAcquisitionService._get_coverage_url(settings, "depth_model_20m_lat", bbox) + + assert url.startswith("https://") + assert "request=GetCoverage" in url + assert "version=1.0.0" in url + assert "crs=EPSG%3A4326" in url or "crs=EPSG:4326" in url + width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox) + assert 1 <= width <= MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE + assert 1 <= height <= MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE + + +def test_source_module_never_disables_tls_verification() -> None: + source = ( + Path(__file__).resolve().parents[1] / "app" / "services" / "mdk_bathymetry_acquisition_service.py" + ).read_text(encoding="utf-8") + + assert "_create_unverified_context" not in source + assert "CERT_NONE" not in source + assert "check_hostname = False" not in source diff --git a/backend/tests/test_model_asset_catalog.py b/backend/tests/test_model_asset_catalog.py index d92b5a9a..a5c7be7f 100644 --- a/backend/tests/test_model_asset_catalog.py +++ b/backend/tests/test_model_asset_catalog.py @@ -119,7 +119,7 @@ def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) - assert asset.size_bytes == len(b"local model") assert len(asset.sha256) == 64 assert asset.active is True - assert asset.status == "available" + assert asset.status == "approved" assert asset.will_download_models is False @@ -134,6 +134,25 @@ def test_model_asset_catalog_resolves_known_asset(tmp_path: Path) -> None: assert asset.model_path == str(model_file) +def test_model_asset_catalog_only_exposes_explicit_active_asset_in_runtime(tmp_path: Path) -> None: + active_file = tmp_path / "approved-building-detector.pt" + active_file.write_bytes(b"approved") + (tmp_path / "training-smoke.pt").write_bytes(b"experiment") + (tmp_path / "partial-checkpoint.pt").write_bytes(b"partial") + settings = Settings( + yolo_models_dir=str(tmp_path), + yolo_model_path=str(active_file), + yolo_enabled=True, + ) + + response = ModelAssetCatalogService.list_assets(settings=settings) + + assert response.total == 1 + assert response.items[0].filename == active_file.name + assert response.items[0].active is True + assert response.items[0].status == "approved" + + def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None: settings = Settings(yolo_models_dir=str(tmp_path), yolo_enabled=True) diff --git a/backend/tests/test_post_rc_regional_official_vector.py b/backend/tests/test_post_rc_regional_official_vector.py index c1fdd9c9..945e07b3 100644 --- a/backend/tests/test_post_rc_regional_official_vector.py +++ b/backend/tests/test_post_rc_regional_official_vector.py @@ -119,6 +119,72 @@ def test_regional_product_registry_is_explicit_and_source_specific() -> None: assert products["urbis_buildings"]["coverage_zones"] == ["brussels"] assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0." assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"] + # urbis_street_axes is live-validated against the UrbIS WFS capabilities: + # urbisvector:StreetAxes exposes INSPIRE_ID and LineString geometry. The + # same capabilities document advertises no hydrography feature type, so + # Brussels surface water intentionally stays not_configured. + assert products["urbis_street_axes"]["coverage_zones"] == ["brussels"] + assert products["urbis_street_axes"]["collection"] == "urbisvector:StreetAxes" + assert products["urbis_street_axes"]["geometry_types"] == [ + "LineString", + "MultiLineString", + ] + assert products["urbis_street_axes"]["theme"] == "roads" + assert products["urbis_land_cover_blocks"]["collection"] == "urbisvector:Blocks" + assert products["urbis_land_cover_blocks"]["theme"] == "space_occupation" + assert products["urbis_forest_parks"]["theme"] == "forest" + assert products["urbis_water_surfaces"]["theme"] == "water" + + +def test_urbis_land_cover_products_filter_only_documented_block_classes() -> None: + scope_wgs84 = Polygon( + [(4.35, 50.84), (4.36, 50.84), (4.36, 50.85), (4.35, 50.85), (4.35, 50.84)] + ) + scope_metric = Polygon([_TO_LAMBERT72.transform(x, y) for x, y in scope_wgs84.exterior.coords]) + min_x, min_y, max_x, max_y = scope_metric.bounds + + def block(block_type: str): + return { + "type": "Feature", + "id": f"Blocks.{block_type}", + "geometry": { + "type": "Polygon", + "coordinates": [[ + [min_x + 10, min_y + 10], + [min_x + 100, min_y + 10], + [min_x + 100, min_y + 100], + [min_x + 10, min_y + 100], + [min_x + 10, min_y + 10], + ]], + }, + "properties": { + "INSPIRE_ID": f"https://databrussels.be/id/block/{block_type}", + "TYPE": block_type, + }, + } + + forest_product = OfficialVectorAcquisitionService._product("urbis_forest_parks") + water_product = OfficialVectorAcquisitionService._product("urbis_water_surfaces") + land_cover_product = OfficialVectorAcquisitionService._product("urbis_land_cover_blocks") + + assert OfficialVectorAcquisitionService._normalize_regional_feature( + forest_product, block("FO"), scope_metric, "brussels" + ) is not None + assert OfficialVectorAcquisitionService._normalize_regional_feature( + forest_product, block("CB"), scope_metric, "brussels" + ) is None + assert OfficialVectorAcquisitionService._normalize_regional_feature( + water_product, block("WB"), scope_metric, "brussels" + ) is not None + assert OfficialVectorAcquisitionService._normalize_regional_feature( + water_product, block("GB"), scope_metric, "brussels" + ) is None + normalized = OfficialVectorAcquisitionService._normalize_regional_feature( + land_cover_product, block("CB"), scope_metric, "brussels" + ) + assert normalized is not None + assert normalized["properties"]["TYPE"] == "CB" + assert normalized["properties"]["clipped_area_ha"] > 0 def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None: diff --git a/backend/tests/test_rc4_national_coverage.py b/backend/tests/test_rc4_national_coverage.py index d06420c0..9701bbd6 100644 --- a/backend/tests/test_rc4_national_coverage.py +++ b/backend/tests/test_rc4_national_coverage.py @@ -404,8 +404,8 @@ def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbo assert "Belgium and North Sea Workbench" in focus assert "nationalProject" in workspace_hook - assert "data.areas.length > 0" in workspace_hook - assert "dataset.status === 'ready'" in workspace_hook + assert "return nationalProject.id" in workspace_hook + assert "NATIONAL_WORKSPACE_REGION" in workspace_hook assert "externalApi.resolveCoverage" in coverage_hook assert "coverage.outside_supported_scope" in map_workspace assert "coverageStatusLabel" in map_workspace diff --git a/backend/tests/test_run_state_consistency.py b/backend/tests/test_run_state_consistency.py new file mode 100644 index 00000000..7b14e2e8 --- /dev/null +++ b/backend/tests/test_run_state_consistency.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from app.core.config import Settings +from app.core.errors import AppError +from app.models import AnalysisRun, Dataset, Job, Project +from app.services.detection_service import DetectionService +from app.services.job_service import JobService +from app.services.segmentation_service import SegmentationService + + +class FakeSession: + """Minimal session double without rollback support, mirroring existing test doubles.""" + + def __init__(self, objects=None) -> None: + self.objects = objects or {} + self.added = [] + self.commits = 0 + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def add(self, item) -> None: + self.added.append(item) + if getattr(item, "id", None) is not None: + self.objects[(item.__class__, item.id)] = item + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + pass + + +def _project_and_dataset(): + project_id = uuid4() + dataset_id = uuid4() + project = Project(id=project_id, name="Mol") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="ortho.tif", + dataset_type="raster", + source="user_upload", + storage_path="storage/uploads/ortho.tif", + ) + db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) + return db, project_id, dataset_id + + +def _statuses(db: FakeSession) -> tuple[list[str], list[str]]: + runs = [item.status for item in db.added if isinstance(item, AnalysisRun)] + jobs = [item.status for item in db.added if isinstance(item, Job)] + return runs, jobs + + +def test_invalid_fixture_detections_mark_run_and_job_failed() -> None: + db, project_id, dataset_id = _project_and_dataset() + + with pytest.raises(AppError) as exc_info: + DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="manual-fixture-detector", + confidence_threshold=0.5, + parameters_json={"fixture_mode": True, "fixture_detections": "not-a-list"}, + settings=Settings(_env_file=None), + ) + + assert exc_info.value.code == "INVALID_FIXTURE_DETECTIONS" + run_statuses, job_statuses = _statuses(db) + assert run_statuses and all(status == "failed" for status in run_statuses) + assert job_statuses and all(status == "failed" for status in job_statuses) + + +def test_invalid_fixture_segmentations_mark_run_and_job_failed() -> None: + db, project_id, dataset_id = _project_and_dataset() + + with pytest.raises(AppError) as exc_info: + SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="fixture-segmenter", + confidence_threshold=0.5, + parameters_json={"fixture_mode": True, "fixture_segmentations": "not-a-list"}, + settings=Settings(_env_file=None), + ) + + assert exc_info.value.code == "INVALID_FIXTURE_SEGMENTATIONS" + run_statuses, job_statuses = _statuses(db) + assert run_statuses and all(status == "failed" for status in run_statuses) + assert job_statuses and all(status == "failed" for status in job_statuses) + + +def test_unexpected_error_in_sync_job_marks_job_failed() -> None: + project_id = uuid4() + db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Mol")}) + + def exploding_operation(): + raise RuntimeError("unexpected internal failure") + + with pytest.raises(RuntimeError): + JobService.run_sync_job( + db=db, + project_id=project_id, + job_type="test.unexpected", + parameters={}, + operation=exploding_operation, + ) + + jobs = [item for item in db.added if isinstance(item, Job)] + assert jobs + final_job = jobs[-1] + assert final_job.status == "failed" + assert "Unexpected internal error" in (final_job.error_message or "") + + +def test_app_error_in_sync_job_still_marks_job_failed() -> None: + project_id = uuid4() + db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Mol")}) + + def failing_operation(): + raise AppError(code="SOME_DOMAIN_ERROR", message="Bounded failure", status_code=422) + + with pytest.raises(AppError): + JobService.run_sync_job( + db=db, + project_id=project_id, + job_type="test.bounded", + parameters={}, + operation=failing_operation, + ) + + jobs = [item for item in db.added if isinstance(item, Job)] + assert jobs + assert jobs[-1].status == "failed" + assert jobs[-1].error_message == "Bounded failure" diff --git a/backend/tests/test_segmentation_configured_models.py b/backend/tests/test_segmentation_configured_models.py new file mode 100644 index 00000000..0073b865 --- /dev/null +++ b/backend/tests/test_segmentation_configured_models.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +import json +from pathlib import Path +from uuid import uuid4 + +import pytest +from geoalchemy2.shape import to_shape + +from app.core.config import Settings +from app.models import Dataset, Project, Segmentation +from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon +from app.services.model_registry_service import ModelRegistryService +from app.services.segmentation_service import SegmentationService + +ROOT = Path(__file__).resolve().parents[2] + + +class FakeSession: + def __init__(self, objects=None) -> None: + self.objects = objects or {} + self.added = [] + self.commits = 0 + self.refreshes = [] + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def add(self, item) -> None: + self.added.append(item) + if getattr(item, "id", None) is not None: + self.objects[(item.__class__, item.id)] = item + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + self.refreshes.append(item) + + +class AvailableSegAdapter: + def __init__(self, settings: Settings) -> None: + self.settings = settings + + @staticmethod + def dependencies_available() -> bool: + return True + + def load_model(self, model_path: Path): + return object() + + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]: + return [ + { + "class_name": "building", + "confidence": 0.91, + "points": [[10.0, 20.0], [30.0, 20.0], [30.0, 40.0], [10.0, 40.0]], + "bbox": [10.0, 20.0, 30.0, 40.0], + "properties": {"class_id": 0}, + } + ] + + +class ClassAgnosticSamAdapter(AvailableSegAdapter): + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]: + return [ + { + "class_name": "segment", + "confidence": None, + "points": [[5.0, 5.0], [25.0, 5.0], [25.0, 25.0], [5.0, 25.0]], + "bbox": [5.0, 5.0, 25.0, 25.0], + "properties": {"class_id": -1}, + } + ] + + +class MissingDependencySegAdapter(AvailableSegAdapter): + @staticmethod + def dependencies_available() -> bool: + return False + + +def _project_and_dataset(dataset_type: str = "raster"): + project_id = uuid4() + dataset_id = uuid4() + project = Project(id=project_id, name="Mol") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="ortho.tif", + dataset_type=dataset_type, + source="user_upload", + storage_path="storage/uploads/ortho.tif", + ) + db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) + return db, project_id, dataset_id + + +def _settings(tmp_path: Path, **overrides) -> Settings: + values = { + "yolo_seg_enabled": True, + "yolo_seg_model_path": str(tmp_path / "seg.pt"), + "sam_enabled": True, + "sam_model_path": str(tmp_path / "sam.pt"), + "yolo_max_tiles": 4, + } + values.update(overrides) + return Settings(**values) + + +def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: + tiles = [] + for index in range(tile_count): + tile_path = tmp_path / f"tile_{index:04d}.tif" + tile_path.write_bytes(b"fixture") + tiles.append( + { + "path": str(tile_path), + "pixel_window": [0, 0, 100, 100], + "bounds": [4.0, 51.0, 5.0, 52.0], + "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], + "index": index, + } + ) + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "tile_set_id": "tiles-fixture", + "source_dataset_id": str(uuid4()), + "source_raster_id": str(uuid4()), + "tile_size": 100, + "overlap": 0, + "count": tile_count, + "tiles": tiles, + } + ), + encoding="utf-8", + ) + return manifest_path + + +def test_segmentation_models_report_not_configured_when_disabled(tmp_path: Path) -> None: + settings = _settings(tmp_path, yolo_seg_enabled=False, sam_enabled=False) + + models = { + model.model_id: model + for model in ModelRegistryService.list_segmentation_model_capabilities(settings=settings) + } + + assert models["yolo-seg-configured"].configured is False + assert models["yolo-seg-configured"].status == "not_configured" + assert models["sam-configured"].configured is False + assert models["sam-configured"].status == "not_configured" + + +def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> None: + (tmp_path / "seg.pt").write_bytes(b"weights") + (tmp_path / "sam.pt").write_bytes(b"weights") + settings = _settings(tmp_path) + + models = { + model.model_id: model + for model in ModelRegistryService.list_segmentation_model_capabilities( + settings=settings, + yolo_seg_adapter_class=MissingDependencySegAdapter, + sam_adapter_class=MissingDependencySegAdapter, + ) + } + + assert models["yolo-seg-configured"].status == "dependency_unavailable" + assert models["sam-configured"].status == "dependency_unavailable" + + +def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None: + (tmp_path / "seg.pt").write_bytes(b"weights") + (tmp_path / "sam.pt").write_bytes(b"weights") + settings = _settings(tmp_path) + + models = { + model.model_id: model + for model in ModelRegistryService.list_segmentation_model_capabilities( + settings=settings, + yolo_seg_adapter_class=AvailableSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + } + + assert models["yolo-seg-configured"].configured is True + assert models["yolo-seg-configured"].status == "configured" + assert models["sam-configured"].configured is True + assert models["sam-configured"].status == "configured" + + +def test_segmentation_dependency_check_uses_real_imports_not_find_spec() -> None: + source = (ROOT / "backend" / "app" / "services" / "segmentation_adapter.py").read_text(encoding="utf-8") + + assert 'find_spec("ultralytics")' not in source + assert "import ultralytics" in source + assert "import torch" in source + + +def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None: + (tmp_path / "seg.pt").write_bytes(b"weights") + db, project_id, dataset_id = _project_and_dataset() + settings = _settings(tmp_path) + + with pytest.raises(Exception) as exc_info: + SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-seg-configured", + confidence_threshold=0.5, + settings=settings, + yolo_seg_adapter_class=AvailableSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + + assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED" + + +def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None: + (tmp_path / "seg.pt").write_bytes(b"weights") + db, project_id, dataset_id = _project_and_dataset() + settings = _settings(tmp_path) + manifest_path = _manifest(tmp_path) + + response = SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-seg-configured", + confidence_threshold=0.5, + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_seg_adapter_class=AvailableSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + + assert response.status == "success" + assert response.segmentation_count == 1 + persisted = [item for item in db.added if isinstance(item, Segmentation)] + assert len(persisted) == 1 + segmentation = persisted[0] + assert segmentation.class_name == "building" + assert segmentation.confidence == pytest.approx(0.91) + geometry = to_shape(segmentation.geometry) + assert geometry.geom_type == "MultiPolygon" + min_x, min_y, max_x, max_y = geometry.bounds + assert 4.0 <= min_x <= 5.0 + assert 51.0 <= min_y <= 52.0 + assert max_x <= 5.0 + assert max_y <= 52.0 + assert segmentation.area_m2 is not None and segmentation.area_m2 > 0 + assert segmentation.provenance_json["inference"] == "local" + assert segmentation.provenance_json["model_id"] == "yolo-seg-configured" + + +def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None: + (tmp_path / "sam.pt").write_bytes(b"weights") + db, project_id, dataset_id = _project_and_dataset() + settings = _settings(tmp_path) + manifest_path = _manifest(tmp_path) + + response = SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="sam-configured", + confidence_threshold=0.5, + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_seg_adapter_class=AvailableSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + + assert response.status == "success" + assert response.segmentation_count == 1 + persisted = [item for item in db.added if isinstance(item, Segmentation)] + assert persisted[0].class_name == "segment" + assert persisted[0].confidence is None + + +def test_unconfigured_segmentation_run_fails_closed(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + settings = _settings(tmp_path, yolo_seg_enabled=False) + manifest_path = _manifest(tmp_path) + + response = SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-seg-configured", + confidence_threshold=0.5, + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_seg_adapter_class=AvailableSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + + assert response.status == "failed" + assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE" + assert not [item for item in db.added if isinstance(item, Segmentation)] + + +def test_pixel_points_to_epsg4326_polygon_uses_tile_transform() -> None: + tile = { + "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], + "bounds": [4.0, 51.0, 5.0, 52.0], + "pixel_window": [0, 0, 100, 100], + } + + polygon = pixel_points_to_epsg4326_polygon( + points=[[0.0, 0.0], [100.0, 0.0], [100.0, 100.0], [0.0, 100.0]], + tile=tile, + crs="EPSG:4326", + ) + + min_x, min_y, max_x, max_y = polygon.bounds + assert min_x == pytest.approx(4.0) + assert max_x == pytest.approx(5.0) + assert min_y == pytest.approx(51.0) + assert max_y == pytest.approx(52.0) + + +def test_pixel_points_to_epsg4326_polygon_rejects_degenerate_input() -> None: + tile = {"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01]} + + with pytest.raises(Exception) as exc_info: + pixel_points_to_epsg4326_polygon(points=[[0.0, 0.0], [1.0, 1.0]], tile=tile) + + assert getattr(exc_info.value, "code", None) == "SEGMENTATION_INVALID_MASK" diff --git a/backend/tests/test_sprint177_mol_primary_focus.py b/backend/tests/test_sprint177_mol_primary_focus.py index 260e2892..124abf9d 100644 --- a/backend/tests/test_sprint177_mol_primary_focus.py +++ b/backend/tests/test_sprint177_mol_primary_focus.py @@ -4,7 +4,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -def test_frontend_declares_mol_as_primary_operating_focus() -> None: +def test_frontend_declares_national_scope_as_primary_operating_focus() -> None: focus = (ROOT / "frontend" / "src" / "config" / "primaryFocus.ts").read_text( encoding="utf-8" ) @@ -23,19 +23,17 @@ def test_frontend_declares_mol_as_primary_operating_focus() -> None: / "WorkbenchNavigation.tsx" ).read_text(encoding="utf-8") - assert "PRIMARY_FOCUS_LABEL = 'Mol'" in focus - assert "PRIMARY_FOCUS_REGION = 'Mol, Kempen'" in focus - assert "[5.1167, 51.1919]" in focus - assert "isPrimaryFocusProjectData" in focus - assert "isPrimaryFocusProjectData(project, data.datasets)" in project_hook + assert "NATIONAL_WORKSPACE_PROJECT_NAME = 'Belgium and North Sea Workbench'" in focus + assert "NATIONAL_WORKSPACE_REGION = 'Belgie en Belgische Noordzee'" in focus + assert "NATIONAL_MAP_CENTER" in focus + assert "return nationalProject.id" in project_hook assert "hasMappedAnalysisContext(data)" in project_hook assert "dataset.dataset_type === 'raster'" in project_hook assert "dataset.dataset_type === 'vector' || dataset.dataset_type === 'geojson'" in project_hook - assert "const primaryContext = inspectedCandidates.find" in project_hook - assert "PRIMARY_FOCUS_AREA_NAME" in project_hook - assert "PRIMARY_FOCUS_AREA_GEOJSON" in project_hook - assert "4.35,51.28" not in project_hook - assert "center: PRIMARY_FOCUS_CENTER" in map_source + assert "PRIMARY_FOCUS_AREA_NAME" not in project_hook + assert "PRIMARY_FOCUS_AREA_GEOJSON" not in project_hook + assert "center: NATIONAL_MAP_CENTER" in map_source + assert "zoom: NATIONAL_MAP_ZOOM" in map_source assert "GeoIntel" in navigation assert "Atlas Workbench" in navigation diff --git a/backend/tests/test_sprint181_mol_municipality_workspace.py b/backend/tests/test_sprint181_mol_municipality_workspace.py index 1220be59..c87cee37 100644 --- a/backend/tests/test_sprint181_mol_municipality_workspace.py +++ b/backend/tests/test_sprint181_mol_municipality_workspace.py @@ -142,7 +142,7 @@ def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> assert db.refreshes == 0 -def test_municipality_workspace_is_wired_into_runtime_and_frontend_priority() -> None: +def test_municipality_workspace_remains_a_regression_fixture_without_frontend_priority() -> None: readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") focus = (ROOT / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8") @@ -154,7 +154,8 @@ def test_municipality_workspace_is_wired_into_runtime_and_frontend_priority() -> assert "py_compile scripts/provision_mol_municipality_workspace.py" in readiness assert "COPY scripts/provision_mol_municipality_workspace.py" in dockerfile assert "PRIMARY_FOCUS_MUNICIPALITY_PROJECT_NAME = 'Mol Municipality Workbench'" in focus - assert "items.find(isPrimaryFocusMunicipalityProject)" in project_hook + assert "items.find(isPrimaryFocusMunicipalityProject)" not in project_hook + assert "return nationalProject.id" in project_hook assert "datasets.find(isPrimaryFocusMunicipalityBoundaryDataset)" in dataset_hook assert "featureCollectionBounds(featureCollection)" in map_source assert "useMemo(() => getFeatureCollectionBBox(mapFeatureCollection)" in map_workspace diff --git a/backend/tests/test_sprint193_end_user_workbench.py b/backend/tests/test_sprint193_end_user_workbench.py index 82d3dac6..72b05234 100644 --- a/backend/tests/test_sprint193_end_user_workbench.py +++ b/backend/tests/test_sprint193_end_user_workbench.py @@ -8,13 +8,15 @@ def read(path: str) -> str: return (ROOT / path).read_text(encoding="utf-8") -def test_regional_workspace_is_automatic_and_map_has_one_scope_selector() -> None: +def test_national_workspace_is_automatic_and_map_has_one_scope_selector() -> None: project_hook = read("frontend/src/hooks/useProjectWorkspace.ts") map_workspace = read("frontend/src/components/map/MapWorkspace.tsx") + national_check = project_hook.index("const nationalProject") regional_check = project_hook.index("const regionalProject") - municipality_check = project_hook.index("const municipalityProject") - assert regional_check < municipality_check + assert national_check < regional_check + assert "return nationalProject.id" in project_hook + assert "const municipalityProject" not in project_hook assert 'aria-label="Regio"' not in map_workspace assert 'aria-label="Ingeladen regiobereik"' in map_workspace assert "Snel naar een gemeente (optioneel)" in map_workspace @@ -56,7 +58,8 @@ def test_configured_yolo_and_active_asset_are_selected_without_hiding_limitation assert "asset.active" in hook assert "getYoloPreflight" in hook assert 'aria-label="Status gebouwdetectie"' in lab - assert "resultaten blijven controleplichtig" in lab + assert "Nog niet nationaal gevalideerd" in lab + assert "vereisen lokale referentiedata en QA" in lab assert "Modelkalibratie voor beheerders" in lab diff --git a/backend/tests/test_sprint194_regional_timeseries.py b/backend/tests/test_sprint194_regional_timeseries.py index c6e44fe7..a0caf8bc 100644 --- a/backend/tests/test_sprint194_regional_timeseries.py +++ b/backend/tests/test_sprint194_regional_timeseries.py @@ -249,7 +249,7 @@ def test_end_user_dataset_sources_are_human_readable() -> None: assert "department_omgeving_land_use: 'Departement Omgeving'" in display assert "statbel: 'Statbel'" in display assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace - assert "resultDataset ? getDatasetSourceDisplayName(resultDataset)" in workspace + assert "getDatasetSourceDisplayName(resultDataset)" in workspace assert "Snel naar een gemeente (optioneel)" in workspace assert "latestDatasetBySeries" in catalog assert "Historische meetmomenten" in catalog diff --git a/backend/tests/test_sprint235_bathymetry_profiles.py b/backend/tests/test_sprint235_bathymetry_profiles.py index 93979132..1455a959 100644 --- a/backend/tests/test_sprint235_bathymetry_profiles.py +++ b/backend/tests/test_sprint235_bathymetry_profiles.py @@ -174,7 +174,10 @@ def test_bathymetry_source_registry_is_honest_and_nationally_extensible() -> Non assert by_key["vha_inland_profiles"]["integration_status"] == "operational" assert by_key["vha_inland_profiles"]["acquisition_supported"] is True assert by_key["mdk_bcp_bathymetry"]["vertical_reference"] == "LAT" - assert by_key["mdk_bcp_bathymetry"]["acquisition_supported"] is False + # Bounded MDK acquisition now exists but stays fail-closed until the + # operator enables it explicitly with a live-validated coverage id. + assert by_key["mdk_bcp_bathymetry"]["acquisition_supported"] is True + assert by_key["mdk_bcp_bathymetry"]["configured"] is False assert by_key["spw_walloon_waterway_bathymetry"]["vertical_reference"] == "mDNG" assert by_key["spw_walloon_waterway_bathymetry"]["license_note"].startswith("CC BY 4.0") diff --git a/backend/tests/test_sprint236_bathymetry_expansion.py b/backend/tests/test_sprint236_bathymetry_expansion.py index b4f6f64e..a856e283 100644 --- a/backend/tests/test_sprint236_bathymetry_expansion.py +++ b/backend/tests/test_sprint236_bathymetry_expansion.py @@ -411,7 +411,9 @@ def test_expansion_scripts_are_packaged_and_readiness_checked() -> None: for item in BathymetryProfileAcquisitionService.list_sources() } assert sources["mdk_bcp_bathymetry"]["integration_status"] == "probe_only" - assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is False + # Bounded acquisition is implemented but remains disabled by default. + assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is True + assert sources["mdk_bcp_bathymetry"]["configured"] is False assert "EL_wcs" in sources["mdk_bcp_bathymetry"]["service_url"] diff --git a/backend/tests/test_sprint240_official_flemish_themes.py b/backend/tests/test_sprint240_official_flemish_themes.py index 6ce0dc85..1fedc81d 100644 --- a/backend/tests/test_sprint240_official_flemish_themes.py +++ b/backend/tests/test_sprint240_official_flemish_themes.py @@ -134,6 +134,10 @@ def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() - "spw_picc_water_surfaces", "urbis_buildings", "urbis_cadastral_parcels", + "urbis_street_axes", + "urbis_land_cover_blocks", + "urbis_forest_parks", + "urbis_water_surfaces", } == set(vector) assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative" assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline" @@ -401,7 +405,7 @@ def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypa assert products_response.status_code == 200 assert set(products_response.json()) == {"data"} - assert products_response.json()["data"]["total"] == 8 + assert products_response.json()["data"]["total"] == 12 assert acquire_response.status_code == 200 assert set(acquire_response.json()) == {"data"} assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire" diff --git a/deploy/unraid/geointel-unraid-template.xml b/deploy/unraid/geointel-unraid-template.xml index 6f7d72cf..a82ae6e3 100644 --- a/deploy/unraid/geointel-unraid-template.xml +++ b/deploy/unraid/geointel-unraid-template.xml @@ -90,6 +90,12 @@ https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs 20 4 + false + + EPSG:4326 + 0.25 + 120 + 160 true https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs 100 @@ -110,6 +116,18 @@ 1000 0.5 1 + false + + yolo-seg-configured + Configured YOLO segmentation + + false + + sam-configured + Configured SAM segmentation + + 300 + 0.5 true http://host.docker.internal:11434 qwen3.5:9b diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index 0b6d561f..2b018c8b 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -94,6 +94,15 @@ MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/servic MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20 MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4 +# Bounded MDK acquisition stays fail-closed until the readiness probe reports +# "reachable" and an advertised coverage id is configured explicitly. +MDK_BATHYMETRY_ACQUISITION_ENABLED=false +MDK_BATHYMETRY_COVERAGE_ID= +MDK_BATHYMETRY_REQUEST_CRS=EPSG:4326 +MDK_BATHYMETRY_MAX_BBOX_DEG2=0.25 +MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS=120 +MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB=160 + # Allowlisted Departement Omgeving policy rasters. Regional requests are # transferred as fixed 10 km WCS tiles before exact Area clipping. THEMATIC_RASTER_ENABLED=true @@ -120,6 +129,21 @@ YOLO_MAX_DETECTIONS=1000 YOLO_DUPLICATE_IOU_THRESHOLD=0.5 YOLO_BATCH_SIZE=1 +# Local segmentation models. GeoIntel never downloads model weights +# automatically; point these to existing local files to enable inference. +YOLO_SEG_ENABLED=false +YOLO_SEG_MODEL_PATH= +YOLO_SEG_MODEL_ID=yolo-seg-configured +YOLO_SEG_MODEL_DISPLAY_NAME=Configured YOLO segmentation +YOLO_SEG_MODEL_VERSION= +SAM_ENABLED=false +SAM_MODEL_PATH= +SAM_MODEL_ID=sam-configured +SAM_MODEL_DISPLAY_NAME=Configured SAM segmentation +SAM_MODEL_VERSION= +SEGMENTATION_MAX_MASKS_PER_TILE=300 +SEGMENTATION_DUPLICATE_IOU_THRESHOLD=0.5 + # Local Ollama assistant. The all-in-one container reaches the Unraid host # through Docker's host-gateway mapping; no Ollama port is exposed by GeoIntel. OLLAMA_ENABLED=true diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index 26e05f16..16bd27a7 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -80,6 +80,12 @@ MDK_BATHYMETRY_PROBE_ENABLED="${MDK_BATHYMETRY_PROBE_ENABLED:-true}" MDK_BATHYMETRY_WCS_URL="${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs}" MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20}" MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4}" +MDK_BATHYMETRY_ACQUISITION_ENABLED="${MDK_BATHYMETRY_ACQUISITION_ENABLED:-false}" +MDK_BATHYMETRY_COVERAGE_ID="${MDK_BATHYMETRY_COVERAGE_ID:-}" +MDK_BATHYMETRY_REQUEST_CRS="${MDK_BATHYMETRY_REQUEST_CRS:-EPSG:4326}" +MDK_BATHYMETRY_MAX_BBOX_DEG2="${MDK_BATHYMETRY_MAX_BBOX_DEG2:-0.25}" +MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS="${MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS:-120}" +MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB="${MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB:-160}" THEMATIC_RASTER_ENABLED="${THEMATIC_RASTER_ENABLED:-true}" THEMATIC_RASTER_WCS_URL="${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}" THEMATIC_RASTER_MIN_SIDE_M="${THEMATIC_RASTER_MIN_SIDE_M:-100}" @@ -100,6 +106,18 @@ YOLO_MAX_TILES="${YOLO_MAX_TILES:-100}" YOLO_MAX_DETECTIONS="${YOLO_MAX_DETECTIONS:-1000}" YOLO_DUPLICATE_IOU_THRESHOLD="${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5}" YOLO_BATCH_SIZE="${YOLO_BATCH_SIZE:-1}" +YOLO_SEG_ENABLED="${YOLO_SEG_ENABLED:-false}" +YOLO_SEG_MODEL_PATH="${YOLO_SEG_MODEL_PATH:-}" +YOLO_SEG_MODEL_ID="${YOLO_SEG_MODEL_ID:-yolo-seg-configured}" +YOLO_SEG_MODEL_DISPLAY_NAME="${YOLO_SEG_MODEL_DISPLAY_NAME:-Configured YOLO segmentation}" +YOLO_SEG_MODEL_VERSION="${YOLO_SEG_MODEL_VERSION:-}" +SAM_ENABLED="${SAM_ENABLED:-false}" +SAM_MODEL_PATH="${SAM_MODEL_PATH:-}" +SAM_MODEL_ID="${SAM_MODEL_ID:-sam-configured}" +SAM_MODEL_DISPLAY_NAME="${SAM_MODEL_DISPLAY_NAME:-Configured SAM segmentation}" +SAM_MODEL_VERSION="${SAM_MODEL_VERSION:-}" +SEGMENTATION_MAX_MASKS_PER_TILE="${SEGMENTATION_MAX_MASKS_PER_TILE:-300}" +SEGMENTATION_DUPLICATE_IOU_THRESHOLD="${SEGMENTATION_DUPLICATE_IOU_THRESHOLD:-0.5}" OLLAMA_ENABLED="${OLLAMA_ENABLED:-true}" OLLAMA_BASE_URL="${OLLAMA_BASE_URL:-http://host.docker.internal:11434}" OLLAMA_DEFAULT_MODEL="${OLLAMA_DEFAULT_MODEL:-qwen3.5:9b}" @@ -248,6 +266,12 @@ docker run -d \ -e MDK_BATHYMETRY_WCS_URL="$MDK_BATHYMETRY_WCS_URL" \ -e MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="$MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS" \ -e MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="$MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB" \ + -e MDK_BATHYMETRY_ACQUISITION_ENABLED="$MDK_BATHYMETRY_ACQUISITION_ENABLED" \ + -e MDK_BATHYMETRY_COVERAGE_ID="$MDK_BATHYMETRY_COVERAGE_ID" \ + -e MDK_BATHYMETRY_REQUEST_CRS="$MDK_BATHYMETRY_REQUEST_CRS" \ + -e MDK_BATHYMETRY_MAX_BBOX_DEG2="$MDK_BATHYMETRY_MAX_BBOX_DEG2" \ + -e MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS="$MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS" \ + -e MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB="$MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB" \ -e THEMATIC_RASTER_ENABLED="$THEMATIC_RASTER_ENABLED" \ -e THEMATIC_RASTER_WCS_URL="$THEMATIC_RASTER_WCS_URL" \ -e THEMATIC_RASTER_MIN_SIDE_M="$THEMATIC_RASTER_MIN_SIDE_M" \ @@ -268,6 +292,18 @@ docker run -d \ -e YOLO_MAX_DETECTIONS="$YOLO_MAX_DETECTIONS" \ -e YOLO_DUPLICATE_IOU_THRESHOLD="$YOLO_DUPLICATE_IOU_THRESHOLD" \ -e YOLO_BATCH_SIZE="$YOLO_BATCH_SIZE" \ + -e YOLO_SEG_ENABLED="$YOLO_SEG_ENABLED" \ + -e YOLO_SEG_MODEL_PATH="$YOLO_SEG_MODEL_PATH" \ + -e YOLO_SEG_MODEL_ID="$YOLO_SEG_MODEL_ID" \ + -e YOLO_SEG_MODEL_DISPLAY_NAME="$YOLO_SEG_MODEL_DISPLAY_NAME" \ + -e YOLO_SEG_MODEL_VERSION="$YOLO_SEG_MODEL_VERSION" \ + -e SAM_ENABLED="$SAM_ENABLED" \ + -e SAM_MODEL_PATH="$SAM_MODEL_PATH" \ + -e SAM_MODEL_ID="$SAM_MODEL_ID" \ + -e SAM_MODEL_DISPLAY_NAME="$SAM_MODEL_DISPLAY_NAME" \ + -e SAM_MODEL_VERSION="$SAM_MODEL_VERSION" \ + -e SEGMENTATION_MAX_MASKS_PER_TILE="$SEGMENTATION_MAX_MASKS_PER_TILE" \ + -e SEGMENTATION_DUPLICATE_IOU_THRESHOLD="$SEGMENTATION_DUPLICATE_IOU_THRESHOLD" \ -e OLLAMA_ENABLED="$OLLAMA_ENABLED" \ -e OLLAMA_BASE_URL="$OLLAMA_BASE_URL" \ -e OLLAMA_DEFAULT_MODEL="$OLLAMA_DEFAULT_MODEL" \ diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index 9cf1d9f8..10d0a5e3 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -93,6 +93,24 @@ services: YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000} YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5} YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1} + YOLO_SEG_ENABLED: ${YOLO_SEG_ENABLED:-false} + YOLO_SEG_MODEL_PATH: ${YOLO_SEG_MODEL_PATH:-} + YOLO_SEG_MODEL_ID: ${YOLO_SEG_MODEL_ID:-yolo-seg-configured} + YOLO_SEG_MODEL_DISPLAY_NAME: ${YOLO_SEG_MODEL_DISPLAY_NAME:-Configured YOLO segmentation} + YOLO_SEG_MODEL_VERSION: ${YOLO_SEG_MODEL_VERSION:-} + SAM_ENABLED: ${SAM_ENABLED:-false} + SAM_MODEL_PATH: ${SAM_MODEL_PATH:-} + SAM_MODEL_ID: ${SAM_MODEL_ID:-sam-configured} + SAM_MODEL_DISPLAY_NAME: ${SAM_MODEL_DISPLAY_NAME:-Configured SAM segmentation} + SAM_MODEL_VERSION: ${SAM_MODEL_VERSION:-} + SEGMENTATION_MAX_MASKS_PER_TILE: ${SEGMENTATION_MAX_MASKS_PER_TILE:-300} + SEGMENTATION_DUPLICATE_IOU_THRESHOLD: ${SEGMENTATION_DUPLICATE_IOU_THRESHOLD:-0.5} + MDK_BATHYMETRY_ACQUISITION_ENABLED: ${MDK_BATHYMETRY_ACQUISITION_ENABLED:-false} + MDK_BATHYMETRY_COVERAGE_ID: ${MDK_BATHYMETRY_COVERAGE_ID:-} + MDK_BATHYMETRY_REQUEST_CRS: ${MDK_BATHYMETRY_REQUEST_CRS:-EPSG:4326} + MDK_BATHYMETRY_MAX_BBOX_DEG2: ${MDK_BATHYMETRY_MAX_BBOX_DEG2:-0.25} + MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS: ${MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS:-120} + MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB: ${MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB:-160} OLLAMA_ENABLED: ${OLLAMA_ENABLED:-true} OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-http://host.docker.internal:11434} OLLAMA_DEFAULT_MODEL: ${OLLAMA_DEFAULT_MODEL:-qwen3.5:9b} diff --git a/docker-compose.yml b/docker-compose.yml index 51cbdb33..afa019a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,12 @@ services: MDK_BATHYMETRY_WCS_URL: ${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs} MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS: ${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20} MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB: ${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4} + MDK_BATHYMETRY_ACQUISITION_ENABLED: ${MDK_BATHYMETRY_ACQUISITION_ENABLED:-false} + MDK_BATHYMETRY_COVERAGE_ID: ${MDK_BATHYMETRY_COVERAGE_ID:-} + MDK_BATHYMETRY_REQUEST_CRS: ${MDK_BATHYMETRY_REQUEST_CRS:-EPSG:4326} + MDK_BATHYMETRY_MAX_BBOX_DEG2: ${MDK_BATHYMETRY_MAX_BBOX_DEG2:-0.25} + MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS: ${MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS:-120} + MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB: ${MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB:-160} THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true} THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs} THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100} @@ -112,6 +118,19 @@ services: YOLO_MAX_DETECTIONS: ${YOLO_MAX_DETECTIONS:-1000} YOLO_DUPLICATE_IOU_THRESHOLD: ${YOLO_DUPLICATE_IOU_THRESHOLD:-0.5} YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1} + YOLO_SEG_ENABLED: ${YOLO_SEG_ENABLED:-false} + YOLO_SEG_MODEL_PATH: ${YOLO_SEG_MODEL_PATH:-} + YOLO_SEG_MODEL_ID: ${YOLO_SEG_MODEL_ID:-yolo-seg-configured} + YOLO_SEG_MODEL_DISPLAY_NAME: ${YOLO_SEG_MODEL_DISPLAY_NAME:-Configured YOLO segmentation} + YOLO_SEG_MODEL_VERSION: ${YOLO_SEG_MODEL_VERSION:-} + SAM_ENABLED: ${SAM_ENABLED:-false} + SAM_MODEL_PATH: ${SAM_MODEL_PATH:-} + SAM_MODEL_ID: ${SAM_MODEL_ID:-sam-configured} + SAM_MODEL_DISPLAY_NAME: ${SAM_MODEL_DISPLAY_NAME:-Configured SAM segmentation} + SAM_MODEL_VERSION: ${SAM_MODEL_VERSION:-} + SEGMENTATION_MAX_MASKS_PER_TILE: ${SEGMENTATION_MAX_MASKS_PER_TILE:-300} + SEGMENTATION_DUPLICATE_IOU_THRESHOLD: ${SEGMENTATION_DUPLICATE_IOU_THRESHOLD:-0.5} + GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP: ${GEOINTEL_RECONCILE_INTERRUPTED_RUNS_ON_STARTUP:-true} OLLAMA_ENABLED: ${OLLAMA_ENABLED:-false} OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-http://host.docker.internal:11434} OLLAMA_DEFAULT_MODEL: ${OLLAMA_DEFAULT_MODEL:-qwen3.5:9b} diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 8f8d5500..4ab13474 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -123,7 +123,7 @@ a canonical operational workspace without depending on its position among newer operator or benchmark projects: ```text -GET /api/v1/projects?name=Kempen%20Regional%20Workbench&limit=1 +GET /api/v1/projects?name=Belgium%20and%20North%20Sea%20Workbench&limit=1 GET /api/v1/projects?status=archived&limit=50 ``` @@ -135,7 +135,7 @@ Request: { "name": "Geel building detection demo", "description": "Detect buildings and validate against GRB", - "region": "Kempen" + "region": "Belgium and Belgian North Sea" } ``` @@ -1202,13 +1202,15 @@ Returns object-detection model capability descriptors. ### GET `/api/v1/detection/model-assets` -Returns local runtime model files discovered in the configured model directory. -This is a read-only catalog. GeoIntel never downloads, creates, mutates or -deletes model weights from this endpoint. +Returns governed local runtime model files. This is a read-only catalog. +GeoIntel never downloads, creates, mutates or deletes model weights from this +endpoint. -The backend scans `YOLO_MODELS_DIR` (default `/app/models`) and reports -supported local model files such as `.pt`, `.onnx` and `.engine`. The active -model is the file matching `YOLO_MODEL_PATH`. +The backend scans `YOLO_MODELS_DIR` (default `/app/models`). When +`YOLO_MODEL_PATH` resolves to an existing file, production catalog output is +restricted to that explicitly approved active model. When no active model is +configured, supported `.pt`, `.onnx` and `.engine` files remain visible for +development/operator discovery but cannot make the configured detector ready. Response data: @@ -1226,8 +1228,8 @@ Response data: "size_bytes": 123456, "sha256": "sha256hex", "active": true, - "status": "available", - "limitation_message": "Local runtime model asset. GeoIntel will not download or mutate model weights.", + "status": "approved", + "limitation_message": "Approved local runtime model asset. GeoIntel will not download or mutate model weights.", "will_download_models": false } ], @@ -2217,9 +2219,11 @@ sets an explicit Ollama context window and returns Returns the governed bathymetry source registry in the canonical envelope. VHA inland profiles are `operational`. MDK Belgian Continental Shelf is -`probe_only`. The pinned SPW Walloon bathymetry archive is `operational` -through a bounded, explicit operator import. There is no browser-side source -fetch and no arbitrary source URL. +`not_configured` by default and becomes `operational` only after the operator +explicitly enables bounded acquisition and pins a coverage identifier. The +pinned SPW Walloon bathymetry archive is `operational` through a bounded, +explicit operator import. There is no browser-side source fetch and no +arbitrary source URL. ### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness` @@ -2227,9 +2231,24 @@ Runs one bounded, read-only WCS 1.0.0 `GetCapabilities` request with mandatory system TLS verification and a configured response-size limit. Status is one of `disabled`, `invalid_configuration`, `tls_error`, `endpoint_unavailable`, `invalid_capabilities` or `reachable`. A reachable -response lists coverage identifiers, advertised formats and CRS values, but -always returns `acquisition_supported=false`. There is no insecure TLS -fallback and no `GetCoverage` request. +response lists coverage identifiers, advertised formats and CRS values. There +is no insecure TLS fallback and this readiness endpoint never performs a +`GetCoverage` request. + +### POST `/api/v1/projects/{project_id}/datasets/bathymetry/mdk/acquire` + +Runs one explicit, bounded WCS 1.0.0 `GetCoverage` acquisition as a synchronous +Job. The request contains an EPSG:4326 `bbox`, optional persisted `area_id` and +`force_refresh`. Acquisition is fail-closed unless +`MDK_BATHYMETRY_ACQUISITION_ENABLED=true`, a coverage identifier is explicitly +configured, the strict-TLS readiness probe is reachable and that identifier is +advertised by the live capabilities document. + +The configured bbox-area, response-size, timeout and pixel-dimension limits are +always enforced. A successful GeoTIFF is validated and imported through the +existing Dataset raster flow with request hash, response hash, acquisition +time, MDK attribution and the `LAT` vertical reference in provenance. No depth +values are synthesized and no water volume is inferred. ### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire` diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index e8e9f0d9..55fd4a57 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11011,3 +11011,31 @@ Validation: passed (12 tests); - live redeployment and a repeated Belgium-scale rectangle follow on the immutable patch revision. +## 2026-07-22 - National scope and governed Brussels land cover + +Implemented: + +- made the persisted Belgium/North Sea project the unconditional startup + workspace and changed the initial basemap extent and form defaults from Mol + to Belgium plus its legally labelled maritime scope; +- retained Mol/Kempen provisioning and tests strictly as golden regression + evidence instead of product routing; +- live-validated the UrbIS WFS `Blocks` contract and added bounded land-cover, + FO/GB forest/park and WB permanent-water products with clipped PostGIS area + metrics and source-class provenance; +- restricted the production model picker to the explicit active model asset + and surfaced that the current building model is locally, not nationally, + validated. + +Validated during implementation: + +- backend compile and frontend typecheck passed; +- 31 frontend unit tests passed; +- focused national coverage, model catalog, project lifecycle and Mol golden + regression tests passed; +- live resolver checks covered Brussels, Wallonia and all three Belgian + maritime legal zones. WALOUS, Walloon flood analytics and multi-epoch marine + bathymetry remain real open source-integration work and were not simulated. +- the final repository readiness gate passed with 1,084 backend tests, 31 + frontend unit tests, compile, typecheck, production build, Alembic head + `202607160001` and all script syntax/contract checks. diff --git a/docs/DATA_COVERAGE_STATUS.md b/docs/DATA_COVERAGE_STATUS.md index 57c5b373..32da1763 100644 --- a/docs/DATA_COVERAGE_STATUS.md +++ b/docs/DATA_COVERAGE_STATUS.md @@ -1,6 +1,6 @@ # GeoIntel data coverage status -Status date: 2026-07-21 +Status date: 2026-07-22 This document is the operational interpretation of the source registry. It does not replace the legal/source provenance stored with each Dataset. @@ -27,7 +27,7 @@ coverage or historical dates. | Belgium | NGI administrative boundaries; Statbel population/statistical sectors | Statbel population 2021-2025 | | Flanders | GRB buildings, roads, water and parcels; DHMV terrain/surface; VMM flood scenarios; BWK/Natura 2000; DOV soil; policy rasters for space, open space, accessibility and services; agriculture and orthophoto where governed | Population 2021-2025; land-use/land-cover series where retained; agriculture editions; historical maps/orthophotos where the selected product has a real observation date | | Wallonia | Bounded PICC buildings, roads and hydrography; governed SPW bed-elevation/bathymetry products | No general cross-theme regional history yet | -| Brussels | Bounded UrbIS buildings, street axes and cadastral parcels | No general cross-theme regional history yet | +| Brussels | Bounded UrbIS buildings, street axes, cadastral parcels and Land Cover blocks; official FO/GB blocks provide forest/park area and WB blocks provide permanent water area | No general cross-theme regional history yet; the live WFS has no per-feature observation date | | Belgian North Sea | RBINS reporting units; Marine Spatial Plan 2026-2034; governed MDK bathymetry only when runtime acquisition is explicitly configured | No multi-epoch bathymetry or marine-plan trend yet | Mol and the Kempen are golden regression areas. Their persisted partitions are @@ -46,22 +46,18 @@ applicable bounded official source or reports the theme as unsupported. retain their model scenario semantics separately from observed floods. Official record: `https://geoportail.wallonie.be/catalogue/14084108-2c7b-4091-b62d-ff0fc235213a.html`. -3. Add the public UrbIS Land Cover product (regional situation 2024) for - Brussels through its official WFS/download contract. Keep it separate from - cadastral parcels and buildings. Product specification: - `https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_FR20240401.pdf`. -4. Add a common Belgium-wide topographic baseline with normalized theme +3. Add a common Belgium-wide topographic baseline with normalized theme semantics across NGI, Flanders, Wallonia and Brussels. -5. Govern comparable Walloon and Brussels historical editions before exposing +4. Govern comparable Walloon and Brussels historical editions before exposing evolution for buildings, roads, land cover, soil, elevation or flood risk. -6. Add nationally comparable land-cover history with explicit class crosswalks +5. Add nationally comparable land-cover history with explicit class crosswalks and uncertainty; never compare incompatible legends silently. -7. Add multi-epoch marine bathymetry and survey-footprint metadata before +6. Add multi-epoch marine bathymetry and survey-footprint metadata before presenting seabed evolution. -8. Expand persisted raster partition manifests beyond the regression regions +7. Expand persisted raster partition manifests beyond the regression regions only where repeated use justifies caching; bounded acquisition remains the default for one-off selections. -9. Add source freshness probes only for publishers with stable official edition +8. Add source freshness probes only for publishers with stable official edition contracts. Do not infer a new observation from an import or HTTP date. ## Acceptance rules for a new source diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index 557be05c..a1983147 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -15,9 +15,12 @@ runtime source of truth. - Buildings, population, terrain, imagery, nature, agriculture, soil and flood themes may report `partial`, `not_configured` or `unsupported` outside the materialized source partitions. The UI and exports retain that state. -- Belgian North Sea planning/reporting boundaries are materialized. Continuous - authoritative bathymetry acquisition remains `not_configured`; VHA profile - observations are not presented as a seabed model or water volume. +- Belgian North Sea planning/reporting boundaries are materialized. Bounded + strict-TLS MDK WCS acquisition is implemented but stays disabled until the + operator enables `MDK_BATHYMETRY_ACQUISITION_ENABLED` with a coverage id that + the live readiness probe advertises. Until then the theme reports + `not_configured`; VHA profile observations are not presented as a seabed + model or water volume. - Official endpoints can be temporarily unavailable. Bounded acquisition fails closed and never substitutes fixture or fabricated production data. @@ -47,8 +50,11 @@ runtime source of truth. trained general model for all Belgian objects or themes. - PyTorch and Ultralytics are present only in the AI image. No model weights auto-download. A missing local model reports unavailable. -- Real segmentation models remain placeholders; fixture segmentation is - explicit-only. No SAM or YOLO-seg dependency is installed. +- Local YOLO-seg and SAM segmentation are implemented through the ultralytics + interface but stay `not_configured` until the operator points + `YOLO_SEG_MODEL_PATH`/`SAM_MODEL_PATH` to existing local weights and enables + them explicitly. GeoIntel never downloads segmentation weights automatically; + fixture segmentation remains explicit-only. ## Operations diff --git a/docs/TODO.md b/docs/TODO.md index 9801504d..d4d840ee 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -57,9 +57,20 @@ Dit is het enige actuele afwerkingsbord. De lange sprint- en voorbereidingslijsten verderop blijven bewaard als historisch bewijs, maar zijn geen open productroadmap meer. -- [x] Open automatisch de volledige Kempen-werkruimte met Mol als snel - selecteerbaar werkgebied; een technische project- of regioselectie is niet - vereist. +- [x] Open onvoorwaardelijk de nationale `Belgium and North Sea Workbench` + wanneer die bestaat en start de kaart op Belgische schaal. Mol en de Kempen + blijven alleen snel selecteerbare regressiegebieden. +- [x] Maak UrbIS Land Cover begrensd operationeel voor Brussel: alle Blocks als + landbedekking, FO/GB als bos en park en WB als permanent water, met echte + PostGIS-oppervlaktemetrics en broncodes. +- [ ] Implementeer begrensde WALOUS 2018/2020/2023 rasteracquisitie in + EPSG:3812 met officiële klassen, vergelijkbaarheidscontract en pixelbudget. +- [ ] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk + scenario-/juridisch contract; gebruik WMS alleen als context tenzij + analytische pixels of vectorgeometrie officieel beschikbaar zijn. +- [ ] Bouw een Belgische AI-validatiematrix met gelabelde golden AOIs in elk + gewest en aan de kust. Tot die matrix slaagt blijft de huidige YOLO-status + expliciet lokaal gevalideerd en controleplichtig. - [x] Kies een begrijpbaar datathema, teken een rechthoek of gebruik het volledige werkgebied en analyseer alle relevante thema's uit PostGIS. Een getekende selectie mag ontbrekende operationele bronproducten begrensd @@ -109,6 +120,21 @@ geen open productroadmap meer. Workbench met een compacte navigatierail, vaste contextbalk, taakgerichte schermen en gevalideerde desktop-, ultrawide- en mobiele layouts. +Post-V1 afwerkingspass (2026-07-19): + +- [x] Lokale YOLO-seg- en SAM-segmentatie via de ultralytics-interface: echte + adapters, dynamisch modelregister, tegelmanifest-inferentie, georeferentie + van maskpolygonen, dedupe, geodetische oppervlakte en persistentie. Blijft + fail-closed zonder lokale modelgewichten (`YOLO_SEG_*`/`SAM_*` env). +- [x] Begrensde MDK-bathymetrie-acquisitie (WCS GetCoverage) achter de + bestaande strict-TLS readiness-probe: expliciete opt-in, geadverteerd + coverage-id verplicht, bbox-limiet, GeoTIFF-validatie, LAT-provenance. + Endpoint: `POST /datasets/bathymetry/mdk/acquire`. +- [x] UrbIS-wegassen (`urbisvector:StreetAxes`, live gevalideerd tegen de + UrbIS WFS-capabilities) als begrensd Brussels roads-product met + lengte-metrics; de WFS adverteert geen hydrografielaag, dus Brussels + oppervlaktewater blijft eerlijk `not_configured`. + Bewuste, niet-blokkerende grenzen: - De begrensde SPW-rasterflow maakt Waalse waterbodemhoogte in mDNG diff --git a/frontend/README.md b/frontend/README.md index db464335..4f714937 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,8 +1,12 @@ # GeoIntel Frontend (Sprint 4) -React + TypeScript + MapLibre workbench for regional geographic analysis. +React + TypeScript + MapLibre workbench for Belgium and the Belgian North Sea. -The persisted `Kempen Regional Workbench` is the automatic operational data context. Its datasets are loaded once for the complete official 28-municipality Vlaamse vervoerregio; the operator chooses Mol, another municipality or the complete region as a spatial work-area filter. The primary map no longer asks the user to choose a technical project or region before data becomes usable. Regional population and modern forest snapshots use the same current/evolution flow as Mol, while explicit project selection stays available under advanced management. +The persisted `Belgium and North Sea Workbench` is the unconditional primary +data context whenever it exists. The map opens at national extent and supports +bounded selections across Flanders, Wallonia, Brussels and the legally labelled +Belgian maritime zones. Mol and the Kempen remain selectable golden regression +areas, but are never used as an implicit product boundary or startup fallback. The Status workspace includes one compact `Actualiteit en versiecontrole` surface. It separates sources that are current, due for a catalogue review, @@ -29,7 +33,13 @@ as historical observations. The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanalyse`, `Downloads`, `Status` and `Beheer`. Internal benchmark projects, raw dataset metadata, provider capabilities, model registry details and QA evidence remain accessible through labelled advanced disclosures instead of competing with the normal workflow. -Detection defaults to the configured local YOLO asset and automatically selects an available raster and active model asset where possible. The model registry and preflight remain honest when PyTorch, Ultralytics, a local model file or a tile manifest is unavailable. The active building profile is operational but remains review-required: its current coverage-aligned benchmark is approximately precision 0.614, recall 0.606 and F1 0.607 over seven positive AOIs, with zero detections in all three pure-empty controls. A reviewed challenger remains inactive because it produced two detections in empty Postel forest. +Detection defaults to the explicitly configured local YOLO asset and +automatically selects an available raster where possible. In production the +asset catalog exposes only the file matching `YOLO_MODEL_PATH`; training, +partial and smoke checkpoints remain on disk but do not become end-user model +choices. The current building benchmark covers seven Mol/Kempen AOIs and is +shown as local validation, not as proof of national model quality. Every other +Belgian or maritime context requires local reference QA before release. Map-driven building analysis uses the documented footprint-IoU `0.25` and distinguishes model candidates from verified buildings. It shows persisted diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 57c1395f..fae95342 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -842,7 +842,7 @@ function App(): JSX.Element { > {activeWorkspace !== 'map' ?
-

{selectedProject?.region ?? 'Mol, Kempen'}

+

{selectedProject?.region ?? 'Belgie en Belgische Noordzee'}

{activeWorkspaceItem.label}

diff --git a/frontend/src/components/GeoMap.tsx b/frontend/src/components/GeoMap.tsx index 2ae9d12c..5caf24dd 100644 --- a/frontend/src/components/GeoMap.tsx +++ b/frontend/src/components/GeoMap.tsx @@ -1,7 +1,7 @@ import { useEffect, useRef, useState } from 'react' import maplibregl from 'maplibre-gl' import 'maplibre-gl/dist/maplibre-gl.css' -import { PRIMARY_FOCUS_CENTER } from '../config/primaryFocus' +import { NATIONAL_MAP_CENTER, NATIONAL_MAP_ZOOM } from '../config/primaryFocus' import { featureCollectionBounds } from '../lib/geojsonBounds' import type { MapImageOverlay, MapViewportState, VectorSelectionBBox } from '../types' @@ -221,8 +221,8 @@ function GeoMap({ const map = new maplibregl.Map({ container: containerRef.current, style: defaultMapStyle(), - center: PRIMARY_FOCUS_CENTER, - zoom: 11, + center: NATIONAL_MAP_CENTER, + zoom: NATIONAL_MAP_ZOOM, attributionControl: false, }) const resizeObserver = new ResizeObserver(() => { diff --git a/frontend/src/components/datasets/SourceCatalogPanel.tsx b/frontend/src/components/datasets/SourceCatalogPanel.tsx index 561f6d14..563cdf34 100644 --- a/frontend/src/components/datasets/SourceCatalogPanel.tsx +++ b/frontend/src/components/datasets/SourceCatalogPanel.tsx @@ -289,7 +289,7 @@ export function SourceCatalogPanel({ {Number(latestBuildingsRegister.source_metadata?.['building_unit_count'] ?? 0).toLocaleString('nl-BE')} eenheden · {' '} {Number(latestBuildingsRegister.source_metadata?.['linked_address_count'] ?? 0).toLocaleString('nl-BE')} gekoppelde adressen -

Registerstatus en geaggregeerde koppelingen voor Mol; adreslabels en persoonsgegevens worden niet in de kaartlaag getoond.

+

Registerstatus en geaggregeerde koppelingen binnen de werkelijk ingeladen dekking; adreslabels en persoonsgegevens worden niet in de kaartlaag getoond.

) : null} {dhmvDatasets.length > 0 ? ( diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx index 23938534..3d1e5fe3 100644 --- a/frontend/src/components/detection/DetectionLab.tsx +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -297,9 +297,9 @@ export function DetectionLab({

{yoloRuntimeReady ? `${yoloPreflight?.runtime.cuda_available ? 'GPU' : 'CPU'} · lokaal model gevonden` : 'Controleer de modelconfiguratie onder beheer.'}

- Gevalideerde kwaliteit + Validatiescope {selectedOperatorProfile ? `F1 ${selectedOperatorProfile.f1.toFixed(3)}` : 'Nog niet gekoppeld'} -

{selectedOperatorProfile ? `${selectedOperatorProfile.positiveSampleCount} testgebieden · resultaten blijven controleplichtig` : 'Kies het goedgekeurde lokale profiel.'}

+

{selectedOperatorProfile ? selectedOperatorProfile.validationScope : 'Kies een modelprofiel met gedocumenteerd evaluatiebewijs.'}

0 ? 'ai-user-summary-card ai-user-summary-card-ready' : 'ai-user-summary-card'}> Beschikbare luchtbeelden @@ -310,6 +310,15 @@ export function DetectionLab({

{detectionQualityInterpretation(selectedOperatorProfile?.f1)}

+ {selectedOperatorProfile && !selectedOperatorProfile.nationallyValidated ? ( +
+ Nog niet nationaal gevalideerd +

+ Dit model is operationeel voor gecontroleerde beeldanalyse, maar de gemeten kwaliteit geldt alleen voor {selectedOperatorProfile.validationScope}. + Resultaten elders in Belgie of op zee vereisen lokale referentiedata en QA voordat ze als betrouwbaar kunnen worden vrijgegeven. +

+
+ ) : null} Getoond op de kaart {selectedArea?.name ?? 'Geen gebied geselecteerd'} - {selectedArea?.area_m2 ? `${(selectedArea.area_m2 / 1_000_000).toLocaleString('nl-BE', { maximumFractionDigits: 2 })} km2` : 'Kies een gemeente of de volledige regio op de kaart.'} + {selectedArea?.area_m2 ? `${(selectedArea.area_m2 / 1_000_000).toLocaleString('nl-BE', { maximumFractionDigits: 2 })} km2` : 'Kies een gemeente, gewest, zeezone of teken een eigen selectie.'}
@@ -136,7 +139,7 @@ export function AreaPanel({ setSearchQuery(event.target.value) setPage(1) }} - placeholder="Bijvoorbeeld Mol" + placeholder="Bijvoorbeeld Brussel, Namen of Noordzee" />
diff --git a/frontend/src/components/project/ProjectPanel.tsx b/frontend/src/components/project/ProjectPanel.tsx index 5050cf89..538d6d3c 100644 --- a/frontend/src/components/project/ProjectPanel.tsx +++ b/frontend/src/components/project/ProjectPanel.tsx @@ -2,19 +2,23 @@ import type { FormEvent } from 'react' import type { ProjectCreate, ProjectRead } from '../../types' const TECHNICAL_PROJECT_PATTERN = /^(GeoIntel Detection Quality Matrix|GeoIntel hard-negative|GeoIntel training|Mol Building QA)/i +const NATIONAL_PROJECT_NAME = 'Belgium and North Sea Workbench' const REGIONAL_PROJECT_NAME = 'Kempen Regional Workbench' const LEGACY_MOL_PROJECT_NAME = 'Mol Municipality Workbench' -const PROTECTED_PROJECT_NAMES = new Set([REGIONAL_PROJECT_NAME, LEGACY_MOL_PROJECT_NAME]) +const PROTECTED_PROJECT_NAMES = new Set([NATIONAL_PROJECT_NAME, REGIONAL_PROJECT_NAME, LEGACY_MOL_PROJECT_NAME]) function isTechnicalProject(project: ProjectRead): boolean { return TECHNICAL_PROJECT_PATTERN.test(project.name) } function isAdvancedProject(project: ProjectRead): boolean { - return isTechnicalProject(project) || project.name === LEGACY_MOL_PROJECT_NAME + return isTechnicalProject(project) || project.name === LEGACY_MOL_PROJECT_NAME || project.name === REGIONAL_PROJECT_NAME } function projectDisplayName(project: ProjectRead): string { + if (project.name === NATIONAL_PROJECT_NAME) { + return 'Belgie en Belgische Noordzee' + } if (project.name === REGIONAL_PROJECT_NAME) { return 'Kempen · volledige regionale werkruimte' } @@ -126,7 +130,7 @@ export function ProjectPanel({