diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5e8822..6cdca413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ # Changelog +## Sprint 203 Governed hydrology and historical imagery (2026-07-15) + +- Added an explicit Waterinfo KiWIS operator for annual water-level and + discharge station histories. It filters against the persisted Area, retains + raw JSON/checksums and imports each real station/year through DatasetService. +- Added numeric `mean` selection aggregation for station measurements while + keeping every station in an independent temporal series with an explicit + point-versus-area/volume limitation. +- Added a fixed official orthophoto product registry covering current, annual + 2012-2025, older winter periods, RGB 1979-1990 and panchromatic 1971. +- Added historical raster temporal provenance, a constrained browser PNG + endpoint and a MapLibre image overlay/product selector. +- Kept configured-YOLO/current-GRB QA exclusive to the most-recent product; + historical imagery never produces fake current-state quality metrics. +- Converted BWK/Natura 2000, agricultural parcels, Buildings Register, DHMV + and bathymetry into an ordered acceptance-criteria backlog. + ## Sprint 202 Source intelligence, full evolution metrics and local assistant (2026-07-15) - Extended temporal comparisons with exact persisted-Area filtering, every diff --git a/backend/README.md b/backend/README.md index ba1e17e9..567ef067 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1121,16 +1121,40 @@ is never presented as a complete result. ## Bounded official orthophoto acquisition -`POST /api/v1/projects/{project_id}/datasets/orthophoto/acquire` accepts an -explicit EPSG:4326 map rectangle and stores the official Digitaal Vlaanderen -`OMWRGBMRVL`/`Ortho` response as a canonical EPSG:31370 raster Dataset. The +`GET /api/v1/projects/{project_id}/datasets/orthophoto/products` lists the +governed product allowlist. `POST .../datasets/orthophoto/acquire` accepts an +explicit EPSG:4326 map rectangle plus `product_key` and stores the official +Digitaal Vlaanderen WMS response as a canonical EPSG:31370 raster Dataset. The default safety envelope is 128-1,024 m per side, 1 m/pixel, 32 MiB and a 24-hour exact-request cache. It runs synchronously behind the existing Job abstraction and never during startup. +Available products cover the most recent winter image, annual winter mosaics +for 2012-2025, three older winter periods, RGB 1979-1990 and panchromatic 1971. +Historical products persist validity metadata and are deliberately excluded +from configured-YOLO/current-GRB QA. `GET .../datasets/{dataset_id}/raster/image` +is the constrained binary PNG endpoint used by the MapLibre image overlay. + Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`, `ORTHOPHOTO_WMS_LAYER`, `ORTHOPHOTO_RESOLUTION_M`, `ORTHOPHOTO_MIN_SIDE_M`, `ORTHOPHOTO_MAX_SIDE_M`, `ORTHOPHOTO_TIMEOUT_SECONDS`, `ORTHOPHOTO_MAX_RESPONSE_MB` and `ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile unless a separately verified deployment/model profile requires a change. + +## Waterinfo station histories + +Run the explicit operator after the regional workspace and Mol Area exist: + +```bash +docker exec geointel python /app/scripts/provision_waterinfo_station_history.py \ + --project-name "Kempen Regional Workbench" \ + --area-name "Gemeente Mol" \ + --from-year 2013 --to-year 2025 +``` + +The command retains raw KiWIS JSON/checksums and imports only real annual +observations through the canonical dataset upload API. Every station has its +own temporal-series key. Water levels and discharges remain Point measurements; +they are never averaged across stations or presented as municipal water volume. +Use `--fetch-only` to prepare and audit artifacts without persistence. diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index 49da8350..325738bf 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -6,10 +6,10 @@ from typing import Any from uuid import UUID from uuid import UUID as _UUID -from fastapi import APIRouter, Depends, File, Form, HTTPException, Query +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response from fastapi import UploadFile from sqlalchemy.orm import Session -from app.models import Area +from app.models import Area, Project from app.core.errors import AppError from app.db.session import get_db @@ -143,6 +143,14 @@ def acquire_bounded_orthophoto( return envelope(job) +@router.get("/datasets/orthophoto/products", response_model=dict) +def list_orthophoto_products(project_id: UUID, db: Session = Depends(get_db)): + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + items = OrthophotoAcquisitionService.list_products() + return envelope({"items": items, "total": len(items)}) + + @router.get("/datasets", response_model=dict) def list_datasets( project_id: UUID, @@ -426,6 +434,20 @@ def raster_preview_readiness( return envelope(RasterOperationsService.preview(db, dataset_id)) +@router.get("/datasets/{dataset_id}/raster/image") +def raster_orthophoto_image( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + content = OrthophotoAcquisitionService.render_png(db, project_id, dataset_id) + return Response( + content=content, + media_type="image/png", + headers={"Cache-Control": "private, max-age=86400"}, + ) + + @router.get("/datasets/{dataset_id}/raster/stats", response_model=dict) def raster_stats( project_id: UUID, diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 0802c5bf..d56c90d2 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -32,7 +32,7 @@ from .segmentation import ( ) from .health import HealthResponse, SystemCapabilities from .job import JobCreate, JobList, JobRead, JobStatus -from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult +from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult, OrthophotoProductRead from .external import ( ExternalFetchRequest, ExternalFetchResponse, @@ -134,6 +134,7 @@ __all__ = [ "JobStatus", "OrthophotoAcquireRequest", "OrthophotoAcquisitionResult", + "OrthophotoProductRead", "VectorBBoxResponse", "VectorClipRequest", "VectorBufferRequest", diff --git a/backend/app/schemas/orthophoto.py b/backend/app/schemas/orthophoto.py index 7fe51bb4..8043e18b 100644 --- a/backend/app/schemas/orthophoto.py +++ b/backend/app/schemas/orthophoto.py @@ -10,13 +10,31 @@ from .operations import VectorSelectionBBox class OrthophotoAcquireRequest(BaseModel): bbox: VectorSelectionBBox area_id: UUID | None = None + product_key: str = "most_recent" force_refresh: bool = False +class OrthophotoProductRead(BaseModel): + key: str + display_name: str + observation_label: str + temporal_granularity: str + native_resolution_m: float + supports_detection: bool + color_mode: str + catalog_url: str + limitation_message: str + + class OrthophotoAcquisitionResult(BaseModel): output_dataset_id: UUID reused: bool provider: str + product_key: str + display_name: str + observation_label: str + temporal_granularity: str + supports_detection: bool layer: str width: int height: int diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py index 9d7c4fcd..153040f7 100644 --- a/backend/app/services/dataset_service.py +++ b/backend/app/services/dataset_service.py @@ -422,6 +422,11 @@ class DatasetService: source_metadata: dict[str, Any], provenance_metadata: dict[str, Any], area_id: UUID | None = None, + temporal_series_key: str | None = None, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_granularity: str | None = None, source_version: str | None = None, content_type: str = "image/tiff", ) -> DatasetCreateResponse: @@ -438,6 +443,14 @@ class DatasetService: safe_filename = DatasetService._validate_upload_filename(filename) if DatasetService._extension_for_path(safe_filename) not in DatasetService.RASTER_EXTENSIONS: raise AppError(code="INVALID_UPLOAD", message="Raster artifacts require a GeoTIFF filename", status_code=415) + temporal = DatasetService._validate_temporal_metadata( + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + ) dataset_id = uuid.uuid4() storage_info = StorageService.persist_dataset_file( @@ -462,7 +475,7 @@ class DatasetService: source_metadata=source_metadata, provenance_metadata=provenance_metadata, imported_at=datetime.now(timezone.utc), - source_version=source_version, + **temporal, storage_path=storage_info["storage_path"], original_filename=storage_info["original_filename"], stored_filename=storage_info["stored_filename"], @@ -483,6 +496,9 @@ class DatasetService: version=1, storage_path=dataset.storage_path, source_version=dataset.source_version, + observed_at=dataset.observed_at, + valid_from=dataset.valid_from, + valid_to=dataset.valid_to, checksum_sha256=dataset.checksum_sha256, source_metadata=dataset.source_metadata, provenance_metadata=dataset.provenance_metadata, diff --git a/backend/app/services/geo_assistant_service.py b/backend/app/services/geo_assistant_service.py index 4cc5f5ac..8415492e 100644 --- a/backend/app/services/geo_assistant_service.py +++ b/backend/app/services/geo_assistant_service.py @@ -349,6 +349,7 @@ class GeoAssistantService: "measurement_quality": ( "schatting" if summary["is_estimate"] else "exact_binnen_bronrepresentatie" ), + "warning": summary.get("warning"), } ) if dataset.id not in source_dataset_ids: diff --git a/backend/app/services/orthophoto_acquisition_service.py b/backend/app/services/orthophoto_acquisition_service.py index c8e84ea6..d6108097 100644 --- a/backend/app/services/orthophoto_acquisition_service.py +++ b/backend/app/services/orthophoto_acquisition_service.py @@ -1,9 +1,11 @@ from __future__ import annotations import hashlib +import io import json import math import warnings +from dataclasses import dataclass from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, Callable @@ -20,21 +22,169 @@ from shapely.ops import transform as shapely_transform from app.core.config import Settings, get_settings from app.core.errors import AppError from app.models import Area, Dataset, Project -from app.schemas.orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult +from app.schemas.orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult, OrthophotoProductRead from app.services.dataset_service import DatasetService +@dataclass(frozen=True) +class OrthophotoProduct: + key: str + display_name: str + observation_label: str + temporal_granularity: str + native_resolution_m: float + wms_url: str + layer: str + catalog_url: str + limitation_message: str + supports_detection: bool = False + color_mode: str = "rgb" + observed_at: datetime | None = None + valid_from: datetime | None = None + valid_to: datetime | None = None + + class OrthophotoAcquisitionService: PROVIDER = "digitaal_vlaanderen_orthophoto" ATTRIBUTION = "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen" CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen" LIMITATION = "Meest recente samengestelde winterorthofoto op het moment van de aanvraag; geen historische opnamedatum per pixel." + HISTORICAL_WINTER_WMS_URL = "https://geo.api.vlaanderen.be/OMW/wms" + HISTORICAL_WINTER_CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/wmts-orthofotomozaiek-middenschalig-winteropnamen" + HISTORICAL_SUMMER_WMS_URL = "https://geo.api.vlaanderen.be/OKZ/wms" + HISTORICAL_SUMMER_CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-kleinschalig-zomeropnamen" + + @staticmethod + def _products(settings: Settings) -> dict[str, OrthophotoProduct]: + products: list[OrthophotoProduct] = [ + OrthophotoProduct( + key="most_recent", + display_name="Meest recente winterluchtbeeld", + observation_label="Meest recent beschikbaar", + temporal_granularity="snapshot", + native_resolution_m=0.15, + wms_url=settings.orthophoto_wms_url, + layer=settings.orthophoto_wms_layer, + catalog_url=OrthophotoAcquisitionService.CATALOG_URL, + limitation_message=OrthophotoAcquisitionService.LIMITATION, + supports_detection=True, + ) + ] + for year in range(2025, 2011, -1): + products.append( + OrthophotoProduct( + key=str(year), + display_name=f"Winterluchtbeeld {year}", + observation_label=str(year), + temporal_granularity="year", + native_resolution_m=0.15 if year >= 2022 else 0.25, + wms_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_WMS_URL, + layer=f"OMWRGB{year % 100:02d}VL", + catalog_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_CATALOG_URL, + limitation_message=( + "Officiële samengestelde winterorthofoto voor deze jaargang; de exacte opnamedatum kan per tegel verschillen. " + "Historische beelden worden niet met de actuele GRB-toestand gevalideerd." + ), + observed_at=datetime(year, 1, 1, tzinfo=UTC), + valid_from=datetime(year, 1, 1, tzinfo=UTC), + valid_to=datetime(year, 12, 31, 23, 59, 59, tzinfo=UTC), + ) + ) + for key, start_year, end_year, layer in ( + ("2008_2011", 2008, 2011, "OMWRGB08_11VL"), + ("2005_2007", 2005, 2007, "OMWRGB05_07VL"), + ("2000_2003", 2000, 2003, "OMWRGB00_03VL"), + ): + products.append( + OrthophotoProduct( + key=key, + display_name=f"Winterluchtbeeld {start_year}-{end_year}", + observation_label=f"{start_year}-{end_year}", + temporal_granularity="period", + native_resolution_m=0.25, + wms_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_WMS_URL, + layer=layer, + catalog_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_CATALOG_URL, + limitation_message=( + "Officiële samengestelde winterorthofoto uit een meerjarige opnameperiode; dit is geen exacte jaaropname. " + "Historische beelden worden niet met de actuele GRB-toestand gevalideerd." + ), + observed_at=datetime(start_year, 1, 1, tzinfo=UTC), + valid_from=datetime(start_year, 1, 1, tzinfo=UTC), + valid_to=datetime(end_year, 12, 31, 23, 59, 59, tzinfo=UTC), + ) + ) + products.extend( + [ + OrthophotoProduct( + key="1979_1990", + display_name="Zomerluchtbeeld 1979-1990", + observation_label="1979-1990", + temporal_granularity="period", + native_resolution_m=1.0, + wms_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_WMS_URL, + layer="OKZRGB79_90VL", + catalog_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_CATALOG_URL, + limitation_message="Kleinschalig RGB-mozaïek uit meerdere zomervluchten tussen 1979 en 1990; geen exacte jaartoestand.", + observed_at=datetime(1979, 1, 1, tzinfo=UTC), + valid_from=datetime(1979, 1, 1, tzinfo=UTC), + valid_to=datetime(1990, 12, 31, 23, 59, 59, tzinfo=UTC), + ), + OrthophotoProduct( + key="1971", + display_name="Zomerluchtbeeld 1971", + observation_label="1971", + temporal_granularity="year", + native_resolution_m=1.0, + wms_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_WMS_URL, + layer="OKZPAN71VL", + catalog_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_CATALOG_URL, + limitation_message="Kleinschalig panchromatisch mozaïek uit 1971; zwart-wit en niet geschikt voor het huidige RGB-detectiemodel.", + color_mode="panchromatic", + observed_at=datetime(1971, 1, 1, tzinfo=UTC), + valid_from=datetime(1971, 1, 1, tzinfo=UTC), + valid_to=datetime(1971, 12, 31, 23, 59, 59, tzinfo=UTC), + ), + ] + ) + return {product.key: product for product in products} + + @staticmethod + def list_products(settings: Settings | None = None) -> list[dict[str, Any]]: + resolved_settings = settings or get_settings() + return [ + OrthophotoProductRead( + key=product.key, + display_name=product.display_name, + observation_label=product.observation_label, + temporal_granularity=product.temporal_granularity, + native_resolution_m=product.native_resolution_m, + supports_detection=product.supports_detection, + color_mode=product.color_mode, + catalog_url=product.catalog_url, + limitation_message=product.limitation_message, + ).model_dump() + for product in OrthophotoAcquisitionService._products(resolved_settings).values() + ] + + @staticmethod + def _product(product_key: str, settings: Settings) -> OrthophotoProduct: + product = OrthophotoAcquisitionService._products(settings).get(product_key.strip().lower()) + if product is None: + raise AppError( + code="ORTHOPHOTO_PRODUCT_NOT_SUPPORTED", + message="Select an orthophoto product from the official product registry", + details={"product_key": product_key}, + status_code=422, + ) + return product @staticmethod def _prepared_request( payload: OrthophotoAcquireRequest, settings: Settings, ) -> dict[str, Any]: + product = OrthophotoAcquisitionService._product(payload.product_key, settings) if payload.bbox.crs.upper() != "EPSG:4326": raise AppError(code="INVALID_CRS", message="Orthophoto selection bbox must use EPSG:4326", status_code=400) min_x = float(payload.bbox.min_x) @@ -68,8 +218,9 @@ class OrthophotoAcquisitionService: bbox_31370 = [float(value) for value in lambert_bounds] request_identity = { "provider": OrthophotoAcquisitionService.PROVIDER, - "wms_url": settings.orthophoto_wms_url, - "layer": settings.orthophoto_wms_layer, + "product_key": product.key, + "wms_url": product.wms_url, + "layer": product.layer, "bbox_epsg4326": [round(value, 8) for value in bbox_4326], "bbox_epsg31370": [round(value, 3) for value in bbox_31370], "width": width, @@ -77,11 +228,18 @@ class OrthophotoAcquisitionService: "resolution_m": settings.orthophoto_resolution_m, } request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode("utf-8")).hexdigest() + spatial_identity = { + "bbox_epsg4326": request_identity["bbox_epsg4326"], + "width": width, + "height": height, + "resolution_m": settings.orthophoto_resolution_m, + } + spatial_hash = hashlib.sha256(json.dumps(spatial_identity, sort_keys=True).encode("utf-8")).hexdigest() params = { "SERVICE": "WMS", "VERSION": "1.3.0", "REQUEST": "GetMap", - "LAYERS": settings.orthophoto_wms_layer, + "LAYERS": product.layer, "STYLES": "", "FORMAT": "image/tiff", "CRS": "EPSG:31370", @@ -91,8 +249,10 @@ class OrthophotoAcquisitionService: } return { **request_identity, + "product": product, + "spatial_hash": spatial_hash, "request_hash": request_hash, - "request_url": f"{settings.orthophoto_wms_url}?{urlencode(params)}", + "request_url": f"{product.wms_url}?{urlencode(params)}", "params": params, "bbox_epsg4326": bbox_4326, "bbox_epsg31370": bbox_31370, @@ -124,8 +284,14 @@ class OrthophotoAcquisitionService: ) @staticmethod - def _cached_dataset(db, project_id: UUID, filename: str, settings: Settings) -> Dataset | None: - if settings.orthophoto_cache_ttl_hours <= 0: + def _cached_dataset( + db, + project_id: UUID, + filename: str, + settings: Settings, + product: OrthophotoProduct, + ) -> Dataset | None: + if product.key == "most_recent" and settings.orthophoto_cache_ttl_hours <= 0: return None candidate = ( db.query(Dataset) @@ -145,7 +311,7 @@ class OrthophotoAcquisitionService: return None if imported_at.tzinfo is None: imported_at = imported_at.replace(tzinfo=UTC) - if datetime.now(UTC) - imported_at > timedelta(hours=settings.orthophoto_cache_ttl_hours): + if product.key == "most_recent" and datetime.now(UTC) - imported_at > timedelta(hours=settings.orthophoto_cache_ttl_hours): return None return candidate @@ -196,7 +362,9 @@ class OrthophotoAcquisitionService: with warnings.catch_warnings(): warnings.simplefilter("ignore", NotGeoreferencedWarning) with source_memory.open() as source: - if source.width != prepared["width"] or source.height != prepared["height"] or source.count < 3: + product: OrthophotoProduct = prepared["product"] + minimum_band_count = 1 if product.color_mode == "panchromatic" else 3 + if source.width != prepared["width"] or source.height != prepared["height"] or source.count < minimum_band_count: raise AppError( code="ORTHOPHOTO_PROVIDER_INVALID_RESPONSE", message="Official orthophoto dimensions or RGB bands do not match the bounded request", @@ -216,7 +384,7 @@ class OrthophotoAcquisitionService: with output_memory.open(**profile) as output: output.write(image) output.update_tags( - source="Digitaal Vlaanderen OMWRGBMRVL WMS Ortho layer", + source=f"Digitaal Vlaanderen WMS {product.layer}", source_url=prepared["request_url"], attribution=OrthophotoAcquisitionService.ATTRIBUTION, acquisition="explicit_bounded_map_selection", @@ -245,44 +413,74 @@ class OrthophotoAcquisitionService: if not resolved_settings.orthophoto_enabled: raise AppError(code="ORTHOPHOTO_NOT_CONFIGURED", message="Official orthophoto acquisition is disabled", status_code=503) prepared = OrthophotoAcquisitionService._prepared_request(payload, resolved_settings) + product: OrthophotoProduct = prepared["product"] OrthophotoAcquisitionService._validate_area_scope(db, project_id, payload.area_id, prepared["bbox_epsg4326"]) - filename = f"orthofoto_selectie_{prepared['request_hash'][:12]}.tif" + filename = f"orthofoto_{product.key}_{prepared['request_hash'][:12]}.tif" - cached = None if payload.force_refresh else OrthophotoAcquisitionService._cached_dataset(db, project_id, filename, resolved_settings) + cached = None if payload.force_refresh else OrthophotoAcquisitionService._cached_dataset( + db, + project_id, + filename, + resolved_settings, + product, + ) if cached is not None: return OrthophotoAcquisitionResult( output_dataset_id=cached.id, reused=True, provider=OrthophotoAcquisitionService.PROVIDER, - layer=resolved_settings.orthophoto_wms_layer, + product_key=product.key, + display_name=product.display_name, + observation_label=product.observation_label, + temporal_granularity=product.temporal_granularity, + supports_detection=product.supports_detection, + layer=product.layer, width=prepared["width"], height=prepared["height"], resolution_m=resolved_settings.orthophoto_resolution_m, bbox_epsg4326=prepared["bbox_epsg4326"], bbox_epsg31370=prepared["bbox_epsg31370"], attribution=OrthophotoAcquisitionService.ATTRIBUTION, - limitation_message=OrthophotoAcquisitionService.LIMITATION, + limitation_message=product.limitation_message, ).model_dump(mode="json") raw_content, response_content_type = OrthophotoAcquisitionService._fetch(prepared["request_url"], resolved_settings, opener) geotiff_content = OrthophotoAcquisitionService._georeference_tiff(raw_content, prepared) acquired_at = datetime.now(UTC) + observed_at = product.observed_at or acquired_at dataset = DatasetService.import_raster_bytes( db, project_id=project_id, area_id=payload.area_id, filename=filename, content=geotiff_content, - source="Digitaal Vlaanderen OMWRGBMRVL WMS", + source=f"Digitaal Vlaanderen WMS {product.layer}", source_name=OrthophotoAcquisitionService.PROVIDER, - source_version=f"most_recent_at_{acquired_at.date().isoformat()}", + temporal_series_key=f"digitaal-vlaanderen:orthophoto:{prepared['spatial_hash'][:24]}", + observed_at=observed_at, + valid_from=product.valid_from or observed_at, + valid_to=product.valid_to, + temporal_granularity=product.temporal_granularity, + source_version=( + f"most_recent_at_{acquired_at.date().isoformat()}" + if product.key == "most_recent" + else product.key + ), content_type="image/tiff", source_metadata={ "provider": OrthophotoAcquisitionService.PROVIDER, "service": "WMS", "service_version": "1.3.0", - "layer": resolved_settings.orthophoto_wms_layer, - "catalog_url": OrthophotoAcquisitionService.CATALOG_URL, + "product_key": product.key, + "product_display_name": product.display_name, + "observation_label": product.observation_label, + "observation_date_precision": product.temporal_granularity, + "native_resolution_m": product.native_resolution_m, + "requested_resolution_m": resolved_settings.orthophoto_resolution_m, + "color_mode": product.color_mode, + "supports_detection": product.supports_detection, + "layer": product.layer, + "catalog_url": product.catalog_url, "attribution": OrthophotoAcquisitionService.ATTRIBUTION, "license_note": "Gebruik volgens het gebruiksrecht geografische webdiensten van Digitaal Vlaanderen.", }, @@ -290,6 +488,7 @@ class OrthophotoAcquisitionService: "acquisition": "explicit_bounded_map_selection", "acquired_at": acquired_at.isoformat(), "request_hash": prepared["request_hash"], + "spatial_hash": prepared["spatial_hash"], "request_url": prepared["request_url"], "response_content_type": response_content_type, "bbox_epsg4326": prepared["bbox_epsg4326"], @@ -297,19 +496,70 @@ class OrthophotoAcquisitionService: "width": prepared["width"], "height": prepared["height"], "resolution_m": resolved_settings.orthophoto_resolution_m, - "limitation_message": OrthophotoAcquisitionService.LIMITATION, + "limitation_message": product.limitation_message, }, ) return OrthophotoAcquisitionResult( output_dataset_id=dataset.id, reused=False, provider=OrthophotoAcquisitionService.PROVIDER, - layer=resolved_settings.orthophoto_wms_layer, + product_key=product.key, + display_name=product.display_name, + observation_label=product.observation_label, + temporal_granularity=product.temporal_granularity, + supports_detection=product.supports_detection, + layer=product.layer, width=prepared["width"], height=prepared["height"], resolution_m=resolved_settings.orthophoto_resolution_m, bbox_epsg4326=prepared["bbox_epsg4326"], bbox_epsg31370=prepared["bbox_epsg31370"], attribution=OrthophotoAcquisitionService.ATTRIBUTION, - limitation_message=OrthophotoAcquisitionService.LIMITATION, + limitation_message=product.limitation_message, ).model_dump(mode="json") + + @staticmethod + def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1600) -> bytes: + dataset = db.get(Dataset, dataset_id) + if ( + dataset is None + or dataset.project_id != project_id + or dataset.source_name != OrthophotoAcquisitionService.PROVIDER + or dataset.status != "ready" + or not dataset.storage_path + or not Path(dataset.storage_path).is_file() + ): + raise AppError(code="ORTHOPHOTO_NOT_FOUND", message="Orthophoto dataset not found", status_code=404) + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.enums import Resampling + except ImportError as exc: + raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Raster preview dependencies are unavailable", status_code=503) from exc + + try: + with rasterio.open(dataset.storage_path) as source: + scale = min(1.0, max_dimension / max(source.width, source.height)) + width = max(1, round(source.width * scale)) + height = max(1, round(source.height * scale)) + indexes = [1] if source.count == 1 else list(range(1, min(source.count, 3) + 1)) + pixels = source.read(indexes, out_shape=(len(indexes), height, width), resampling=Resampling.bilinear) + if pixels.dtype != np.uint8: + pixels = np.clip(pixels, 0, 255).astype(np.uint8) + if len(indexes) == 1: + image = Image.fromarray(pixels[0]) + else: + image = Image.fromarray(np.moveaxis(pixels[:3], 0, 2)) + output = io.BytesIO() + image.save(output, format="PNG", optimize=True) + return output.getvalue() + except AppError: + raise + except Exception as exc: + raise AppError( + code="ORTHOPHOTO_PREVIEW_FAILED", + message="The persisted orthophoto could not be rendered", + details={"reason": str(exc)}, + status_code=500, + ) from exc diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index 1e656213..9b6e2fad 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -23,6 +23,7 @@ FULL_AREA_CLIPPED_OPERATOR_TOOLS = { "provision_official_landuse_timeseries.py", "provision_regional_grb_buildings.py", "provision_regional_grb_context.py", + "provision_waterinfo_station_history.py", } @@ -439,7 +440,7 @@ class VectorFeatureService: length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*metric_filter).scalar() divisor = 1_000.0 if unit == "km" else 1.0 metric_value = float(length_m or 0.0) / divisor - elif method in {"sum", "area_weighted_sum"}: + elif method in {"sum", "mean", "area_weighted_sum"}: property_name = str(config.get("property") or "").strip() if not property_name: raise AppError( @@ -457,8 +458,9 @@ class VectorFeatureService: ) coverage_ratio = intersection_area / func.nullif(source_area, 0.0) value_expression = numeric_value * coverage_ratio + aggregate_function = func.avg if method == "mean" else func.sum aggregate_value = ( - db.query(func.coalesce(func.sum(value_expression), 0.0)) + db.query(func.coalesce(aggregate_function(value_expression), 0.0)) .filter(*selection_filter) .filter(VectorFeature.properties_json.op("->>")(property_name).isnot(None)) .scalar() diff --git a/backend/tests/test_sprint196_map_orthophoto_analysis.py b/backend/tests/test_sprint196_map_orthophoto_analysis.py index 4d7033fe..1f23285c 100644 --- a/backend/tests/test_sprint196_map_orthophoto_analysis.py +++ b/backend/tests/test_sprint196_map_orthophoto_analysis.py @@ -86,7 +86,13 @@ class FakeImageResponse: return self.content[:limit] -def _selection_payload(*, side_m: float = 512.0, force_refresh: bool = True, area_id=None) -> OrthophotoAcquireRequest: +def _selection_payload( + *, + side_m: float = 512.0, + force_refresh: bool = True, + area_id=None, + product_key: str = "most_recent", +) -> OrthophotoAcquireRequest: west, south = 199_000.0, 210_000.0 transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) min_lon, min_lat = transformer.transform(west, south) @@ -100,6 +106,7 @@ def _selection_payload(*, side_m: float = 512.0, force_refresh: bool = True, are "crs": "EPSG:4326", }, area_id=area_id, + product_key=product_key, force_refresh=force_refresh, ) @@ -136,6 +143,25 @@ def test_orthophoto_request_is_bounded_and_uses_official_wms_contract() -> None: assert len(prepared["request_hash"]) == 64 +def test_orthophoto_product_registry_exposes_only_governed_official_layers() -> None: + settings = Settings(_env_file=None) + products = OrthophotoAcquisitionService.list_products(settings) + keys = [item["key"] for item in products] + + assert keys[0] == "most_recent" + assert {"2025", "2012", "2008_2011", "2000_2003", "1979_1990", "1971"}.issubset(keys) + assert next(item for item in products if item["key"] == "most_recent")["supports_detection"] is True + assert all(item["supports_detection"] is False for item in products if item["key"] != "most_recent") + + prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings) + assert prepared["params"]["LAYERS"] == "OKZPAN71VL" + assert prepared["wms_url"] == "https://geo.api.vlaanderen.be/OKZ/wms" + + with pytest.raises(AppError) as exc_info: + OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="arbitrary-layer"), settings) + assert exc_info.value.code == "ORTHOPHOTO_PRODUCT_NOT_SUPPORTED" + + @pytest.mark.parametrize( ("side_m", "expected_code"), [(64.0, "ORTHOPHOTO_SELECTION_TOO_SMALL"), (1_200.0, "ORTHOPHOTO_SELECTION_TOO_LARGE")], @@ -240,7 +266,7 @@ def test_orthophoto_acquisition_reuses_fresh_exact_request_without_provider_call cached = Dataset( id=uuid4(), project_id=project_id, - name=f"orthofoto_selectie_{prepared['request_hash'][:12]}.tif", + name=f"orthofoto_most_recent_{prepared['request_hash'][:12]}.tif", dataset_type="raster", source="Digitaal Vlaanderen", source_name="digitaal_vlaanderen_orthophoto", @@ -277,6 +303,54 @@ def test_orthophoto_provider_rejects_non_image_response() -> None: assert exc_info.value.code == "ORTHOPHOTO_PROVIDER_INVALID_RESPONSE" +def test_historical_orthophoto_persists_temporal_product_provenance(tmp_path) -> None: + project_id = uuid4() + payload = _selection_payload(product_key="2020") + db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) + settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0) + prepared = OrthophotoAcquisitionService._prepared_request(payload, settings) + response = FakeImageResponse(_source_tiff(prepared["width"], prepared["height"])) + + result = OrthophotoAcquisitionService.acquire( + db, + project_id, + payload, + settings=settings, + opener=lambda *_args, **_kwargs: response, + ) + + dataset = next(row for row in db.added if isinstance(row, Dataset)) + assert result["product_key"] == "2020" + assert result["supports_detection"] is False + assert dataset.observed_at.year == 2020 + assert dataset.temporal_granularity == "year" + assert dataset.source_metadata["layer"] == "OMWRGB20VL" + assert dataset.source_metadata["product_key"] == "2020" + assert dataset.provenance_metadata["spatial_hash"] == prepared["spatial_hash"] + + +def test_persisted_orthophoto_renders_browser_png(tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + path = tmp_path / "ortho.tif" + path.write_bytes(_source_tiff(32, 24)) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="ortho.tif", + dataset_type="raster", + source="Digitaal Vlaanderen", + source_name="digitaal_vlaanderen_orthophoto", + status="ready", + storage_path=str(path), + ) + db = FakeSession({(Dataset, dataset_id): dataset}) + + png = OrthophotoAcquisitionService.render_png(db, project_id, dataset_id) + + assert png.startswith(b"\x89PNG\r\n\x1a\n") + + def test_orthophoto_endpoint_returns_canonical_job_envelope(monkeypatch) -> None: project_id = uuid4() output_dataset_id = uuid4() @@ -307,6 +381,22 @@ def test_orthophoto_endpoint_returns_canonical_job_envelope(monkeypatch) -> None assert any(isinstance(row, Job) for row in db.added) +def test_orthophoto_product_endpoint_returns_canonical_envelope() -> None: + project_id = uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) + app.dependency_overrides[get_db] = lambda: db + try: + response = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/orthophoto/products") + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + body = response.json() + assert set(body) == {"data"} + assert body["data"]["total"] == len(body["data"]["items"]) + assert body["data"]["items"][0]["key"] == "most_recent" + + def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() -> None: app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8") hook_source = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8") diff --git a/backend/tests/test_sprint201_semantic_selection_metrics.py b/backend/tests/test_sprint201_semantic_selection_metrics.py index e01b8a1f..dce17eb5 100644 --- a/backend/tests/test_sprint201_semantic_selection_metrics.py +++ b/backend/tests/test_sprint201_semantic_selection_metrics.py @@ -114,6 +114,36 @@ def test_population_keeps_configured_metric_and_adds_sector_count() -> None: } +def test_station_measurement_uses_numeric_mean_without_area_extrapolation() -> None: + dataset = themed_dataset("water", method="mean") + dataset.source_name = "waterinfo" + dataset.source_metadata.update( + { + "semantic_metrics": False, + "selection_aggregation": { + "metric_key": "water_level", + "method": "mean", + "property": "annual_mean_water_level_m", + "label": "Jaargemiddelde waterstand", + "unit": "m", + "warning": "Puntmeting; geen gebiedsdekkend watervolume.", + }, + } + ) + + result = VectorFeatureService.summarize_features_by_bbox( + SequenceScalarSession([30.455]), + dataset=dataset, + bbox=BBOX, + total_feature_count=1, + ) + + assert result["metric_value"] == 30.455 + assert result["aggregation_method"] == "mean" + assert result["metric_unit"] == "m" + assert result["warning"] == "Puntmeting; geen gebiedsdekkend watervolume." + + def test_future_regional_imports_persist_semantic_aggregation_configuration() -> None: buildings = (ROOT / "scripts/provision_regional_grb_buildings.py").read_text(encoding="utf-8") context = (ROOT / "scripts/provision_regional_grb_context.py").read_text(encoding="utf-8") diff --git a/backend/tests/test_sprint203_waterinfo_history.py b/backend/tests/test_sprint203_waterinfo_history.py new file mode 100644 index 00000000..59c77c58 --- /dev/null +++ b/backend/tests/test_sprint203_waterinfo_history.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +from shapely.geometry import box + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_operator(): + script_path = ROOT / "scripts" / "provision_waterinfo_station_history.py" + spec = importlib.util.spec_from_file_location("waterinfo_history_operator", script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class JsonResponse: + ok = True + status_code = 200 + text = "" + + def __init__(self, payload): + self.payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self.payload + + +class JsonSession: + def __init__(self, payloads): + self.payloads = iter(payloads) + self.calls = [] + + def get(self, url, *, params, timeout): + self.calls.append((url, params, timeout)) + return JsonResponse(next(self.payloads)) + + +def test_waterinfo_station_discovery_filters_exact_area_and_uses_annual_group() -> None: + module = load_operator() + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.1, 51.2]}, + "properties": {"ts_id": 5319042, "station_no": "L10_089", "station_name": "Mol/ScheppelijkeNete"}, + }, + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [6.0, 52.0]}, + "properties": {"ts_id": 999, "station_no": "outside", "station_name": "Outside"}, + }, + ], + } + session = JsonSession([payload]) + + raw, stations = module.discover_station_series( + session, + module.PARAMETERS["water_level"], + box(5.0, 51.0, 5.3, 51.4), + timeout=30, + ) + + assert raw == payload + assert [item["ts_id"] for item in stations] == ["5319042"] + assert session.calls[0][1]["timeseriesgroup_id"] == "192784" + assert session.calls[0][1]["request"] == "getTimeseriesValueLayer" + + +def test_waterinfo_annual_values_reject_invalid_sentinel_and_keep_real_zero() -> None: + module = load_operator() + payload = [ + { + "ts_id": 5319042, + "data": [ + ["2013-01-01T00:00:00.000+01:00", 30.46], + ["2014-01-01T00:00:00.000+01:00", -9999], + ["2015-01-01T00:00:00.000+01:00", 0.0], + ["2026-01-01T00:00:00.000+01:00", 99.0], + ], + } + ] + session = JsonSession([payload]) + + raw, values = module.fetch_annual_values(session, "5319042", from_year=2013, to_year=2025, timeout=30) + + assert raw == payload + assert values == {2013: 30.46, 2015: 0.0} + assert session.calls[0][1]["request"] == "getTimeseriesValues" + + +def test_waterinfo_snapshot_and_series_keep_station_identity_and_honest_metric() -> None: + module = load_operator() + parameter = module.PARAMETERS["water_level"] + station = { + "ts_id": "5319042", + "geometry": {"type": "Point", "coordinates": [5.1, 51.2]}, + "properties": { + "station_id": "123", + "station_no": "L10_089", + "station_name": "Mol/ScheppelijkeNete", + "ts_unitsymbol": "m", + }, + } + + snapshot = module.build_snapshot(parameter, station, 2025, 30.455) + + feature = snapshot["features"][0] + assert module.series_key(parameter, station) == "waterinfo:water_level:annual:l10-089" + assert feature["geometry"]["type"] == "Point" + assert feature["properties"]["annual_mean_water_level_m"] == 30.455 + assert feature["properties"]["timeseries_id"] == "5319042" + assert "volume" in parameter.limitation + + +def test_waterinfo_operator_is_packaged_and_readiness_checked() -> None: + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + vector_service = (ROOT / "backend" / "app" / "services" / "vector_feature_service.py").read_text(encoding="utf-8") + + assert "py_compile scripts/provision_waterinfo_station_history.py" in readiness + assert "COPY scripts/provision_waterinfo_station_history.py" in dockerfile + assert '"provision_waterinfo_station_history.py"' in vector_service + assert '"sum", "mean", "area_weighted_sum"' in vector_service diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index eef96bdf..ad1ca8f2 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -77,6 +77,7 @@ COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_ COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py +COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index c83885bb..6891ee37 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -191,15 +191,24 @@ Response: `DatasetRead` with extracted metadata if supported. Vector uploads remain stored as original files and are also persisted into `vector_features` as queryable PostGIS state. +### GET `/api/v1/projects/{project_id}/datasets/orthophoto/products` + +Return the governed Digitaal Vlaanderen orthophoto product allowlist in the +canonical envelope. Every product reports its key, display/observation label, +temporal granularity, native resolution, colour mode, catalogue URL, +limitations and whether current configured-YOLO detection is allowed. + ### POST `/api/v1/projects/{project_id}/datasets/orthophoto/acquire` -Explicitly acquire a bounded most-recent winter orthophoto selection from the -official Digitaal Vlaanderen `OMWRGBMRVL` WMS `Ortho` layer. +Explicitly acquire a bounded orthophoto selection from a governed official +Digitaal Vlaanderen WMS product. Arbitrary WMS URLs and layer names are not +accepted. ```json { "bbox": {"min_x": 5.10, "min_y": 51.17, "max_x": 5.11, "max_y": 51.18, "crs": "EPSG:4326"}, "area_id": "optional-project-area-uuid", + "product_key": "most_recent", "force_refresh": false } ``` @@ -207,7 +216,8 @@ official Digitaal Vlaanderen `OMWRGBMRVL` WMS `Ortho` layer. The canonical envelope contains a synchronous Job. Its `output_dataset_id` identifies the raster Dataset; `result_json` contains provider, layer, pixel dimensions, EPSG:4326/EPSG:31370 bounds, sampling resolution, attribution, -cache reuse and limitation text. +cache reuse and limitation text. Historical products also persist their +observation/validity period and a spatially scoped temporal-series key. Safety contract: @@ -218,8 +228,11 @@ Safety contract: reuse; - WMS bytes are georeferenced to EPSG:31370 and persisted only through `DatasetService`; no fetch runs on startup; -- this is the latest mosaic available at request time, not a historical - observation date for every pixel. +- only `most_recent` can enter the current configured-YOLO plus GRB-QA path; + historical products are visual evidence and are never validated against the + current GRB state; +- product periods such as `1979_1990` remain explicitly multi-year and are not + presented as exact annual observations. ### GET `/api/v1/projects/{project_id}/datasets` @@ -274,6 +287,13 @@ If preview dependencies are unavailable: - code: `RASTER_PROCESSING_UNAVAILABLE` - message: `Raster preview unavailable...` +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image` + +Return a persisted orthophoto Dataset as a bounded browser-safe PNG. This is an +explicit binary non-envelope endpoint used by the MapLibre image source. It +accepts only ready datasets from the governed orthophoto provider and never +reads arbitrary filesystem paths. + ### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/clip` Clip raster by selected area. Returns a `202`-style accepted job payload through the job wrapper (`jobs` create/read flow). diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 9e92d779..9bd50077 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8438,3 +8438,33 @@ Next: - Integrate Waterinfo/VMM station observations as point time series and add historical orthophoto acquisition, while preserving their spatial and methodological limitations. + +## Sprint 203 - Governed Waterinfo history and historical orthophotos (2026-07-15) + +Implemented: +- Added `provision_waterinfo_station_history.py` using the documented Waterinfo + KiWIS annual water-level/discharge groups. The operator filters station + points against the exact persisted Area, retains raw JSON plus SHA256 + manifests and writes only through the canonical upload API. +- Persisted one Point Dataset per station/year and kept station identities in + separate temporal series. Added backend `mean` aggregation without combining + stations or inferring area-wide water level/volume. +- Added a governed orthophoto allowlist for the most-recent product, annual + winter mosaics 2012-2025, older winter periods, RGB 1979-1990 and + panchromatic 1971. Arbitrary WMS URLs/layers remain impossible. +- Added raster temporal metadata and a constrained PNG rendering endpoint for + persisted orthophoto datasets. The map can select and display official + historical imagery over the same bounded rectangle. +- Historical products explicitly bypass configured-YOLO and current-GRB QA; + only `most_recent` retains that path. +- Updated source inventory behavior and wrote a governed source backlog with + acceptance criteria for BWK/Natura 2000, agricultural parcels, Buildings + Register, DHMV and bathymetry. + +Validation evidence: +- Focused Waterinfo/orthophoto/semantic metric tests and frontend typecheck/build + passed before the full repository gate. + +Remaining operational step: +- Deploy the all-in-one image, provision the real Mol Waterinfo station series + and verify one historical image acquisition/overlay through the LAN browser. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index a9bbce4c..3303fc84 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -214,9 +214,6 @@ These sources are available from their public authorities but are not silently treated as loaded GeoIntel data. The Source inventory labels them separately until a governed operator import, provenance record and validation pass exist. -- Historical orthophotos (Digitaal Vlaanderen): 1971 and 1979-1990 through - the `OKZ` WMS, with additional dated mosaics as separate products. Suitable - for visual/image evolution after a bounded acquisition contract is added. - Biologische Waarderingskaart / Natura 2000 (INBO), state 2025: suitable for habitat, biotope and ecological-value analysis, not a continuous annual series. @@ -229,9 +226,35 @@ until a governed operator import, provenance record and validation pass exist. - DHMV II DTM/DSM (Digitaal Vlaanderen): 1 m/5 m elevation based on 2013-2015 LiDAR, suitable for elevation, slope and drainage. It does not provide water depth. -- Waterinfo/VMM: station time series for water level, flow and precipitation. - These can describe hydrological state, but do not provide area-wide water - volume without a compatible bottom profile/bathymetry model. + +## Governed Waterinfo station history + +`scripts/provision_waterinfo_station_history.py` uses the public Waterinfo +KiWIS query service only after an explicit operator command. It discovers the +documented annual water-level (`192784`) and discharge (`192895`) groups, +filters station points against the exact persisted Area and retains the raw +station/value JSON plus SHA256 manifests. + +Each station and year becomes one immutable reference Dataset through the +normal upload API. Series keys include the station identity; different stations +are never averaged into one municipal value. Selection aggregation is a numeric +mean over the selected station records and remains labelled as a point +measurement. Water level or discharge does not establish area-wide water +volume without compatible depth, profile and coverage data. + +## Governed historical orthophotos + +The bounded map acquisition registry includes the official annual winter +mosaics for 2012-2025, period products for 2000-2003, 2005-2007 and 2008-2011, +the RGB 1979-1990 summer mosaic and the panchromatic 1971 mosaic. Requests stay +within the configured 128-1,024 m safety envelope and are persisted as +EPSG:31370 raster Datasets with explicit product, layer, observation period, +attribution and request hashes. + +Historical mosaics are for visual comparison only in this phase. Current YOLO +building detection and current GRB QA remain restricted to `most_recent`, since +validating an old image against today's building state would produce dishonest +quality metrics. ## OSM diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index 11fa8485..24eae66d 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -198,6 +198,21 @@ Metadata: - nodata - resolution +Temporal orthophoto rasters additionally require a governed product key, WMS +layer, observation label, `observed_at`, optional `valid_from`/`valid_to`, +temporal granularity, request/spatial hash, attribution and a limitation that +states whether the product is annual, multi-year or merely most recent. + +### Hydrological station observations + +Waterinfo observations are persisted as EPSG:4326 Point features, one station +and one observation year per immutable Dataset. Required properties are the +station/timeseries identity, measurement type, numeric annual value, reported +unit, observation year, provider/owner and attribution. Different station +series may not be merged into one area-wide value. Point water level and +discharge may not be converted to water volume without governed compatible +depth/profile data. + ### Vector Ondersteund: diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 84533234..42ff547b 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -99,6 +99,16 @@ Mask files are provenance/debug artifacts. QA, map display and GeoJSON output mu Do not delete originals automatically. Derived outputs may be cleaned through explicit cache management. +## Official temporal source artifacts + +Waterinfo raw station layers, timeseries responses and checksum manifests live +under `storage/operator-data/waterinfo//`. These are immutable source +evidence; queryable annual Point snapshots are normal Dataset/vector_feature +records. Bounded orthophotos are normal raster Dataset files. Their WMS URL, +product/layer, request/spatial hash, temporal validity and limitations are held +in source/provenance metadata. Browser PNG rendering is derived on request and +does not replace the stored GeoTIFF. + Offline demo export artifacts can be inspected and cleaned with: ```bash diff --git a/docs/TODO.md b/docs/TODO.md index 974ff2bb..e4c8fb2c 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -22,12 +22,55 @@ - [x] Add a source inventory that separates loaded data from audited official follow-up sources. - [x] Add a local Ollama question window grounded in persisted GeoIntel metrics and installed server models. - [ ] Add a governed depth/bathymetry source before exposing water volume; never infer volume from 2D GRB water geometry. -- [ ] Integrate one governed hydrology source (Waterinfo/VMM station series) without presenting point measurements as area-wide water volume. -- [ ] Add bounded historical orthophoto acquisition and visual change analysis after validating layer/year coverage. +- [x] Integrate governed Waterinfo/VMM annual station series without presenting point measurements as area-wide water volume. +- [x] Add bounded historical orthophoto acquisition for official 1971-2025 products with a map overlay and no current-GRB QA on old imagery. - [ ] Add BWK/Natura 2000 and annual agricultural-use parcels through explicit provider/operator contracts. - [ ] Extend the official 1778/1873/1969 historical buildings, water and roads series from Mol to the approved regional scope with partitioned source audits. - [x] Connect a drawn rectangle to bounded official orthophoto acquisition, local configured-YOLO detection and persisted GRB QA. +## Governed source expansion backlog + +Implement these in order. Every source must use an explicit operator/provider +contract, retain raw checksummed evidence, persist through DatasetService and +pass live Mol validation before regional expansion. + +### P1 - Biologische Waarderingskaart / Natura 2000 + +- [ ] Confirm the official downloadable service/layer/version and licence for the 2025 state. +- [ ] Define stable habitat/biotope/value fields and keep BWK valuation separate from Natura 2000 habitat classification. +- [ ] Clip in EPSG:31370 to Mol, transform to EPSG:4326 and persist valid polygonal `vector_features` with source feature ids. +- [ ] Add hectare metrics per governed class plus unknown/unmapped-class diagnostics; do not invent an annual trend from one state. +- [ ] Add source manifest, unit tests, live row/count/geometry audit and map/source-inventory presentation. + +### P2 - Annual agricultural-use parcels + +- [ ] Inventory official annual editions and document schema/code-list changes before selecting a comparable year range. +- [ ] Retain annual source files and crop code lists; normalize only fields whose meaning is stable across editions. +- [ ] Persist separate annual Datasets with hectares by crop/use class and an explicit unstable-parcel-identity limitation. +- [ ] Compare area/category totals over time; do not claim parcel lineage where identifiers or boundaries changed. +- [ ] Validate Mol first, then partition all 28 Kempen municipalities with completeness/checksum manifests. + +### P3 - Buildings and Addresses Register + +- [ ] Define a governed snapshot operator and relation mapping between building unit, building object, address and GRB geometry. +- [ ] Keep register lifecycle/status semantics separate from GRB footprint geometry and document match confidence/unmatched rows. +- [ ] Add current building-status/address metrics without exposing personal data or treating addresses as households/population. +- [ ] Add Mol reconciliation tests and a live mismatch report before regional import. + +### P4 - Digitaal Hoogtemodel Vlaanderen + +- [ ] Select official DTM/DSM product, service and native resolution; retain acquisition date and vertical reference. +- [ ] Add bounded raster acquisition/clip storage with nodata, CRS, resolution and checksum validation. +- [ ] Implement governed elevation, relief and slope statistics in metres/degrees; keep drainage interpretation explicitly derived. +- [ ] Do not label terrain/surface height as water depth and do not enable volume from DHMV alone. + +### P5 - Water depth / bathymetry + +- [ ] Identify an authoritative source with compatible spatial coverage, vertical datum, date and uncertainty; otherwise keep volume unavailable. +- [ ] Define waterbody linkage, surface elevation, bottom elevation and uncertainty propagation before adding any volume metric. +- [ ] Validate coverage gaps and prohibit extrapolation outside measured/profiled waterbodies. +- [ ] Add independent GIS review and golden-volume fixtures before exposing the result to users or Ollama. + This file now starts with the current implementation status. Older preparation/backlog sections are preserved below as historical planning context and should not be treated as the live sprint board without checking `docs/CODEX_EXECUTION_LOG.md`. ## Release hardening status diff --git a/frontend/README.md b/frontend/README.md index 20f91f69..2679e7a6 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -404,6 +404,13 @@ their real observation ranges. A collapsed follow-up catalogue distinguishes official sources that exist from datasets that are already persisted in the active project. +The Map result panel also loads a governed orthophoto product list. The most +recent product retains the configured-YOLO plus current-GRB QA action. Official +historical years/periods use the same bounded rectangle, persist as raster +Datasets and render as MapLibre image overlays, but expose no misleading +current-state AI/QA action. The Sources inventory moves Waterinfo and historical +orthophotos from follow-up to loaded evidence only after such datasets exist. + Evolution mode compares the exact selected persisted Area when the full municipality/region action is used. It shows the selected before/after values, all compatible supporting metrics and a chart/table for every observation in diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index a6b8d754..27caea52 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1018,6 +1018,10 @@ function App(): JSX.Element { orthophotoAnalysisRunning={mapOrthophotoAnalysis.running} orthophotoAnalysisQuality={mapOrthophotoAnalysis.lastQuality} orthophotoAnalysisDetectionCount={mapOrthophotoAnalysis.lastDetectionCount} + orthophotoProducts={mapOrthophotoAnalysis.products} + selectedOrthophotoProductKey={mapOrthophotoAnalysis.selectedProductKey} + orthophotoResult={mapOrthophotoAnalysis.lastResult} + orthophotoImageUrl={mapOrthophotoAnalysis.imageUrl} availableMapDatasets={availableMapDatasets} selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''} selectedFeature={selectedMapFeature} @@ -1039,6 +1043,7 @@ function App(): JSX.Element { onRunMapSelectionQa={runMapSelectionQa} onOpenMapSelectionQualityEvidence={openMapSelectionQualityEvidence} onRunOrthophotoAnalysis={mapOrthophotoAnalysis.run} + onSelectOrthophotoProduct={mapOrthophotoAnalysis.setSelectedProductKey} onClearQualityEvidence={clearQualityEvidenceGeoJson} /> ) : null} diff --git a/frontend/src/components/GeoMap.tsx b/frontend/src/components/GeoMap.tsx index 60a34175..e6a5384f 100644 --- a/frontend/src/components/GeoMap.tsx +++ b/frontend/src/components/GeoMap.tsx @@ -3,7 +3,7 @@ import maplibregl from 'maplibre-gl' import 'maplibre-gl/dist/maplibre-gl.css' import { PRIMARY_FOCUS_CENTER } from '../config/primaryFocus' import { featureCollectionBounds } from '../lib/geojsonBounds' -import type { MapViewportState, VectorSelectionBBox } from '../types' +import type { MapImageOverlay, MapViewportState, VectorSelectionBBox } from '../types' interface GeoMapProps { data: GeoJSON.FeatureCollection | null @@ -13,6 +13,7 @@ interface GeoMapProps { selectedFeature?: GeoJSON.Feature | null selectionData?: GeoJSON.FeatureCollection | null qaEvidenceData?: GeoJSON.FeatureCollection | null + imageOverlay?: MapImageOverlay | null selectionBbox?: { min_x: number; min_y: number; max_x: number; max_y: number } | null bboxSelectionMode?: boolean visible?: boolean @@ -153,6 +154,7 @@ function GeoMap({ selectedFeature = null, selectionData = null, qaEvidenceData = null, + imageOverlay = null, selectionBbox = null, bboxSelectionMode = false, visible = true, @@ -346,6 +348,43 @@ function GeoMap({ } }, []) + useEffect(() => { + const map = mapRef.current + if (!map || !mapStyleReady || !map.isStyleLoaded()) { + return + } + if (map.getLayer('bounded-orthophoto')) { + map.removeLayer('bounded-orthophoto') + } + if (map.getSource('bounded-orthophoto')) { + map.removeSource('bounded-orthophoto') + } + if (!imageOverlay) { + return + } + const [minX, minY, maxX, maxY] = imageOverlay.bbox + map.addSource('bounded-orthophoto', { + type: 'image', + url: imageOverlay.url, + coordinates: [ + [minX, maxY], + [maxX, maxY], + [maxX, minY], + [minX, minY], + ], + }) + const beforeLayer = ['area-fill', 'dataset-fill', 'selection-bbox-fill'].find((layerId) => map.getLayer(layerId)) + map.addLayer( + { + id: 'bounded-orthophoto', + type: 'raster', + source: 'bounded-orthophoto', + paint: { 'raster-opacity': imageOverlay.opacity ?? 0.88 }, + }, + beforeLayer, + ) + }, [imageOverlay, mapStyleReady]) + useEffect(() => { const map = mapRef.current if (!map || !mapStyleReady || !map.isStyleLoaded()) { diff --git a/frontend/src/components/datasets/SourceCatalogPanel.tsx b/frontend/src/components/datasets/SourceCatalogPanel.tsx index 70bad2d2..fcb8cfe1 100644 --- a/frontend/src/components/datasets/SourceCatalogPanel.tsx +++ b/frontend/src/components/datasets/SourceCatalogPanel.tsx @@ -15,13 +15,15 @@ const THEME_LABELS: Record = { const AVAILABLE_SOURCES = [ { + key: 'historical_orthophoto', name: 'Historische orthofoto’s', owner: 'Digitaal Vlaanderen', - coverage: '1971 en 1979-1990; aanvullende jaargangen bestaan afzonderlijk', - value: 'Visuele evolutie en toekomstige beeldvergelijking', + coverage: '1971, 1979-1990, 2000-2025', + value: 'Visuele evolutie via begrensde officiële luchtbeelden', url: 'https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-kleinschalig-zomeropnamen', }, { + key: 'bwk', name: 'Biologische Waarderingskaart / Natura 2000', owner: 'INBO', coverage: 'Toestand 2025', @@ -29,6 +31,7 @@ const AVAILABLE_SOURCES = [ url: 'https://www.vlaanderen.be/datavindplaats/catalogus/biologische-waarderingskaart-en-natura-2000-habitatkaart-toestand-2025', }, { + key: 'agriculture', name: 'Landbouwgebruikspercelen', owner: 'Agentschap Landbouw en Zeevisserij', coverage: 'Jaarlijkse bestanden', @@ -36,6 +39,7 @@ const AVAILABLE_SOURCES = [ url: 'https://www.vlaanderen.be/datavindplaats/catalogus/open-geodata-landbouwgebruikspercelen', }, { + key: 'buildings_register', name: 'Gebouwen- en adressenregister', owner: 'Digitaal Vlaanderen', coverage: 'Continu geactualiseerd', @@ -43,6 +47,7 @@ const AVAILABLE_SOURCES = [ url: 'https://www.vlaanderen.be/datavindplaats/catalogus/gebouwen-en-adressenregister', }, { + key: 'elevation', name: 'Digitaal Hoogtemodel Vlaanderen II', owner: 'Digitaal Vlaanderen', coverage: 'LiDAR-opname 2013-2015, DTM/DSM 1 m en 5 m', @@ -50,6 +55,7 @@ const AVAILABLE_SOURCES = [ url: 'https://www.vlaanderen.be/digitaal-vlaanderen/onze-diensten-en-platformen/earth-observation-data-science-eodas/het-digitaal-hoogtemodel/digitaal-hoogtemodel-vlaanderen-ii', }, { + key: 'waterinfo', name: 'Waterinfo en VMM-metingen', owner: 'Vlaamse Milieumaatschappij', coverage: 'Meetpunten en tijdreeksen voor waterstand, debiet en neerslag', @@ -91,6 +97,17 @@ function timelineSummary( export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.Element { const ready = datasets.filter((dataset) => dataset.status === 'ready') + const waterinfoDatasets = ready.filter((dataset) => dataset.source_name === 'waterinfo') + const historicalOrthophotos = ready.filter( + (dataset) => + dataset.source_name === 'digitaal_vlaanderen_orthophoto' && + String(dataset.source_metadata?.['product_key'] ?? 'most_recent') !== 'most_recent', + ) + const pendingSources = AVAILABLE_SOURCES.filter((source) => { + if (source.key === 'waterinfo') return waterinfoDatasets.length === 0 + if (source.key === 'historical_orthophoto') return historicalOrthophotos.length === 0 + return true + }) const themes = Object.keys(THEME_LABELS).map((theme) => { const matches = ready.filter((dataset) => datasetTheme(dataset) === theme) const temporal = matches.filter((dataset) => dataset.temporal_series_key && dataset.observed_at) @@ -148,10 +165,29 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E ))} + {waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 ? ( +
+ {waterinfoDatasets.length > 0 ? ( +
+ Waterinfo meetreeksen + {new Set(waterinfoDatasets.map((dataset) => dataset.temporal_series_key).filter(Boolean)).size} stationsreeksen · {waterinfoDatasets.length} jaarmetingen +

Puntmetingen blijven afzonderlijk per station en worden niet als gebiedsgemiddelde of watervolume voorgesteld.

+
+ ) : null} + {historicalOrthophotos.length > 0 ? ( +
+ Historische luchtbeelden + {historicalOrthophotos.length} begrensde kaartselecties bewaard +

Officiële jaargangen en periodes zijn via de kaart beschikbaar zonder actuele GRB-validatie.

+
+ ) : null} +
+ ) : null} +
Officiële bronnen die hierna kunnen worden ingeladen
- {AVAILABLE_SOURCES.map((source) => ( + {pendingSources.map((source) => (
{source.name} diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index 01d6f492..bcd4e44f 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from 'react' import GeoMap from '../GeoMap' -import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types' +import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types' import { featureCollectionBounds } from '../../lib/geojsonBounds' import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights' import { useTemporalComparison } from '../../hooks/useTemporalComparison' @@ -459,6 +459,10 @@ interface MapWorkspaceProps { orthophotoAnalysisRunning: boolean orthophotoAnalysisQuality: DetectionQaResult | null orthophotoAnalysisDetectionCount: number | null + orthophotoProducts: OrthophotoProductRead[] + selectedOrthophotoProductKey: string + orthophotoResult: OrthophotoAcquisitionResult | null + orthophotoImageUrl: string | null availableMapDatasets: DatasetCreateResponse[] selectedMapDatasetId: string onSelectMapArea: (areaId: string) => void @@ -479,6 +483,7 @@ interface MapWorkspaceProps { onRunMapSelectionQa: (candidateDataset?: DatasetCreateResponse | null) => Promise onOpenMapSelectionQualityEvidence: () => void onRunOrthophotoAnalysis: (bbox: VectorSelectionBBox) => Promise + onSelectOrthophotoProduct: (productKey: string) => void onClearQualityEvidence?: () => void } @@ -534,6 +539,10 @@ export function MapWorkspace({ orthophotoAnalysisRunning, orthophotoAnalysisQuality, orthophotoAnalysisDetectionCount, + orthophotoProducts, + selectedOrthophotoProductKey, + orthophotoResult, + orthophotoImageUrl, availableMapDatasets, selectedMapDatasetId, onSelectMapArea, @@ -554,6 +563,7 @@ export function MapWorkspace({ onRunMapSelectionQa, onOpenMapSelectionQualityEvidence, onRunOrthophotoAnalysis, + onSelectOrthophotoProduct, onClearQualityEvidence, }: MapWorkspaceProps): JSX.Element { const [advancedMode, setAdvancedMode] = useState(false) @@ -615,6 +625,15 @@ export function MapWorkspace({ const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0] const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id] const analysisOverlayActive = mapContentMode === 'analysis' && analysisLayerAvailable && Boolean(mapFeatureCollection) + const selectedOrthophotoProduct = orthophotoProducts.find((item) => item.key === selectedOrthophotoProductKey) ?? null + const orthophotoImageOverlay = orthophotoResult && orthophotoImageUrl && orthophotoResult.bbox_epsg4326.length === 4 + ? { + url: orthophotoImageUrl, + bbox: orthophotoResult.bbox_epsg4326 as [number, number, number, number], + label: orthophotoResult.display_name, + opacity: 0.9, + } + : null const activeThemeDataset = themeDatasetMap[activeTheme.id] const activeScopeProject = projects.find((project) => project.id === selectedProjectId) ?? null const activeScopeLabel = activeScopeProject ? operationalScopeProjectLabel(activeScopeProject) : 'Werkgebied' @@ -1173,6 +1192,7 @@ export function MapWorkspace({ areaData={areaFeatureCollection} selectedFeature={selectedFeature} selectionData={analysisMode === 'current' ? mapSelectionResult?.geojson ?? null : null} + imageOverlay={orthophotoImageOverlay} selectionBbox={mapSelectionBbox} bboxSelectionMode={bboxSelectionMode} visible={mapLayerVisible} @@ -1188,6 +1208,7 @@ export function MapWorkspace({ />
Werkgebied + {orthophotoImageOverlay ? {orthophotoImageOverlay.label} : null} {analysisOverlayActive ? ( <> AI-kandidaten @@ -1233,9 +1254,25 @@ export function MapWorkspace({
Beeldanalyse - Gebouwen herkennen op luchtbeeld - Officieel luchtbeeld, lokaal AI-model en automatische controle met GRB. + {selectedOrthophotoProduct?.supports_detection ? 'Gebouwen herkennen op luchtbeeld' : 'Historisch luchtbeeld bekijken'} + + {selectedOrthophotoProduct?.supports_detection + ? 'Officieel luchtbeeld, lokaal AI-model en automatische controle met GRB.' + : 'Officieel historisch mozaïek. Geen vergelijking met de actuele GRB-toestand.'} +
+