From cc1b905ed3da708319bea687c2a779def7547da1 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 22 Jul 2026 08:02:58 +0200 Subject: [PATCH] fix: close Walloon terrain provisioning --- CHANGELOG.md | 17 ++++++++--- backend/app/services/spw_terrain_service.py | 33 ++++++++++++++------- backend/tests/test_spw_terrain_service.py | 15 ++++++++++ docs/CODEX_EXECUTION_LOG.md | 33 +++++++++++++++++++++ docs/KNOWN_LIMITATIONS.md | 4 +++ docs/TODO.md | 10 ++++--- scripts/provision_spw_terrain_source.py | 23 ++++++++++++-- 7 files changed, 113 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b95831e..a5f56cec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,18 @@ official view-class crosswalk normalizes stacked codes to the existing 11-class series, while source value `0` is explicitly treated as background nodata and never contributes to area metrics. +- Exposed governed WALOUS rasters to the map evolution workspace and scoped + temporal-series choices to the selected persisted Area. The Wallonia golden + area now shows one unambiguous 2018/2020/2023 series instead of unrelated + acquisitions from other geometries. - Added bounded Walloon terrain acquisition from the official SPW MNT 2021-2022 1 m GeoTIFF. Operator provisioning validates archive bounds, safe extraction, CRS, resolution, band count, elevation samples and checksums; selection persistence and terrain metrics retain DNG/EPSG:5710 instead of incorrectly labelling Walloon elevations as TAW. +- Made terrain readiness fail closed on both the source raster and its valid + SHA-256 sidecar, wrote provisioning evidence atomically, and corrected the + persisted acquisition interval to the canonical `period` granularity. - Extended detection-model capabilities with machine-readable training scope, validation scope, validated regions, national-validation status and the operator-review requirement. The configured local model remains bound to @@ -43,10 +50,12 @@ - Added the current legal SPW Walloon flood-hazard polygons as a bounded authoritative vector product with class-aware hectare metrics and canonical persistence. No WMS pixels or modeled depths are fabricated. -- Kept the Walloon DTM and MDK North Sea depth model honest: the published DTM - artifacts require an explicit large-storage partition plan, while the MDK - endpoint still fails strict hostname validation. Neither is presented as a - measured analytical layer. +- Selected the official 1 m Walloon MNT after a live capacity audit and kept + the officially listed circa 198 GB 0.5 m distribution outside V1 because it + adds no required analytical capability and would require still more working + space during extraction. The MDK endpoint still fails strict hostname + validation and remains fail-closed rather than being presented as measured + North Sea bathymetry. - Extended the bounded all-in-one PostGIS recovery wait to 15 minutes and the immutable deploy health gate to 16 minutes. This prevents a large persistent data directory from being terminated mid-recovery by the former two- and diff --git a/backend/app/services/spw_terrain_service.py b/backend/app/services/spw_terrain_service.py index bab664e2..002bbcea 100644 --- a/backend/app/services/spw_terrain_service.py +++ b/backend/app/services/spw_terrain_service.py @@ -56,12 +56,27 @@ class SpwTerrainService: def _source_path(settings: Settings) -> Path: return Path(settings.spw_terrain_source_dir) / SpwTerrainService.SOURCE_FILENAME + @staticmethod + def _source_sha256(settings: Settings) -> str | None: + checksum_path = ( + Path(settings.spw_terrain_source_dir) + / SpwTerrainService.SOURCE_SHA256_FILENAME + ) + if not checksum_path.is_file(): + return None + parts = checksum_path.read_text(encoding="ascii").strip().split() + digest = parts[0].lower() if parts else "" + if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): + return None + return digest + @staticmethod def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]: resolved = settings or get_settings() configured = ( resolved.spw_terrain_enabled and SpwTerrainService._source_path(resolved).is_file() + and SpwTerrainService._source_sha256(resolved) is not None ) product = SpwTerrainProductRead( key=SpwTerrainService.PRODUCT_KEY, @@ -338,12 +353,16 @@ class SpwTerrainService: status_code=503, ) source_path = SpwTerrainService._source_path(resolved) - if not source_path.is_file(): + source_sha256 = SpwTerrainService._source_sha256(resolved) + if not source_path.is_file() or source_sha256 is None: raise AppError( code="SPW_TERRAIN_SOURCE_NOT_PROVISIONED", - message="The official SPW MNT source archive has not been provisioned on this runtime", + message="The official SPW MNT source and checksum have not been provisioned on this runtime", details={ "expected_path": str(source_path), + "expected_checksum_path": str( + source_path.with_name(SpwTerrainService.SOURCE_SHA256_FILENAME) + ), "operator_command": "python scripts/provision_spw_terrain_source.py", }, status_code=503, @@ -390,14 +409,6 @@ class SpwTerrainService: content, validation = SpwTerrainService._read_source_window( source_path, scope, resolution, resolved ) - source_sha256_path = source_path.with_name( - SpwTerrainService.SOURCE_SHA256_FILENAME - ) - source_sha256 = ( - source_sha256_path.read_text(encoding="ascii").strip().split()[0] - if source_sha256_path.is_file() - else None - ) acquired_at = datetime.now(UTC) dataset = DatasetService.import_raster_bytes( db, @@ -410,7 +421,7 @@ class SpwTerrainService: observed_at=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC), valid_from=datetime(2021, 2, 19, tzinfo=UTC), valid_to=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC), - temporal_granularity="acquisition_period", + temporal_granularity="period", source_version="RELIEF_WALLONIE_MNT_1M_2021_2022", source_metadata={ "provider": SpwTerrainService.PROVIDER, diff --git a/backend/tests/test_spw_terrain_service.py b/backend/tests/test_spw_terrain_service.py index db70da02..95371335 100644 --- a/backend/tests/test_spw_terrain_service.py +++ b/backend/tests/test_spw_terrain_service.py @@ -102,11 +102,23 @@ def make_source(path: Path) -> list[float]: return [min_lon, min_lat, max_lon, max_lat] +def write_source_checksum(source_dir: Path) -> str: + digest = "a" * 64 + (source_dir / SpwTerrainService.SOURCE_SHA256_FILENAME).write_text( + f"{digest} {SpwTerrainService.SOURCE_FILENAME}\n", encoding="ascii" + ) + return digest + + def test_spw_terrain_registry_reports_real_source_state(tmp_path: Path) -> None: before = SpwTerrainService.list_products(settings=settings(tmp_path))[0] assert before["status"] == "source_not_provisioned" make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME) + without_checksum = SpwTerrainService.list_products(settings=settings(tmp_path))[0] + assert without_checksum["configured"] is False + + write_source_checksum(tmp_path) after = SpwTerrainService.list_products(settings=settings(tmp_path))[0] assert after["configured"] is True @@ -119,6 +131,7 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance( tmp_path: Path, monkeypatch ) -> None: bbox = make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME) + source_digest = write_source_checksum(tmp_path) project = Project(id=uuid4(), name="Belgium") captured = {} @@ -150,6 +163,8 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance( assert captured["source_metadata"]["vertical_unit_label"] == "m DNG" assert captured["source_metadata"]["coverage_zones"] == ["wallonia"] assert captured["provenance_metadata"]["resampling"] == "bilinear" + assert captured["provenance_metadata"]["source_sha256"] == source_digest + assert captured["temporal_granularity"] == "period" assert captured["valid_from"].date().isoformat() == "2021-02-19" with rasterio.MemoryFile(captured["content"]) as memory: with memory.open() as derived: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index f6f7c0c9..ba0557b0 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11173,3 +11173,36 @@ Known governed boundaries remain explicit: WALOUS covers Wallonia rather than all Belgium, no unstable 2018 artifact is accepted without a verifiable source checksum, and the current local building model is not represented as nationally trained or validated. + +## 2026-07-22 - WALOUS 2018, Walloon terrain and evidence-scope closure + +Implemented and verified: + +- accepted the stable official WALOUS 2018 distribution only after validating + the 1,122,785,133-byte archive (`21e514...c9a`) and extracted raster + (`a788cf...56b2`), then materialized one area-scoped 2018/2020/2023 series; +- exposed that governed series in the evolution workspace and removed foreign + Area acquisitions from its choices; the live Wallonia golden Area shows + exactly three official moments and one unambiguous series; +- provisioned the complete official SPW MNT 2021-2022 1 m source. The archive + was 43,904,242,006 bytes with SHA-256 `04f3ca45821dc2866d854e75011eed1acefcfead9a5263e5a9760e29ded21f7c`; + the extracted EPSG:3812 Float32 raster is 44,014,505,895 bytes, 253,085 by + 146,727 cells, uses nodata -9999 and SHA-256 + `027f90fd304b683cdbf9a0c735152769cf093eb38599e7d5d9dc8792040a9f61`; +- removed the 40.89 GiB terrain download archive after checksum validation and + retained the source raster, checksum sidecar and atomic provisioning report; +- made terrain availability fail closed until both raster and valid SHA-256 + sidecar exist, and persisted the official 2021-02-19/2022-03-05 observation + interval using the canonical `period` granularity; +- constrained AI capability claims to the actual Mol/Kempen evidence through + machine-readable training/validation scope, `nationally_validated=false` and + mandatory operator review. No national training result was fabricated. + +Pre-release evidence: + +- the complete readiness gate passed with 1,103 backend tests and 34 frontend + tests, backend compile, API/documentation contract audit, frontend typecheck, + production build, Alembic head `202607160001` and all script checks; +- live browser acceptance showed the exact WALOUS 2018-2023 series and no + console warnings or errors. Final terrain Dataset and immutable-image evidence + are recorded after deployment below. diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index f2413b77..e8005c53 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -12,6 +12,10 @@ runtime source of truth. persisted NGI/RBINS editions. Detailed themes are federated by jurisdiction; an operational Flemish source does not imply equivalent Walloon or Brussels coverage. +- Land cover remains source-correct by jurisdiction: WALOUS 2018/2020/2023 + covers Wallonia, Flemish products cover Flanders and UrbIS Land Cover covers + Brussels. GeoIntel does not relabel WALOUS as a national source or silently + merge incompatible regional legends. - Buildings, population, terrain, imagery, nature, agriculture, soil and flood themes may report `partial`, `not_configured` or `unsupported` outside the materialized source partitions. The UI and exports retain that state. diff --git a/docs/TODO.md b/docs/TODO.md index ddba335b..5c36c137 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -68,11 +68,13 @@ geen open productroadmap meer. pixelbudget, automatische tijdreeksmaterialisatie en evolutiemetrics. De vaste officiële 2018-GeoTIFF-distributie is live checksum-gevalideerd; bronwaarde `0` wordt expliciet als achtergrond/nodata behandeld. -- [ ] Rond de lopende live provisioning van het officiële SPW MNT 2021-2022 - op 1 m in de operatorcache af en - bied begrensde terreinacquisitie/analyse in DNG aan. De 0,5 m-distributie blijft +- [x] Rond de live provisioning van het officiële SPW MNT 2021-2022 op 1 m in + de operatorcache af en bied begrensde terreinacquisitie/analyse in DNG aan. + Het bronraster en de checksum-sidecar zijn gevalideerd en het downloadarchief + is na veilige extractie verwijderd. De 0,5 m-distributie blijft buiten V1 omdat zij geen noodzakelijke analysecapaciteit toevoegt tegenover de - gevalideerde 1 m-bron en circa 213 GB bronopslag vraagt. + gevalideerde 1 m-bron en officieel als circa 198 GB download wordt aangeboden, + exclusief extra werkruimte tijdens extractie. - [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk scenario-/juridisch contract; gebruik WMS alleen als context tenzij analytische pixels of vectorgeometrie officieel beschikbaar zijn. diff --git a/scripts/provision_spw_terrain_source.py b/scripts/provision_spw_terrain_source.py index 37f3b421..2cd3e14b 100644 --- a/scripts/provision_spw_terrain_source.py +++ b/scripts/provision_spw_terrain_source.py @@ -32,6 +32,17 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def write_text_atomic(path: Path, content: str, *, encoding: str) -> None: + temporary = path.with_suffix(path.suffix + ".part") + temporary.unlink(missing_ok=True) + try: + temporary.write_text(content, encoding=encoding) + temporary.replace(path) + except Exception: + temporary.unlink(missing_ok=True) + raise + + def download(destination: Path) -> str: temporary = destination.with_suffix(destination.suffix + ".part") temporary.unlink(missing_ok=True) @@ -202,8 +213,10 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict: archive.unlink(missing_ok=True) validation = validate_raster(target) source_digest = sha256_file(target) - target.with_suffix(".sha256").write_text( - f"{source_digest} {target.name}\n", encoding="ascii" + write_text_atomic( + target.with_suffix(".sha256"), + f"{source_digest} {target.name}\n", + encoding="ascii", ) validation.update( { @@ -214,7 +227,11 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict: } ) report_path = destination / "provisioning-report.json" - report_path.write_text(json.dumps(validation, indent=2) + "\n", encoding="utf-8") + write_text_atomic( + report_path, + json.dumps(validation, indent=2) + "\n", + encoding="utf-8", + ) print(f"SPW MNT: ready ({target.stat().st_size / 1024 / 1024 / 1024:.1f} GiB)") print(f"Provisioning report: {report_path}") return validation