fix: close Walloon terrain provisioning
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-22 08:02:58 +02:00
parent 6808f3a12e
commit cc1b905ed3
7 changed files with 113 additions and 22 deletions
+13 -4
View File
@@ -15,11 +15,18 @@
official view-class crosswalk normalizes stacked codes to the existing official view-class crosswalk normalizes stacked codes to the existing
11-class series, while source value `0` is explicitly treated as background 11-class series, while source value `0` is explicitly treated as background
nodata and never contributes to area metrics. nodata and never contributes to area metrics.
- Exposed governed WALOUS rasters to the map evolution workspace and scoped
temporal-series choices to the selected persisted Area. The Wallonia golden
area now shows one unambiguous 2018/2020/2023 series instead of unrelated
acquisitions from other geometries.
- Added bounded Walloon terrain acquisition from the official SPW MNT - Added bounded Walloon terrain acquisition from the official SPW MNT
2021-2022 1 m GeoTIFF. Operator provisioning validates archive bounds, safe 2021-2022 1 m GeoTIFF. Operator provisioning validates archive bounds, safe
extraction, CRS, resolution, band count, elevation samples and checksums; extraction, CRS, resolution, band count, elevation samples and checksums;
selection persistence and terrain metrics retain DNG/EPSG:5710 instead of selection persistence and terrain metrics retain DNG/EPSG:5710 instead of
incorrectly labelling Walloon elevations as TAW. incorrectly labelling Walloon elevations as TAW.
- Made terrain readiness fail closed on both the source raster and its valid
SHA-256 sidecar, wrote provisioning evidence atomically, and corrected the
persisted acquisition interval to the canonical `period` granularity.
- Extended detection-model capabilities with machine-readable training scope, - Extended detection-model capabilities with machine-readable training scope,
validation scope, validated regions, national-validation status and the validation scope, validated regions, national-validation status and the
operator-review requirement. The configured local model remains bound to operator-review requirement. The configured local model remains bound to
@@ -43,10 +50,12 @@
- Added the current legal SPW Walloon flood-hazard polygons as a bounded - Added the current legal SPW Walloon flood-hazard polygons as a bounded
authoritative vector product with class-aware hectare metrics and canonical authoritative vector product with class-aware hectare metrics and canonical
persistence. No WMS pixels or modeled depths are fabricated. persistence. No WMS pixels or modeled depths are fabricated.
- Kept the Walloon DTM and MDK North Sea depth model honest: the published DTM - Selected the official 1 m Walloon MNT after a live capacity audit and kept
artifacts require an explicit large-storage partition plan, while the MDK the officially listed circa 198 GB 0.5 m distribution outside V1 because it
endpoint still fails strict hostname validation. Neither is presented as a adds no required analytical capability and would require still more working
measured analytical layer. space during extraction. The MDK endpoint still fails strict hostname
validation and remains fail-closed rather than being presented as measured
North Sea bathymetry.
- Extended the bounded all-in-one PostGIS recovery wait to 15 minutes and the - Extended the bounded all-in-one PostGIS recovery wait to 15 minutes and the
immutable deploy health gate to 16 minutes. This prevents a large persistent immutable deploy health gate to 16 minutes. This prevents a large persistent
data directory from being terminated mid-recovery by the former two- and data directory from being terminated mid-recovery by the former two- and
+22 -11
View File
@@ -56,12 +56,27 @@ class SpwTerrainService:
def _source_path(settings: Settings) -> Path: def _source_path(settings: Settings) -> Path:
return Path(settings.spw_terrain_source_dir) / SpwTerrainService.SOURCE_FILENAME return Path(settings.spw_terrain_source_dir) / SpwTerrainService.SOURCE_FILENAME
@staticmethod
def _source_sha256(settings: Settings) -> str | None:
checksum_path = (
Path(settings.spw_terrain_source_dir)
/ SpwTerrainService.SOURCE_SHA256_FILENAME
)
if not checksum_path.is_file():
return None
parts = checksum_path.read_text(encoding="ascii").strip().split()
digest = parts[0].lower() if parts else ""
if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
return None
return digest
@staticmethod @staticmethod
def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]: def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]:
resolved = settings or get_settings() resolved = settings or get_settings()
configured = ( configured = (
resolved.spw_terrain_enabled resolved.spw_terrain_enabled
and SpwTerrainService._source_path(resolved).is_file() and SpwTerrainService._source_path(resolved).is_file()
and SpwTerrainService._source_sha256(resolved) is not None
) )
product = SpwTerrainProductRead( product = SpwTerrainProductRead(
key=SpwTerrainService.PRODUCT_KEY, key=SpwTerrainService.PRODUCT_KEY,
@@ -338,12 +353,16 @@ class SpwTerrainService:
status_code=503, status_code=503,
) )
source_path = SpwTerrainService._source_path(resolved) source_path = SpwTerrainService._source_path(resolved)
if not source_path.is_file(): source_sha256 = SpwTerrainService._source_sha256(resolved)
if not source_path.is_file() or source_sha256 is None:
raise AppError( raise AppError(
code="SPW_TERRAIN_SOURCE_NOT_PROVISIONED", code="SPW_TERRAIN_SOURCE_NOT_PROVISIONED",
message="The official SPW MNT source archive has not been provisioned on this runtime", message="The official SPW MNT source and checksum have not been provisioned on this runtime",
details={ details={
"expected_path": str(source_path), "expected_path": str(source_path),
"expected_checksum_path": str(
source_path.with_name(SpwTerrainService.SOURCE_SHA256_FILENAME)
),
"operator_command": "python scripts/provision_spw_terrain_source.py", "operator_command": "python scripts/provision_spw_terrain_source.py",
}, },
status_code=503, status_code=503,
@@ -390,14 +409,6 @@ class SpwTerrainService:
content, validation = SpwTerrainService._read_source_window( content, validation = SpwTerrainService._read_source_window(
source_path, scope, resolution, resolved source_path, scope, resolution, resolved
) )
source_sha256_path = source_path.with_name(
SpwTerrainService.SOURCE_SHA256_FILENAME
)
source_sha256 = (
source_sha256_path.read_text(encoding="ascii").strip().split()[0]
if source_sha256_path.is_file()
else None
)
acquired_at = datetime.now(UTC) acquired_at = datetime.now(UTC)
dataset = DatasetService.import_raster_bytes( dataset = DatasetService.import_raster_bytes(
db, db,
@@ -410,7 +421,7 @@ class SpwTerrainService:
observed_at=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC), observed_at=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
valid_from=datetime(2021, 2, 19, tzinfo=UTC), valid_from=datetime(2021, 2, 19, tzinfo=UTC),
valid_to=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC), valid_to=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
temporal_granularity="acquisition_period", temporal_granularity="period",
source_version="RELIEF_WALLONIE_MNT_1M_2021_2022", source_version="RELIEF_WALLONIE_MNT_1M_2021_2022",
source_metadata={ source_metadata={
"provider": SpwTerrainService.PROVIDER, "provider": SpwTerrainService.PROVIDER,
+15
View File
@@ -102,11 +102,23 @@ def make_source(path: Path) -> list[float]:
return [min_lon, min_lat, max_lon, max_lat] return [min_lon, min_lat, max_lon, max_lat]
def write_source_checksum(source_dir: Path) -> str:
digest = "a" * 64
(source_dir / SpwTerrainService.SOURCE_SHA256_FILENAME).write_text(
f"{digest} {SpwTerrainService.SOURCE_FILENAME}\n", encoding="ascii"
)
return digest
def test_spw_terrain_registry_reports_real_source_state(tmp_path: Path) -> None: def test_spw_terrain_registry_reports_real_source_state(tmp_path: Path) -> None:
before = SpwTerrainService.list_products(settings=settings(tmp_path))[0] before = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
assert before["status"] == "source_not_provisioned" assert before["status"] == "source_not_provisioned"
make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME) make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
without_checksum = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
assert without_checksum["configured"] is False
write_source_checksum(tmp_path)
after = SpwTerrainService.list_products(settings=settings(tmp_path))[0] after = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
assert after["configured"] is True assert after["configured"] is True
@@ -119,6 +131,7 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance(
tmp_path: Path, monkeypatch tmp_path: Path, monkeypatch
) -> None: ) -> None:
bbox = make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME) bbox = make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
source_digest = write_source_checksum(tmp_path)
project = Project(id=uuid4(), name="Belgium") project = Project(id=uuid4(), name="Belgium")
captured = {} captured = {}
@@ -150,6 +163,8 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance(
assert captured["source_metadata"]["vertical_unit_label"] == "m DNG" assert captured["source_metadata"]["vertical_unit_label"] == "m DNG"
assert captured["source_metadata"]["coverage_zones"] == ["wallonia"] assert captured["source_metadata"]["coverage_zones"] == ["wallonia"]
assert captured["provenance_metadata"]["resampling"] == "bilinear" assert captured["provenance_metadata"]["resampling"] == "bilinear"
assert captured["provenance_metadata"]["source_sha256"] == source_digest
assert captured["temporal_granularity"] == "period"
assert captured["valid_from"].date().isoformat() == "2021-02-19" assert captured["valid_from"].date().isoformat() == "2021-02-19"
with rasterio.MemoryFile(captured["content"]) as memory: with rasterio.MemoryFile(captured["content"]) as memory:
with memory.open() as derived: with memory.open() as derived:
+33
View File
@@ -11173,3 +11173,36 @@ Known governed boundaries remain explicit: WALOUS covers Wallonia rather than
all Belgium, no unstable 2018 artifact is accepted without a verifiable source all Belgium, no unstable 2018 artifact is accepted without a verifiable source
checksum, and the current local building model is not represented as nationally checksum, and the current local building model is not represented as nationally
trained or validated. trained or validated.
## 2026-07-22 - WALOUS 2018, Walloon terrain and evidence-scope closure
Implemented and verified:
- accepted the stable official WALOUS 2018 distribution only after validating
the 1,122,785,133-byte archive (`21e514...c9a`) and extracted raster
(`a788cf...56b2`), then materialized one area-scoped 2018/2020/2023 series;
- exposed that governed series in the evolution workspace and removed foreign
Area acquisitions from its choices; the live Wallonia golden Area shows
exactly three official moments and one unambiguous series;
- provisioned the complete official SPW MNT 2021-2022 1 m source. The archive
was 43,904,242,006 bytes with SHA-256 `04f3ca45821dc2866d854e75011eed1acefcfead9a5263e5a9760e29ded21f7c`;
the extracted EPSG:3812 Float32 raster is 44,014,505,895 bytes, 253,085 by
146,727 cells, uses nodata -9999 and SHA-256
`027f90fd304b683cdbf9a0c735152769cf093eb38599e7d5d9dc8792040a9f61`;
- removed the 40.89 GiB terrain download archive after checksum validation and
retained the source raster, checksum sidecar and atomic provisioning report;
- made terrain availability fail closed until both raster and valid SHA-256
sidecar exist, and persisted the official 2021-02-19/2022-03-05 observation
interval using the canonical `period` granularity;
- constrained AI capability claims to the actual Mol/Kempen evidence through
machine-readable training/validation scope, `nationally_validated=false` and
mandatory operator review. No national training result was fabricated.
Pre-release evidence:
- the complete readiness gate passed with 1,103 backend tests and 34 frontend
tests, backend compile, API/documentation contract audit, frontend typecheck,
production build, Alembic head `202607160001` and all script checks;
- live browser acceptance showed the exact WALOUS 2018-2023 series and no
console warnings or errors. Final terrain Dataset and immutable-image evidence
are recorded after deployment below.
+4
View File
@@ -12,6 +12,10 @@ runtime source of truth.
persisted NGI/RBINS editions. Detailed themes are federated by jurisdiction; persisted NGI/RBINS editions. Detailed themes are federated by jurisdiction;
an operational Flemish source does not imply equivalent Walloon or Brussels an operational Flemish source does not imply equivalent Walloon or Brussels
coverage. coverage.
- Land cover remains source-correct by jurisdiction: WALOUS 2018/2020/2023
covers Wallonia, Flemish products cover Flanders and UrbIS Land Cover covers
Brussels. GeoIntel does not relabel WALOUS as a national source or silently
merge incompatible regional legends.
- Buildings, population, terrain, imagery, nature, agriculture, soil and flood - Buildings, population, terrain, imagery, nature, agriculture, soil and flood
themes may report `partial`, `not_configured` or `unsupported` outside the themes may report `partial`, `not_configured` or `unsupported` outside the
materialized source partitions. The UI and exports retain that state. materialized source partitions. The UI and exports retain that state.
+6 -4
View File
@@ -68,11 +68,13 @@ geen open productroadmap meer.
pixelbudget, automatische tijdreeksmaterialisatie en evolutiemetrics. De vaste pixelbudget, automatische tijdreeksmaterialisatie en evolutiemetrics. De vaste
officiële 2018-GeoTIFF-distributie is live checksum-gevalideerd; bronwaarde `0` officiële 2018-GeoTIFF-distributie is live checksum-gevalideerd; bronwaarde `0`
wordt expliciet als achtergrond/nodata behandeld. wordt expliciet als achtergrond/nodata behandeld.
- [ ] Rond de lopende live provisioning van het officiële SPW MNT 2021-2022 - [x] Rond de live provisioning van het officiële SPW MNT 2021-2022 op 1 m in
op 1 m in de operatorcache af en de operatorcache af en bied begrensde terreinacquisitie/analyse in DNG aan.
bied begrensde terreinacquisitie/analyse in DNG aan. De 0,5 m-distributie blijft Het bronraster en de checksum-sidecar zijn gevalideerd en het downloadarchief
is na veilige extractie verwijderd. De 0,5 m-distributie blijft
buiten V1 omdat zij geen noodzakelijke analysecapaciteit toevoegt tegenover de buiten V1 omdat zij geen noodzakelijke analysecapaciteit toevoegt tegenover de
gevalideerde 1 m-bron en circa 213 GB bronopslag vraagt. gevalideerde 1 m-bron en officieel als circa 198 GB download wordt aangeboden,
exclusief extra werkruimte tijdens extractie.
- [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk - [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk
scenario-/juridisch contract; gebruik WMS alleen als context tenzij scenario-/juridisch contract; gebruik WMS alleen als context tenzij
analytische pixels of vectorgeometrie officieel beschikbaar zijn. analytische pixels of vectorgeometrie officieel beschikbaar zijn.
+20 -3
View File
@@ -32,6 +32,17 @@ def sha256_file(path: Path) -> str:
return digest.hexdigest() return digest.hexdigest()
def write_text_atomic(path: Path, content: str, *, encoding: str) -> None:
temporary = path.with_suffix(path.suffix + ".part")
temporary.unlink(missing_ok=True)
try:
temporary.write_text(content, encoding=encoding)
temporary.replace(path)
except Exception:
temporary.unlink(missing_ok=True)
raise
def download(destination: Path) -> str: def download(destination: Path) -> str:
temporary = destination.with_suffix(destination.suffix + ".part") temporary = destination.with_suffix(destination.suffix + ".part")
temporary.unlink(missing_ok=True) temporary.unlink(missing_ok=True)
@@ -202,8 +213,10 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict:
archive.unlink(missing_ok=True) archive.unlink(missing_ok=True)
validation = validate_raster(target) validation = validate_raster(target)
source_digest = sha256_file(target) source_digest = sha256_file(target)
target.with_suffix(".sha256").write_text( write_text_atomic(
f"{source_digest} {target.name}\n", encoding="ascii" target.with_suffix(".sha256"),
f"{source_digest} {target.name}\n",
encoding="ascii",
) )
validation.update( validation.update(
{ {
@@ -214,7 +227,11 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict:
} }
) )
report_path = destination / "provisioning-report.json" report_path = destination / "provisioning-report.json"
report_path.write_text(json.dumps(validation, indent=2) + "\n", encoding="utf-8") write_text_atomic(
report_path,
json.dumps(validation, indent=2) + "\n",
encoding="utf-8",
)
print(f"SPW MNT: ready ({target.stat().st_size / 1024 / 1024 / 1024:.1f} GiB)") print(f"SPW MNT: ready ({target.stat().st_size / 1024 / 1024 / 1024:.1f} GiB)")
print(f"Provisioning report: {report_path}") print(f"Provisioning report: {report_path}")
return validation return validation