fix: close Walloon terrain provisioning
This commit is contained in:
+13
-4
@@ -15,11 +15,18 @@
|
||||
official view-class crosswalk normalizes stacked codes to the existing
|
||||
11-class series, while source value `0` is explicitly treated as background
|
||||
nodata and never contributes to area metrics.
|
||||
- Exposed governed WALOUS rasters to the map evolution workspace and scoped
|
||||
temporal-series choices to the selected persisted Area. The Wallonia golden
|
||||
area now shows one unambiguous 2018/2020/2023 series instead of unrelated
|
||||
acquisitions from other geometries.
|
||||
- Added bounded Walloon terrain acquisition from the official SPW MNT
|
||||
2021-2022 1 m GeoTIFF. Operator provisioning validates archive bounds, safe
|
||||
extraction, CRS, resolution, band count, elevation samples and checksums;
|
||||
selection persistence and terrain metrics retain DNG/EPSG:5710 instead of
|
||||
incorrectly labelling Walloon elevations as TAW.
|
||||
- Made terrain readiness fail closed on both the source raster and its valid
|
||||
SHA-256 sidecar, wrote provisioning evidence atomically, and corrected the
|
||||
persisted acquisition interval to the canonical `period` granularity.
|
||||
- Extended detection-model capabilities with machine-readable training scope,
|
||||
validation scope, validated regions, national-validation status and the
|
||||
operator-review requirement. The configured local model remains bound to
|
||||
@@ -43,10 +50,12 @@
|
||||
- Added the current legal SPW Walloon flood-hazard polygons as a bounded
|
||||
authoritative vector product with class-aware hectare metrics and canonical
|
||||
persistence. No WMS pixels or modeled depths are fabricated.
|
||||
- Kept the Walloon DTM and MDK North Sea depth model honest: the published DTM
|
||||
artifacts require an explicit large-storage partition plan, while the MDK
|
||||
endpoint still fails strict hostname validation. Neither is presented as a
|
||||
measured analytical layer.
|
||||
- Selected the official 1 m Walloon MNT after a live capacity audit and kept
|
||||
the officially listed circa 198 GB 0.5 m distribution outside V1 because it
|
||||
adds no required analytical capability and would require still more working
|
||||
space during extraction. The MDK endpoint still fails strict hostname
|
||||
validation and remains fail-closed rather than being presented as measured
|
||||
North Sea bathymetry.
|
||||
- Extended the bounded all-in-one PostGIS recovery wait to 15 minutes and the
|
||||
immutable deploy health gate to 16 minutes. This prevents a large persistent
|
||||
data directory from being terminated mid-recovery by the former two- and
|
||||
|
||||
@@ -56,12 +56,27 @@ class SpwTerrainService:
|
||||
def _source_path(settings: Settings) -> Path:
|
||||
return Path(settings.spw_terrain_source_dir) / SpwTerrainService.SOURCE_FILENAME
|
||||
|
||||
@staticmethod
|
||||
def _source_sha256(settings: Settings) -> str | None:
|
||||
checksum_path = (
|
||||
Path(settings.spw_terrain_source_dir)
|
||||
/ SpwTerrainService.SOURCE_SHA256_FILENAME
|
||||
)
|
||||
if not checksum_path.is_file():
|
||||
return None
|
||||
parts = checksum_path.read_text(encoding="ascii").strip().split()
|
||||
digest = parts[0].lower() if parts else ""
|
||||
if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
|
||||
return None
|
||||
return digest
|
||||
|
||||
@staticmethod
|
||||
def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]:
|
||||
resolved = settings or get_settings()
|
||||
configured = (
|
||||
resolved.spw_terrain_enabled
|
||||
and SpwTerrainService._source_path(resolved).is_file()
|
||||
and SpwTerrainService._source_sha256(resolved) is not None
|
||||
)
|
||||
product = SpwTerrainProductRead(
|
||||
key=SpwTerrainService.PRODUCT_KEY,
|
||||
@@ -338,12 +353,16 @@ class SpwTerrainService:
|
||||
status_code=503,
|
||||
)
|
||||
source_path = SpwTerrainService._source_path(resolved)
|
||||
if not source_path.is_file():
|
||||
source_sha256 = SpwTerrainService._source_sha256(resolved)
|
||||
if not source_path.is_file() or source_sha256 is None:
|
||||
raise AppError(
|
||||
code="SPW_TERRAIN_SOURCE_NOT_PROVISIONED",
|
||||
message="The official SPW MNT source archive has not been provisioned on this runtime",
|
||||
message="The official SPW MNT source and checksum have not been provisioned on this runtime",
|
||||
details={
|
||||
"expected_path": str(source_path),
|
||||
"expected_checksum_path": str(
|
||||
source_path.with_name(SpwTerrainService.SOURCE_SHA256_FILENAME)
|
||||
),
|
||||
"operator_command": "python scripts/provision_spw_terrain_source.py",
|
||||
},
|
||||
status_code=503,
|
||||
@@ -390,14 +409,6 @@ class SpwTerrainService:
|
||||
content, validation = SpwTerrainService._read_source_window(
|
||||
source_path, scope, resolution, resolved
|
||||
)
|
||||
source_sha256_path = source_path.with_name(
|
||||
SpwTerrainService.SOURCE_SHA256_FILENAME
|
||||
)
|
||||
source_sha256 = (
|
||||
source_sha256_path.read_text(encoding="ascii").strip().split()[0]
|
||||
if source_sha256_path.is_file()
|
||||
else None
|
||||
)
|
||||
acquired_at = datetime.now(UTC)
|
||||
dataset = DatasetService.import_raster_bytes(
|
||||
db,
|
||||
@@ -410,7 +421,7 @@ class SpwTerrainService:
|
||||
observed_at=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
|
||||
valid_from=datetime(2021, 2, 19, tzinfo=UTC),
|
||||
valid_to=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
|
||||
temporal_granularity="acquisition_period",
|
||||
temporal_granularity="period",
|
||||
source_version="RELIEF_WALLONIE_MNT_1M_2021_2022",
|
||||
source_metadata={
|
||||
"provider": SpwTerrainService.PROVIDER,
|
||||
|
||||
@@ -102,11 +102,23 @@ def make_source(path: Path) -> list[float]:
|
||||
return [min_lon, min_lat, max_lon, max_lat]
|
||||
|
||||
|
||||
def write_source_checksum(source_dir: Path) -> str:
|
||||
digest = "a" * 64
|
||||
(source_dir / SpwTerrainService.SOURCE_SHA256_FILENAME).write_text(
|
||||
f"{digest} {SpwTerrainService.SOURCE_FILENAME}\n", encoding="ascii"
|
||||
)
|
||||
return digest
|
||||
|
||||
|
||||
def test_spw_terrain_registry_reports_real_source_state(tmp_path: Path) -> None:
|
||||
before = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
||||
assert before["status"] == "source_not_provisioned"
|
||||
|
||||
make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
|
||||
without_checksum = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
||||
assert without_checksum["configured"] is False
|
||||
|
||||
write_source_checksum(tmp_path)
|
||||
after = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
||||
|
||||
assert after["configured"] is True
|
||||
@@ -119,6 +131,7 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
bbox = make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
|
||||
source_digest = write_source_checksum(tmp_path)
|
||||
project = Project(id=uuid4(), name="Belgium")
|
||||
captured = {}
|
||||
|
||||
@@ -150,6 +163,8 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance(
|
||||
assert captured["source_metadata"]["vertical_unit_label"] == "m DNG"
|
||||
assert captured["source_metadata"]["coverage_zones"] == ["wallonia"]
|
||||
assert captured["provenance_metadata"]["resampling"] == "bilinear"
|
||||
assert captured["provenance_metadata"]["source_sha256"] == source_digest
|
||||
assert captured["temporal_granularity"] == "period"
|
||||
assert captured["valid_from"].date().isoformat() == "2021-02-19"
|
||||
with rasterio.MemoryFile(captured["content"]) as memory:
|
||||
with memory.open() as derived:
|
||||
|
||||
@@ -11173,3 +11173,36 @@ Known governed boundaries remain explicit: WALOUS covers Wallonia rather than
|
||||
all Belgium, no unstable 2018 artifact is accepted without a verifiable source
|
||||
checksum, and the current local building model is not represented as nationally
|
||||
trained or validated.
|
||||
|
||||
## 2026-07-22 - WALOUS 2018, Walloon terrain and evidence-scope closure
|
||||
|
||||
Implemented and verified:
|
||||
|
||||
- accepted the stable official WALOUS 2018 distribution only after validating
|
||||
the 1,122,785,133-byte archive (`21e514...c9a`) and extracted raster
|
||||
(`a788cf...56b2`), then materialized one area-scoped 2018/2020/2023 series;
|
||||
- exposed that governed series in the evolution workspace and removed foreign
|
||||
Area acquisitions from its choices; the live Wallonia golden Area shows
|
||||
exactly three official moments and one unambiguous series;
|
||||
- provisioned the complete official SPW MNT 2021-2022 1 m source. The archive
|
||||
was 43,904,242,006 bytes with SHA-256 `04f3ca45821dc2866d854e75011eed1acefcfead9a5263e5a9760e29ded21f7c`;
|
||||
the extracted EPSG:3812 Float32 raster is 44,014,505,895 bytes, 253,085 by
|
||||
146,727 cells, uses nodata -9999 and SHA-256
|
||||
`027f90fd304b683cdbf9a0c735152769cf093eb38599e7d5d9dc8792040a9f61`;
|
||||
- removed the 40.89 GiB terrain download archive after checksum validation and
|
||||
retained the source raster, checksum sidecar and atomic provisioning report;
|
||||
- made terrain availability fail closed until both raster and valid SHA-256
|
||||
sidecar exist, and persisted the official 2021-02-19/2022-03-05 observation
|
||||
interval using the canonical `period` granularity;
|
||||
- constrained AI capability claims to the actual Mol/Kempen evidence through
|
||||
machine-readable training/validation scope, `nationally_validated=false` and
|
||||
mandatory operator review. No national training result was fabricated.
|
||||
|
||||
Pre-release evidence:
|
||||
|
||||
- the complete readiness gate passed with 1,103 backend tests and 34 frontend
|
||||
tests, backend compile, API/documentation contract audit, frontend typecheck,
|
||||
production build, Alembic head `202607160001` and all script checks;
|
||||
- live browser acceptance showed the exact WALOUS 2018-2023 series and no
|
||||
console warnings or errors. Final terrain Dataset and immutable-image evidence
|
||||
are recorded after deployment below.
|
||||
|
||||
@@ -12,6 +12,10 @@ runtime source of truth.
|
||||
persisted NGI/RBINS editions. Detailed themes are federated by jurisdiction;
|
||||
an operational Flemish source does not imply equivalent Walloon or Brussels
|
||||
coverage.
|
||||
- Land cover remains source-correct by jurisdiction: WALOUS 2018/2020/2023
|
||||
covers Wallonia, Flemish products cover Flanders and UrbIS Land Cover covers
|
||||
Brussels. GeoIntel does not relabel WALOUS as a national source or silently
|
||||
merge incompatible regional legends.
|
||||
- Buildings, population, terrain, imagery, nature, agriculture, soil and flood
|
||||
themes may report `partial`, `not_configured` or `unsupported` outside the
|
||||
materialized source partitions. The UI and exports retain that state.
|
||||
|
||||
+6
-4
@@ -68,11 +68,13 @@ geen open productroadmap meer.
|
||||
pixelbudget, automatische tijdreeksmaterialisatie en evolutiemetrics. De vaste
|
||||
officiële 2018-GeoTIFF-distributie is live checksum-gevalideerd; bronwaarde `0`
|
||||
wordt expliciet als achtergrond/nodata behandeld.
|
||||
- [ ] Rond de lopende live provisioning van het officiële SPW MNT 2021-2022
|
||||
op 1 m in de operatorcache af en
|
||||
bied begrensde terreinacquisitie/analyse in DNG aan. De 0,5 m-distributie blijft
|
||||
- [x] Rond de live provisioning van het officiële SPW MNT 2021-2022 op 1 m in
|
||||
de operatorcache af en bied begrensde terreinacquisitie/analyse in DNG aan.
|
||||
Het bronraster en de checksum-sidecar zijn gevalideerd en het downloadarchief
|
||||
is na veilige extractie verwijderd. De 0,5 m-distributie blijft
|
||||
buiten V1 omdat zij geen noodzakelijke analysecapaciteit toevoegt tegenover de
|
||||
gevalideerde 1 m-bron en circa 213 GB bronopslag vraagt.
|
||||
gevalideerde 1 m-bron en officieel als circa 198 GB download wordt aangeboden,
|
||||
exclusief extra werkruimte tijdens extractie.
|
||||
- [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk
|
||||
scenario-/juridisch contract; gebruik WMS alleen als context tenzij
|
||||
analytische pixels of vectorgeometrie officieel beschikbaar zijn.
|
||||
|
||||
@@ -32,6 +32,17 @@ def sha256_file(path: Path) -> str:
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_text_atomic(path: Path, content: str, *, encoding: str) -> None:
|
||||
temporary = path.with_suffix(path.suffix + ".part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
try:
|
||||
temporary.write_text(content, encoding=encoding)
|
||||
temporary.replace(path)
|
||||
except Exception:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def download(destination: Path) -> str:
|
||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
@@ -202,8 +213,10 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict:
|
||||
archive.unlink(missing_ok=True)
|
||||
validation = validate_raster(target)
|
||||
source_digest = sha256_file(target)
|
||||
target.with_suffix(".sha256").write_text(
|
||||
f"{source_digest} {target.name}\n", encoding="ascii"
|
||||
write_text_atomic(
|
||||
target.with_suffix(".sha256"),
|
||||
f"{source_digest} {target.name}\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
validation.update(
|
||||
{
|
||||
@@ -214,7 +227,11 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict:
|
||||
}
|
||||
)
|
||||
report_path = destination / "provisioning-report.json"
|
||||
report_path.write_text(json.dumps(validation, indent=2) + "\n", encoding="utf-8")
|
||||
write_text_atomic(
|
||||
report_path,
|
||||
json.dumps(validation, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"SPW MNT: ready ({target.stat().st_size / 1024 / 1024 / 1024:.1f} GiB)")
|
||||
print(f"Provisioning report: {report_path}")
|
||||
return validation
|
||||
|
||||
Reference in New Issue
Block a user