fix: close Walloon terrain provisioning
This commit is contained in:
+13
-4
@@ -15,11 +15,18 @@
|
|||||||
official view-class crosswalk normalizes stacked codes to the existing
|
official view-class crosswalk normalizes stacked codes to the existing
|
||||||
11-class series, while source value `0` is explicitly treated as background
|
11-class series, while source value `0` is explicitly treated as background
|
||||||
nodata and never contributes to area metrics.
|
nodata and never contributes to area metrics.
|
||||||
|
- Exposed governed WALOUS rasters to the map evolution workspace and scoped
|
||||||
|
temporal-series choices to the selected persisted Area. The Wallonia golden
|
||||||
|
area now shows one unambiguous 2018/2020/2023 series instead of unrelated
|
||||||
|
acquisitions from other geometries.
|
||||||
- Added bounded Walloon terrain acquisition from the official SPW MNT
|
- Added bounded Walloon terrain acquisition from the official SPW MNT
|
||||||
2021-2022 1 m GeoTIFF. Operator provisioning validates archive bounds, safe
|
2021-2022 1 m GeoTIFF. Operator provisioning validates archive bounds, safe
|
||||||
extraction, CRS, resolution, band count, elevation samples and checksums;
|
extraction, CRS, resolution, band count, elevation samples and checksums;
|
||||||
selection persistence and terrain metrics retain DNG/EPSG:5710 instead of
|
selection persistence and terrain metrics retain DNG/EPSG:5710 instead of
|
||||||
incorrectly labelling Walloon elevations as TAW.
|
incorrectly labelling Walloon elevations as TAW.
|
||||||
|
- Made terrain readiness fail closed on both the source raster and its valid
|
||||||
|
SHA-256 sidecar, wrote provisioning evidence atomically, and corrected the
|
||||||
|
persisted acquisition interval to the canonical `period` granularity.
|
||||||
- Extended detection-model capabilities with machine-readable training scope,
|
- Extended detection-model capabilities with machine-readable training scope,
|
||||||
validation scope, validated regions, national-validation status and the
|
validation scope, validated regions, national-validation status and the
|
||||||
operator-review requirement. The configured local model remains bound to
|
operator-review requirement. The configured local model remains bound to
|
||||||
@@ -43,10 +50,12 @@
|
|||||||
- Added the current legal SPW Walloon flood-hazard polygons as a bounded
|
- Added the current legal SPW Walloon flood-hazard polygons as a bounded
|
||||||
authoritative vector product with class-aware hectare metrics and canonical
|
authoritative vector product with class-aware hectare metrics and canonical
|
||||||
persistence. No WMS pixels or modeled depths are fabricated.
|
persistence. No WMS pixels or modeled depths are fabricated.
|
||||||
- Kept the Walloon DTM and MDK North Sea depth model honest: the published DTM
|
- Selected the official 1 m Walloon MNT after a live capacity audit and kept
|
||||||
artifacts require an explicit large-storage partition plan, while the MDK
|
the officially listed circa 198 GB 0.5 m distribution outside V1 because it
|
||||||
endpoint still fails strict hostname validation. Neither is presented as a
|
adds no required analytical capability and would require still more working
|
||||||
measured analytical layer.
|
space during extraction. The MDK endpoint still fails strict hostname
|
||||||
|
validation and remains fail-closed rather than being presented as measured
|
||||||
|
North Sea bathymetry.
|
||||||
- Extended the bounded all-in-one PostGIS recovery wait to 15 minutes and the
|
- Extended the bounded all-in-one PostGIS recovery wait to 15 minutes and the
|
||||||
immutable deploy health gate to 16 minutes. This prevents a large persistent
|
immutable deploy health gate to 16 minutes. This prevents a large persistent
|
||||||
data directory from being terminated mid-recovery by the former two- and
|
data directory from being terminated mid-recovery by the former two- and
|
||||||
|
|||||||
@@ -56,12 +56,27 @@ class SpwTerrainService:
|
|||||||
def _source_path(settings: Settings) -> Path:
|
def _source_path(settings: Settings) -> Path:
|
||||||
return Path(settings.spw_terrain_source_dir) / SpwTerrainService.SOURCE_FILENAME
|
return Path(settings.spw_terrain_source_dir) / SpwTerrainService.SOURCE_FILENAME
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _source_sha256(settings: Settings) -> str | None:
|
||||||
|
checksum_path = (
|
||||||
|
Path(settings.spw_terrain_source_dir)
|
||||||
|
/ SpwTerrainService.SOURCE_SHA256_FILENAME
|
||||||
|
)
|
||||||
|
if not checksum_path.is_file():
|
||||||
|
return None
|
||||||
|
parts = checksum_path.read_text(encoding="ascii").strip().split()
|
||||||
|
digest = parts[0].lower() if parts else ""
|
||||||
|
if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
|
||||||
|
return None
|
||||||
|
return digest
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]:
|
def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]:
|
||||||
resolved = settings or get_settings()
|
resolved = settings or get_settings()
|
||||||
configured = (
|
configured = (
|
||||||
resolved.spw_terrain_enabled
|
resolved.spw_terrain_enabled
|
||||||
and SpwTerrainService._source_path(resolved).is_file()
|
and SpwTerrainService._source_path(resolved).is_file()
|
||||||
|
and SpwTerrainService._source_sha256(resolved) is not None
|
||||||
)
|
)
|
||||||
product = SpwTerrainProductRead(
|
product = SpwTerrainProductRead(
|
||||||
key=SpwTerrainService.PRODUCT_KEY,
|
key=SpwTerrainService.PRODUCT_KEY,
|
||||||
@@ -338,12 +353,16 @@ class SpwTerrainService:
|
|||||||
status_code=503,
|
status_code=503,
|
||||||
)
|
)
|
||||||
source_path = SpwTerrainService._source_path(resolved)
|
source_path = SpwTerrainService._source_path(resolved)
|
||||||
if not source_path.is_file():
|
source_sha256 = SpwTerrainService._source_sha256(resolved)
|
||||||
|
if not source_path.is_file() or source_sha256 is None:
|
||||||
raise AppError(
|
raise AppError(
|
||||||
code="SPW_TERRAIN_SOURCE_NOT_PROVISIONED",
|
code="SPW_TERRAIN_SOURCE_NOT_PROVISIONED",
|
||||||
message="The official SPW MNT source archive has not been provisioned on this runtime",
|
message="The official SPW MNT source and checksum have not been provisioned on this runtime",
|
||||||
details={
|
details={
|
||||||
"expected_path": str(source_path),
|
"expected_path": str(source_path),
|
||||||
|
"expected_checksum_path": str(
|
||||||
|
source_path.with_name(SpwTerrainService.SOURCE_SHA256_FILENAME)
|
||||||
|
),
|
||||||
"operator_command": "python scripts/provision_spw_terrain_source.py",
|
"operator_command": "python scripts/provision_spw_terrain_source.py",
|
||||||
},
|
},
|
||||||
status_code=503,
|
status_code=503,
|
||||||
@@ -390,14 +409,6 @@ class SpwTerrainService:
|
|||||||
content, validation = SpwTerrainService._read_source_window(
|
content, validation = SpwTerrainService._read_source_window(
|
||||||
source_path, scope, resolution, resolved
|
source_path, scope, resolution, resolved
|
||||||
)
|
)
|
||||||
source_sha256_path = source_path.with_name(
|
|
||||||
SpwTerrainService.SOURCE_SHA256_FILENAME
|
|
||||||
)
|
|
||||||
source_sha256 = (
|
|
||||||
source_sha256_path.read_text(encoding="ascii").strip().split()[0]
|
|
||||||
if source_sha256_path.is_file()
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
acquired_at = datetime.now(UTC)
|
acquired_at = datetime.now(UTC)
|
||||||
dataset = DatasetService.import_raster_bytes(
|
dataset = DatasetService.import_raster_bytes(
|
||||||
db,
|
db,
|
||||||
@@ -410,7 +421,7 @@ class SpwTerrainService:
|
|||||||
observed_at=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
|
observed_at=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
|
||||||
valid_from=datetime(2021, 2, 19, tzinfo=UTC),
|
valid_from=datetime(2021, 2, 19, tzinfo=UTC),
|
||||||
valid_to=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
|
valid_to=datetime(2022, 3, 5, 23, 59, 59, tzinfo=UTC),
|
||||||
temporal_granularity="acquisition_period",
|
temporal_granularity="period",
|
||||||
source_version="RELIEF_WALLONIE_MNT_1M_2021_2022",
|
source_version="RELIEF_WALLONIE_MNT_1M_2021_2022",
|
||||||
source_metadata={
|
source_metadata={
|
||||||
"provider": SpwTerrainService.PROVIDER,
|
"provider": SpwTerrainService.PROVIDER,
|
||||||
|
|||||||
@@ -102,11 +102,23 @@ def make_source(path: Path) -> list[float]:
|
|||||||
return [min_lon, min_lat, max_lon, max_lat]
|
return [min_lon, min_lat, max_lon, max_lat]
|
||||||
|
|
||||||
|
|
||||||
|
def write_source_checksum(source_dir: Path) -> str:
|
||||||
|
digest = "a" * 64
|
||||||
|
(source_dir / SpwTerrainService.SOURCE_SHA256_FILENAME).write_text(
|
||||||
|
f"{digest} {SpwTerrainService.SOURCE_FILENAME}\n", encoding="ascii"
|
||||||
|
)
|
||||||
|
return digest
|
||||||
|
|
||||||
|
|
||||||
def test_spw_terrain_registry_reports_real_source_state(tmp_path: Path) -> None:
|
def test_spw_terrain_registry_reports_real_source_state(tmp_path: Path) -> None:
|
||||||
before = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
before = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
||||||
assert before["status"] == "source_not_provisioned"
|
assert before["status"] == "source_not_provisioned"
|
||||||
|
|
||||||
make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
|
make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
|
||||||
|
without_checksum = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
||||||
|
assert without_checksum["configured"] is False
|
||||||
|
|
||||||
|
write_source_checksum(tmp_path)
|
||||||
after = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
after = SpwTerrainService.list_products(settings=settings(tmp_path))[0]
|
||||||
|
|
||||||
assert after["configured"] is True
|
assert after["configured"] is True
|
||||||
@@ -119,6 +131,7 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance(
|
|||||||
tmp_path: Path, monkeypatch
|
tmp_path: Path, monkeypatch
|
||||||
) -> None:
|
) -> None:
|
||||||
bbox = make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
|
bbox = make_source(tmp_path / SpwTerrainService.SOURCE_FILENAME)
|
||||||
|
source_digest = write_source_checksum(tmp_path)
|
||||||
project = Project(id=uuid4(), name="Belgium")
|
project = Project(id=uuid4(), name="Belgium")
|
||||||
captured = {}
|
captured = {}
|
||||||
|
|
||||||
@@ -150,6 +163,8 @@ def test_spw_terrain_acquisition_persists_bounded_dng_raster_and_provenance(
|
|||||||
assert captured["source_metadata"]["vertical_unit_label"] == "m DNG"
|
assert captured["source_metadata"]["vertical_unit_label"] == "m DNG"
|
||||||
assert captured["source_metadata"]["coverage_zones"] == ["wallonia"]
|
assert captured["source_metadata"]["coverage_zones"] == ["wallonia"]
|
||||||
assert captured["provenance_metadata"]["resampling"] == "bilinear"
|
assert captured["provenance_metadata"]["resampling"] == "bilinear"
|
||||||
|
assert captured["provenance_metadata"]["source_sha256"] == source_digest
|
||||||
|
assert captured["temporal_granularity"] == "period"
|
||||||
assert captured["valid_from"].date().isoformat() == "2021-02-19"
|
assert captured["valid_from"].date().isoformat() == "2021-02-19"
|
||||||
with rasterio.MemoryFile(captured["content"]) as memory:
|
with rasterio.MemoryFile(captured["content"]) as memory:
|
||||||
with memory.open() as derived:
|
with memory.open() as derived:
|
||||||
|
|||||||
@@ -11173,3 +11173,36 @@ Known governed boundaries remain explicit: WALOUS covers Wallonia rather than
|
|||||||
all Belgium, no unstable 2018 artifact is accepted without a verifiable source
|
all Belgium, no unstable 2018 artifact is accepted without a verifiable source
|
||||||
checksum, and the current local building model is not represented as nationally
|
checksum, and the current local building model is not represented as nationally
|
||||||
trained or validated.
|
trained or validated.
|
||||||
|
|
||||||
|
## 2026-07-22 - WALOUS 2018, Walloon terrain and evidence-scope closure
|
||||||
|
|
||||||
|
Implemented and verified:
|
||||||
|
|
||||||
|
- accepted the stable official WALOUS 2018 distribution only after validating
|
||||||
|
the 1,122,785,133-byte archive (`21e514...c9a`) and extracted raster
|
||||||
|
(`a788cf...56b2`), then materialized one area-scoped 2018/2020/2023 series;
|
||||||
|
- exposed that governed series in the evolution workspace and removed foreign
|
||||||
|
Area acquisitions from its choices; the live Wallonia golden Area shows
|
||||||
|
exactly three official moments and one unambiguous series;
|
||||||
|
- provisioned the complete official SPW MNT 2021-2022 1 m source. The archive
|
||||||
|
was 43,904,242,006 bytes with SHA-256 `04f3ca45821dc2866d854e75011eed1acefcfead9a5263e5a9760e29ded21f7c`;
|
||||||
|
the extracted EPSG:3812 Float32 raster is 44,014,505,895 bytes, 253,085 by
|
||||||
|
146,727 cells, uses nodata -9999 and SHA-256
|
||||||
|
`027f90fd304b683cdbf9a0c735152769cf093eb38599e7d5d9dc8792040a9f61`;
|
||||||
|
- removed the 40.89 GiB terrain download archive after checksum validation and
|
||||||
|
retained the source raster, checksum sidecar and atomic provisioning report;
|
||||||
|
- made terrain availability fail closed until both raster and valid SHA-256
|
||||||
|
sidecar exist, and persisted the official 2021-02-19/2022-03-05 observation
|
||||||
|
interval using the canonical `period` granularity;
|
||||||
|
- constrained AI capability claims to the actual Mol/Kempen evidence through
|
||||||
|
machine-readable training/validation scope, `nationally_validated=false` and
|
||||||
|
mandatory operator review. No national training result was fabricated.
|
||||||
|
|
||||||
|
Pre-release evidence:
|
||||||
|
|
||||||
|
- the complete readiness gate passed with 1,103 backend tests and 34 frontend
|
||||||
|
tests, backend compile, API/documentation contract audit, frontend typecheck,
|
||||||
|
production build, Alembic head `202607160001` and all script checks;
|
||||||
|
- live browser acceptance showed the exact WALOUS 2018-2023 series and no
|
||||||
|
console warnings or errors. Final terrain Dataset and immutable-image evidence
|
||||||
|
are recorded after deployment below.
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ runtime source of truth.
|
|||||||
persisted NGI/RBINS editions. Detailed themes are federated by jurisdiction;
|
persisted NGI/RBINS editions. Detailed themes are federated by jurisdiction;
|
||||||
an operational Flemish source does not imply equivalent Walloon or Brussels
|
an operational Flemish source does not imply equivalent Walloon or Brussels
|
||||||
coverage.
|
coverage.
|
||||||
|
- Land cover remains source-correct by jurisdiction: WALOUS 2018/2020/2023
|
||||||
|
covers Wallonia, Flemish products cover Flanders and UrbIS Land Cover covers
|
||||||
|
Brussels. GeoIntel does not relabel WALOUS as a national source or silently
|
||||||
|
merge incompatible regional legends.
|
||||||
- Buildings, population, terrain, imagery, nature, agriculture, soil and flood
|
- Buildings, population, terrain, imagery, nature, agriculture, soil and flood
|
||||||
themes may report `partial`, `not_configured` or `unsupported` outside the
|
themes may report `partial`, `not_configured` or `unsupported` outside the
|
||||||
materialized source partitions. The UI and exports retain that state.
|
materialized source partitions. The UI and exports retain that state.
|
||||||
|
|||||||
+6
-4
@@ -68,11 +68,13 @@ geen open productroadmap meer.
|
|||||||
pixelbudget, automatische tijdreeksmaterialisatie en evolutiemetrics. De vaste
|
pixelbudget, automatische tijdreeksmaterialisatie en evolutiemetrics. De vaste
|
||||||
officiële 2018-GeoTIFF-distributie is live checksum-gevalideerd; bronwaarde `0`
|
officiële 2018-GeoTIFF-distributie is live checksum-gevalideerd; bronwaarde `0`
|
||||||
wordt expliciet als achtergrond/nodata behandeld.
|
wordt expliciet als achtergrond/nodata behandeld.
|
||||||
- [ ] Rond de lopende live provisioning van het officiële SPW MNT 2021-2022
|
- [x] Rond de live provisioning van het officiële SPW MNT 2021-2022 op 1 m in
|
||||||
op 1 m in de operatorcache af en
|
de operatorcache af en bied begrensde terreinacquisitie/analyse in DNG aan.
|
||||||
bied begrensde terreinacquisitie/analyse in DNG aan. De 0,5 m-distributie blijft
|
Het bronraster en de checksum-sidecar zijn gevalideerd en het downloadarchief
|
||||||
|
is na veilige extractie verwijderd. De 0,5 m-distributie blijft
|
||||||
buiten V1 omdat zij geen noodzakelijke analysecapaciteit toevoegt tegenover de
|
buiten V1 omdat zij geen noodzakelijke analysecapaciteit toevoegt tegenover de
|
||||||
gevalideerde 1 m-bron en circa 213 GB bronopslag vraagt.
|
gevalideerde 1 m-bron en officieel als circa 198 GB download wordt aangeboden,
|
||||||
|
exclusief extra werkruimte tijdens extractie.
|
||||||
- [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk
|
- [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk
|
||||||
scenario-/juridisch contract; gebruik WMS alleen als context tenzij
|
scenario-/juridisch contract; gebruik WMS alleen als context tenzij
|
||||||
analytische pixels of vectorgeometrie officieel beschikbaar zijn.
|
analytische pixels of vectorgeometrie officieel beschikbaar zijn.
|
||||||
|
|||||||
@@ -32,6 +32,17 @@ def sha256_file(path: Path) -> str:
|
|||||||
return digest.hexdigest()
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def write_text_atomic(path: Path, content: str, *, encoding: str) -> None:
|
||||||
|
temporary = path.with_suffix(path.suffix + ".part")
|
||||||
|
temporary.unlink(missing_ok=True)
|
||||||
|
try:
|
||||||
|
temporary.write_text(content, encoding=encoding)
|
||||||
|
temporary.replace(path)
|
||||||
|
except Exception:
|
||||||
|
temporary.unlink(missing_ok=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
def download(destination: Path) -> str:
|
def download(destination: Path) -> str:
|
||||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||||
temporary.unlink(missing_ok=True)
|
temporary.unlink(missing_ok=True)
|
||||||
@@ -202,8 +213,10 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict:
|
|||||||
archive.unlink(missing_ok=True)
|
archive.unlink(missing_ok=True)
|
||||||
validation = validate_raster(target)
|
validation = validate_raster(target)
|
||||||
source_digest = sha256_file(target)
|
source_digest = sha256_file(target)
|
||||||
target.with_suffix(".sha256").write_text(
|
write_text_atomic(
|
||||||
f"{source_digest} {target.name}\n", encoding="ascii"
|
target.with_suffix(".sha256"),
|
||||||
|
f"{source_digest} {target.name}\n",
|
||||||
|
encoding="ascii",
|
||||||
)
|
)
|
||||||
validation.update(
|
validation.update(
|
||||||
{
|
{
|
||||||
@@ -214,7 +227,11 @@ def provision(destination: Path, force: bool, keep_archive: bool) -> dict:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
report_path = destination / "provisioning-report.json"
|
report_path = destination / "provisioning-report.json"
|
||||||
report_path.write_text(json.dumps(validation, indent=2) + "\n", encoding="utf-8")
|
write_text_atomic(
|
||||||
|
report_path,
|
||||||
|
json.dumps(validation, indent=2) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
print(f"SPW MNT: ready ({target.stat().st_size / 1024 / 1024 / 1024:.1f} GiB)")
|
print(f"SPW MNT: ready ({target.stat().st_size / 1024 / 1024 / 1024:.1f} GiB)")
|
||||||
print(f"Provisioning report: {report_path}")
|
print(f"Provisioning report: {report_path}")
|
||||||
return validation
|
return validation
|
||||||
|
|||||||
Reference in New Issue
Block a user