Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
375 lines
14 KiB
Python
375 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import zipfile
|
|
|
|
import pytest
|
|
from shapely.geometry import Polygon
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
|
|
def load_script(name: str):
|
|
path = SCRIPTS / name
|
|
module_name = f"test_{path.stem}_sprint227"
|
|
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
assert spec is not None
|
|
assert spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[module_name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
PREFLIGHT = load_script("statbel_population_preflight.py")
|
|
OPERATOR = load_script("provision_mol_population_history.py")
|
|
POPULATION_URL = (
|
|
"https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/"
|
|
"OPENDATA_SECTOREN_2025_NEW.zip"
|
|
)
|
|
GEOMETRY_URL = (
|
|
"https://statbel.fgov.be/sites/default/files/files/opendata/Statistische%20sectoren/"
|
|
"sh_statbel_statistical_sectors_31370_20250101.geojson.zip"
|
|
)
|
|
|
|
|
|
def population_archive(
|
|
*,
|
|
duplicate: bool = False,
|
|
invalid_total: bool = False,
|
|
missing_column: bool = False,
|
|
unexpected_non_spatial: bool = False,
|
|
unsafe_member: bool = False,
|
|
) -> bytes:
|
|
headers = ["CD_REFNIS", "CD_SECTOR", "TOTAL", "TX_DESCR_SECTOR_NL", "TX_DESCR_NL"]
|
|
if missing_column:
|
|
headers.remove("TOTAL")
|
|
rows = [
|
|
["13025", "13024A00-", "120", "Mol centrum", "Mol"],
|
|
["13025", "13025A01-", "bad" if invalid_total else "80", "Mol rand", "Mol"],
|
|
["13025", "13025ZZZZ", "3", "Niet te lokaliseren in een sector", "Mol"],
|
|
["13008", "13008A00-", "40", "Geel centrum", "Geel"],
|
|
]
|
|
if duplicate:
|
|
rows.append(["13025", "13024A00-", "1", "Dubbel", "Mol"])
|
|
if unexpected_non_spatial:
|
|
rows.append(["13025", "13025B00-", "2", "Ontbrekende geometrie", "Mol"])
|
|
lines = ["|".join(headers)]
|
|
for row in rows:
|
|
values = row if not missing_column else [row[0], row[1], row[3], row[4]]
|
|
lines.append("|".join(values))
|
|
buffer = io.BytesIO()
|
|
with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
|
archive.writestr("OPENDATA_SECTOREN_2025_NEW.txt", "\n".join(lines))
|
|
if unsafe_member:
|
|
archive.writestr("../escape.txt", "unsafe")
|
|
return buffer.getvalue()
|
|
|
|
|
|
def square_feature(
|
|
sector_code: str,
|
|
x: float,
|
|
*,
|
|
date: str = "2025-01-01",
|
|
municipality_code: str | None = None,
|
|
) -> dict:
|
|
coordinates = [[
|
|
[x, 200000],
|
|
[x + 100, 200000],
|
|
[x + 100, 200100],
|
|
[x, 200100],
|
|
[x, 200000],
|
|
]]
|
|
return {
|
|
"type": "Feature",
|
|
"properties": {
|
|
"cd_sector": sector_code,
|
|
"cd_munty_refnis": municipality_code or sector_code[:5],
|
|
"dt_situation": date,
|
|
"ms_area_ha": 1.0,
|
|
"tx_sector_descr_nl": sector_code,
|
|
},
|
|
"geometry": {"type": "Polygon", "coordinates": coordinates},
|
|
}
|
|
|
|
|
|
def geometry_archive(
|
|
*,
|
|
crs: str = "urn:ogc:def:crs:EPSG::31370",
|
|
date: str = "2025-01-01",
|
|
municipality_mismatch: bool = False,
|
|
repairable_invalid: bool = False,
|
|
include_crs_token_in_member: bool = True,
|
|
) -> bytes:
|
|
payload = {
|
|
"type": "FeatureCollection",
|
|
"name": "sh_statbel_statistical_sectors_31370_20250101",
|
|
"crs": {"type": "name", "properties": {"name": crs}},
|
|
"features": [
|
|
square_feature(
|
|
"13024A00-",
|
|
150000,
|
|
date=date,
|
|
municipality_code="13008" if municipality_mismatch else "13025",
|
|
),
|
|
square_feature("13025A01-", 150200, date=date),
|
|
square_feature("13008A00-", 150400, date=date),
|
|
],
|
|
}
|
|
if repairable_invalid:
|
|
payload["features"][0]["geometry"] = {
|
|
"type": "MultiPolygon",
|
|
"coordinates": [
|
|
[[[150000, 200000], [150100, 200000], [150100, 200100], [150000, 200100], [150000, 200000]]],
|
|
[[[150100, 200000], [150200, 200000], [150200, 200100], [150100, 200100], [150100, 200000]]],
|
|
],
|
|
}
|
|
payload["features"][0]["properties"]["ms_area_ha"] = 2.0
|
|
buffer = io.BytesIO()
|
|
member_stem = (
|
|
"sh_statbel_statistical_sectors_31370_20250101"
|
|
if include_crs_token_in_member
|
|
else "sh_statbel_statistical_sectors_20250101"
|
|
)
|
|
with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
|
archive.writestr(
|
|
f"{member_stem}.geojson/{member_stem}.geojson",
|
|
json.dumps(payload),
|
|
)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def baseline_snapshot(path: Path, *, total: int = 198) -> Path:
|
|
scope = PREFLIGHT.GEOGRAPHIC_SCOPES["mol"]
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"observation_year": 2024,
|
|
"member_nis_codes": list(scope.nis_codes),
|
|
"features": [
|
|
{"type": "Feature", "properties": {"source_feature_id": "13024A00-", "population_total": total - 80}},
|
|
{"type": "Feature", "properties": {"source_feature_id": "13025A01-", "population_total": 80}},
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return path
|
|
|
|
|
|
def validate(tmp_path: Path, **overrides):
|
|
values = {
|
|
"year": 2025,
|
|
"layout": "new",
|
|
"population_content": population_archive(),
|
|
"population_url": POPULATION_URL,
|
|
"geometry_content": geometry_archive(),
|
|
"geometry_url": GEOMETRY_URL,
|
|
"scope": PREFLIGHT.GEOGRAPHIC_SCOPES["mol"],
|
|
"baseline_snapshot": baseline_snapshot(tmp_path / "baseline.geojson"),
|
|
}
|
|
values.update(overrides)
|
|
return PREFLIGHT.validate_statbel_release(**values)
|
|
|
|
|
|
def test_preflight_reconciles_spatial_and_unlocated_population(tmp_path: Path) -> None:
|
|
result = validate(tmp_path)
|
|
manifest = result.manifest
|
|
|
|
assert manifest["status"] == "passed"
|
|
assert manifest["import_eligible"] is True
|
|
assert manifest["release"] == {
|
|
"year": 2025,
|
|
"population_layout": "new",
|
|
"geometry_date": "2025-01-01",
|
|
"license": "CC BY 4.0",
|
|
}
|
|
assert manifest["national_accounting"] == {
|
|
"population_row_count": 4,
|
|
"geometry_feature_count": 3,
|
|
"spatial_population_total": 240,
|
|
"unlocated_row_count": 1,
|
|
"unlocated_population_total": 3,
|
|
"population_total": 243,
|
|
}
|
|
assert manifest["scope_accounting"]["spatial_sector_count"] == 2
|
|
assert manifest["scope_accounting"]["spatial_population_total"] == 200
|
|
assert manifest["scope_accounting"]["unlocated_population_total"] == 3
|
|
assert manifest["scope_accounting"]["accounted_population_total"] == 203
|
|
assert manifest["baseline"]["annual_change_ratio"] == pytest.approx(200 / 198 - 1)
|
|
assert len(manifest["artifacts"]["population"]["archive_sha256"]) == 64
|
|
assert len(manifest["schemas"]["geometry_schema_sha256"]) == 64
|
|
|
|
|
|
def test_preflight_supports_the_complete_national_scope() -> None:
|
|
result = PREFLIGHT.validate_statbel_release(
|
|
year=2025,
|
|
layout="new",
|
|
population_content=population_archive(),
|
|
population_url=POPULATION_URL,
|
|
geometry_content=geometry_archive(),
|
|
geometry_url=GEOMETRY_URL,
|
|
scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"],
|
|
)
|
|
|
|
accounting = result.manifest["scope_accounting"]
|
|
assert accounting["scope_key"] == "belgium"
|
|
assert accounting["member_count"] == 2
|
|
assert accounting["member_nis_codes"] == ["13008", "13025"]
|
|
assert accounting["spatial_population_total"] == 240
|
|
assert accounting["unlocated_population_total"] == 3
|
|
assert accounting["accounted_population_total"] == 243
|
|
|
|
|
|
def test_preflight_accepts_real_archive_member_without_repeated_crs_token(tmp_path: Path) -> None:
|
|
result = validate(
|
|
tmp_path,
|
|
geometry_content=geometry_archive(include_crs_token_in_member=False),
|
|
)
|
|
|
|
assert result.manifest["status"] == "passed"
|
|
assert result.manifest["artifacts"]["geometry"]["member"].endswith(
|
|
"/sh_statbel_statistical_sectors_20250101.geojson"
|
|
)
|
|
|
|
|
|
def test_national_preflight_rejects_an_unscoped_baseline(tmp_path: Path) -> None:
|
|
with pytest.raises(PREFLIGHT.StatbelPreflightError) as exc_info:
|
|
PREFLIGHT.validate_statbel_release(
|
|
year=2025,
|
|
layout="new",
|
|
population_content=population_archive(),
|
|
population_url=POPULATION_URL,
|
|
geometry_content=geometry_archive(),
|
|
geometry_url=GEOMETRY_URL,
|
|
scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"],
|
|
baseline_snapshot=baseline_snapshot(tmp_path / "baseline.geojson"),
|
|
)
|
|
|
|
assert exc_info.value.code == "STATBEL_BASELINE_SCOPE_MISMATCH"
|
|
|
|
|
|
def test_preflight_reports_bounded_topology_repairs(tmp_path: Path) -> None:
|
|
result = validate(tmp_path, geometry_content=geometry_archive(repairable_invalid=True))
|
|
|
|
assert result.manifest["schemas"]["geometry_repair_count"] == 1
|
|
assert result.manifest["schemas"]["geometry_repaired_sector_codes"] == ["13024A00-"]
|
|
assert result.geometry.payload["features"][0]["geometry"]["type"] == "Polygon"
|
|
|
|
|
|
def test_preflight_accepts_official_slash_geometry_date(tmp_path: Path) -> None:
|
|
result = validate(tmp_path, geometry_content=geometry_archive(date="2025/01/01"))
|
|
|
|
assert len(result.geometry.payload["features"]) == 3
|
|
assert result.manifest["import_eligible"] is True
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("population_kwargs", "error_code"),
|
|
[
|
|
({"missing_column": True}, "STATBEL_POPULATION_SCHEMA_MISMATCH"),
|
|
({"duplicate": True}, "STATBEL_POPULATION_DUPLICATE_SECTOR"),
|
|
({"invalid_total": True}, "STATBEL_POPULATION_TOTAL_REJECTED"),
|
|
({"unexpected_non_spatial": True}, "STATBEL_JOIN_GEOMETRY_MISSING"),
|
|
({"unsafe_member": True}, "STATBEL_ARCHIVE_MEMBER_REJECTED"),
|
|
],
|
|
)
|
|
def test_preflight_rejects_population_contract_breaks(
|
|
tmp_path: Path,
|
|
population_kwargs: dict,
|
|
error_code: str,
|
|
) -> None:
|
|
with pytest.raises(PREFLIGHT.StatbelPreflightError) as exc_info:
|
|
validate(tmp_path, population_content=population_archive(**population_kwargs))
|
|
|
|
assert exc_info.value.code == error_code
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("geometry_kwargs", "error_code"),
|
|
[
|
|
({"crs": "EPSG:4326"}, "STATBEL_GEOMETRY_CRS_REJECTED"),
|
|
({"date": "2026-01-01"}, "STATBEL_GEOMETRY_DATE_MISMATCH"),
|
|
({"date": "2025/13/01"}, "STATBEL_GEOMETRY_DATE_MISMATCH"),
|
|
({"municipality_mismatch": True}, "STATBEL_JOIN_MUNICIPALITY_MISMATCH"),
|
|
],
|
|
)
|
|
def test_preflight_rejects_geometry_contract_breaks(
|
|
tmp_path: Path,
|
|
geometry_kwargs: dict,
|
|
error_code: str,
|
|
) -> None:
|
|
with pytest.raises(PREFLIGHT.StatbelPreflightError) as exc_info:
|
|
validate(tmp_path, geometry_content=geometry_archive(**geometry_kwargs))
|
|
|
|
assert exc_info.value.code == error_code
|
|
|
|
|
|
def test_preflight_rejects_excessive_population_change(tmp_path: Path) -> None:
|
|
baseline = baseline_snapshot(tmp_path / "low-baseline.geojson", total=100)
|
|
|
|
with pytest.raises(PREFLIGHT.StatbelPreflightError) as exc_info:
|
|
validate(tmp_path, baseline_snapshot=baseline)
|
|
|
|
assert exc_info.value.code == "STATBEL_POPULATION_CHANGE_REVIEW_REQUIRED"
|
|
assert exc_info.value.details["annual_change_ratio"] == pytest.approx(1.0)
|
|
|
|
|
|
def test_operator_stages_raw_artifacts_manifest_and_accounted_snapshot(tmp_path: Path) -> None:
|
|
scope = OPERATOR.GEOGRAPHIC_SCOPES["mol"]
|
|
boundary = Polygon([(-180, -90), (180, -90), (180, 90), (-180, 90)])
|
|
|
|
path, manifest_path, manifest = OPERATOR.stage_release(
|
|
year=2025,
|
|
population_content=population_archive(),
|
|
geometry_content=geometry_archive(),
|
|
output_dir=tmp_path,
|
|
boundary=boundary,
|
|
scope=scope,
|
|
)
|
|
|
|
snapshot = json.loads(path.read_text(encoding="utf-8"))
|
|
assert len(snapshot["features"]) == 2
|
|
assert snapshot["spatial_population_total"] == 200
|
|
assert snapshot["unlocated_population_total"] == 3
|
|
assert snapshot["accounted_population_total"] == 203
|
|
assert manifest_path.is_file()
|
|
assert Path(manifest["artifacts"]["population"]["retained_path"]).is_file()
|
|
assert Path(manifest["artifacts"]["geometry"]["retained_path"]).is_file()
|
|
assert OPERATOR.load_preflight_manifest(manifest_path, path, 2025, scope)["import_eligible"] is True
|
|
|
|
original_snapshot = path.read_bytes()
|
|
path.write_bytes(original_snapshot + b"\n")
|
|
with pytest.raises(RuntimeError, match="does not authorize"):
|
|
OPERATOR.load_preflight_manifest(manifest_path, path, 2025, scope)
|
|
|
|
path.write_bytes(original_snapshot)
|
|
geometry_path = Path(manifest["artifacts"]["geometry"]["retained_path"])
|
|
geometry_path.write_bytes(geometry_path.read_bytes() + b"tampered")
|
|
with pytest.raises(RuntimeError, match="geometry archive"):
|
|
OPERATOR.load_preflight_manifest(manifest_path, path, 2025, scope)
|
|
|
|
|
|
def test_population_rows_no_longer_silently_skip_invalid_values() -> None:
|
|
scope = OPERATOR.GEOGRAPHIC_SCOPES["mol"]
|
|
|
|
with pytest.raises(RuntimeError, match="invalid code or TOTAL"):
|
|
OPERATOR.population_rows(population_archive(invalid_total=True), scope)
|
|
|
|
|
|
def test_preflight_is_packaged_and_release_checked() -> None:
|
|
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
|
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
|
|
|
|
assert "COPY scripts/statbel_population_preflight.py" in dockerfile
|
|
assert "py_compile scripts/statbel_population_preflight.py" in readiness
|