Harden Statbel population import compatibility
This commit is contained in:
@@ -15,6 +15,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from hashlib import sha256
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
@@ -30,9 +31,15 @@ from requests.adapters import HTTPAdapter
|
||||
from shapely.geometry import mapping, shape
|
||||
from shapely.ops import transform
|
||||
from shapely.validation import make_valid
|
||||
from urllib.parse import unquote, urlsplit
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope
|
||||
from statbel_population_preflight import (
|
||||
StatbelPreflightError,
|
||||
validate_statbel_release,
|
||||
write_manifest,
|
||||
)
|
||||
|
||||
|
||||
MUNICIPALITY_NAME = "Mol"
|
||||
@@ -57,6 +64,7 @@ POPULATION_URLS = {
|
||||
2024: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2024.zip",
|
||||
2025: "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/OPENDATA_SECTOREN_2025_NEW.zip",
|
||||
}
|
||||
POPULATION_LAYOUTS = {year: ("new" if year == 2025 else "standard") for year in POPULATION_URLS}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -182,16 +190,35 @@ def population_rows(content: bytes, scope: GeographicScope) -> dict[str, dict[st
|
||||
text = raw.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode("cp1252")
|
||||
required = {"CD_REFNIS", "CD_SECTOR", "TOTAL", "TX_DESCR_SECTOR_NL", "TX_DESCR_NL"}
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter="|")
|
||||
missing_columns = sorted(required - set(reader.fieldnames or ()))
|
||||
if missing_columns:
|
||||
raise RuntimeError(f"Statbel population table is missing required columns: {', '.join(missing_columns)}")
|
||||
return scoped_population_rows(list(reader), scope)
|
||||
|
||||
|
||||
def scoped_population_rows(source_rows: list[dict[str, Any]], scope: GeographicScope) -> dict[str, dict[str, Any]]:
|
||||
members = {member.nis_code: member.name for member in scope.members}
|
||||
rows: dict[str, dict[str, Any]] = {}
|
||||
for row in csv.DictReader(io.StringIO(text), delimiter="|"):
|
||||
seen: set[str] = set()
|
||||
for row_number, row in enumerate(source_rows, start=2):
|
||||
nis_code = str(row.get("CD_REFNIS") or "").strip()
|
||||
sector_code = str(row.get("CD_SECTOR") or "").strip().upper()
|
||||
total_raw = str(row.get("TOTAL") if row.get("TOTAL") is not None else "").strip()
|
||||
if (
|
||||
len(nis_code) != 5
|
||||
or not nis_code.isdigit()
|
||||
or len(sector_code) != 9
|
||||
or not sector_code[:5].isdigit()
|
||||
or not total_raw.isdigit()
|
||||
):
|
||||
raise RuntimeError(f"Statbel population row {row_number} has invalid code or TOTAL values")
|
||||
if sector_code in seen:
|
||||
raise RuntimeError(f"Statbel population table contains duplicate sector {sector_code}")
|
||||
seen.add(sector_code)
|
||||
if nis_code not in members:
|
||||
continue
|
||||
sector_code = str(row.get("CD_SECTOR") or "").strip()
|
||||
total_raw = str(row.get("TOTAL") or "").strip()
|
||||
if not sector_code or not total_raw or not total_raw.isdigit():
|
||||
continue
|
||||
rows[sector_code] = {
|
||||
"population_total": int(total_raw),
|
||||
"sector_name_nl": row.get("TX_DESCR_SECTOR_NL"),
|
||||
@@ -210,6 +237,7 @@ def build_snapshot(
|
||||
population: dict[str, dict[str, Any]],
|
||||
boundary,
|
||||
scope: GeographicScope,
|
||||
preflight_manifest: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
member_codes = set(scope.nis_codes)
|
||||
@@ -245,8 +273,20 @@ def build_snapshot(
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
features.append({"type": "Feature", "id": sector_code, "geometry": mapping(geometry), "properties": combined})
|
||||
if missing_population:
|
||||
raise RuntimeError(f"Statbel geometry has {missing_population} sectors without population rows for {year}")
|
||||
if not features:
|
||||
raise RuntimeError(f"No joined population sectors were produced for {year}")
|
||||
spatial_population_total = sum(int(feature["properties"]["population_total"]) for feature in features)
|
||||
accounting = (preflight_manifest or {}).get("scope_accounting") or {}
|
||||
if accounting:
|
||||
expected_count = int(accounting.get("spatial_sector_count") or 0)
|
||||
expected_total = int(accounting.get("spatial_population_total") or -1)
|
||||
if len(features) != expected_count or spatial_population_total != expected_total:
|
||||
raise RuntimeError(
|
||||
f"Derived snapshot accounting differs from the passed Statbel preflight for {year}: "
|
||||
f"features {len(features)}/{expected_count}, population {spatial_population_total}/{expected_total}"
|
||||
)
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"Statbel population by statistical sector - {scope.display_name} {year}",
|
||||
@@ -258,10 +298,145 @@ def build_snapshot(
|
||||
"geometry_clipped_to_area": True,
|
||||
"observation_year": year,
|
||||
"missing_population_sector_count": missing_population,
|
||||
"spatial_population_total": spatial_population_total,
|
||||
"unlocated_population_row_count": int(accounting.get("unlocated_row_count") or 0),
|
||||
"unlocated_population_total": int(accounting.get("unlocated_population_total") or 0),
|
||||
"accounted_population_total": int(accounting.get("accounted_population_total") or spatial_population_total),
|
||||
"population_accounting_limitation": (
|
||||
"Statbel ZZZZ rows cannot be mapped and are excluded from spatial selection metrics."
|
||||
),
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
digest = sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_bytes_atomic(path: Path, content: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_bytes(content)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def write_text_atomic(path: Path, content: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(content, encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def snapshot_path(output_dir: Path, scope: GeographicScope, year: int) -> Path:
|
||||
return output_dir / f"{scope.key.replace('-', '_')}_statbel_population_{year}.geojson"
|
||||
|
||||
|
||||
def preflight_manifest_path(output_dir: Path, scope: GeographicScope, year: int) -> Path:
|
||||
return output_dir / f"{scope.key.replace('-', '_')}_statbel_population_{year}.preflight.json"
|
||||
|
||||
|
||||
def previous_snapshot_path(output_dir: Path, scope: GeographicScope, year: int) -> Path | None:
|
||||
candidates = [snapshot_path(output_dir, scope, candidate) for candidate in POPULATION_URLS if candidate < year]
|
||||
available = [path for path in candidates if path.is_file()]
|
||||
return max(available, key=lambda path: int(path.stem.rsplit("_", 1)[-1])) if available else None
|
||||
|
||||
|
||||
def load_preflight_manifest(path: Path, snapshot: Path, year: int, scope: GeographicScope) -> dict[str, Any]:
|
||||
try:
|
||||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError(f"Statbel preflight manifest is unreadable at {path}") from exc
|
||||
release = manifest.get("release") or {}
|
||||
accounting = manifest.get("scope_accounting") or {}
|
||||
derived = (manifest.get("artifacts") or {}).get("derived_snapshot") or {}
|
||||
if (
|
||||
manifest.get("status") != "passed"
|
||||
or manifest.get("import_eligible") is not True
|
||||
or int(release.get("year") or 0) != year
|
||||
or accounting.get("scope_key") != scope.key
|
||||
or derived.get("sha256") != sha256_path(snapshot)
|
||||
):
|
||||
raise RuntimeError(f"Statbel preflight manifest at {path} does not authorize the retained snapshot")
|
||||
for artifact_name in ("population", "geometry"):
|
||||
artifact = (manifest.get("artifacts") or {}).get(artifact_name) or {}
|
||||
retained_path = Path(str(artifact.get("retained_path") or ""))
|
||||
if (
|
||||
not retained_path.is_file()
|
||||
or artifact.get("archive_sha256") != sha256_path(retained_path)
|
||||
or int(artifact.get("archive_size_bytes") or -1) != retained_path.stat().st_size
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Statbel preflight manifest at {path} does not authorize the retained {artifact_name} archive"
|
||||
)
|
||||
return manifest
|
||||
|
||||
|
||||
def stage_release(
|
||||
*,
|
||||
year: int,
|
||||
population_content: bytes,
|
||||
geometry_content: bytes,
|
||||
output_dir: Path,
|
||||
boundary,
|
||||
scope: GeographicScope,
|
||||
) -> tuple[Path, Path, dict[str, Any]]:
|
||||
layout = POPULATION_LAYOUTS[year]
|
||||
population_url = POPULATION_URLS[year]
|
||||
geometry_url = SECTOR_URL.format(year=year)
|
||||
result = validate_statbel_release(
|
||||
year=year,
|
||||
layout=layout,
|
||||
population_content=population_content,
|
||||
population_url=population_url,
|
||||
geometry_content=geometry_content,
|
||||
geometry_url=geometry_url,
|
||||
scope=scope,
|
||||
baseline_snapshot=previous_snapshot_path(output_dir, scope, year),
|
||||
)
|
||||
raw_dir = output_dir / "raw" / str(year)
|
||||
population_archive_path = raw_dir / Path(unquote(urlsplit(population_url).path)).name
|
||||
geometry_archive_path = raw_dir / Path(unquote(urlsplit(geometry_url).path)).name
|
||||
write_bytes_atomic(population_archive_path, population_content)
|
||||
write_bytes_atomic(geometry_archive_path, geometry_content)
|
||||
|
||||
manifest = dict(result.manifest)
|
||||
manifest["artifacts"] = {
|
||||
**manifest["artifacts"],
|
||||
"population": {
|
||||
**manifest["artifacts"]["population"],
|
||||
"retained_path": str(population_archive_path),
|
||||
},
|
||||
"geometry": {
|
||||
**manifest["artifacts"]["geometry"],
|
||||
"retained_path": str(geometry_archive_path),
|
||||
},
|
||||
}
|
||||
population = scoped_population_rows(list(result.population.rows.values()), scope)
|
||||
path = snapshot_path(output_dir, scope, year)
|
||||
snapshot = build_snapshot(
|
||||
year,
|
||||
result.geometry.payload,
|
||||
population,
|
||||
boundary,
|
||||
scope,
|
||||
preflight_manifest=manifest,
|
||||
)
|
||||
write_text_atomic(path, json.dumps(snapshot, ensure_ascii=False, separators=(",", ":")))
|
||||
manifest["artifacts"]["derived_snapshot"] = {
|
||||
"retained_path": str(path),
|
||||
"size_bytes": path.stat().st_size,
|
||||
"sha256": sha256_path(path),
|
||||
"feature_count": len(snapshot["features"]),
|
||||
}
|
||||
manifest_path = preflight_manifest_path(output_dir, scope, year)
|
||||
write_manifest(manifest_path, manifest)
|
||||
return path, manifest_path, manifest
|
||||
|
||||
|
||||
def locate_workspace(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
@@ -292,8 +467,11 @@ def upload_snapshot(
|
||||
path: Path,
|
||||
timeout: int,
|
||||
scope: GeographicScope,
|
||||
preflight_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
observed_at = f"{year}-01-01T00:00:00Z"
|
||||
preflight = load_preflight_manifest(preflight_path, path, year, scope)
|
||||
accounting = preflight["scope_accounting"]
|
||||
source_metadata = {
|
||||
"provider": "Statbel",
|
||||
"authority_level": "authoritative",
|
||||
@@ -309,6 +487,11 @@ def upload_snapshot(
|
||||
"observation_date_precision": "year",
|
||||
"identity_stable": False,
|
||||
"identity_limitation": "Statistical-sector codes and boundaries can change between annual editions.",
|
||||
"population_layout": preflight["release"]["population_layout"],
|
||||
"population_accounting": accounting,
|
||||
"spatial_population_limitation": (
|
||||
"ZZZZ population rows have no geometry and are excluded from spatial selection metrics."
|
||||
),
|
||||
"selection_aggregation": {
|
||||
"method": "area_weighted_sum",
|
||||
"property": "population_total",
|
||||
@@ -325,6 +508,12 @@ def upload_snapshot(
|
||||
"geometry_clipped_to_area": True,
|
||||
"sector_geometry_url": SECTOR_URL.format(year=year),
|
||||
"population_url": POPULATION_URLS[year],
|
||||
"population_layout": preflight["release"]["population_layout"],
|
||||
"preflight_manifest_path": str(preflight_path),
|
||||
"preflight_manifest_sha256": sha256_path(preflight_path),
|
||||
"population_archive_sha256": preflight["artifacts"]["population"]["archive_sha256"],
|
||||
"sector_archive_sha256": preflight["artifacts"]["geometry"]["archive_sha256"],
|
||||
"derived_snapshot_sha256": preflight["artifacts"]["derived_snapshot"]["sha256"],
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
with path.open("rb") as handle:
|
||||
@@ -373,28 +562,52 @@ def main() -> int:
|
||||
try:
|
||||
boundary_path = resolve_boundary_path(args, scope)
|
||||
boundary = load_boundary(boundary_path, scope)
|
||||
prepared: list[tuple[int, Path, int]] = []
|
||||
prepared: list[dict[str, Any]] = []
|
||||
with build_session() as source_session:
|
||||
for year in years:
|
||||
path = output_dir / f"{scope.key.replace('-', '_')}_statbel_population_{year}.geojson"
|
||||
path = snapshot_path(output_dir, scope, year)
|
||||
manifest_path = preflight_manifest_path(output_dir, scope, year)
|
||||
preflight_status = "passed"
|
||||
if args.force or not path.exists():
|
||||
sectors_response = source_session.get(SECTOR_URL.format(year=year), timeout=args.request_timeout)
|
||||
sectors_response.raise_for_status()
|
||||
population_response = source_session.get(POPULATION_URLS[year], timeout=args.request_timeout)
|
||||
population_response.raise_for_status()
|
||||
snapshot = build_snapshot(
|
||||
year,
|
||||
zip_member_json(sectors_response.content),
|
||||
population_rows(population_response.content, scope),
|
||||
boundary,
|
||||
scope,
|
||||
path, manifest_path, _manifest = stage_release(
|
||||
year=year,
|
||||
population_content=population_response.content,
|
||||
geometry_content=sectors_response.content,
|
||||
output_dir=output_dir,
|
||||
boundary=boundary,
|
||||
scope=scope,
|
||||
)
|
||||
path.write_text(json.dumps(snapshot, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
|
||||
elif manifest_path.is_file():
|
||||
load_preflight_manifest(manifest_path, path, year, scope)
|
||||
else:
|
||||
preflight_status = "legacy_existing_only"
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
prepared.append((year, path, len(payload.get("features") or [])))
|
||||
prepared.append(
|
||||
{
|
||||
"year": year,
|
||||
"path": path,
|
||||
"manifest_path": manifest_path if manifest_path.is_file() else None,
|
||||
"feature_count": len(payload.get("features") or []),
|
||||
"preflight_status": preflight_status,
|
||||
}
|
||||
)
|
||||
|
||||
if args.fetch_only:
|
||||
results = [{"year": year, "path": str(path), "feature_count": count, "status": "prepared"} for year, path, count in prepared]
|
||||
results = [
|
||||
{
|
||||
"year": item["year"],
|
||||
"path": str(item["path"]),
|
||||
"preflight_manifest_path": str(item["manifest_path"]) if item["manifest_path"] else None,
|
||||
"preflight_status": item["preflight_status"],
|
||||
"feature_count": item["feature_count"],
|
||||
"status": "prepared" if item["preflight_status"] == "passed" else "legacy_cached",
|
||||
}
|
||||
for item in prepared
|
||||
]
|
||||
else:
|
||||
base_url = args.base_url.rstrip("/")
|
||||
with requests.Session() as api_session:
|
||||
@@ -405,7 +618,9 @@ def main() -> int:
|
||||
area_name,
|
||||
args.import_timeout,
|
||||
)
|
||||
for year, path, count in prepared:
|
||||
for item in prepared:
|
||||
year = int(item["year"])
|
||||
path = item["path"]
|
||||
observed_at = f"{year}-01-01T00:00:00+00:00"
|
||||
dataset = next(
|
||||
(
|
||||
@@ -419,6 +634,10 @@ def main() -> int:
|
||||
if dataset:
|
||||
results.append({"year": year, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "existing"})
|
||||
continue
|
||||
if item["preflight_status"] != "passed" or item["manifest_path"] is None:
|
||||
raise RuntimeError(
|
||||
f"Statbel {year} has only a legacy cached snapshot; rerun with --force to create preflight evidence before import"
|
||||
)
|
||||
dataset = upload_snapshot(
|
||||
api_session,
|
||||
base_url,
|
||||
@@ -428,10 +647,14 @@ def main() -> int:
|
||||
path,
|
||||
args.import_timeout,
|
||||
scope,
|
||||
item["manifest_path"],
|
||||
)
|
||||
results.append({"year": year, "dataset_id": dataset["id"], "feature_count": dataset.get("feature_count"), "status": "imported"})
|
||||
except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError, zipfile.BadZipFile) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
payload = {"status": "error", "message": str(exc)}
|
||||
if isinstance(exc, StatbelPreflightError):
|
||||
payload.update({"error_code": exc.code, "details": exc.details})
|
||||
print(json.dumps(payload, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
|
||||
@@ -45,6 +45,7 @@ ${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_municipality_workspace.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_context_layers.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/statbel_population_preflight.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||
|
||||
@@ -0,0 +1,814 @@
|
||||
"""Fail-closed compatibility preflight for a staged Statbel population release.
|
||||
|
||||
The preflight reads local official ZIP artifacts only. It validates source
|
||||
identities, archive safety, schemas, CRS, sector joins and population
|
||||
accounting before an operator may pass derived GeoJSON to DatasetService.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
import zipfile
|
||||
|
||||
from shapely.geometry import mapping, shape
|
||||
from shapely.ops import unary_union
|
||||
from shapely.validation import make_valid
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope
|
||||
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
DEFAULT_MAX_ANNUAL_CHANGE_RATIO = 0.05
|
||||
MAX_ARCHIVE_MEMBERS = 64
|
||||
MAX_POPULATION_ARCHIVE_BYTES = 10 * 1024 * 1024
|
||||
MAX_POPULATION_UNCOMPRESSED_BYTES = 30 * 1024 * 1024
|
||||
MAX_GEOMETRY_ARCHIVE_BYTES = 100 * 1024 * 1024
|
||||
MAX_GEOMETRY_UNCOMPRESSED_BYTES = 400 * 1024 * 1024
|
||||
MAX_COMPRESSION_RATIO = 100.0
|
||||
REQUIRED_POPULATION_FIELDS = (
|
||||
"CD_REFNIS",
|
||||
"CD_SECTOR",
|
||||
"TOTAL",
|
||||
"TX_DESCR_SECTOR_NL",
|
||||
"TX_DESCR_NL",
|
||||
)
|
||||
REQUIRED_GEOMETRY_FIELDS = (
|
||||
"cd_sector",
|
||||
"cd_munty_refnis",
|
||||
"dt_situation",
|
||||
"ms_area_ha",
|
||||
)
|
||||
SECTOR_CODE_PATTERN = re.compile(r"^[0-9]{5}[A-Z0-9-]{4}$")
|
||||
MUNICIPALITY_CODE_PATTERN = re.compile(r"^[0-9]{5}$")
|
||||
POPULATION_MEMBER_PATTERN = re.compile(
|
||||
r"^OPENDATA_SECTOREN_(20[0-9]{2})(?:_(NEW|OLD))?\.(?:txt|csv)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
GEOMETRY_MEMBER_PATTERN = re.compile(
|
||||
r"^sh_statbel_statistical_sectors_31370_(20[0-9]{2})0101\.geojson$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
POPULATION_SOURCE_PATTERN = re.compile(
|
||||
r"^/sites/default/files/files/opendata/bevolking/sectoren/"
|
||||
r"OPENDATA_SECTOREN_(20[0-9]{2})(?:_(NEW|OLD))?\.zip$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
GEOMETRY_SOURCE_PATTERN = re.compile(
|
||||
r"^/sites/default/files/files/opendata/Statistische%20sectoren/"
|
||||
r"sh_statbel_statistical_sectors_31370_(20[0-9]{2})0101\.geojson\.zip$",
|
||||
)
|
||||
ALLOWED_CRS_NAMES = {"EPSG:31370", "urn:ogc:def:crs:EPSG::31370"}
|
||||
|
||||
|
||||
class StatbelPreflightError(RuntimeError):
|
||||
def __init__(self, code: str, message: str, *, details: dict[str, Any] | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.details = details or {}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PopulationArchiveData:
|
||||
member_name: str
|
||||
layout: str
|
||||
columns: tuple[str, ...]
|
||||
rows: dict[str, dict[str, Any]]
|
||||
population_total: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GeometryArchiveData:
|
||||
member_name: str
|
||||
crs: str
|
||||
property_columns: tuple[str, ...]
|
||||
payload: dict[str, Any]
|
||||
municipality_by_sector: dict[str, str]
|
||||
repaired_sector_codes: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StatbelPreflightResult:
|
||||
manifest: dict[str, Any]
|
||||
population: PopulationArchiveData
|
||||
geometry: GeometryArchiveData
|
||||
|
||||
|
||||
def _fail(code: str, message: str, **details: Any) -> None:
|
||||
raise StatbelPreflightError(code, message, details=details)
|
||||
|
||||
|
||||
def _sha256_bytes(content: bytes) -> str:
|
||||
return sha256(content).hexdigest()
|
||||
|
||||
|
||||
def _schema_fingerprint(values: tuple[str, ...]) -> str:
|
||||
return sha256(json.dumps(values, ensure_ascii=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def _layout_from_variant(variant: str | None) -> str:
|
||||
if variant is None:
|
||||
return "standard"
|
||||
return variant.lower()
|
||||
|
||||
|
||||
def _validate_source_url(url: str, pattern: re.Pattern[str], *, code: str) -> re.Match[str]:
|
||||
parsed = urlsplit(url)
|
||||
match = pattern.fullmatch(parsed.path)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname != "statbel.fgov.be"
|
||||
or parsed.port not in {None, 443}
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or not match
|
||||
):
|
||||
_fail(code, "Source URL is outside the approved official Statbel release path.", url=url)
|
||||
return match
|
||||
|
||||
|
||||
def validate_population_source_url(url: str, year: int, layout: str) -> None:
|
||||
match = _validate_source_url(url, POPULATION_SOURCE_PATTERN, code="STATBEL_POPULATION_URL_REJECTED")
|
||||
url_year = int(match.group(1))
|
||||
url_layout = _layout_from_variant(match.group(2))
|
||||
if url_year != year or url_layout != layout:
|
||||
_fail(
|
||||
"STATBEL_POPULATION_URL_EDITION_MISMATCH",
|
||||
"Population source URL does not match the requested year and layout.",
|
||||
expected_year=year,
|
||||
actual_year=url_year,
|
||||
expected_layout=layout,
|
||||
actual_layout=url_layout,
|
||||
)
|
||||
|
||||
|
||||
def validate_geometry_source_url(url: str, year: int) -> None:
|
||||
match = _validate_source_url(url, GEOMETRY_SOURCE_PATTERN, code="STATBEL_GEOMETRY_URL_REJECTED")
|
||||
url_year = int(match.group(1))
|
||||
if url_year != year:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_URL_EDITION_MISMATCH",
|
||||
"Geometry source URL does not match the requested population year.",
|
||||
expected_year=year,
|
||||
actual_year=url_year,
|
||||
)
|
||||
|
||||
|
||||
def _safe_archive_members(
|
||||
content: bytes,
|
||||
*,
|
||||
compressed_limit: int,
|
||||
uncompressed_limit: int,
|
||||
artifact: str,
|
||||
) -> list[zipfile.ZipInfo]:
|
||||
if not content or len(content) > compressed_limit:
|
||||
_fail(
|
||||
"STATBEL_ARCHIVE_SIZE_REJECTED",
|
||||
f"{artifact} archive exceeds the bounded compressed size.",
|
||||
size_bytes=len(content),
|
||||
limit_bytes=compressed_limit,
|
||||
)
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||||
members = archive.infolist()
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise StatbelPreflightError("STATBEL_ARCHIVE_INVALID", f"{artifact} archive is not a valid ZIP.") from exc
|
||||
if not members or len(members) > MAX_ARCHIVE_MEMBERS:
|
||||
_fail(
|
||||
"STATBEL_ARCHIVE_MEMBER_COUNT_REJECTED",
|
||||
f"{artifact} archive has an unexpected member count.",
|
||||
member_count=len(members),
|
||||
)
|
||||
total_size = 0
|
||||
for member in members:
|
||||
path = PurePosixPath(member.filename.replace("\\", "/"))
|
||||
if path.is_absolute() or ".." in path.parts or member.flag_bits & 0x1:
|
||||
_fail(
|
||||
"STATBEL_ARCHIVE_MEMBER_REJECTED",
|
||||
f"{artifact} archive contains an unsafe member.",
|
||||
member=member.filename,
|
||||
)
|
||||
total_size += member.file_size
|
||||
if member.file_size and member.compress_size == 0:
|
||||
_fail("STATBEL_ARCHIVE_RATIO_REJECTED", f"{artifact} archive has an invalid compression ratio.")
|
||||
if member.compress_size and member.file_size / member.compress_size > MAX_COMPRESSION_RATIO:
|
||||
_fail(
|
||||
"STATBEL_ARCHIVE_RATIO_REJECTED",
|
||||
f"{artifact} archive exceeds the allowed compression ratio.",
|
||||
member=member.filename,
|
||||
)
|
||||
if total_size > uncompressed_limit:
|
||||
_fail(
|
||||
"STATBEL_ARCHIVE_UNCOMPRESSED_SIZE_REJECTED",
|
||||
f"{artifact} archive exceeds the bounded uncompressed size.",
|
||||
size_bytes=total_size,
|
||||
limit_bytes=uncompressed_limit,
|
||||
)
|
||||
return members
|
||||
|
||||
|
||||
def _decode_population_table(raw: bytes) -> str:
|
||||
try:
|
||||
return raw.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
return raw.decode("cp1252")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise StatbelPreflightError(
|
||||
"STATBEL_POPULATION_ENCODING_REJECTED",
|
||||
"Population table is neither UTF-8 nor Windows-1252 text.",
|
||||
) from exc
|
||||
|
||||
|
||||
def parse_population_archive(content: bytes, year: int, layout: str) -> PopulationArchiveData:
|
||||
members = _safe_archive_members(
|
||||
content,
|
||||
compressed_limit=MAX_POPULATION_ARCHIVE_BYTES,
|
||||
uncompressed_limit=MAX_POPULATION_UNCOMPRESSED_BYTES,
|
||||
artifact="Population",
|
||||
)
|
||||
table_members = [member for member in members if member.filename.lower().endswith((".txt", ".csv"))]
|
||||
if len(table_members) != 1:
|
||||
_fail(
|
||||
"STATBEL_POPULATION_MEMBER_AMBIGUOUS",
|
||||
"Population archive must contain exactly one TXT or CSV table.",
|
||||
table_member_count=len(table_members),
|
||||
)
|
||||
member = table_members[0]
|
||||
match = POPULATION_MEMBER_PATTERN.fullmatch(PurePosixPath(member.filename).name)
|
||||
if not match:
|
||||
_fail(
|
||||
"STATBEL_POPULATION_MEMBER_REJECTED",
|
||||
"Population table filename does not follow the official Statbel edition contract.",
|
||||
member=member.filename,
|
||||
)
|
||||
member_year = int(match.group(1))
|
||||
member_layout = _layout_from_variant(match.group(2))
|
||||
if member_year != year or member_layout != layout:
|
||||
_fail(
|
||||
"STATBEL_POPULATION_MEMBER_EDITION_MISMATCH",
|
||||
"Population table member does not match the requested year and layout.",
|
||||
expected_year=year,
|
||||
actual_year=member_year,
|
||||
expected_layout=layout,
|
||||
actual_layout=member_layout,
|
||||
)
|
||||
if member_layout == "old" or (year == 2025 and member_layout != "new"):
|
||||
_fail(
|
||||
"STATBEL_LAYOUT_NOT_CURRENT",
|
||||
"The selected population layout is transition evidence and is not import eligible.",
|
||||
year=year,
|
||||
layout=member_layout,
|
||||
)
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||||
text = _decode_population_table(archive.read(member))
|
||||
reader = csv.DictReader(io.StringIO(text), delimiter="|")
|
||||
columns = tuple(reader.fieldnames or ())
|
||||
missing_columns = sorted(set(REQUIRED_POPULATION_FIELDS) - set(columns))
|
||||
if missing_columns:
|
||||
_fail(
|
||||
"STATBEL_POPULATION_SCHEMA_MISMATCH",
|
||||
"Population table is missing required columns.",
|
||||
missing_columns=missing_columns,
|
||||
columns=list(columns),
|
||||
)
|
||||
|
||||
rows: dict[str, dict[str, Any]] = {}
|
||||
population_total = 0
|
||||
for row_number, source_row in enumerate(reader, start=2):
|
||||
row = {str(key): value for key, value in source_row.items() if key is not None}
|
||||
municipality_code = str(row.get("CD_REFNIS") or "").strip()
|
||||
sector_code = str(row.get("CD_SECTOR") or "").strip().upper()
|
||||
total_raw = str(row.get("TOTAL") or "").strip()
|
||||
if (
|
||||
not MUNICIPALITY_CODE_PATTERN.fullmatch(municipality_code)
|
||||
or not SECTOR_CODE_PATTERN.fullmatch(sector_code)
|
||||
):
|
||||
_fail(
|
||||
"STATBEL_POPULATION_SECTOR_CODE_REJECTED",
|
||||
"Population row contains an invalid municipality or sector code.",
|
||||
row_number=row_number,
|
||||
municipality_code=municipality_code,
|
||||
sector_code=sector_code,
|
||||
)
|
||||
if not total_raw.isdigit():
|
||||
_fail(
|
||||
"STATBEL_POPULATION_TOTAL_REJECTED",
|
||||
"Population TOTAL must be a non-negative integer.",
|
||||
row_number=row_number,
|
||||
sector_code=sector_code,
|
||||
value=total_raw,
|
||||
)
|
||||
if sector_code in rows:
|
||||
_fail(
|
||||
"STATBEL_POPULATION_DUPLICATE_SECTOR",
|
||||
"Population table contains duplicate sector codes.",
|
||||
sector_code=sector_code,
|
||||
)
|
||||
total = int(total_raw)
|
||||
rows[sector_code] = {**row, "CD_REFNIS": municipality_code, "CD_SECTOR": sector_code, "TOTAL": total}
|
||||
population_total += total
|
||||
if not rows or population_total <= 0:
|
||||
_fail("STATBEL_POPULATION_EMPTY", "Population table contains no usable population accounting.")
|
||||
return PopulationArchiveData(
|
||||
member_name=member.filename,
|
||||
layout=member_layout,
|
||||
columns=columns,
|
||||
rows=rows,
|
||||
population_total=population_total,
|
||||
)
|
||||
|
||||
|
||||
def _crs_name(payload: dict[str, Any]) -> str:
|
||||
crs = payload.get("crs")
|
||||
if not isinstance(crs, dict):
|
||||
return ""
|
||||
properties = crs.get("properties")
|
||||
return str(properties.get("name") or "") if isinstance(properties, dict) else ""
|
||||
|
||||
|
||||
def _polygonal_part(geometry):
|
||||
if geometry.geom_type in {"Polygon", "MultiPolygon"}:
|
||||
return geometry
|
||||
polygonal = [part for part in getattr(geometry, "geoms", ()) if part.geom_type in {"Polygon", "MultiPolygon"}]
|
||||
return unary_union(polygonal) if polygonal else geometry
|
||||
|
||||
|
||||
def parse_geometry_archive(content: bytes, year: int) -> GeometryArchiveData:
|
||||
members = _safe_archive_members(
|
||||
content,
|
||||
compressed_limit=MAX_GEOMETRY_ARCHIVE_BYTES,
|
||||
uncompressed_limit=MAX_GEOMETRY_UNCOMPRESSED_BYTES,
|
||||
artifact="Geometry",
|
||||
)
|
||||
geometry_members = [member for member in members if member.filename.lower().endswith(".geojson")]
|
||||
if len(geometry_members) != 1:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_MEMBER_AMBIGUOUS",
|
||||
"Geometry archive must contain exactly one GeoJSON dataset.",
|
||||
geometry_member_count=len(geometry_members),
|
||||
)
|
||||
member = geometry_members[0]
|
||||
match = GEOMETRY_MEMBER_PATTERN.fullmatch(PurePosixPath(member.filename).name)
|
||||
if not match or int(match.group(1)) != year:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_MEMBER_EDITION_MISMATCH",
|
||||
"Geometry member does not match the requested January 1 edition.",
|
||||
expected_year=year,
|
||||
member=member.filename,
|
||||
)
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as archive:
|
||||
try:
|
||||
payload = json.loads(archive.read(member).decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise StatbelPreflightError(
|
||||
"STATBEL_GEOMETRY_JSON_REJECTED",
|
||||
"Geometry member is not valid UTF-8 GeoJSON.",
|
||||
) from exc
|
||||
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection":
|
||||
_fail("STATBEL_GEOMETRY_TYPE_REJECTED", "Geometry artifact is not a GeoJSON FeatureCollection.")
|
||||
crs = _crs_name(payload)
|
||||
if crs not in ALLOWED_CRS_NAMES:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_CRS_REJECTED",
|
||||
"Geometry artifact must explicitly declare EPSG:31370.",
|
||||
actual_crs=crs or None,
|
||||
)
|
||||
features = payload.get("features")
|
||||
if not isinstance(features, list) or not features:
|
||||
_fail("STATBEL_GEOMETRY_EMPTY", "Geometry artifact contains no features.")
|
||||
|
||||
municipality_by_sector: dict[str, str] = {}
|
||||
repaired_sector_codes: list[str] = []
|
||||
property_columns: set[str] = set()
|
||||
expected_date = f"{year}-01-01"
|
||||
for feature_number, feature in enumerate(features, start=1):
|
||||
if not isinstance(feature, dict) or feature.get("type") != "Feature":
|
||||
_fail("STATBEL_GEOMETRY_FEATURE_REJECTED", "Geometry artifact contains an invalid feature.")
|
||||
properties = feature.get("properties")
|
||||
if not isinstance(properties, dict):
|
||||
_fail("STATBEL_GEOMETRY_PROPERTIES_REJECTED", "Geometry feature has no property object.")
|
||||
property_columns.update(str(key) for key in properties)
|
||||
missing = sorted(set(REQUIRED_GEOMETRY_FIELDS) - set(properties))
|
||||
if missing:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_SCHEMA_MISMATCH",
|
||||
"Geometry feature is missing required properties.",
|
||||
feature_number=feature_number,
|
||||
missing_columns=missing,
|
||||
)
|
||||
sector_code = str(properties.get("cd_sector") or "").strip().upper()
|
||||
municipality_code = str(properties.get("cd_munty_refnis") or "").strip()
|
||||
if (
|
||||
not SECTOR_CODE_PATTERN.fullmatch(sector_code)
|
||||
or not MUNICIPALITY_CODE_PATTERN.fullmatch(municipality_code)
|
||||
):
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_SECTOR_CODE_REJECTED",
|
||||
"Geometry feature contains an invalid municipality or sector code.",
|
||||
sector_code=sector_code,
|
||||
municipality_code=municipality_code,
|
||||
)
|
||||
if sector_code in municipality_by_sector:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_DUPLICATE_SECTOR",
|
||||
"Geometry artifact contains duplicate sector codes.",
|
||||
sector_code=sector_code,
|
||||
)
|
||||
if str(properties.get("dt_situation") or "") != expected_date:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_DATE_MISMATCH",
|
||||
"Geometry situation date does not match the population reference year.",
|
||||
sector_code=sector_code,
|
||||
expected_date=expected_date,
|
||||
actual_date=properties.get("dt_situation"),
|
||||
)
|
||||
geometry_payload = feature.get("geometry")
|
||||
if not isinstance(geometry_payload, dict) or geometry_payload.get("type") not in {"Polygon", "MultiPolygon"}:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_SHAPE_REJECTED",
|
||||
"Sector geometry must be a Polygon or MultiPolygon.",
|
||||
sector_code=sector_code,
|
||||
)
|
||||
try:
|
||||
geometry = shape(geometry_payload)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise StatbelPreflightError(
|
||||
"STATBEL_GEOMETRY_SHAPE_REJECTED",
|
||||
f"Sector {sector_code} has unreadable geometry.",
|
||||
) from exc
|
||||
if geometry.is_empty or geometry.area <= 0:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_INVALID",
|
||||
"Sector geometry must be non-empty and have positive area.",
|
||||
sector_code=sector_code,
|
||||
)
|
||||
if not geometry.is_valid:
|
||||
repaired = _polygonal_part(make_valid(geometry))
|
||||
area_delta = abs(repaired.area - geometry.area)
|
||||
if (
|
||||
repaired.is_empty
|
||||
or not repaired.is_valid
|
||||
or repaired.geom_type not in {"Polygon", "MultiPolygon"}
|
||||
or repaired.area <= 0
|
||||
or area_delta > max(0.01, geometry.area * 0.000001)
|
||||
):
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_REPAIR_REJECTED",
|
||||
"Invalid sector geometry cannot be repaired without changing its polygonal meaning.",
|
||||
sector_code=sector_code,
|
||||
original_geometry_type=geometry.geom_type,
|
||||
repaired_geometry_type=repaired.geom_type,
|
||||
original_area=geometry.area,
|
||||
repaired_area=repaired.area,
|
||||
)
|
||||
geometry = repaired
|
||||
feature["geometry"] = mapping(geometry)
|
||||
repaired_sector_codes.append(sector_code)
|
||||
try:
|
||||
declared_area_ha = float(properties.get("ms_area_ha"))
|
||||
except (TypeError, ValueError):
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_AREA_REJECTED",
|
||||
"Sector ms_area_ha must be numeric.",
|
||||
sector_code=sector_code,
|
||||
)
|
||||
calculated_area_ha = geometry.area / 10_000
|
||||
tolerance = max(0.01, declared_area_ha * 0.001)
|
||||
if declared_area_ha <= 0 or abs(calculated_area_ha - declared_area_ha) > tolerance:
|
||||
_fail(
|
||||
"STATBEL_GEOMETRY_AREA_MISMATCH",
|
||||
"Declared sector area does not match EPSG:31370 geometry area.",
|
||||
sector_code=sector_code,
|
||||
declared_area_ha=declared_area_ha,
|
||||
calculated_area_ha=calculated_area_ha,
|
||||
)
|
||||
municipality_by_sector[sector_code] = municipality_code
|
||||
return GeometryArchiveData(
|
||||
member_name=member.filename,
|
||||
crs=crs,
|
||||
property_columns=tuple(sorted(property_columns)),
|
||||
payload=payload,
|
||||
municipality_by_sector=municipality_by_sector,
|
||||
repaired_sector_codes=tuple(repaired_sector_codes),
|
||||
)
|
||||
|
||||
|
||||
def _validate_scope(scope: GeographicScope, geometry: GeometryArchiveData, population: PopulationArchiveData) -> None:
|
||||
requested = set(scope.nis_codes)
|
||||
if not requested or any(not MUNICIPALITY_CODE_PATTERN.fullmatch(value) for value in requested):
|
||||
_fail("STATBEL_SCOPE_REJECTED", "Approved geographic scope contains invalid NIS codes.")
|
||||
geometry_codes = set(geometry.municipality_by_sector.values())
|
||||
population_codes = {str(row["CD_REFNIS"]) for row in population.rows.values()}
|
||||
missing = sorted(code for code in requested if code not in geometry_codes or code not in population_codes)
|
||||
if missing:
|
||||
_fail(
|
||||
"STATBEL_SCOPE_COVERAGE_MISSING",
|
||||
"Candidate release does not cover every municipality in the approved scope.",
|
||||
missing_nis_codes=missing,
|
||||
)
|
||||
|
||||
|
||||
def _baseline_summary(path: Path | None, *, year: int, scope: GeographicScope) -> dict[str, Any] | None:
|
||||
if path is None:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise StatbelPreflightError(
|
||||
"STATBEL_BASELINE_REJECTED",
|
||||
"Baseline snapshot is not readable GeoJSON evidence.",
|
||||
) from exc
|
||||
baseline_year = int(payload.get("observation_year") or 0)
|
||||
if baseline_year <= 0 or baseline_year >= year:
|
||||
_fail(
|
||||
"STATBEL_BASELINE_YEAR_REJECTED",
|
||||
"Baseline observation year must precede the candidate release.",
|
||||
baseline_year=baseline_year,
|
||||
candidate_year=year,
|
||||
)
|
||||
if set(str(value) for value in payload.get("member_nis_codes") or []) != set(scope.nis_codes):
|
||||
_fail("STATBEL_BASELINE_SCOPE_MISMATCH", "Baseline snapshot does not use the same approved scope.")
|
||||
features = payload.get("features")
|
||||
if not isinstance(features, list) or not features:
|
||||
_fail("STATBEL_BASELINE_EMPTY", "Baseline snapshot contains no population features.")
|
||||
sector_codes: set[str] = set()
|
||||
spatial_total = 0
|
||||
for feature in features:
|
||||
properties = feature.get("properties") if isinstance(feature, dict) else None
|
||||
if not isinstance(properties, dict):
|
||||
_fail("STATBEL_BASELINE_SCHEMA_MISMATCH", "Baseline feature has no property object.")
|
||||
sector_code = str(properties.get("source_feature_id") or properties.get("cd_sector") or "").strip()
|
||||
total = properties.get("population_total")
|
||||
if sector_code in sector_codes or not isinstance(total, int) or total < 0:
|
||||
_fail("STATBEL_BASELINE_SCHEMA_MISMATCH", "Baseline population evidence is not unique and numeric.")
|
||||
sector_codes.add(sector_code)
|
||||
spatial_total += total
|
||||
return {
|
||||
"path": str(path),
|
||||
"year": baseline_year,
|
||||
"spatial_sector_count": len(sector_codes),
|
||||
"spatial_population_total": spatial_total,
|
||||
}
|
||||
|
||||
|
||||
def validate_statbel_release(
|
||||
*,
|
||||
year: int,
|
||||
layout: str,
|
||||
population_content: bytes,
|
||||
population_url: str,
|
||||
geometry_content: bytes,
|
||||
geometry_url: str,
|
||||
scope: GeographicScope,
|
||||
baseline_snapshot: Path | None = None,
|
||||
max_annual_change_ratio: float = DEFAULT_MAX_ANNUAL_CHANGE_RATIO,
|
||||
) -> StatbelPreflightResult:
|
||||
if year < 2000 or year > datetime.now(timezone.utc).year + 1:
|
||||
_fail("STATBEL_YEAR_REJECTED", "Candidate population year is outside the supported review range.", year=year)
|
||||
if layout not in {"standard", "new"}:
|
||||
_fail("STATBEL_LAYOUT_REJECTED", "Candidate layout must be standard or new.", layout=layout)
|
||||
if max_annual_change_ratio <= 0 or max_annual_change_ratio > 0.25:
|
||||
_fail("STATBEL_CHANGE_LIMIT_REJECTED", "Annual population change limit must be greater than 0 and at most 25%.")
|
||||
validate_population_source_url(population_url, year, layout)
|
||||
validate_geometry_source_url(geometry_url, year)
|
||||
population = parse_population_archive(population_content, year, layout)
|
||||
geometry = parse_geometry_archive(geometry_content, year)
|
||||
_validate_scope(scope, geometry, population)
|
||||
|
||||
population_codes = set(population.rows)
|
||||
geometry_codes = set(geometry.municipality_by_sector)
|
||||
municipality_mismatches = sorted(
|
||||
code
|
||||
for code in population_codes & geometry_codes
|
||||
if str(population.rows[code]["CD_REFNIS"]) != geometry.municipality_by_sector[code]
|
||||
)
|
||||
if municipality_mismatches:
|
||||
_fail(
|
||||
"STATBEL_JOIN_MUNICIPALITY_MISMATCH",
|
||||
"Population and geometry assign one or more sectors to different reference municipalities.",
|
||||
count=len(municipality_mismatches),
|
||||
examples=municipality_mismatches[:10],
|
||||
)
|
||||
geometry_without_population = sorted(geometry_codes - population_codes)
|
||||
if geometry_without_population:
|
||||
_fail(
|
||||
"STATBEL_JOIN_POPULATION_MISSING",
|
||||
"One or more sector geometries have no population row.",
|
||||
count=len(geometry_without_population),
|
||||
examples=geometry_without_population[:10],
|
||||
)
|
||||
population_without_geometry = sorted(population_codes - geometry_codes)
|
||||
unexpected_non_spatial = [code for code in population_without_geometry if not code.endswith("ZZZZ")]
|
||||
if unexpected_non_spatial:
|
||||
_fail(
|
||||
"STATBEL_JOIN_GEOMETRY_MISSING",
|
||||
"Population rows without geometry must use the explicit ZZZZ unlocated-sector contract.",
|
||||
count=len(unexpected_non_spatial),
|
||||
examples=unexpected_non_spatial[:10],
|
||||
)
|
||||
|
||||
national_spatial_total = sum(int(population.rows[code]["TOTAL"]) for code in geometry_codes)
|
||||
national_unlocated_total = sum(int(population.rows[code]["TOTAL"]) for code in population_without_geometry)
|
||||
if national_spatial_total + national_unlocated_total != population.population_total:
|
||||
_fail("STATBEL_TOTAL_RECONCILIATION_FAILED", "National population accounting does not reconcile.")
|
||||
requested = set(scope.nis_codes)
|
||||
scope_spatial_codes = {
|
||||
code for code in geometry_codes if geometry.municipality_by_sector[code] in requested
|
||||
}
|
||||
scope_unlocated_codes = {
|
||||
code for code in population_without_geometry if str(population.rows[code]["CD_REFNIS"]) in requested
|
||||
}
|
||||
scope_spatial_total = sum(int(population.rows[code]["TOTAL"]) for code in scope_spatial_codes)
|
||||
scope_unlocated_total = sum(int(population.rows[code]["TOTAL"]) for code in scope_unlocated_codes)
|
||||
|
||||
baseline = _baseline_summary(baseline_snapshot, year=year, scope=scope)
|
||||
if baseline:
|
||||
baseline_total = int(baseline["spatial_population_total"])
|
||||
if baseline_total <= 0:
|
||||
_fail("STATBEL_BASELINE_TOTAL_REJECTED", "Baseline population total must be positive.")
|
||||
years = year - int(baseline["year"])
|
||||
annual_change_ratio = (scope_spatial_total / baseline_total) ** (1 / years) - 1
|
||||
baseline["candidate_spatial_population_total"] = scope_spatial_total
|
||||
baseline["annual_change_ratio"] = annual_change_ratio
|
||||
baseline["max_annual_change_ratio"] = max_annual_change_ratio
|
||||
if abs(annual_change_ratio) > max_annual_change_ratio:
|
||||
_fail(
|
||||
"STATBEL_POPULATION_CHANGE_REVIEW_REQUIRED",
|
||||
"Candidate spatial population change exceeds the configured annual review limit.",
|
||||
baseline_year=baseline["year"],
|
||||
candidate_year=year,
|
||||
annual_change_ratio=annual_change_ratio,
|
||||
max_annual_change_ratio=max_annual_change_ratio,
|
||||
)
|
||||
|
||||
manifest = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"status": "passed",
|
||||
"import_eligible": True,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"release": {
|
||||
"year": year,
|
||||
"population_layout": layout,
|
||||
"geometry_date": f"{year}-01-01",
|
||||
"license": "CC BY 4.0",
|
||||
},
|
||||
"artifacts": {
|
||||
"population": {
|
||||
"source_url": population_url,
|
||||
"archive_size_bytes": len(population_content),
|
||||
"archive_sha256": _sha256_bytes(population_content),
|
||||
"member": population.member_name,
|
||||
},
|
||||
"geometry": {
|
||||
"source_url": geometry_url,
|
||||
"archive_size_bytes": len(geometry_content),
|
||||
"archive_sha256": _sha256_bytes(geometry_content),
|
||||
"member": geometry.member_name,
|
||||
},
|
||||
},
|
||||
"schemas": {
|
||||
"population_columns": list(population.columns),
|
||||
"population_schema_sha256": _schema_fingerprint(population.columns),
|
||||
"geometry_property_columns": list(geometry.property_columns),
|
||||
"geometry_schema_sha256": _schema_fingerprint(geometry.property_columns),
|
||||
"geometry_crs": geometry.crs,
|
||||
"geometry_repair_count": len(geometry.repaired_sector_codes),
|
||||
"geometry_repaired_sector_codes": list(geometry.repaired_sector_codes),
|
||||
},
|
||||
"national_accounting": {
|
||||
"population_row_count": len(population.rows),
|
||||
"geometry_feature_count": len(geometry_codes),
|
||||
"spatial_population_total": national_spatial_total,
|
||||
"unlocated_row_count": len(population_without_geometry),
|
||||
"unlocated_population_total": national_unlocated_total,
|
||||
"population_total": population.population_total,
|
||||
},
|
||||
"scope_accounting": {
|
||||
"scope_key": scope.key,
|
||||
"scope_display_name": scope.display_name,
|
||||
"member_count": len(scope.members),
|
||||
"member_nis_codes": list(scope.nis_codes),
|
||||
"spatial_sector_count": len(scope_spatial_codes),
|
||||
"spatial_population_total": scope_spatial_total,
|
||||
"unlocated_row_count": len(scope_unlocated_codes),
|
||||
"unlocated_population_total": scope_unlocated_total,
|
||||
"accounted_population_total": scope_spatial_total + scope_unlocated_total,
|
||||
},
|
||||
"join_accounting": {
|
||||
"geometry_without_population_count": 0,
|
||||
"population_without_geometry_count": len(population_without_geometry),
|
||||
"population_without_geometry_contract": "sector_code_suffix_ZZZZ",
|
||||
},
|
||||
"baseline": baseline,
|
||||
"checks": [
|
||||
"official_source_identity",
|
||||
"archive_safety",
|
||||
"population_schema",
|
||||
"geometry_schema",
|
||||
"geometry_crs_and_validity",
|
||||
"sector_join",
|
||||
"population_total_reconciliation",
|
||||
"approved_scope_coverage",
|
||||
"baseline_change_limit" if baseline else "baseline_not_supplied",
|
||||
],
|
||||
"limitations": [
|
||||
"ZZZZ population rows have no map geometry and are excluded from spatial selection metrics.",
|
||||
"Statistical-sector identity and boundaries are not assumed stable across editions.",
|
||||
"Repairable source topology errors are normalized with make_valid and reported in this manifest.",
|
||||
"A passed technical preflight is not an instruction to replace an existing Dataset.",
|
||||
],
|
||||
}
|
||||
return StatbelPreflightResult(manifest=manifest, population=population, geometry=geometry)
|
||||
|
||||
|
||||
def write_manifest(path: Path, manifest: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Validate a staged official Statbel population release.")
|
||||
parser.add_argument("--year", type=int, required=True)
|
||||
parser.add_argument("--layout", choices=("standard", "new"), required=True)
|
||||
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default="kempen-transport-region")
|
||||
parser.add_argument("--population-archive", type=Path, required=True)
|
||||
parser.add_argument("--population-url", required=True)
|
||||
parser.add_argument("--geometry-archive", type=Path, required=True)
|
||||
parser.add_argument("--geometry-url", required=True)
|
||||
parser.add_argument("--baseline-snapshot", type=Path)
|
||||
parser.add_argument("--max-annual-change-percent", type=float, default=5.0)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||
try:
|
||||
result = validate_statbel_release(
|
||||
year=args.year,
|
||||
layout=args.layout,
|
||||
population_content=args.population_archive.read_bytes(),
|
||||
population_url=args.population_url,
|
||||
geometry_content=args.geometry_archive.read_bytes(),
|
||||
geometry_url=args.geometry_url,
|
||||
scope=scope,
|
||||
baseline_snapshot=args.baseline_snapshot,
|
||||
max_annual_change_ratio=args.max_annual_change_percent / 100,
|
||||
)
|
||||
manifest = dict(result.manifest)
|
||||
manifest["artifacts"] = {
|
||||
**manifest["artifacts"],
|
||||
"population": {
|
||||
**manifest["artifacts"]["population"],
|
||||
"retained_path": str(args.population_archive),
|
||||
},
|
||||
"geometry": {
|
||||
**manifest["artifacts"]["geometry"],
|
||||
"retained_path": str(args.geometry_archive),
|
||||
},
|
||||
}
|
||||
write_manifest(args.output, manifest)
|
||||
except (OSError, StatbelPreflightError) as exc:
|
||||
if isinstance(exc, StatbelPreflightError):
|
||||
payload = {"status": "error", "error_code": exc.code, "message": exc.message, "details": exc.details}
|
||||
else:
|
||||
payload = {"status": "error", "error_code": "STATBEL_PREFLIGHT_IO_ERROR", "message": str(exc)}
|
||||
print(json.dumps(payload, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"import_eligible": True,
|
||||
"year": args.year,
|
||||
"layout": args.layout,
|
||||
"scope": scope.key,
|
||||
"manifest_path": str(args.output),
|
||||
"scope_accounting": manifest["scope_accounting"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user