446 lines
17 KiB
Python
446 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
from hashlib import sha256
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
|
|
def load_script(name: str):
|
|
path = SCRIPTS / name
|
|
module_name = f"test_{path.stem}_sprint228"
|
|
spec = importlib.util.spec_from_file_location(module_name, path)
|
|
assert spec is not None
|
|
assert spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[module_name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
MANAGER = load_script("manage_statbel_population_release.py")
|
|
OPERATOR = load_script("provision_mol_population_history.py")
|
|
|
|
|
|
def arguments(tmp_path: Path, **overrides) -> argparse.Namespace:
|
|
values = {
|
|
"action": "plan",
|
|
"project_id": "00000000-0000-0000-0000-000000000001",
|
|
"scope": "kempen-transport-region",
|
|
"api_url": "http://127.0.0.1:8000/api/v1",
|
|
"confirm_edition": None,
|
|
"confirm_layout": None,
|
|
"confirm_plan_sha256": None,
|
|
"confirm_review_sha256": None,
|
|
"approve": False,
|
|
"reviewer": None,
|
|
"review_note": "",
|
|
"plan_path": None,
|
|
"review_path": None,
|
|
"output_root": tmp_path / "operator-data" / "regional-timeseries",
|
|
"scope_output_root": tmp_path / "operator-data" / "geographic-scopes",
|
|
"evidence_root": tmp_path / "operator-evidence" / "statbel-population-refresh",
|
|
"refresh_catalog": False,
|
|
"request_timeout": 300,
|
|
"api_timeout": 180,
|
|
"import_timeout": 3600,
|
|
}
|
|
values.update(overrides)
|
|
return argparse.Namespace(**values)
|
|
|
|
|
|
def catalog_item(*, remote: str = "2026", local: str | None = "2025", catalog_hash: str = "a" * 64) -> dict:
|
|
return {
|
|
"source_name": "statbel",
|
|
"status": "available",
|
|
"reachable": True,
|
|
"error_code": None,
|
|
"remote_version": remote,
|
|
"local_source_version": local,
|
|
"remote_title": f"Bevolking per statistische sector {remote} (nieuwe REDEGEO-sectorindeling)",
|
|
"message": "De officiele catalogus bevestigt de nieuwe REDEGEO-sectorindeling.",
|
|
"matched_layers": ["population_txt_current", "landing_page", "cc_by_4_0"],
|
|
"capabilities_sha256": catalog_hash,
|
|
"metadata_identifier": f"NodeID{remote}",
|
|
"metadata_url": f"https://statbel.fgov.be/nl/open-data/bevolking-statistische-sector-{remote}",
|
|
"checked_at": "2026-07-17T08:00:00Z",
|
|
}
|
|
|
|
|
|
def decision(args: argparse.Namespace, *, remote: str = "2026", local: str | None = "2025") -> dict:
|
|
return MANAGER.fetch_release_decision_from_item(args, catalog_item(remote=remote, local=local))
|
|
|
|
|
|
def release_2026():
|
|
return MANAGER.release_from_catalog_item(catalog_item())
|
|
|
|
|
|
def write_staged_artifacts(args: argparse.Namespace, release) -> None:
|
|
scope = MANAGER.GEOGRAPHIC_SCOPES[args.scope]
|
|
output_dir = MANAGER.population_output_dir(args)
|
|
raw_dir = output_dir / "raw" / str(release.year)
|
|
raw_dir.mkdir(parents=True, exist_ok=True)
|
|
population_path = raw_dir / "OPENDATA_SECTOREN_2026_NEW.zip"
|
|
geometry_path = raw_dir / "sh_statbel_statistical_sectors_31370_20260101.geojson.zip"
|
|
population_path.write_bytes(b"official population archive")
|
|
geometry_path.write_bytes(b"official geometry archive")
|
|
snapshot = MANAGER.snapshot_path(output_dir, scope, release.year)
|
|
snapshot.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"status": "passed",
|
|
"import_eligible": True,
|
|
"release": {"year": 2026, "population_layout": "new"},
|
|
"artifacts": {
|
|
"population": {
|
|
"source_url": release.population_url,
|
|
"archive_sha256": sha256(population_path.read_bytes()).hexdigest(),
|
|
"archive_size_bytes": population_path.stat().st_size,
|
|
"retained_path": str(population_path),
|
|
},
|
|
"geometry": {
|
|
"source_url": release.geometry_url,
|
|
"archive_sha256": sha256(geometry_path.read_bytes()).hexdigest(),
|
|
"archive_size_bytes": geometry_path.stat().st_size,
|
|
"retained_path": str(geometry_path),
|
|
},
|
|
"derived_snapshot": {
|
|
"sha256": sha256(snapshot.read_bytes()).hexdigest(),
|
|
"size_bytes": snapshot.stat().st_size,
|
|
"feature_count": 700,
|
|
},
|
|
},
|
|
"scope_accounting": {
|
|
"scope_key": scope.key,
|
|
"spatial_sector_count": 700,
|
|
"spatial_population_total": 510000,
|
|
"unlocated_population_total": 300,
|
|
"accounted_population_total": 510300,
|
|
},
|
|
"national_accounting": {"population_total": 12000000},
|
|
"baseline": {"year": 2025, "annual_change_ratio": 0.007},
|
|
"schemas": {"geometry_repair_count": 2},
|
|
}
|
|
manifest_path = MANAGER.preflight_manifest_path(output_dir, scope, release.year)
|
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
|
|
|
|
|
def staged_plan(args: argparse.Namespace, release, release_decision: dict) -> tuple[Path, dict]:
|
|
write_staged_artifacts(args, release)
|
|
result = {
|
|
"status": "ok",
|
|
"scope": args.scope,
|
|
"snapshots": [
|
|
{
|
|
"year": release.year,
|
|
"status": "prepared",
|
|
"preflight_status": "passed",
|
|
"feature_count": 700,
|
|
}
|
|
],
|
|
}
|
|
plan = MANAGER.build_staged_plan(args, release_decision, release, result)
|
|
path = MANAGER.default_plan_path(args, release.year)
|
|
MANAGER.write_json(path, plan)
|
|
return path, plan
|
|
|
|
|
|
def test_future_release_config_is_strict_and_previous_snapshot_is_discovered(tmp_path: Path) -> None:
|
|
release = OPERATOR.resolve_release_config(
|
|
2026,
|
|
layout="new",
|
|
population_url=(
|
|
"https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/"
|
|
"OPENDATA_SECTOREN_2026_NEW.zip"
|
|
),
|
|
geometry_url=(
|
|
"https://statbel.fgov.be/sites/default/files/files/opendata/Statistische%20sectoren/"
|
|
"sh_statbel_statistical_sectors_31370_20260101.geojson.zip"
|
|
),
|
|
)
|
|
scope = OPERATOR.GEOGRAPHIC_SCOPES["mol"]
|
|
baseline = OPERATOR.snapshot_path(tmp_path, scope, 2026)
|
|
baseline.write_text("{}", encoding="utf-8")
|
|
|
|
assert release.year == 2026
|
|
assert release.layout == "new"
|
|
assert OPERATOR.previous_snapshot_path(tmp_path, scope, 2027) == baseline
|
|
with pytest.raises(ValueError, match="supplied together"):
|
|
OPERATOR.resolve_release_config(2026, layout="new")
|
|
|
|
|
|
def test_population_workspace_pagination_reads_every_dataset() -> None:
|
|
rows = [{"id": index} for index in range(401)]
|
|
|
|
class Response:
|
|
ok = True
|
|
status_code = 200
|
|
text = ""
|
|
|
|
def __init__(self, payload: dict) -> None:
|
|
self.payload = payload
|
|
|
|
def json(self) -> dict:
|
|
return {"data": self.payload}
|
|
|
|
class Session:
|
|
def get(self, _url: str, *, params: dict, timeout: int):
|
|
assert timeout == 30
|
|
offset = int(params["offset"])
|
|
limit = int(params["limit"])
|
|
return Response({"items": rows[offset : offset + limit], "total": len(rows)})
|
|
|
|
assert OPERATOR.list_paginated_items(Session(), "http://backend/datasets", timeout=30) == rows
|
|
|
|
|
|
def test_population_archive_download_is_bounded_before_streaming() -> None:
|
|
class Response:
|
|
url = (
|
|
"https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/"
|
|
"OPENDATA_SECTOREN_2026_NEW.zip"
|
|
)
|
|
headers = {"Content-Length": str(OPERATOR.MAX_POPULATION_ARCHIVE_BYTES + 1)}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def raise_for_status(self) -> None:
|
|
return None
|
|
|
|
def iter_content(self, *, chunk_size: int):
|
|
raise AssertionError(f"download should fail before streaming {chunk_size}")
|
|
|
|
class Session:
|
|
def get(self, *_args, **_kwargs):
|
|
return Response()
|
|
|
|
with pytest.raises(RuntimeError, match="download limit"):
|
|
OPERATOR.download_archive(
|
|
Session(),
|
|
url=Response.url,
|
|
year=2026,
|
|
layout="new",
|
|
max_bytes=OPERATOR.MAX_POPULATION_ARCHIVE_BYTES,
|
|
timeout=30,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("remote", "local", "expected"),
|
|
[
|
|
("2026", "2025", "update_available"),
|
|
("2025", "2025", "current"),
|
|
("2026", None, "not_loaded"),
|
|
("2025", "2026", "blocked_remote_older"),
|
|
],
|
|
)
|
|
def test_catalog_decision_orders_remote_and_local_editions(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
remote: str,
|
|
local: str | None,
|
|
expected: str,
|
|
) -> None:
|
|
args = arguments(tmp_path)
|
|
monkeypatch.setattr(
|
|
MANAGER,
|
|
"api_data",
|
|
lambda *_args, **_kwargs: {"items": [catalog_item(remote=remote, local=local)]},
|
|
)
|
|
|
|
result = MANAGER.fetch_release_decision(args, refresh=True)
|
|
|
|
assert result["status"] == expected
|
|
assert result["release"]["year"] == int(remote)
|
|
assert result["automatic_download"] is False
|
|
assert result["automatic_import"] is False
|
|
|
|
|
|
def test_operator_commands_separate_staging_from_apply(tmp_path: Path) -> None:
|
|
args = arguments(tmp_path)
|
|
release = release_2026()
|
|
|
|
stage = MANAGER.build_operator_command(args, release, fetch_only=True)
|
|
apply = MANAGER.build_operator_command(args, release, fetch_only=False)
|
|
|
|
assert "--force" in stage
|
|
assert "--fetch-only" in stage
|
|
assert "--force" not in apply
|
|
assert "--fetch-only" not in apply
|
|
assert stage[stage.index("--population-layout") + 1] == "new"
|
|
assert stage[stage.index("--population-url") + 1] == release.population_url
|
|
assert stage[stage.index("--geometry-url") + 1] == release.geometry_url
|
|
|
|
|
|
def test_staged_plan_and_review_require_exact_hashes(tmp_path: Path) -> None:
|
|
args = arguments(tmp_path)
|
|
release = release_2026()
|
|
release_decision = decision(args)
|
|
plan_path, plan = staged_plan(args, release, release_decision)
|
|
args.confirm_plan_sha256 = plan["plan_sha256"]
|
|
|
|
loaded_path, loaded = MANAGER.load_staged_plan(args, release)
|
|
assert loaded_path == plan_path
|
|
assert loaded["evidence"]["scope_accounting"]["accounted_population_total"] == 510300
|
|
|
|
args.approve = True
|
|
args.reviewer = "GeoIntel operator"
|
|
args.review_note = "Schema, totalen en ZZZZ-accounting nagekeken."
|
|
review = MANAGER.build_review_evidence(args, plan_path, plan)
|
|
review_path = MANAGER.default_review_path(args, release.year)
|
|
MANAGER.write_json(review_path, review)
|
|
args.confirm_review_sha256 = review["review_sha256"]
|
|
|
|
loaded_review_path, loaded_review = MANAGER.load_review_evidence(args, release, plan)
|
|
assert loaded_review_path == review_path
|
|
assert loaded_review["status"] == "approved"
|
|
assert "scope_and_national_accounting" in loaded_review["reviewed_checks"]
|
|
|
|
|
|
def test_tampered_source_or_review_is_rejected(tmp_path: Path) -> None:
|
|
args = arguments(tmp_path)
|
|
release = release_2026()
|
|
release_decision = decision(args)
|
|
plan_path, plan = staged_plan(args, release, release_decision)
|
|
args.confirm_plan_sha256 = plan["plan_sha256"]
|
|
population_path = Path(plan["evidence"]["population_archive"]["retained_path"])
|
|
population_path.write_bytes(population_path.read_bytes() + b"tampered")
|
|
|
|
with pytest.raises(RuntimeError, match="population archive"):
|
|
MANAGER.load_staged_plan(args, release)
|
|
|
|
write_staged_artifacts(args, release)
|
|
args.approve = True
|
|
args.reviewer = "Operator"
|
|
review = MANAGER.build_review_evidence(args, plan_path, plan)
|
|
review_path = MANAGER.default_review_path(args, release.year)
|
|
MANAGER.write_json(review_path, review)
|
|
review_payload = json.loads(review_path.read_text(encoding="utf-8"))
|
|
review_payload["reviewer"] = "Someone else"
|
|
review_path.write_text(json.dumps(review_payload), encoding="utf-8")
|
|
args.confirm_review_sha256 = review["review_sha256"]
|
|
|
|
with pytest.raises(RuntimeError, match="checksum"):
|
|
MANAGER.load_review_evidence(args, release, plan)
|
|
|
|
|
|
def test_catalog_drift_and_outside_evidence_path_are_rejected(tmp_path: Path) -> None:
|
|
args = arguments(tmp_path)
|
|
original = decision(args)
|
|
checked_later = json.loads(json.dumps(original))
|
|
checked_later["catalog_identity"]["catalog_checked_at"] = "2026-07-19T01:00:00Z"
|
|
|
|
MANAGER.require_catalog_unchanged(
|
|
{"release": original["release"], "catalog_identity": original["catalog_identity"]},
|
|
checked_later,
|
|
)
|
|
|
|
changed = json.loads(json.dumps(original))
|
|
changed["catalog_identity"]["capabilities_sha256"] = "b" * 64
|
|
|
|
with pytest.raises(RuntimeError, match="changed"):
|
|
MANAGER.require_catalog_unchanged(
|
|
{"release": original["release"], "catalog_identity": original["catalog_identity"]},
|
|
changed,
|
|
)
|
|
with pytest.raises(RuntimeError, match="outside"):
|
|
MANAGER.governed_evidence_path(args, tmp_path / "outside.json")
|
|
|
|
|
|
def test_apply_flow_requires_approved_review_and_writes_evidence(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
args = arguments(
|
|
tmp_path,
|
|
action="apply",
|
|
confirm_edition="2026",
|
|
confirm_layout="new",
|
|
)
|
|
release = release_2026()
|
|
release_decision = decision(args)
|
|
plan_path, plan = staged_plan(args, release, release_decision)
|
|
args.confirm_plan_sha256 = plan["plan_sha256"]
|
|
args.approve = True
|
|
args.reviewer = "GeoIntel operator"
|
|
review = MANAGER.build_review_evidence(args, plan_path, plan)
|
|
review_path = MANAGER.default_review_path(args, release.year)
|
|
MANAGER.write_json(review_path, review)
|
|
args.confirm_review_sha256 = review["review_sha256"]
|
|
final_decision = json.loads(json.dumps(release_decision))
|
|
final_decision["status"] = "current"
|
|
final_decision["local_source_version"] = "2026"
|
|
decisions = iter((release_decision, final_decision))
|
|
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
|
|
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
|
|
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: next(decisions))
|
|
monkeypatch.setattr(
|
|
MANAGER,
|
|
"run_operator",
|
|
lambda *_args, **_kwargs: {
|
|
"status": "ok",
|
|
"snapshots": [{"year": 2026, "status": "imported", "dataset_id": "dataset-2026", "feature_count": 700}],
|
|
},
|
|
)
|
|
|
|
assert MANAGER.main() == 0
|
|
applied_path = plan_path.with_name("applied-evidence.json")
|
|
applied = json.loads(applied_path.read_text(encoding="utf-8"))
|
|
assert applied["dataset_id"] == "dataset-2026"
|
|
assert applied["review_sha256"] == review["review_sha256"]
|
|
assert len(applied["applied_evidence_sha256"]) == 64
|
|
|
|
|
|
def test_current_release_cannot_be_staged(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
args = arguments(tmp_path, action="stage", confirm_edition="2025", confirm_layout="new")
|
|
current = decision(args, remote="2025", local="2025")
|
|
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
|
|
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
|
|
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: current)
|
|
|
|
assert MANAGER.main() == 1
|
|
assert "not safely stageable: current" in capsys.readouterr().err
|
|
|
|
|
|
def test_plan_action_is_read_only_for_current_release(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
capsys: pytest.CaptureFixture[str],
|
|
) -> None:
|
|
args = arguments(tmp_path, action="plan")
|
|
current = decision(args, remote="2025", local="2025")
|
|
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
|
|
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
|
|
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: current)
|
|
|
|
assert MANAGER.main() == 0
|
|
output = json.loads(capsys.readouterr().out)
|
|
assert output["status"] == "ok"
|
|
assert output["decision"]["status"] == "current"
|
|
assert not args.evidence_root.exists()
|
|
|
|
|
|
def test_release_manager_is_packaged_and_release_checked() -> None:
|
|
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
|
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
|
|
|
|
assert "COPY scripts/manage_statbel_population_release.py" in dockerfile
|
|
assert "py_compile scripts/manage_statbel_population_release.py" in readiness
|