Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
50dc6692a0fdb7e37db3d535fb67e3a9a46f772b
geointel/backend/tests/test_sprint228_statbel_release_management.py
T
Jens faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
Details
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
Details
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
Details
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
Details
Initial public release
2026-08-31 21:56:53 +02:00

446 lines
17 KiB
Python
Raw Blame History

from __future__ import annotations
import argparse
from hashlib import sha256
import importlib.util
import json
from pathlib import Path
import sys
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
def load_script(name: str):
path = SCRIPTS / name
module_name = f"test_{path.stem}_sprint228"
spec = importlib.util.spec_from_file_location(module_name, path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
MANAGER = load_script("manage_statbel_population_release.py")
OPERATOR = load_script("provision_mol_population_history.py")
def arguments(tmp_path: Path, **overrides) -> argparse.Namespace:
values = {
"action": "plan",
"project_id": "00000000-0000-0000-0000-000000000001",
"scope": "kempen-transport-region",
"api_url": "http://127.0.0.1:8000/api/v1",
"confirm_edition": None,
"confirm_layout": None,
"confirm_plan_sha256": None,
"confirm_review_sha256": None,
"approve": False,
"reviewer": None,
"review_note": "",
"plan_path": None,
"review_path": None,
"output_root": tmp_path / "operator-data" / "regional-timeseries",
"scope_output_root": tmp_path / "operator-data" / "geographic-scopes",
"evidence_root": tmp_path / "operator-evidence" / "statbel-population-refresh",
"refresh_catalog": False,
"request_timeout": 300,
"api_timeout": 180,
"import_timeout": 3600,
}
values.update(overrides)
return argparse.Namespace(**values)
def catalog_item(*, remote: str = "2026", local: str | None = "2025", catalog_hash: str = "a" * 64) -> dict:
return {
"source_name": "statbel",
"status": "available",
"reachable": True,
"error_code": None,
"remote_version": remote,
"local_source_version": local,
"remote_title": f"Bevolking per statistische sector {remote} (nieuwe REDEGEO-sectorindeling)",
"message": "De officiele catalogus bevestigt de nieuwe REDEGEO-sectorindeling.",
"matched_layers": ["population_txt_current", "landing_page", "cc_by_4_0"],
"capabilities_sha256": catalog_hash,
"metadata_identifier": f"NodeID{remote}",
"metadata_url": f"https://statbel.fgov.be/nl/open-data/bevolking-statistische-sector-{remote}",
"checked_at": "2026-07-17T08:00:00Z",
}
def decision(args: argparse.Namespace, *, remote: str = "2026", local: str | None = "2025") -> dict:
return MANAGER.fetch_release_decision_from_item(args, catalog_item(remote=remote, local=local))
def release_2026():
return MANAGER.release_from_catalog_item(catalog_item())
def write_staged_artifacts(args: argparse.Namespace, release) -> None:
scope = MANAGER.GEOGRAPHIC_SCOPES[args.scope]
output_dir = MANAGER.population_output_dir(args)
raw_dir = output_dir / "raw" / str(release.year)
raw_dir.mkdir(parents=True, exist_ok=True)
population_path = raw_dir / "OPENDATA_SECTOREN_2026_NEW.zip"
geometry_path = raw_dir / "sh_statbel_statistical_sectors_31370_20260101.geojson.zip"
population_path.write_bytes(b"official population archive")
geometry_path.write_bytes(b"official geometry archive")
snapshot = MANAGER.snapshot_path(output_dir, scope, release.year)
snapshot.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
manifest = {
"schema_version": 1,
"status": "passed",
"import_eligible": True,
"release": {"year": 2026, "population_layout": "new"},
"artifacts": {
"population": {
"source_url": release.population_url,
"archive_sha256": sha256(population_path.read_bytes()).hexdigest(),
"archive_size_bytes": population_path.stat().st_size,
"retained_path": str(population_path),
},
"geometry": {
"source_url": release.geometry_url,
"archive_sha256": sha256(geometry_path.read_bytes()).hexdigest(),
"archive_size_bytes": geometry_path.stat().st_size,
"retained_path": str(geometry_path),
},
"derived_snapshot": {
"sha256": sha256(snapshot.read_bytes()).hexdigest(),
"size_bytes": snapshot.stat().st_size,
"feature_count": 700,
},
},
"scope_accounting": {
"scope_key": scope.key,
"spatial_sector_count": 700,
"spatial_population_total": 510000,
"unlocated_population_total": 300,
"accounted_population_total": 510300,
},
"national_accounting": {"population_total": 12000000},
"baseline": {"year": 2025, "annual_change_ratio": 0.007},
"schemas": {"geometry_repair_count": 2},
}
manifest_path = MANAGER.preflight_manifest_path(output_dir, scope, release.year)
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
def staged_plan(args: argparse.Namespace, release, release_decision: dict) -> tuple[Path, dict]:
write_staged_artifacts(args, release)
result = {
"status": "ok",
"scope": args.scope,
"snapshots": [
{
"year": release.year,
"status": "prepared",
"preflight_status": "passed",
"feature_count": 700,
}
],
}
plan = MANAGER.build_staged_plan(args, release_decision, release, result)
path = MANAGER.default_plan_path(args, release.year)
MANAGER.write_json(path, plan)
return path, plan
def test_future_release_config_is_strict_and_previous_snapshot_is_discovered(tmp_path: Path) -> None:
release = OPERATOR.resolve_release_config(
2026,
layout="new",
population_url=(
"https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/"
"OPENDATA_SECTOREN_2026_NEW.zip"
),
geometry_url=(
"https://statbel.fgov.be/sites/default/files/files/opendata/Statistische%20sectoren/"
"sh_statbel_statistical_sectors_31370_20260101.geojson.zip"
),
)
scope = OPERATOR.GEOGRAPHIC_SCOPES["mol"]
baseline = OPERATOR.snapshot_path(tmp_path, scope, 2026)
baseline.write_text("{}", encoding="utf-8")
assert release.year == 2026
assert release.layout == "new"
assert OPERATOR.previous_snapshot_path(tmp_path, scope, 2027) == baseline
with pytest.raises(ValueError, match="supplied together"):
OPERATOR.resolve_release_config(2026, layout="new")
def test_population_workspace_pagination_reads_every_dataset() -> None:
rows = [{"id": index} for index in range(401)]
class Response:
ok = True
status_code = 200
text = ""
def __init__(self, payload: dict) -> None:
self.payload = payload
def json(self) -> dict:
return {"data": self.payload}
class Session:
def get(self, _url: str, *, params: dict, timeout: int):
assert timeout == 30
offset = int(params["offset"])
limit = int(params["limit"])
return Response({"items": rows[offset : offset + limit], "total": len(rows)})
assert OPERATOR.list_paginated_items(Session(), "http://backend/datasets", timeout=30) == rows
def test_population_archive_download_is_bounded_before_streaming() -> None:
class Response:
url = (
"https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/"
"OPENDATA_SECTOREN_2026_NEW.zip"
)
headers = {"Content-Length": str(OPERATOR.MAX_POPULATION_ARCHIVE_BYTES + 1)}
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def raise_for_status(self) -> None:
return None
def iter_content(self, *, chunk_size: int):
raise AssertionError(f"download should fail before streaming {chunk_size}")
class Session:
def get(self, *_args, **_kwargs):
return Response()
with pytest.raises(RuntimeError, match="download limit"):
OPERATOR.download_archive(
Session(),
url=Response.url,
year=2026,
layout="new",
max_bytes=OPERATOR.MAX_POPULATION_ARCHIVE_BYTES,
timeout=30,
)
@pytest.mark.parametrize(
("remote", "local", "expected"),
[
("2026", "2025", "update_available"),
("2025", "2025", "current"),
("2026", None, "not_loaded"),
("2025", "2026", "blocked_remote_older"),
],
)
def test_catalog_decision_orders_remote_and_local_editions(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
remote: str,
local: str | None,
expected: str,
) -> None:
args = arguments(tmp_path)
monkeypatch.setattr(
MANAGER,
"api_data",
lambda *_args, **_kwargs: {"items": [catalog_item(remote=remote, local=local)]},
)
result = MANAGER.fetch_release_decision(args, refresh=True)
assert result["status"] == expected
assert result["release"]["year"] == int(remote)
assert result["automatic_download"] is False
assert result["automatic_import"] is False
def test_operator_commands_separate_staging_from_apply(tmp_path: Path) -> None:
args = arguments(tmp_path)
release = release_2026()
stage = MANAGER.build_operator_command(args, release, fetch_only=True)
apply = MANAGER.build_operator_command(args, release, fetch_only=False)
assert "--force" in stage
assert "--fetch-only" in stage
assert "--force" not in apply
assert "--fetch-only" not in apply
assert stage[stage.index("--population-layout") + 1] == "new"
assert stage[stage.index("--population-url") + 1] == release.population_url
assert stage[stage.index("--geometry-url") + 1] == release.geometry_url
def test_staged_plan_and_review_require_exact_hashes(tmp_path: Path) -> None:
args = arguments(tmp_path)
release = release_2026()
release_decision = decision(args)
plan_path, plan = staged_plan(args, release, release_decision)
args.confirm_plan_sha256 = plan["plan_sha256"]
loaded_path, loaded = MANAGER.load_staged_plan(args, release)
assert loaded_path == plan_path
assert loaded["evidence"]["scope_accounting"]["accounted_population_total"] == 510300
args.approve = True
args.reviewer = "GeoIntel operator"
args.review_note = "Schema, totalen en ZZZZ-accounting nagekeken."
review = MANAGER.build_review_evidence(args, plan_path, plan)
review_path = MANAGER.default_review_path(args, release.year)
MANAGER.write_json(review_path, review)
args.confirm_review_sha256 = review["review_sha256"]
loaded_review_path, loaded_review = MANAGER.load_review_evidence(args, release, plan)
assert loaded_review_path == review_path
assert loaded_review["status"] == "approved"
assert "scope_and_national_accounting" in loaded_review["reviewed_checks"]
def test_tampered_source_or_review_is_rejected(tmp_path: Path) -> None:
args = arguments(tmp_path)
release = release_2026()
release_decision = decision(args)
plan_path, plan = staged_plan(args, release, release_decision)
args.confirm_plan_sha256 = plan["plan_sha256"]
population_path = Path(plan["evidence"]["population_archive"]["retained_path"])
population_path.write_bytes(population_path.read_bytes() + b"tampered")
with pytest.raises(RuntimeError, match="population archive"):
MANAGER.load_staged_plan(args, release)
write_staged_artifacts(args, release)
args.approve = True
args.reviewer = "Operator"
review = MANAGER.build_review_evidence(args, plan_path, plan)
review_path = MANAGER.default_review_path(args, release.year)
MANAGER.write_json(review_path, review)
review_payload = json.loads(review_path.read_text(encoding="utf-8"))
review_payload["reviewer"] = "Someone else"
review_path.write_text(json.dumps(review_payload), encoding="utf-8")
args.confirm_review_sha256 = review["review_sha256"]
with pytest.raises(RuntimeError, match="checksum"):
MANAGER.load_review_evidence(args, release, plan)
def test_catalog_drift_and_outside_evidence_path_are_rejected(tmp_path: Path) -> None:
args = arguments(tmp_path)
original = decision(args)
checked_later = json.loads(json.dumps(original))
checked_later["catalog_identity"]["catalog_checked_at"] = "2026-07-19T01:00:00Z"
MANAGER.require_catalog_unchanged(
{"release": original["release"], "catalog_identity": original["catalog_identity"]},
checked_later,
)
changed = json.loads(json.dumps(original))
changed["catalog_identity"]["capabilities_sha256"] = "b" * 64
with pytest.raises(RuntimeError, match="changed"):
MANAGER.require_catalog_unchanged(
{"release": original["release"], "catalog_identity": original["catalog_identity"]},
changed,
)
with pytest.raises(RuntimeError, match="outside"):
MANAGER.governed_evidence_path(args, tmp_path / "outside.json")
def test_apply_flow_requires_approved_review_and_writes_evidence(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
args = arguments(
tmp_path,
action="apply",
confirm_edition="2026",
confirm_layout="new",
)
release = release_2026()
release_decision = decision(args)
plan_path, plan = staged_plan(args, release, release_decision)
args.confirm_plan_sha256 = plan["plan_sha256"]
args.approve = True
args.reviewer = "GeoIntel operator"
review = MANAGER.build_review_evidence(args, plan_path, plan)
review_path = MANAGER.default_review_path(args, release.year)
MANAGER.write_json(review_path, review)
args.confirm_review_sha256 = review["review_sha256"]
final_decision = json.loads(json.dumps(release_decision))
final_decision["status"] = "current"
final_decision["local_source_version"] = "2026"
decisions = iter((release_decision, final_decision))
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: next(decisions))
monkeypatch.setattr(
MANAGER,
"run_operator",
lambda *_args, **_kwargs: {
"status": "ok",
"snapshots": [{"year": 2026, "status": "imported", "dataset_id": "dataset-2026", "feature_count": 700}],
},
)
assert MANAGER.main() == 0
applied_path = plan_path.with_name("applied-evidence.json")
applied = json.loads(applied_path.read_text(encoding="utf-8"))
assert applied["dataset_id"] == "dataset-2026"
assert applied["review_sha256"] == review["review_sha256"]
assert len(applied["applied_evidence_sha256"]) == 64
def test_current_release_cannot_be_staged(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
args = arguments(tmp_path, action="stage", confirm_edition="2025", confirm_layout="new")
current = decision(args, remote="2025", local="2025")
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: current)
assert MANAGER.main() == 1
assert "not safely stageable: current" in capsys.readouterr().err
def test_plan_action_is_read_only_for_current_release(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
) -> None:
args = arguments(tmp_path, action="plan")
current = decision(args, remote="2025", local="2025")
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: current)
assert MANAGER.main() == 0
output = json.loads(capsys.readouterr().out)
assert output["status"] == "ok"
assert output["decision"]["status"] == "current"
assert not args.evidence_root.exists()
def test_release_manager_is_packaged_and_release_checked() -> None:
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
assert "COPY scripts/manage_statbel_population_release.py" in dockerfile
assert "py_compile scripts/manage_statbel_population_release.py" in readiness
Reference in New Issue View Git Blame Copy Permalink