Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
598 lines
26 KiB
Python
598 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""Plan, stage, review and explicitly apply one Statbel population edition.
|
|
|
|
Planning is read-only. Staging downloads official artifacts and runs the
|
|
existing fail-closed preflight without database persistence. Review records an
|
|
explicit named approval. Apply requires both exact evidence hashes and reuses
|
|
the canonical population DatasetService upload path.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
from hashlib import sha256
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
from geographic_scopes import GEOGRAPHIC_SCOPES
|
|
from provision_mol_population_history import (
|
|
PopulationReleaseConfig,
|
|
load_preflight_manifest,
|
|
preflight_manifest_path,
|
|
resolve_release_config,
|
|
sha256_path,
|
|
snapshot_path,
|
|
)
|
|
|
|
|
|
DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1"
|
|
DEFAULT_SCOPE = "kempen-transport-region"
|
|
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/regional-timeseries")
|
|
DEFAULT_SCOPE_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
|
DEFAULT_EVIDENCE_ROOT = Path("/app/storage/operator-evidence/statbel-population-refresh")
|
|
YEAR_PATTERN = re.compile(r"^20[0-9]{2}$")
|
|
SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
|
|
ACTIONABLE_STATUSES = {"update_available", "not_loaded"}
|
|
STABLE_CATALOG_IDENTITY_FIELDS = (
|
|
"metadata_identifier",
|
|
"metadata_url",
|
|
"remote_version",
|
|
"capabilities_sha256",
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Governed Statbel population release: plan, stage, review, then checksum-confirmed apply."
|
|
)
|
|
parser.add_argument("action", choices=("plan", "stage", "review", "apply"))
|
|
parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope")
|
|
parser.add_argument("--scope", choices=tuple(sorted(GEOGRAPHIC_SCOPES)), default=DEFAULT_SCOPE)
|
|
parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL))
|
|
parser.add_argument("--confirm-edition", help="Exact official population year required after plan")
|
|
parser.add_argument("--confirm-layout", choices=("standard", "new"), help="Exact planned REDEGEO layout")
|
|
parser.add_argument("--confirm-plan-sha256", help="Exact staged-plan hash required for review/apply")
|
|
parser.add_argument("--confirm-review-sha256", help="Exact review-evidence hash required for apply")
|
|
parser.add_argument("--approve", action="store_true", help="Explicitly approve the staged evidence during review")
|
|
parser.add_argument("--reviewer", help="Named human/operator approving the staged evidence")
|
|
parser.add_argument("--review-note", default="", help="Optional bounded review note")
|
|
parser.add_argument("--plan-path", type=Path, help="Override the governed staged-plan path")
|
|
parser.add_argument("--review-path", type=Path, help="Override the governed review-evidence path")
|
|
parser.add_argument(
|
|
"--output-root",
|
|
type=Path,
|
|
default=Path(os.environ.get("GEOINTEL_REGIONAL_TIMESERIES_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
|
)
|
|
parser.add_argument(
|
|
"--scope-output-root",
|
|
type=Path,
|
|
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_SCOPE_OUTPUT_ROOT)),
|
|
)
|
|
parser.add_argument(
|
|
"--evidence-root",
|
|
type=Path,
|
|
default=Path(os.environ.get("GEOINTEL_STATBEL_REFRESH_EVIDENCE_ROOT", DEFAULT_EVIDENCE_ROOT)),
|
|
)
|
|
parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short official-catalog cache")
|
|
parser.add_argument("--request-timeout", type=int, default=300)
|
|
parser.add_argument("--api-timeout", type=int, default=180)
|
|
parser.add_argument("--import-timeout", type=int, default=3600)
|
|
return parser.parse_args()
|
|
|
|
|
|
def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]:
|
|
endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}"
|
|
request = Request(
|
|
endpoint,
|
|
headers={"Accept": "application/json", "User-Agent": "GeoIntel-Statbel-release/1.0"},
|
|
)
|
|
try:
|
|
with urlopen(request, timeout=timeout) as response:
|
|
payload = json.load(response)
|
|
except HTTPError as exc:
|
|
body = exc.read().decode("utf-8", errors="replace")
|
|
raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc
|
|
except URLError as exc:
|
|
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
|
|
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
|
|
raise RuntimeError("GeoIntel API response is not a canonical data envelope")
|
|
return payload["data"]
|
|
|
|
|
|
def validate_project_scope(args: argparse.Namespace) -> None:
|
|
project = api_data(args.api_url, f"projects/{args.project_id}", args.api_timeout)
|
|
expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name
|
|
if project.get("name") != expected_name:
|
|
raise RuntimeError(
|
|
f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'"
|
|
)
|
|
|
|
|
|
def _layout_from_catalog_item(item: dict[str, Any], year: int) -> str:
|
|
evidence = f"{item.get('remote_title') or ''} {item.get('message') or ''}".casefold()
|
|
if "nieuwe redegeo-sectorindeling" in evidence:
|
|
return "new"
|
|
if year == 2025:
|
|
raise RuntimeError("The official 2025 release is missing explicit new-REDEGEO evidence")
|
|
if "actuele sectorindeling" in evidence:
|
|
return "standard"
|
|
raise RuntimeError("The official Statbel catalog did not expose a recognized population layout")
|
|
|
|
|
|
def release_from_catalog_item(item: dict[str, Any]) -> PopulationReleaseConfig:
|
|
version = str(item.get("remote_version") or "")
|
|
if not YEAR_PATTERN.fullmatch(version):
|
|
raise RuntimeError("The official Statbel catalog did not provide one valid population year")
|
|
year = int(version)
|
|
layout = _layout_from_catalog_item(item, year)
|
|
suffix = "_NEW" if layout == "new" else ""
|
|
return resolve_release_config(
|
|
year,
|
|
layout=layout,
|
|
population_url=(
|
|
"https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/"
|
|
f"OPENDATA_SECTOREN_{year}{suffix}.zip"
|
|
),
|
|
geometry_url=(
|
|
"https://statbel.fgov.be/sites/default/files/files/opendata/Statistische%20sectoren/"
|
|
f"sh_statbel_statistical_sectors_31370_{year}0101.geojson.zip"
|
|
),
|
|
)
|
|
|
|
|
|
def fetch_release_decision_from_item(args: argparse.Namespace, item: dict[str, Any]) -> dict[str, Any]:
|
|
if (
|
|
item.get("source_name") != "statbel"
|
|
or item.get("status") != "available"
|
|
or item.get("reachable") is not True
|
|
or item.get("error_code")
|
|
or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or ""))
|
|
or not item.get("metadata_identifier")
|
|
):
|
|
raise RuntimeError("The official Statbel catalog is not safely available for release planning")
|
|
required_evidence = {"population_txt_current", "landing_page", "cc_by_4_0"}
|
|
if not required_evidence.issubset(set(item.get("matched_layers") or [])):
|
|
raise RuntimeError("The official Statbel catalog is missing required release evidence")
|
|
release = release_from_catalog_item(item)
|
|
local_version = str(item.get("local_source_version") or "")
|
|
if local_version and not YEAR_PATTERN.fullmatch(local_version):
|
|
status = "blocked_local_version"
|
|
elif not local_version:
|
|
status = "not_loaded"
|
|
elif int(local_version) == release.year:
|
|
status = "current"
|
|
elif int(local_version) < release.year:
|
|
status = "update_available"
|
|
else:
|
|
status = "blocked_remote_older"
|
|
return {
|
|
"schema_version": 1,
|
|
"status": status,
|
|
"project_id": args.project_id,
|
|
"scope": args.scope,
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"local_source_version": local_version or None,
|
|
"release": {
|
|
"year": release.year,
|
|
"layout": release.layout,
|
|
"population_url": release.population_url,
|
|
"geometry_url": release.geometry_url,
|
|
},
|
|
"catalog_identity": {
|
|
"metadata_identifier": item["metadata_identifier"],
|
|
"metadata_url": item.get("metadata_url"),
|
|
"remote_version": str(release.year),
|
|
"capabilities_sha256": item["capabilities_sha256"],
|
|
"catalog_checked_at": item.get("checked_at"),
|
|
},
|
|
"automatic_download": False,
|
|
"automatic_import": False,
|
|
"destructive_replacement": False,
|
|
"next_action": "stage" if status in ACTIONABLE_STATUSES else None,
|
|
}
|
|
|
|
|
|
def fetch_release_decision(args: argparse.Namespace, *, refresh: bool) -> dict[str, Any]:
|
|
query = "true" if refresh else "false"
|
|
report = api_data(
|
|
args.api_url,
|
|
f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={query}",
|
|
args.api_timeout,
|
|
)
|
|
matches = [item for item in report.get("items") or [] if item.get("source_name") == "statbel"]
|
|
if len(matches) != 1:
|
|
raise RuntimeError("Source catalog report did not contain exactly one Statbel population contract")
|
|
return fetch_release_decision_from_item(args, matches[0])
|
|
|
|
|
|
def require_release_confirmation(args: argparse.Namespace, release: PopulationReleaseConfig) -> None:
|
|
if args.confirm_edition != str(release.year):
|
|
raise RuntimeError(f"Explicit --confirm-edition {release.year} is required")
|
|
if args.confirm_layout != release.layout:
|
|
raise RuntimeError(f"Explicit --confirm-layout {release.layout} is required")
|
|
|
|
|
|
def internal_base_url(api_url: str) -> str:
|
|
value = api_url.rstrip("/")
|
|
if value.endswith("/api/v1"):
|
|
value = value[:-7]
|
|
parsed = urlparse(value)
|
|
if parsed.hostname not in {"127.0.0.1", "localhost", "::1"}:
|
|
raise RuntimeError("Stage/apply must run inside GeoIntel against the local backend API")
|
|
return value
|
|
|
|
|
|
def population_output_dir(args: argparse.Namespace) -> Path:
|
|
return args.output_root / args.scope / "population"
|
|
|
|
|
|
def default_plan_path(args: argparse.Namespace, year: int) -> Path:
|
|
return args.evidence_root / args.scope / str(year) / "staged-plan.json"
|
|
|
|
|
|
def default_review_path(args: argparse.Namespace, year: int) -> Path:
|
|
return args.evidence_root / args.scope / str(year) / "review-evidence.json"
|
|
|
|
|
|
def governed_evidence_path(args: argparse.Namespace, path: Path) -> Path:
|
|
if not path.resolve().is_relative_to(args.evidence_root.resolve()):
|
|
raise RuntimeError(f"Release evidence path is outside the governed evidence root: {path}")
|
|
return path
|
|
|
|
|
|
def build_operator_command(
|
|
args: argparse.Namespace,
|
|
release: PopulationReleaseConfig,
|
|
*,
|
|
fetch_only: bool,
|
|
) -> list[str]:
|
|
scope = GEOGRAPHIC_SCOPES[args.scope]
|
|
command = [
|
|
sys.executable,
|
|
str(Path(__file__).resolve().parent / "provision_mol_population_history.py"),
|
|
"--scope",
|
|
scope.key,
|
|
"--base-url",
|
|
internal_base_url(args.api_url),
|
|
"--project-name",
|
|
scope.project_name,
|
|
"--area-name",
|
|
scope.area_name,
|
|
"--years",
|
|
str(release.year),
|
|
"--population-url",
|
|
release.population_url,
|
|
"--geometry-url",
|
|
release.geometry_url,
|
|
"--population-layout",
|
|
release.layout,
|
|
"--scope-output-root",
|
|
str(args.scope_output_root),
|
|
"--output-dir",
|
|
str(population_output_dir(args)),
|
|
"--request-timeout",
|
|
str(args.request_timeout),
|
|
"--import-timeout",
|
|
str(args.import_timeout),
|
|
]
|
|
if fetch_only:
|
|
command.extend(("--force", "--fetch-only"))
|
|
return command
|
|
|
|
|
|
def run_operator(command: list[str], *, action: str) -> dict[str, Any]:
|
|
print(f"Statbel population: {action}...", file=sys.stderr, flush=True)
|
|
completed = subprocess.run(command, check=False, capture_output=True, text=True, encoding="utf-8")
|
|
if completed.returncode != 0:
|
|
detail = completed.stderr.strip() or completed.stdout.strip() or "operator returned no diagnostics"
|
|
raise RuntimeError(f"Statbel population {action} failed with exit {completed.returncode}: {detail[-3000:]}")
|
|
try:
|
|
payload = json.loads(completed.stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError(f"Statbel population {action} returned invalid JSON") from exc
|
|
if payload.get("status") != "ok":
|
|
raise RuntimeError(f"Statbel population {action} did not report success")
|
|
return payload
|
|
|
|
|
|
def canonical_sha256(payload: dict[str, Any], hash_field: str) -> str:
|
|
content = {key: value for key, value in payload.items() if key != hash_field}
|
|
encoded = json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
return sha256(encoded).hexdigest()
|
|
|
|
|
|
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(path.suffix + ".partial")
|
|
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
temporary.replace(path)
|
|
|
|
|
|
def validate_staged_release(
|
|
args: argparse.Namespace,
|
|
release: PopulationReleaseConfig,
|
|
) -> dict[str, Any]:
|
|
scope = GEOGRAPHIC_SCOPES[args.scope]
|
|
output_dir = population_output_dir(args)
|
|
resolved_output = output_dir.resolve()
|
|
snapshot = snapshot_path(output_dir, scope, release.year)
|
|
manifest_path = preflight_manifest_path(output_dir, scope, release.year)
|
|
for path in (snapshot, manifest_path):
|
|
if not path.resolve().is_relative_to(resolved_output):
|
|
raise RuntimeError(f"Staged Statbel evidence is outside the governed output root: {path}")
|
|
manifest = load_preflight_manifest(manifest_path, snapshot, release.year, scope)
|
|
manifest_release = manifest.get("release") or {}
|
|
artifacts = manifest.get("artifacts") or {}
|
|
if (
|
|
manifest_release.get("population_layout") != release.layout
|
|
or (artifacts.get("population") or {}).get("source_url") != release.population_url
|
|
or (artifacts.get("geometry") or {}).get("source_url") != release.geometry_url
|
|
):
|
|
raise RuntimeError("Staged Statbel manifest no longer matches the planned release identity")
|
|
for artifact_name in ("population", "geometry"):
|
|
retained = Path(str((artifacts.get(artifact_name) or {}).get("retained_path") or ""))
|
|
if not retained.resolve().is_relative_to(resolved_output):
|
|
raise RuntimeError(f"Staged {artifact_name} archive is outside the governed output root: {retained}")
|
|
return {
|
|
"manifest_path": str(manifest_path),
|
|
"manifest_sha256": sha256_path(manifest_path),
|
|
"snapshot_path": str(snapshot),
|
|
"snapshot_sha256": (artifacts.get("derived_snapshot") or {}).get("sha256"),
|
|
"snapshot_size_bytes": (artifacts.get("derived_snapshot") or {}).get("size_bytes"),
|
|
"feature_count": (artifacts.get("derived_snapshot") or {}).get("feature_count"),
|
|
"population_archive": artifacts.get("population"),
|
|
"geometry_archive": artifacts.get("geometry"),
|
|
"scope_accounting": manifest.get("scope_accounting"),
|
|
"national_accounting": manifest.get("national_accounting"),
|
|
"baseline": manifest.get("baseline"),
|
|
"geometry_repair_count": (manifest.get("schemas") or {}).get("geometry_repair_count"),
|
|
}
|
|
|
|
|
|
def build_staged_plan(
|
|
args: argparse.Namespace,
|
|
decision: dict[str, Any],
|
|
release: PopulationReleaseConfig,
|
|
operator_result: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
evidence = validate_staged_release(args, release)
|
|
payload: dict[str, Any] = {
|
|
"schema_version": 1,
|
|
"status": "staged",
|
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
"project_id": args.project_id,
|
|
"scope": args.scope,
|
|
"local_source_version_before_apply": decision.get("local_source_version"),
|
|
"release": decision["release"],
|
|
"catalog_identity": decision["catalog_identity"],
|
|
"evidence": evidence,
|
|
"operator_result": operator_result,
|
|
"review_required": True,
|
|
"apply_requires_plan_sha256": True,
|
|
"apply_requires_review_sha256": True,
|
|
"automatic_import": False,
|
|
"destructive_replacement": False,
|
|
"existing_snapshots_retained": True,
|
|
}
|
|
payload["plan_sha256"] = canonical_sha256(payload, "plan_sha256")
|
|
return payload
|
|
|
|
|
|
def _release_from_payload(payload: dict[str, Any]) -> PopulationReleaseConfig:
|
|
release = payload.get("release") or {}
|
|
year = release.get("year") if isinstance(release, dict) else None
|
|
layout = release.get("layout") if isinstance(release, dict) else None
|
|
population_url = release.get("population_url") if isinstance(release, dict) else None
|
|
geometry_url = release.get("geometry_url") if isinstance(release, dict) else None
|
|
if (
|
|
not isinstance(year, int)
|
|
or isinstance(year, bool)
|
|
or not isinstance(layout, str)
|
|
or not isinstance(population_url, str)
|
|
or not isinstance(geometry_url, str)
|
|
or not all((layout, population_url, geometry_url))
|
|
):
|
|
raise RuntimeError("Release evidence is missing required identity fields")
|
|
return resolve_release_config(
|
|
year,
|
|
layout=layout,
|
|
population_url=population_url,
|
|
geometry_url=geometry_url,
|
|
)
|
|
|
|
|
|
def load_staged_plan(
|
|
args: argparse.Namespace,
|
|
release: PopulationReleaseConfig,
|
|
) -> tuple[Path, dict[str, Any]]:
|
|
path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
|
|
if not path.is_file():
|
|
raise RuntimeError(f"Staged Statbel plan is missing: {path}")
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
actual_sha = canonical_sha256(payload, "plan_sha256")
|
|
if payload.get("plan_sha256") != actual_sha:
|
|
raise RuntimeError("Staged Statbel plan checksum is invalid")
|
|
if args.confirm_plan_sha256 != actual_sha:
|
|
raise RuntimeError(f"Explicit --confirm-plan-sha256 {actual_sha} is required")
|
|
if (
|
|
payload.get("status") != "staged"
|
|
or payload.get("project_id") != args.project_id
|
|
or payload.get("scope") != args.scope
|
|
or _release_from_payload(payload) != release
|
|
):
|
|
raise RuntimeError("Staged Statbel plan identity is invalid")
|
|
current_evidence = validate_staged_release(args, release)
|
|
if current_evidence != payload.get("evidence"):
|
|
raise RuntimeError("Staged Statbel artifacts no longer match the approved plan")
|
|
return path, payload
|
|
|
|
|
|
def require_catalog_unchanged(plan: dict[str, Any], decision: dict[str, Any]) -> None:
|
|
planned_identity = plan.get("catalog_identity") or {}
|
|
current_identity = decision.get("catalog_identity") or {}
|
|
identity_changed = any(
|
|
planned_identity.get(field) != current_identity.get(field)
|
|
for field in STABLE_CATALOG_IDENTITY_FIELDS
|
|
)
|
|
if plan.get("release") != decision.get("release") or identity_changed:
|
|
raise RuntimeError("The official Statbel release evidence changed; create and review a new staged plan")
|
|
|
|
|
|
def build_review_evidence(
|
|
args: argparse.Namespace,
|
|
plan_path: Path,
|
|
plan: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
reviewer = str(args.reviewer or "").strip()
|
|
note = str(args.review_note or "").strip()
|
|
if not args.approve or len(reviewer) < 2 or len(reviewer) > 120:
|
|
raise RuntimeError("Review requires --approve and a named --reviewer between 2 and 120 characters")
|
|
if len(note) > 1000:
|
|
raise RuntimeError("Review note must not exceed 1000 characters")
|
|
payload: dict[str, Any] = {
|
|
"schema_version": 1,
|
|
"status": "approved",
|
|
"reviewed_at": datetime.now(timezone.utc).isoformat(),
|
|
"reviewer": reviewer,
|
|
"review_note": note or None,
|
|
"project_id": args.project_id,
|
|
"scope": args.scope,
|
|
"release": plan["release"],
|
|
"staged_plan_path": str(plan_path),
|
|
"staged_plan_sha256": plan["plan_sha256"],
|
|
"reviewed_checks": [
|
|
"official_catalog_identity",
|
|
"source_archive_checksums",
|
|
"population_and_geometry_schemas",
|
|
"scope_and_national_accounting",
|
|
"unlocated_population_accounting",
|
|
"baseline_change_limit",
|
|
"bounded_geometry_repairs",
|
|
"immutable_dataset_apply",
|
|
],
|
|
}
|
|
payload["review_sha256"] = canonical_sha256(payload, "review_sha256")
|
|
return payload
|
|
|
|
|
|
def load_review_evidence(
|
|
args: argparse.Namespace,
|
|
release: PopulationReleaseConfig,
|
|
plan: dict[str, Any],
|
|
) -> tuple[Path, dict[str, Any]]:
|
|
path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
|
|
if not path.is_file():
|
|
raise RuntimeError(f"Approved Statbel review evidence is missing: {path}")
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
actual_sha = canonical_sha256(payload, "review_sha256")
|
|
if payload.get("review_sha256") != actual_sha:
|
|
raise RuntimeError("Statbel review evidence checksum is invalid")
|
|
if args.confirm_review_sha256 != actual_sha:
|
|
raise RuntimeError(f"Explicit --confirm-review-sha256 {actual_sha} is required")
|
|
if (
|
|
payload.get("status") != "approved"
|
|
or payload.get("project_id") != args.project_id
|
|
or payload.get("scope") != args.scope
|
|
or payload.get("release") != plan.get("release")
|
|
or payload.get("staged_plan_sha256") != plan.get("plan_sha256")
|
|
or not str(payload.get("reviewer") or "").strip()
|
|
):
|
|
raise RuntimeError("Statbel review evidence does not authorize this staged plan")
|
|
return path, payload
|
|
|
|
|
|
def _snapshot_result(operator_result: dict[str, Any], year: int) -> dict[str, Any]:
|
|
matches = [item for item in operator_result.get("snapshots") or [] if int(item.get("year") or 0) == year]
|
|
if len(matches) != 1:
|
|
raise RuntimeError("Population operator did not return exactly one result for the approved edition")
|
|
return matches[0]
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
if min(args.request_timeout, args.api_timeout, args.import_timeout) <= 0:
|
|
raise ValueError("All timeout safety limits must be positive")
|
|
validate_project_scope(args)
|
|
refresh = args.refresh_catalog or args.action in {"stage", "review", "apply"}
|
|
decision = fetch_release_decision(args, refresh=refresh)
|
|
release = _release_from_payload(decision)
|
|
|
|
if args.action == "plan":
|
|
print(json.dumps({"status": "ok", "action": "plan", "decision": decision}, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
require_release_confirmation(args, release)
|
|
if args.action == "stage":
|
|
if decision.get("status") not in ACTIONABLE_STATUSES:
|
|
raise RuntimeError(f"Statbel release is not safely stageable: {decision.get('status')}")
|
|
operator_result = run_operator(build_operator_command(args, release, fetch_only=True), action="staging")
|
|
staged_result = _snapshot_result(operator_result, release.year)
|
|
if staged_result.get("status") != "prepared" or staged_result.get("preflight_status") != "passed":
|
|
raise RuntimeError("Population staging did not produce passed preflight evidence")
|
|
plan = build_staged_plan(args, decision, release, operator_result)
|
|
plan_path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
|
|
write_json(plan_path, plan)
|
|
print(json.dumps({"status": "staged", "plan_path": str(plan_path), **plan}, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
plan_path, plan = load_staged_plan(args, release)
|
|
require_catalog_unchanged(plan, decision)
|
|
if args.action == "review":
|
|
review = build_review_evidence(args, plan_path, plan)
|
|
review_path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
|
|
write_json(review_path, review)
|
|
print(json.dumps({"status": "approved", "review_path": str(review_path), **review}, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
review_path, review = load_review_evidence(args, release, plan)
|
|
operator_result = run_operator(build_operator_command(args, release, fetch_only=False), action="applying")
|
|
applied_result = _snapshot_result(operator_result, release.year)
|
|
if applied_result.get("status") not in {"imported", "existing"} or not applied_result.get("dataset_id"):
|
|
raise RuntimeError("Population apply did not return one persisted immutable Dataset")
|
|
final_decision = fetch_release_decision(args, refresh=True)
|
|
if final_decision.get("status") != "current" or final_decision.get("local_source_version") != str(release.year):
|
|
raise RuntimeError("Applied population Dataset did not become the current local Statbel edition")
|
|
evidence = {
|
|
"schema_version": 1,
|
|
"status": "applied",
|
|
"applied_at": datetime.now(timezone.utc).isoformat(),
|
|
"project_id": args.project_id,
|
|
"scope": args.scope,
|
|
"release": plan["release"],
|
|
"staged_plan_path": str(plan_path),
|
|
"staged_plan_sha256": plan["plan_sha256"],
|
|
"review_path": str(review_path),
|
|
"review_sha256": review["review_sha256"],
|
|
"reviewer": review["reviewer"],
|
|
"dataset_id": applied_result["dataset_id"],
|
|
"dataset_status": applied_result["status"],
|
|
"feature_count": applied_result.get("feature_count"),
|
|
"final_catalog_decision": final_decision,
|
|
"existing_snapshots_retained": True,
|
|
}
|
|
evidence["applied_evidence_sha256"] = canonical_sha256(evidence, "applied_evidence_sha256")
|
|
evidence_path = plan_path.with_name("applied-evidence.json")
|
|
write_json(evidence_path, evidence)
|
|
print(json.dumps({"status": "applied", "evidence_path": str(evidence_path), **evidence}, ensure_ascii=False, indent=2))
|
|
return 0
|
|
except (OSError, RuntimeError, ValueError, KeyError, json.JSONDecodeError) as exc:
|
|
print(
|
|
json.dumps({"status": "error", "action": args.action, "message": str(exc)}, ensure_ascii=False),
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|