geointel/scripts/manage_statbel_population_release.py

#!/usr/bin/env python3
"""Plan, stage, review and explicitly apply one Statbel population edition.

Planning is read-only. Staging downloads official artifacts and runs the
existing fail-closed preflight without database persistence. Review records an
explicit named approval. Apply requires both exact evidence hashes and reuses
the canonical population DatasetService upload path.
"""

from __future__ import annotations

import argparse
from datetime import datetime, timezone
from hashlib import sha256
import json
import os
from pathlib import Path
import re
import subprocess
import sys
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urlparse
from urllib.request import Request, urlopen

from geographic_scopes import GEOGRAPHIC_SCOPES
from provision_mol_population_history import (
    PopulationReleaseConfig,
    load_preflight_manifest,
    preflight_manifest_path,
    resolve_release_config,
    sha256_path,
    snapshot_path,
)


DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1"
DEFAULT_SCOPE = "kempen-transport-region"
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/regional-timeseries")
DEFAULT_SCOPE_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
DEFAULT_EVIDENCE_ROOT = Path("/app/storage/operator-evidence/statbel-population-refresh")
YEAR_PATTERN = re.compile(r"^20[0-9]{2}$")
SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
ACTIONABLE_STATUSES = {"update_available", "not_loaded"}


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Governed Statbel population release: plan, stage, review, then checksum-confirmed apply."
    )
    parser.add_argument("action", choices=("plan", "stage", "review", "apply"))
    parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope")
    parser.add_argument("--scope", choices=tuple(sorted(GEOGRAPHIC_SCOPES)), default=DEFAULT_SCOPE)
    parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL))
    parser.add_argument("--confirm-edition", help="Exact official population year required after plan")
    parser.add_argument("--confirm-layout", choices=("standard", "new"), help="Exact planned REDEGEO layout")
    parser.add_argument("--confirm-plan-sha256", help="Exact staged-plan hash required for review/apply")
    parser.add_argument("--confirm-review-sha256", help="Exact review-evidence hash required for apply")
    parser.add_argument("--approve", action="store_true", help="Explicitly approve the staged evidence during review")
    parser.add_argument("--reviewer", help="Named human/operator approving the staged evidence")
    parser.add_argument("--review-note", default="", help="Optional bounded review note")
    parser.add_argument("--plan-path", type=Path, help="Override the governed staged-plan path")
    parser.add_argument("--review-path", type=Path, help="Override the governed review-evidence path")
    parser.add_argument(
        "--output-root",
        type=Path,
        default=Path(os.environ.get("GEOINTEL_REGIONAL_TIMESERIES_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
    )
    parser.add_argument(
        "--scope-output-root",
        type=Path,
        default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_SCOPE_OUTPUT_ROOT)),
    )
    parser.add_argument(
        "--evidence-root",
        type=Path,
        default=Path(os.environ.get("GEOINTEL_STATBEL_REFRESH_EVIDENCE_ROOT", DEFAULT_EVIDENCE_ROOT)),
    )
    parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short official-catalog cache")
    parser.add_argument("--request-timeout", type=int, default=300)
    parser.add_argument("--api-timeout", type=int, default=180)
    parser.add_argument("--import-timeout", type=int, default=3600)
    return parser.parse_args()


def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]:
    endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}"
    request = Request(
        endpoint,
        headers={"Accept": "application/json", "User-Agent": "GeoIntel-Statbel-release/1.0"},
    )
    try:
        with urlopen(request, timeout=timeout) as response:
            payload = json.load(response)
    except HTTPError as exc:
        body = exc.read().decode("utf-8", errors="replace")
        raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc
    except URLError as exc:
        raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
    if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
        raise RuntimeError("GeoIntel API response is not a canonical data envelope")
    return payload["data"]


def validate_project_scope(args: argparse.Namespace) -> None:
    project = api_data(args.api_url, f"projects/{args.project_id}", args.api_timeout)
    expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name
    if project.get("name") != expected_name:
        raise RuntimeError(
            f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'"
        )


def _layout_from_catalog_item(item: dict[str, Any], year: int) -> str:
    evidence = f"{item.get('remote_title') or ''} {item.get('message') or ''}".casefold()
    if "nieuwe redegeo-sectorindeling" in evidence:
        return "new"
    if year == 2025:
        raise RuntimeError("The official 2025 release is missing explicit new-REDEGEO evidence")
    if "actuele sectorindeling" in evidence:
        return "standard"
    raise RuntimeError("The official Statbel catalog did not expose a recognized population layout")


def release_from_catalog_item(item: dict[str, Any]) -> PopulationReleaseConfig:
    version = str(item.get("remote_version") or "")
    if not YEAR_PATTERN.fullmatch(version):
        raise RuntimeError("The official Statbel catalog did not provide one valid population year")
    year = int(version)
    layout = _layout_from_catalog_item(item, year)
    suffix = "_NEW" if layout == "new" else ""
    return resolve_release_config(
        year,
        layout=layout,
        population_url=(
            "https://statbel.fgov.be/sites/default/files/files/opendata/bevolking/sectoren/"
            f"OPENDATA_SECTOREN_{year}{suffix}.zip"
        ),
        geometry_url=(
            "https://statbel.fgov.be/sites/default/files/files/opendata/Statistische%20sectoren/"
            f"sh_statbel_statistical_sectors_31370_{year}0101.geojson.zip"
        ),
    )


def fetch_release_decision_from_item(args: argparse.Namespace, item: dict[str, Any]) -> dict[str, Any]:
    if (
        item.get("source_name") != "statbel"
        or item.get("status") != "available"
        or item.get("reachable") is not True
        or item.get("error_code")
        or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or ""))
        or not item.get("metadata_identifier")
    ):
        raise RuntimeError("The official Statbel catalog is not safely available for release planning")
    required_evidence = {"population_txt_current", "landing_page", "cc_by_4_0"}
    if not required_evidence.issubset(set(item.get("matched_layers") or [])):
        raise RuntimeError("The official Statbel catalog is missing required release evidence")
    release = release_from_catalog_item(item)
    local_version = str(item.get("local_source_version") or "")
    if local_version and not YEAR_PATTERN.fullmatch(local_version):
        status = "blocked_local_version"
    elif not local_version:
        status = "not_loaded"
    elif int(local_version) == release.year:
        status = "current"
    elif int(local_version) < release.year:
        status = "update_available"
    else:
        status = "blocked_remote_older"
    return {
        "schema_version": 1,
        "status": status,
        "project_id": args.project_id,
        "scope": args.scope,
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "local_source_version": local_version or None,
        "release": {
            "year": release.year,
            "layout": release.layout,
            "population_url": release.population_url,
            "geometry_url": release.geometry_url,
        },
        "catalog_identity": {
            "metadata_identifier": item["metadata_identifier"],
            "metadata_url": item.get("metadata_url"),
            "remote_version": str(release.year),
            "capabilities_sha256": item["capabilities_sha256"],
            "catalog_checked_at": item.get("checked_at"),
        },
        "automatic_download": False,
        "automatic_import": False,
        "destructive_replacement": False,
        "next_action": "stage" if status in ACTIONABLE_STATUSES else None,
    }


def fetch_release_decision(args: argparse.Namespace, *, refresh: bool) -> dict[str, Any]:
    query = "true" if refresh else "false"
    report = api_data(
        args.api_url,
        f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={query}",
        args.api_timeout,
    )
    matches = [item for item in report.get("items") or [] if item.get("source_name") == "statbel"]
    if len(matches) != 1:
        raise RuntimeError("Source catalog report did not contain exactly one Statbel population contract")
    return fetch_release_decision_from_item(args, matches[0])


def require_release_confirmation(args: argparse.Namespace, release: PopulationReleaseConfig) -> None:
    if args.confirm_edition != str(release.year):
        raise RuntimeError(f"Explicit --confirm-edition {release.year} is required")
    if args.confirm_layout != release.layout:
        raise RuntimeError(f"Explicit --confirm-layout {release.layout} is required")


def internal_base_url(api_url: str) -> str:
    value = api_url.rstrip("/")
    if value.endswith("/api/v1"):
        value = value[:-7]
    parsed = urlparse(value)
    if parsed.hostname not in {"127.0.0.1", "localhost", "::1"}:
        raise RuntimeError("Stage/apply must run inside GeoIntel against the local backend API")
    return value


def population_output_dir(args: argparse.Namespace) -> Path:
    return args.output_root / args.scope / "population"


def default_plan_path(args: argparse.Namespace, year: int) -> Path:
    return args.evidence_root / args.scope / str(year) / "staged-plan.json"


def default_review_path(args: argparse.Namespace, year: int) -> Path:
    return args.evidence_root / args.scope / str(year) / "review-evidence.json"


def governed_evidence_path(args: argparse.Namespace, path: Path) -> Path:
    if not path.resolve().is_relative_to(args.evidence_root.resolve()):
        raise RuntimeError(f"Release evidence path is outside the governed evidence root: {path}")
    return path


def build_operator_command(
    args: argparse.Namespace,
    release: PopulationReleaseConfig,
    *,
    fetch_only: bool,
) -> list[str]:
    scope = GEOGRAPHIC_SCOPES[args.scope]
    command = [
        sys.executable,
        str(Path(__file__).resolve().parent / "provision_mol_population_history.py"),
        "--scope",
        scope.key,
        "--base-url",
        internal_base_url(args.api_url),
        "--project-name",
        scope.project_name,
        "--area-name",
        scope.area_name,
        "--years",
        str(release.year),
        "--population-url",
        release.population_url,
        "--geometry-url",
        release.geometry_url,
        "--population-layout",
        release.layout,
        "--scope-output-root",
        str(args.scope_output_root),
        "--output-dir",
        str(population_output_dir(args)),
        "--request-timeout",
        str(args.request_timeout),
        "--import-timeout",
        str(args.import_timeout),
    ]
    if fetch_only:
        command.extend(("--force", "--fetch-only"))
    return command


def run_operator(command: list[str], *, action: str) -> dict[str, Any]:
    print(f"Statbel population: {action}...", file=sys.stderr, flush=True)
    completed = subprocess.run(command, check=False, capture_output=True, text=True, encoding="utf-8")
    if completed.returncode != 0:
        detail = completed.stderr.strip() or completed.stdout.strip() or "operator returned no diagnostics"
        raise RuntimeError(f"Statbel population {action} failed with exit {completed.returncode}: {detail[-3000:]}")
    try:
        payload = json.loads(completed.stdout)
    except json.JSONDecodeError as exc:
        raise RuntimeError(f"Statbel population {action} returned invalid JSON") from exc
    if payload.get("status") != "ok":
        raise RuntimeError(f"Statbel population {action} did not report success")
    return payload


def canonical_sha256(payload: dict[str, Any], hash_field: str) -> str:
    content = {key: value for key, value in payload.items() if key != hash_field}
    encoded = json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
    return sha256(encoded).hexdigest()


def write_json(path: Path, payload: dict[str, Any]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_suffix(path.suffix + ".partial")
    temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    temporary.replace(path)


def validate_staged_release(
    args: argparse.Namespace,
    release: PopulationReleaseConfig,
) -> dict[str, Any]:
    scope = GEOGRAPHIC_SCOPES[args.scope]
    output_dir = population_output_dir(args)
    resolved_output = output_dir.resolve()
    snapshot = snapshot_path(output_dir, scope, release.year)
    manifest_path = preflight_manifest_path(output_dir, scope, release.year)
    for path in (snapshot, manifest_path):
        if not path.resolve().is_relative_to(resolved_output):
            raise RuntimeError(f"Staged Statbel evidence is outside the governed output root: {path}")
    manifest = load_preflight_manifest(manifest_path, snapshot, release.year, scope)
    manifest_release = manifest.get("release") or {}
    artifacts = manifest.get("artifacts") or {}
    if (
        manifest_release.get("population_layout") != release.layout
        or (artifacts.get("population") or {}).get("source_url") != release.population_url
        or (artifacts.get("geometry") or {}).get("source_url") != release.geometry_url
    ):
        raise RuntimeError("Staged Statbel manifest no longer matches the planned release identity")
    for artifact_name in ("population", "geometry"):
        retained = Path(str((artifacts.get(artifact_name) or {}).get("retained_path") or ""))
        if not retained.resolve().is_relative_to(resolved_output):
            raise RuntimeError(f"Staged {artifact_name} archive is outside the governed output root: {retained}")
    return {
        "manifest_path": str(manifest_path),
        "manifest_sha256": sha256_path(manifest_path),
        "snapshot_path": str(snapshot),
        "snapshot_sha256": (artifacts.get("derived_snapshot") or {}).get("sha256"),
        "snapshot_size_bytes": (artifacts.get("derived_snapshot") or {}).get("size_bytes"),
        "feature_count": (artifacts.get("derived_snapshot") or {}).get("feature_count"),
        "population_archive": artifacts.get("population"),
        "geometry_archive": artifacts.get("geometry"),
        "scope_accounting": manifest.get("scope_accounting"),
        "national_accounting": manifest.get("national_accounting"),
        "baseline": manifest.get("baseline"),
        "geometry_repair_count": (manifest.get("schemas") or {}).get("geometry_repair_count"),
    }


def build_staged_plan(
    args: argparse.Namespace,
    decision: dict[str, Any],
    release: PopulationReleaseConfig,
    operator_result: dict[str, Any],
) -> dict[str, Any]:
    evidence = validate_staged_release(args, release)
    payload: dict[str, Any] = {
        "schema_version": 1,
        "status": "staged",
        "created_at": datetime.now(timezone.utc).isoformat(),
        "project_id": args.project_id,
        "scope": args.scope,
        "local_source_version_before_apply": decision.get("local_source_version"),
        "release": decision["release"],
        "catalog_identity": decision["catalog_identity"],
        "evidence": evidence,
        "operator_result": operator_result,
        "review_required": True,
        "apply_requires_plan_sha256": True,
        "apply_requires_review_sha256": True,
        "automatic_import": False,
        "destructive_replacement": False,
        "existing_snapshots_retained": True,
    }
    payload["plan_sha256"] = canonical_sha256(payload, "plan_sha256")
    return payload


def _release_from_payload(payload: dict[str, Any]) -> PopulationReleaseConfig:
    release = payload.get("release") or {}
    year = release.get("year") if isinstance(release, dict) else None
    layout = release.get("layout") if isinstance(release, dict) else None
    population_url = release.get("population_url") if isinstance(release, dict) else None
    geometry_url = release.get("geometry_url") if isinstance(release, dict) else None
    if (
        not isinstance(year, int)
        or isinstance(year, bool)
        or not isinstance(layout, str)
        or not isinstance(population_url, str)
        or not isinstance(geometry_url, str)
        or not all((layout, population_url, geometry_url))
    ):
        raise RuntimeError("Release evidence is missing required identity fields")
    return resolve_release_config(
        year,
        layout=layout,
        population_url=population_url,
        geometry_url=geometry_url,
    )


def load_staged_plan(
    args: argparse.Namespace,
    release: PopulationReleaseConfig,
) -> tuple[Path, dict[str, Any]]:
    path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
    if not path.is_file():
        raise RuntimeError(f"Staged Statbel plan is missing: {path}")
    payload = json.loads(path.read_text(encoding="utf-8"))
    actual_sha = canonical_sha256(payload, "plan_sha256")
    if payload.get("plan_sha256") != actual_sha:
        raise RuntimeError("Staged Statbel plan checksum is invalid")
    if args.confirm_plan_sha256 != actual_sha:
        raise RuntimeError(f"Explicit --confirm-plan-sha256 {actual_sha} is required")
    if (
        payload.get("status") != "staged"
        or payload.get("project_id") != args.project_id
        or payload.get("scope") != args.scope
        or _release_from_payload(payload) != release
    ):
        raise RuntimeError("Staged Statbel plan identity is invalid")
    current_evidence = validate_staged_release(args, release)
    if current_evidence != payload.get("evidence"):
        raise RuntimeError("Staged Statbel artifacts no longer match the approved plan")
    return path, payload


def require_catalog_unchanged(plan: dict[str, Any], decision: dict[str, Any]) -> None:
    if plan.get("release") != decision.get("release") or plan.get("catalog_identity") != decision.get("catalog_identity"):
        raise RuntimeError("The official Statbel release evidence changed; create and review a new staged plan")


def build_review_evidence(
    args: argparse.Namespace,
    plan_path: Path,
    plan: dict[str, Any],
) -> dict[str, Any]:
    reviewer = str(args.reviewer or "").strip()
    note = str(args.review_note or "").strip()
    if not args.approve or len(reviewer) < 2 or len(reviewer) > 120:
        raise RuntimeError("Review requires --approve and a named --reviewer between 2 and 120 characters")
    if len(note) > 1000:
        raise RuntimeError("Review note must not exceed 1000 characters")
    payload: dict[str, Any] = {
        "schema_version": 1,
        "status": "approved",
        "reviewed_at": datetime.now(timezone.utc).isoformat(),
        "reviewer": reviewer,
        "review_note": note or None,
        "project_id": args.project_id,
        "scope": args.scope,
        "release": plan["release"],
        "staged_plan_path": str(plan_path),
        "staged_plan_sha256": plan["plan_sha256"],
        "reviewed_checks": [
            "official_catalog_identity",
            "source_archive_checksums",
            "population_and_geometry_schemas",
            "scope_and_national_accounting",
            "unlocated_population_accounting",
            "baseline_change_limit",
            "bounded_geometry_repairs",
            "immutable_dataset_apply",
        ],
    }
    payload["review_sha256"] = canonical_sha256(payload, "review_sha256")
    return payload


def load_review_evidence(
    args: argparse.Namespace,
    release: PopulationReleaseConfig,
    plan: dict[str, Any],
) -> tuple[Path, dict[str, Any]]:
    path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
    if not path.is_file():
        raise RuntimeError(f"Approved Statbel review evidence is missing: {path}")
    payload = json.loads(path.read_text(encoding="utf-8"))
    actual_sha = canonical_sha256(payload, "review_sha256")
    if payload.get("review_sha256") != actual_sha:
        raise RuntimeError("Statbel review evidence checksum is invalid")
    if args.confirm_review_sha256 != actual_sha:
        raise RuntimeError(f"Explicit --confirm-review-sha256 {actual_sha} is required")
    if (
        payload.get("status") != "approved"
        or payload.get("project_id") != args.project_id
        or payload.get("scope") != args.scope
        or payload.get("release") != plan.get("release")
        or payload.get("staged_plan_sha256") != plan.get("plan_sha256")
        or not str(payload.get("reviewer") or "").strip()
    ):
        raise RuntimeError("Statbel review evidence does not authorize this staged plan")
    return path, payload


def _snapshot_result(operator_result: dict[str, Any], year: int) -> dict[str, Any]:
    matches = [item for item in operator_result.get("snapshots") or [] if int(item.get("year") or 0) == year]
    if len(matches) != 1:
        raise RuntimeError("Population operator did not return exactly one result for the approved edition")
    return matches[0]


def main() -> int:
    args = parse_args()
    try:
        if min(args.request_timeout, args.api_timeout, args.import_timeout) <= 0:
            raise ValueError("All timeout safety limits must be positive")
        validate_project_scope(args)
        refresh = args.refresh_catalog or args.action in {"stage", "review", "apply"}
        decision = fetch_release_decision(args, refresh=refresh)
        release = _release_from_payload(decision)

        if args.action == "plan":
            print(json.dumps({"status": "ok", "action": "plan", "decision": decision}, ensure_ascii=False, indent=2))
            return 0

        require_release_confirmation(args, release)
        if args.action == "stage":
            if decision.get("status") not in ACTIONABLE_STATUSES:
                raise RuntimeError(f"Statbel release is not safely stageable: {decision.get('status')}")
            operator_result = run_operator(build_operator_command(args, release, fetch_only=True), action="staging")
            staged_result = _snapshot_result(operator_result, release.year)
            if staged_result.get("status") != "prepared" or staged_result.get("preflight_status") != "passed":
                raise RuntimeError("Population staging did not produce passed preflight evidence")
            plan = build_staged_plan(args, decision, release, operator_result)
            plan_path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
            write_json(plan_path, plan)
            print(json.dumps({"status": "staged", "plan_path": str(plan_path), **plan}, ensure_ascii=False, indent=2))
            return 0

        plan_path, plan = load_staged_plan(args, release)
        require_catalog_unchanged(plan, decision)
        if args.action == "review":
            review = build_review_evidence(args, plan_path, plan)
            review_path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
            write_json(review_path, review)
            print(json.dumps({"status": "approved", "review_path": str(review_path), **review}, ensure_ascii=False, indent=2))
            return 0

        review_path, review = load_review_evidence(args, release, plan)
        operator_result = run_operator(build_operator_command(args, release, fetch_only=False), action="applying")
        applied_result = _snapshot_result(operator_result, release.year)
        if applied_result.get("status") not in {"imported", "existing"} or not applied_result.get("dataset_id"):
            raise RuntimeError("Population apply did not return one persisted immutable Dataset")
        final_decision = fetch_release_decision(args, refresh=True)
        if final_decision.get("status") != "current" or final_decision.get("local_source_version") != str(release.year):
            raise RuntimeError("Applied population Dataset did not become the current local Statbel edition")
        evidence = {
            "schema_version": 1,
            "status": "applied",
            "applied_at": datetime.now(timezone.utc).isoformat(),
            "project_id": args.project_id,
            "scope": args.scope,
            "release": plan["release"],
            "staged_plan_path": str(plan_path),
            "staged_plan_sha256": plan["plan_sha256"],
            "review_path": str(review_path),
            "review_sha256": review["review_sha256"],
            "reviewer": review["reviewer"],
            "dataset_id": applied_result["dataset_id"],
            "dataset_status": applied_result["status"],
            "feature_count": applied_result.get("feature_count"),
            "final_catalog_decision": final_decision,
            "existing_snapshots_retained": True,
        }
        evidence["applied_evidence_sha256"] = canonical_sha256(evidence, "applied_evidence_sha256")
        evidence_path = plan_path.with_name("applied-evidence.json")
        write_json(evidence_path, evidence)
        print(json.dumps({"status": "applied", "evidence_path": str(evidence_path), **evidence}, ensure_ascii=False, indent=2))
        return 0
    except (OSError, RuntimeError, ValueError, KeyError, json.JSONDecodeError) as exc:
        print(
            json.dumps({"status": "error", "action": args.action, "message": str(exc)}, ensure_ascii=False),
            file=sys.stderr,
        )
        return 1


if __name__ == "__main__":
    raise SystemExit(main())