Add governed ALZ release promotion
This commit is contained in:
Codex
@@ -7,6 +7,23 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 229 Governed ALZ definitive release promotion (2026-07-17)
|
||||
|
||||
- Added an operator-only `plan -> stage -> review -> apply` coordinator for
|
||||
future definitive ALZ agricultural-use parcel editions. A provisional v1/v2
|
||||
campaign snapshot remains visible release evidence but is never importable.
|
||||
- Derived the exact archive identity from the allowlisted publication year and
|
||||
date reported by the existing source-catalog probe. Arbitrary provider URLs,
|
||||
current/older releases, catalog drift and changed staged bytes fail closed.
|
||||
- Extended the existing agricultural provisioner to accept exactly one
|
||||
explicitly governed future edition, validate final download identity, keep
|
||||
streamed size bounds and read every paginated workspace Dataset consistently.
|
||||
- Bound archive, normalized GeoJSON, GeoPackage schema/CRS, crop-code list,
|
||||
scope totals and previous-edition deltas into named human review evidence.
|
||||
Apply still creates only an immutable Dataset through DatasetService.
|
||||
- Added focused trust-boundary, tamper, drift, approval, apply and packaging
|
||||
tests without adding an API route, migration, scheduler or frontend action.
|
||||
|
||||
## Sprint 228 Governed Statbel population release promotion (2026-07-17)
|
||||
|
||||
- Added an operator-only `plan -> stage -> review -> apply` coordinator for
|
||||
|
||||
@@ -1219,6 +1219,35 @@ transport region. Every annual source ZIP and crop code list remains under the
|
||||
storage volume. PostGIS computes exact hectares for drawn rectangles and
|
||||
persisted Areas; parcel identities are deliberately unavailable for lineage.
|
||||
|
||||
Future definitive editions use the separate four-phase release coordinator:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py plan \
|
||||
--project-id <KEMPEN_PROJECT_ID> --refresh-catalog
|
||||
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py stage \
|
||||
--project-id <KEMPEN_PROJECT_ID> \
|
||||
--confirm-edition <YYYY-v3_FROM_PLAN>
|
||||
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py review \
|
||||
--project-id <KEMPEN_PROJECT_ID> \
|
||||
--confirm-edition <YYYY-v3_FROM_PLAN> \
|
||||
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
|
||||
--approve --reviewer "<OPERATOR_NAME>"
|
||||
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py apply \
|
||||
--project-id <KEMPEN_PROJECT_ID> \
|
||||
--confirm-edition <YYYY-v3_FROM_PLAN> \
|
||||
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
|
||||
--confirm-review-sha256 <SHA256_FROM_REVIEW>
|
||||
```
|
||||
|
||||
The manager accepts only one catalog-confirmed definitive v3 release. `plan`
|
||||
writes nothing; `stage` downloads and normalizes without PostGIS mutation;
|
||||
`review` binds a named approval; `apply` revalidates catalog, hashes, schema,
|
||||
crop codes, scope accounting and previous-edition deltas before delegating to
|
||||
DatasetService. Provisional v1/v2 snapshots never enter the historical series.
|
||||
|
||||
## Buildings and Addresses Register snapshot
|
||||
|
||||
After the Mol Area and regional GRB buildings have been provisioned, prepare
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
|
||||
def load_script(name: str):
|
||||
path = SCRIPTS / name
|
||||
module_name = f"test_{path.stem}_sprint229"
|
||||
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
OPERATOR = load_script("provision_agricultural_parcel_history.py")
|
||||
MANAGER = load_script("manage_alz_agriculture_release.py")
|
||||
|
||||
|
||||
def arguments(tmp_path: Path, **overrides) -> argparse.Namespace:
|
||||
values = {
|
||||
"action": "plan",
|
||||
"project_id": "00000000-0000-0000-0000-000000000001",
|
||||
"scope": "kempen-transport-region",
|
||||
"api_url": "http://127.0.0.1:8000/api/v1",
|
||||
"confirm_edition": None,
|
||||
"confirm_plan_sha256": None,
|
||||
"confirm_review_sha256": None,
|
||||
"approve": False,
|
||||
"reviewer": None,
|
||||
"review_note": "",
|
||||
"plan_path": None,
|
||||
"review_path": None,
|
||||
"output_root": tmp_path / "operator-evidence" / "agricultural-use-parcels",
|
||||
"evidence_root": tmp_path / "operator-evidence" / "alz-agriculture-refresh",
|
||||
"refresh_catalog": False,
|
||||
"request_timeout": 900,
|
||||
"api_timeout": 180,
|
||||
"import_timeout": 3600,
|
||||
"max_features": 250_000,
|
||||
"max_archive_mb": 250,
|
||||
}
|
||||
values.update(overrides)
|
||||
return argparse.Namespace(**values)
|
||||
|
||||
|
||||
def catalog_item(
|
||||
*,
|
||||
remote: str = "2026-v3",
|
||||
local: str | None = "2025-definitive",
|
||||
catalog_hash: str = "a" * 64,
|
||||
checked_at: str = "2027-03-16T08:00:00Z",
|
||||
) -> dict:
|
||||
year = int(remote[:4])
|
||||
published = "2027-03-15T00:00:00Z" if year == 2026 else "2026-05-13T00:00:00Z"
|
||||
return {
|
||||
"source_name": MANAGER.SOURCE_NAME,
|
||||
"status": "available",
|
||||
"reachable": True,
|
||||
"error_code": None,
|
||||
"remote_version": remote,
|
||||
"remote_published_at": published,
|
||||
"local_source_version": local,
|
||||
"message": f"Definitieve editie {remote}; actuele publicatie {year + 1}-v1 is voorlopig.",
|
||||
"matched_layers": ["definitive_archive", "current_snapshot"],
|
||||
"capabilities_sha256": catalog_hash,
|
||||
"metadata_identifier": "alz-agricultural-use-parcels",
|
||||
"metadata_url": "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen",
|
||||
"checked_at": checked_at,
|
||||
}
|
||||
|
||||
|
||||
def decision(args: argparse.Namespace, *, remote: str = "2026-v3", local: str | None = "2025-definitive") -> dict:
|
||||
return MANAGER.fetch_release_decision_from_item(args, catalog_item(remote=remote, local=local))
|
||||
|
||||
|
||||
def release_2026():
|
||||
return MANAGER.release_from_catalog_item(catalog_item())
|
||||
|
||||
|
||||
def write_staged_artifacts(args: argparse.Namespace, release) -> dict:
|
||||
paths = OPERATOR.artifact_paths(
|
||||
args.output_root,
|
||||
args.scope,
|
||||
release.year,
|
||||
archive_url=release.archive_url,
|
||||
)
|
||||
paths["directory"].mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(paths["archive"], "w") as archive:
|
||||
archive.writestr(f"agpa_{release.year}.gpkg", b"official geopackage")
|
||||
paths["artifact"].write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
||||
codelist = {
|
||||
"year": release.year,
|
||||
"crop_entries": [{"code": "201", "title": "Mais", "group_title": "Mais"}],
|
||||
"code_title_conflicts": {},
|
||||
}
|
||||
paths["codelist"].write_text(json.dumps(codelist), encoding="utf-8")
|
||||
baseline_dir = args.output_root / args.scope / "2025"
|
||||
baseline_dir.mkdir(parents=True, exist_ok=True)
|
||||
(baseline_dir / "agricultural_use_parcels_2025_kempen-transport-region.manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"year": 2025,
|
||||
"scope_key": args.scope,
|
||||
"feature_count": 120_000,
|
||||
"clipped_area_ha": 62_000.0,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"year": release.year,
|
||||
"scope_key": args.scope,
|
||||
"member_nis_codes": ["13025", "13003"],
|
||||
"source_url": release.archive_url,
|
||||
"source_crs": OPERATOR.SOURCE_CRS,
|
||||
"output_crs": OPERATOR.OUTPUT_CRS,
|
||||
"source_archive_sha256": OPERATOR.sha256_file(paths["archive"]),
|
||||
"source_archive_size_bytes": paths["archive"].stat().st_size,
|
||||
"source_feature_count": 180_000,
|
||||
"source_fields": sorted(OPERATOR.STABLE_REQUIRED_FIELDS),
|
||||
"crop_code_list_sha256": OPERATOR.sha256_file(paths["codelist"]),
|
||||
"artifact_sha256": OPERATOR.sha256_file(paths["artifact"]),
|
||||
"feature_count": 121_500,
|
||||
"clipped_feature_count": 800,
|
||||
"clipped_area_ha": 62_500.0,
|
||||
}
|
||||
paths["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return paths
|
||||
|
||||
|
||||
def staged_plan(args: argparse.Namespace, release, release_decision: dict) -> tuple[Path, dict]:
|
||||
write_staged_artifacts(args, release)
|
||||
result = {
|
||||
"status": "ok",
|
||||
"scope": args.scope,
|
||||
"years": [{"year": release.year, "status": "prepared", "feature_count": 121_500}],
|
||||
}
|
||||
plan = MANAGER.build_staged_plan(args, release_decision, release, result)
|
||||
path = MANAGER.default_plan_path(args, release.year)
|
||||
MANAGER.write_json(path, plan)
|
||||
return path, plan
|
||||
|
||||
|
||||
def test_future_release_config_accepts_only_one_exact_official_edition() -> None:
|
||||
url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip"
|
||||
release = OPERATOR.resolve_release_config(2026, archive_url=url)
|
||||
|
||||
assert release.definitive_version == "2026-v3"
|
||||
assert release.archive_url == url
|
||||
future = OPERATOR.resolve_release_config(2027, archive_url=url.replace("2026", "2027"))
|
||||
assert future.definitive_version == "2027-v3"
|
||||
with pytest.raises(ValueError, match="official ALZ URL"):
|
||||
OPERATOR.resolve_release_config(2026, archive_url=url.replace("www.landbouwvlaanderen.be", "example.com"))
|
||||
with pytest.raises(ValueError, match="may not be overridden"):
|
||||
OPERATOR.resolve_release_config(2025, archive_url=url.replace("2026", "2025"))
|
||||
|
||||
|
||||
def test_future_archive_requires_one_selected_year() -> None:
|
||||
url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip"
|
||||
with pytest.raises(ValueError, match="exactly one"):
|
||||
OPERATOR.resolve_release_configs("2025,2026", archive_url=url)
|
||||
|
||||
|
||||
def test_agriculture_workspace_pagination_is_complete_and_total_consistent() -> None:
|
||||
rows = [{"id": index} for index in range(401)]
|
||||
|
||||
class Response:
|
||||
ok = True
|
||||
status_code = 200
|
||||
text = ""
|
||||
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self.payload = payload
|
||||
|
||||
def json(self) -> dict:
|
||||
return {"data": self.payload}
|
||||
|
||||
class Session:
|
||||
drift = False
|
||||
|
||||
def get(self, _url: str, *, params: dict, timeout: int):
|
||||
assert timeout == 30
|
||||
offset = int(params["offset"])
|
||||
total = len(rows) + (1 if self.drift and offset else 0)
|
||||
return Response({"items": rows[offset : offset + int(params["limit"])], "total": total})
|
||||
|
||||
assert OPERATOR.api_items(Session(), "http://backend/datasets", 30) == rows
|
||||
drifting = Session()
|
||||
drifting.drift = True
|
||||
with pytest.raises(RuntimeError, match="total changed"):
|
||||
OPERATOR.api_items(drifting, "http://backend/datasets", 30)
|
||||
|
||||
|
||||
def test_archive_download_rejects_oversize_before_streaming(tmp_path: Path) -> None:
|
||||
url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip"
|
||||
|
||||
class Response:
|
||||
headers = {"content-length": "101"}
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.url = url
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def iter_content(self, *, chunk_size: int):
|
||||
raise AssertionError(f"streaming should not start: {chunk_size}")
|
||||
|
||||
class Session:
|
||||
def get(self, *_args, **_kwargs):
|
||||
return Response()
|
||||
|
||||
with pytest.raises(RuntimeError, match="configured"):
|
||||
OPERATOR.download_archive(Session(), url, tmp_path / "source.zip", timeout=30, max_bytes=100, force=True)
|
||||
|
||||
|
||||
def test_archive_rejects_extracted_size_over_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
archive_path = tmp_path / "source.zip"
|
||||
with zipfile.ZipFile(archive_path, "w") as archive:
|
||||
archive.writestr("agpa_2026.gpkg", b"0123456789")
|
||||
monkeypatch.setattr(OPERATOR, "MAX_EXTRACTED_BYTES", 9)
|
||||
|
||||
with pytest.raises(RuntimeError, match="extracted-size safety limit"):
|
||||
OPERATOR.validate_archive(archive_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("remote", "local", "expected"),
|
||||
[
|
||||
("2026-v3", "2025-definitive", "update_available"),
|
||||
("2025-v3", "2025-definitive", "current"),
|
||||
("2026-v3", None, "not_loaded"),
|
||||
("2025-v3", "2026-definitive", "blocked_remote_older"),
|
||||
],
|
||||
)
|
||||
def test_catalog_decision_orders_only_definitive_editions(
|
||||
tmp_path: Path,
|
||||
remote: str,
|
||||
local: str | None,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert decision(arguments(tmp_path), remote=remote, local=local)["status"] == expected
|
||||
|
||||
|
||||
def test_catalog_decision_keeps_provisional_snapshot_non_importable(tmp_path: Path) -> None:
|
||||
release_decision = decision(arguments(tmp_path))
|
||||
|
||||
assert release_decision["release"]["edition"] == "2026-v3"
|
||||
assert release_decision["provisional_release"] == "2027-v1"
|
||||
assert release_decision["provisional_release_importable"] is False
|
||||
with pytest.raises(RuntimeError, match="definitive YYYY-v3"):
|
||||
MANAGER.fetch_release_decision_from_item(arguments(tmp_path), catalog_item(remote="2026-v1"))
|
||||
|
||||
|
||||
def test_stage_and_apply_commands_are_separate_and_local_only(tmp_path: Path) -> None:
|
||||
args = arguments(tmp_path)
|
||||
release = release_2026()
|
||||
stage = MANAGER.build_operator_command(args, release, fetch_only=True)
|
||||
apply = MANAGER.build_operator_command(args, release, fetch_only=False)
|
||||
|
||||
assert "--force" in stage and "--fetch-only" in stage
|
||||
assert "--force" not in apply and "--fetch-only" not in apply
|
||||
assert stage[stage.index("--archive-url") + 1] == release.archive_url
|
||||
with pytest.raises(RuntimeError, match="inside GeoIntel"):
|
||||
MANAGER.internal_base_url("http://192.168.10.150:1202/api/v1")
|
||||
|
||||
|
||||
def test_staged_plan_binds_source_schema_codelist_scope_and_baseline(tmp_path: Path) -> None:
|
||||
args = arguments(tmp_path)
|
||||
release = release_2026()
|
||||
_, plan = staged_plan(args, release, decision(args))
|
||||
|
||||
evidence = plan["evidence"]
|
||||
assert evidence["feature_count"] == 121_500
|
||||
assert evidence["crop_entry_count"] == 1
|
||||
assert evidence["member_nis_codes"] == ["13025", "13003"]
|
||||
assert evidence["baseline"]["year"] == 2025
|
||||
assert len(evidence["baseline"]["manifest_sha256"]) == 64
|
||||
assert evidence["baseline"]["feature_count_change_ratio"] == 0.0125
|
||||
assert plan["plan_sha256"] == MANAGER.canonical_sha256(plan, "plan_sha256")
|
||||
|
||||
|
||||
def test_modified_staged_bytes_invalidate_plan(tmp_path: Path) -> None:
|
||||
args = arguments(tmp_path)
|
||||
release = release_2026()
|
||||
path, plan = staged_plan(args, release, decision(args))
|
||||
args.plan_path = path
|
||||
args.confirm_plan_sha256 = plan["plan_sha256"]
|
||||
Path(plan["evidence"]["artifact_path"]).write_text("tampered", encoding="utf-8")
|
||||
|
||||
with pytest.raises(RuntimeError, match="incomplete or no longer match"):
|
||||
MANAGER.load_staged_plan(args, release)
|
||||
|
||||
|
||||
def test_plan_and_review_paths_must_remain_governed(tmp_path: Path) -> None:
|
||||
args = arguments(tmp_path, plan_path=tmp_path / "outside.json")
|
||||
with pytest.raises(RuntimeError, match="outside the governed"):
|
||||
MANAGER.governed_evidence_path(args, args.plan_path)
|
||||
|
||||
|
||||
def test_review_requires_named_approval_and_exact_plan_hash(tmp_path: Path) -> None:
|
||||
args = arguments(tmp_path)
|
||||
release = release_2026()
|
||||
path, plan = staged_plan(args, release, decision(args))
|
||||
|
||||
with pytest.raises(RuntimeError, match="--approve"):
|
||||
MANAGER.build_review_evidence(args, path, plan)
|
||||
args.approve = True
|
||||
args.reviewer = "Jens"
|
||||
review = MANAGER.build_review_evidence(args, path, plan)
|
||||
assert review["staged_plan_sha256"] == plan["plan_sha256"]
|
||||
assert review["review_sha256"] == MANAGER.canonical_sha256(review, "review_sha256")
|
||||
|
||||
|
||||
def test_review_tampering_and_catalog_drift_fail_closed(tmp_path: Path) -> None:
|
||||
args = arguments(tmp_path, approve=True, reviewer="Jens")
|
||||
release = release_2026()
|
||||
path, plan = staged_plan(args, release, decision(args))
|
||||
review = MANAGER.build_review_evidence(args, path, plan)
|
||||
review_path = MANAGER.default_review_path(args, release.year)
|
||||
MANAGER.write_json(review_path, review)
|
||||
payload = json.loads(review_path.read_text(encoding="utf-8"))
|
||||
payload["reviewer"] = "changed"
|
||||
MANAGER.write_json(review_path, payload)
|
||||
args.review_path = review_path
|
||||
args.confirm_review_sha256 = review["review_sha256"]
|
||||
|
||||
with pytest.raises(RuntimeError, match="checksum is invalid"):
|
||||
MANAGER.load_review_evidence(args, release, plan)
|
||||
changed = MANAGER.fetch_release_decision_from_item(args, catalog_item(catalog_hash="b" * 64))
|
||||
with pytest.raises(RuntimeError, match="evidence changed"):
|
||||
MANAGER.require_catalog_unchanged(plan, changed)
|
||||
|
||||
|
||||
def test_catalog_check_timestamp_does_not_create_false_drift(tmp_path: Path) -> None:
|
||||
args = arguments(tmp_path)
|
||||
first = MANAGER.fetch_release_decision_from_item(args, catalog_item(checked_at="2027-03-16T08:00:00Z"))
|
||||
second = MANAGER.fetch_release_decision_from_item(args, catalog_item(checked_at="2027-03-16T09:00:00Z"))
|
||||
plan = {"release": first["release"], "catalog_identity": first["catalog_identity"]}
|
||||
|
||||
MANAGER.require_catalog_unchanged(plan, second)
|
||||
assert first["catalog_checked_at"] != second["catalog_checked_at"]
|
||||
|
||||
|
||||
def test_current_edition_cannot_stage_or_write_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
args = arguments(tmp_path, action="stage", confirm_edition="2025-v3")
|
||||
current = decision(args, remote="2025-v3", local="2025-definitive")
|
||||
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
|
||||
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
|
||||
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: current)
|
||||
monkeypatch.setattr(
|
||||
MANAGER,
|
||||
"run_operator",
|
||||
lambda *_args, **_kwargs: pytest.fail("operator must not run for current edition"),
|
||||
)
|
||||
|
||||
assert MANAGER.main() == 1
|
||||
assert not args.evidence_root.exists()
|
||||
|
||||
|
||||
def test_full_apply_requires_review_and_verifies_final_dataset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
args = arguments(tmp_path, action="apply", confirm_edition="2026-v3", approve=True, reviewer="Jens")
|
||||
release = release_2026()
|
||||
update = decision(args)
|
||||
path, plan = staged_plan(args, release, update)
|
||||
review = MANAGER.build_review_evidence(args, path, plan)
|
||||
review_path = MANAGER.default_review_path(args, release.year)
|
||||
MANAGER.write_json(review_path, review)
|
||||
args.confirm_plan_sha256 = plan["plan_sha256"]
|
||||
args.confirm_review_sha256 = review["review_sha256"]
|
||||
current = decision(args, remote="2026-v3", local="2026-definitive")
|
||||
decisions = iter((update, current))
|
||||
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
|
||||
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
|
||||
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: next(decisions))
|
||||
monkeypatch.setattr(
|
||||
MANAGER,
|
||||
"run_operator",
|
||||
lambda *_args, **_kwargs: {
|
||||
"status": "ok",
|
||||
"years": [{"year": 2026, "status": "imported", "dataset_id": "dataset-2026", "feature_count": 121_500}],
|
||||
},
|
||||
)
|
||||
|
||||
assert MANAGER.main() == 0
|
||||
applied = json.loads(path.with_name("applied-evidence.json").read_text(encoding="utf-8"))
|
||||
assert applied["dataset_id"] == "dataset-2026"
|
||||
assert applied["review_sha256"] == review["review_sha256"]
|
||||
assert applied["applied_evidence_sha256"] == MANAGER.canonical_sha256(applied, "applied_evidence_sha256")
|
||||
|
||||
|
||||
def test_manager_is_packaged_and_never_writes_vector_features_directly() -> None:
|
||||
manager = (SCRIPTS / "manage_alz_agriculture_release.py").read_text(encoding="utf-8")
|
||||
operator = (SCRIPTS / "provision_agricultural_parcel_history.py").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "INSERT INTO vector_features" not in manager
|
||||
assert "INSERT INTO vector_features" not in operator
|
||||
assert "/datasets/upload" in operator
|
||||
assert "COPY scripts/manage_alz_agriculture_release.py" in dockerfile
|
||||
assert "py_compile scripts/manage_alz_agriculture_release.py" in readiness
|
||||
@@ -91,6 +91,7 @@ COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_water
|
||||
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
||||
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
||||
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
||||
COPY scripts/manage_alz_agriculture_release.py /app/scripts/manage_alz_agriculture_release.py
|
||||
COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py
|
||||
COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py
|
||||
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
|
||||
|
||||
@@ -457,6 +457,15 @@ human `review` and checksum-confirmed `apply`. No additional public endpoint
|
||||
is introduced. Apply delegates to the existing dataset upload contract and
|
||||
creates a new immutable annual snapshot only after all evidence is unchanged.
|
||||
|
||||
Future definitive ALZ execution likewise remains outside the HTTP request
|
||||
cycle in `scripts/manage_alz_agriculture_release.py`. It derives the exact
|
||||
`agpa_<campaign>_<publication-date>_public.zip` identity from the existing
|
||||
catalog response and accepts only `YYYY-v3`. `stage` is filesystem-only,
|
||||
`review` is a named approval and `apply` requires exact plan/review hashes,
|
||||
revalidates the current catalog and delegates to the existing Dataset upload
|
||||
contract. No ALZ release endpoint, background task or provider URL parameter
|
||||
is added; v1/v2 campaign snapshots remain non-importable.
|
||||
|
||||
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
|
||||
not fetch vector features, raster pixels or models, create jobs/datasets, write
|
||||
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
|
||||
|
||||
@@ -1,3 +1,46 @@
|
||||
## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17)
|
||||
|
||||
Implemented:
|
||||
- Added `scripts/manage_alz_agriculture_release.py` with separate read-only
|
||||
`plan`, filesystem-only `stage`, named `review` and exact-hash `apply`
|
||||
actions for the approved Kempen regional scope.
|
||||
- Derived the only accepted future archive from the allowlisted definitive
|
||||
`YYYY-v3` campaign and publication date exposed by the existing source-
|
||||
catalog probe. Provisional v1/v2 snapshots remain visible but non-importable;
|
||||
arbitrary URLs, current/older releases and catalog drift fail closed.
|
||||
- Extended the existing agricultural provisioner to accept exactly one
|
||||
explicitly governed future edition per run. Final response URLs and streamed
|
||||
archive size are bounded, retained editions cannot be overridden and all
|
||||
workspace API collections are read with stable-total pagination.
|
||||
- Bound archive, normalized GeoJSON, GeoPackage schema/CRS, complete crop-code
|
||||
list, scope counts/area and previous-definitive-edition manifest/deltas into
|
||||
the staged plan. ZIP member/extracted size is bounded. Review and apply
|
||||
require exact SHA-256 values; apply still uses DatasetService and preserves
|
||||
all historical snapshots.
|
||||
- Packaged the manager in the Unraid image/readiness gate and updated source,
|
||||
data, API, persistence, storage and operator documentation. No endpoint,
|
||||
migration, scheduler, browser fetch or frontend behavior changed.
|
||||
|
||||
Validation so far:
|
||||
- 43 tests passed across the original ALZ importer, official catalog probe and
|
||||
new release manager; 58 passed together with the Statbel release suite.
|
||||
- Target Python compilation and Ruff pass. Coverage includes future release
|
||||
identity, v1/v2 exclusion, pagination drift, download bounds, release
|
||||
ordering, stage/apply separation, source/schema/codelist/baseline evidence,
|
||||
path confinement, byte tampering, named review, catalog drift, full mocked
|
||||
apply, current-edition refusal and runtime packaging.
|
||||
- Complete readiness passed with 850 backend tests, 110 documented routes,
|
||||
one Alembic head `202607160001`, frontend typecheck and production build.
|
||||
Static full-chain Alembic SQL and shell syntax checks also passed. Local
|
||||
Docker is not installed in the Codex Windows environment; Compose and live
|
||||
PostGIS validation are therefore deferred to the Tower deployment gate.
|
||||
|
||||
Boundary:
|
||||
- The official page currently advertises definitive `2025-v3` plus
|
||||
provisional `2026-v1`. No newer definitive edition exists, so live stage,
|
||||
review and apply must remain blocked; only a read-only plan and deliberate
|
||||
current-edition refusal may be exercised after deployment.
|
||||
|
||||
## Sprint 228 - Governed Statbel population release promotion (2026-07-17)
|
||||
|
||||
Implemented:
|
||||
|
||||
@@ -283,6 +283,14 @@ which creates the ordinary annual `datasets`, `dataset_versions` and
|
||||
year remains idempotent and previous annual snapshots are never updated or
|
||||
deleted.
|
||||
|
||||
Definitive ALZ release management uses the same persistence boundary and adds
|
||||
no migration or release table. Catalog planning, staged archive/GeoJSON,
|
||||
crop-code evidence and named review live on the filesystem. Only an approved
|
||||
apply invokes the existing Dataset upload service, producing the ordinary
|
||||
annual Dataset, DatasetVersion and vector_features records with
|
||||
`source_version=<year>-definitive`. Earlier annual snapshots are retained and
|
||||
provisional v1/v2 publications cannot create rows.
|
||||
|
||||
## Geometry normalization
|
||||
|
||||
- User-drawn polygons arrive as EPSG:4326.
|
||||
|
||||
+23
-1
@@ -173,7 +173,7 @@ gebeurd.
|
||||
| --- | --- | --- |
|
||||
| GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen |
|
||||
| Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks; officiële DCAT-releaseprobe | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen |
|
||||
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete officiële publicatieprobe; metricvergelijking zonder objectlineage | een nieuwere definitieve v3-editie eerst handmatig beoordelen en daarna via de bestaande begrensde operatorflow toevoegen |
|
||||
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete publicatieprobe en plan-stage-review-apply promotie; metricvergelijking zonder objectlineage | alleen een nieuwere definitieve v3-editie na gestagede schema-/codelijst-/scopecontrole en benoemde review toevoegen |
|
||||
| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald |
|
||||
| landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven |
|
||||
| bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave |
|
||||
@@ -190,6 +190,22 @@ actuele `2026-v1` zichtbaar maar niet updategerechtigd blijft. De probe leest
|
||||
alleen de allowlisted HTML-pagina en linkidentiteiten; ZIP-archieven worden pas
|
||||
door de afzonderlijke operator opgehaald na menselijke editiebevestiging.
|
||||
|
||||
`scripts/manage_alz_agriculture_release.py` beheert zo'n toekomstige
|
||||
definitieve editie in vier afzonderlijke operatorstappen. `plan` vergelijkt
|
||||
read-only de nieuwste v3 met de lokale `*-definitive` Dataset. `stage` vereist
|
||||
de exacte `YYYY-v3`, leidt de archief-URL af uit de gecontroleerde campagne en
|
||||
publicatiedatum en voert de bestaande provisioner uitsluitend met
|
||||
`--force --fetch-only` uit. Het staged plan bindt bronarchief, GeoPackage-
|
||||
schema/CRS, genormaliseerde GeoJSON, gewascodelijst, scope-aantallen en de
|
||||
verschillen met de vorige definitieve editie aan SHA-256.
|
||||
|
||||
`review` vereist een benoemde menselijke goedkeuring van exact dat plan.
|
||||
`apply` vereist de plan- en reviewhash, controleert catalogus en alle bestanden
|
||||
opnieuw en maakt via de bestaande uploadservice hoogstens een nieuwe immutable
|
||||
Dataset. Een v1/v2-snapshot, huidige of oudere editie, cataloguswijziging,
|
||||
gewijzigde bronbyte of evidence buiten de beheerde opslagroot blokkeert de
|
||||
flow. Er is geen scheduler, browserdownload of automatische vervanging.
|
||||
|
||||
De Statbel-probe leest uitsluitend de officiële DCAT Turtle-catalogus en
|
||||
selecteert de nieuwste unieke Nederlandstalige publicatie `Bevolking per
|
||||
statistische sector`. Voor 2025 vereist GeoIntel de nieuwe REDEGEO-indeling;
|
||||
@@ -409,6 +425,12 @@ annual field set, read through the GIS optional dependencies and deleted after
|
||||
the normalized GeoJSON has been built. Geometry is clipped exactly against the
|
||||
persisted scope Area in Lambert 72 and imported only through DatasetService.
|
||||
|
||||
Future definitive editions are accepted only as one explicit release per
|
||||
operator run with an exact official
|
||||
`agpa_<campaign>_<publication-date>_public.zip` identity. Reaching that URL is
|
||||
not sufficient for persistence: the separate release manager must first stage
|
||||
and bind all evidence, receive named approval and revalidate it during apply.
|
||||
|
||||
The regional default creates the series
|
||||
`alz:agricultural-use-parcels:kempen-transport-region`; `--scope mol` creates an
|
||||
independent Mol series. Selection and evolution expose exact intersected
|
||||
|
||||
@@ -307,6 +307,12 @@ official ZIP archive containing a Belgian Lambert 72 GeoPackage. Queryable
|
||||
geometry is clipped against the persisted Area in EPSG:31370 and normalized to
|
||||
EPSG:4326 before canonical `vector_features` persistence.
|
||||
|
||||
A future campaign becomes historical input only when the official publication
|
||||
contract labels it `v3`. Early `v1` and `v2` snapshots are provisional and may
|
||||
not create annual GeoIntel Datasets. The governed release plan binds the exact
|
||||
archive publication date, schema, CRS, crop-code list, scope totals and deltas
|
||||
against the previous definitive edition before named review and apply.
|
||||
|
||||
Stable source fields include `agpakey`, parcel number, declared source area,
|
||||
reference id, spring crop, main crop, official main-crop group, production
|
||||
method and source municipality. All annual source properties remain available,
|
||||
|
||||
@@ -204,6 +204,24 @@ artifact; Dataset and vector_feature rows remain the queryable PostGIS state.
|
||||
The manifest binds source, crop-code list and upload artifact checksums. A
|
||||
checksum conflict with an existing annual Dataset fails closed.
|
||||
|
||||
Future definitive-release decision evidence is stored separately from those
|
||||
source artifacts:
|
||||
|
||||
```text
|
||||
storage/operator-evidence/alz-agriculture-refresh/{scope}/{year}/
|
||||
staged-plan.json
|
||||
review-evidence.json
|
||||
applied-evidence.json
|
||||
```
|
||||
|
||||
The staged plan binds the official publication-page SHA-256 and definitive
|
||||
archive identity to the retained archive, normalized GeoJSON, schema/CRS,
|
||||
crop-code list, scope accounting and previous-edition delta. Review evidence
|
||||
binds a named decision to the exact plan hash. Applied evidence binds both to
|
||||
the resulting immutable Dataset id. Evidence paths outside this root and
|
||||
source artifacts outside `agricultural-use-parcels` are rejected; these files
|
||||
authorize neither deletion nor in-place Dataset replacement.
|
||||
|
||||
Buildings and Addresses Register snapshot evidence lives under:
|
||||
|
||||
```text
|
||||
|
||||
@@ -45,6 +45,7 @@
|
||||
- [x] Add a fail-closed Statbel DCAT publication probe that distinguishes population year, sector-geometry year and the 2025 REDEGEO transition without downloading distributions.
|
||||
- [x] Add a fail-closed Statbel population import preflight with archive/schema/CRS/join/total/baseline checks, retained checksums and explicit ZZZZ accounting before any new Dataset import.
|
||||
- [x] Add an explicit Statbel population plan -> stage -> named review -> checksum-confirmed apply workflow that preserves every prior annual snapshot.
|
||||
- [x] Add an explicit definitive ALZ plan -> stage -> named review -> checksum-confirmed apply workflow while keeping v1/v2 campaign snapshots non-importable.
|
||||
- [ ] Extend catalogue probes only to additional sources that publish a stable official edition contract; do not add background polling or infer releases from HTTP dates alone.
|
||||
|
||||
## Governed source expansion backlog
|
||||
|
||||
@@ -1498,6 +1498,45 @@ Datasets. `--force` refreshes retained evidence but cannot silently replace a
|
||||
conflicting persisted annual checksum. Use `--scope mol` for an independent
|
||||
municipal series.
|
||||
|
||||
### Governed future definitive ALZ release
|
||||
|
||||
Run the four phases only inside the GeoIntel container. The project id must
|
||||
belong to `Kempen Regional Workbench`:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py plan \
|
||||
--project-id <KEMPEN_PROJECT_ID> --refresh-catalog
|
||||
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py stage \
|
||||
--project-id <KEMPEN_PROJECT_ID> \
|
||||
--confirm-edition <YYYY-v3_FROM_PLAN>
|
||||
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py review \
|
||||
--project-id <KEMPEN_PROJECT_ID> \
|
||||
--confirm-edition <YYYY-v3_FROM_PLAN> \
|
||||
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
|
||||
--approve --reviewer "<OPERATOR_NAME>" \
|
||||
--review-note "Schema, gewascodes, scope en jaarverschillen nagekeken"
|
||||
|
||||
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py apply \
|
||||
--project-id <KEMPEN_PROJECT_ID> \
|
||||
--confirm-edition <YYYY-v3_FROM_PLAN> \
|
||||
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
|
||||
--confirm-review-sha256 <SHA256_FROM_REVIEW>
|
||||
```
|
||||
|
||||
`plan` is read-only. `stage` derives the exact archive from the official
|
||||
catalog campaign/publication date, downloads within the 250 MiB ceiling and
|
||||
runs the existing provisioner with `--force --fetch-only`. The staged plan
|
||||
binds archive, GeoJSON, schema, CRS, crop-code list, scope counts and previous
|
||||
definitive-edition manifest hash/deltas. ZIP member count and extracted size
|
||||
are bounded before the GeoPackage is read. `review` imports nothing. `apply` revalidates the
|
||||
catalog and every byte before using the canonical Dataset upload route.
|
||||
|
||||
Only a definitive `YYYY-v3` is eligible. Current or older editions, v1/v2
|
||||
snapshots, changed catalog/source evidence and paths outside the governed
|
||||
roots fail closed. Existing annual Datasets remain immutable and queryable.
|
||||
|
||||
## Buildings and Addresses Register snapshot
|
||||
|
||||
Prepare and audit the current official Mol snapshot without persistence:
|
||||
|
||||
@@ -0,0 +1,616 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plan, stage, review and explicitly apply one definitive ALZ edition.
|
||||
|
||||
Planning reads only the canonical source-catalog report. Staging downloads and
|
||||
prepares one official v3 archive without database persistence. Review records
|
||||
a named approval. Apply requires the exact plan and review hashes and reuses
|
||||
the canonical agricultural DatasetService upload path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES
|
||||
from provision_agricultural_parcel_history import (
|
||||
AgriculturalReleaseConfig,
|
||||
MAX_ARCHIVE_BYTES,
|
||||
OUTPUT_CRS,
|
||||
SOURCE_CRS,
|
||||
STABLE_REQUIRED_FIELDS,
|
||||
artifact_paths,
|
||||
resolve_release_config,
|
||||
reusable_artifact,
|
||||
sha256_file,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1"
|
||||
DEFAULT_SCOPE = "kempen-transport-region"
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/agricultural-use-parcels")
|
||||
DEFAULT_EVIDENCE_ROOT = Path("/app/storage/operator-evidence/alz-agriculture-refresh")
|
||||
SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels"
|
||||
REMOTE_VERSION_PATTERN = re.compile(r"^(20[0-9]{2})-v3$")
|
||||
LOCAL_VERSION_PATTERN = re.compile(r"^(20[0-9]{2})-definitive$")
|
||||
SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
|
||||
PROVISIONAL_VERSION_PATTERN = re.compile(r"\b(20[0-9]{2}-v[12])\b")
|
||||
ACTIONABLE_STATUSES = {"update_available", "not_loaded"}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Governed definitive ALZ release: plan, stage, review, then checksum-confirmed apply."
|
||||
)
|
||||
parser.add_argument("action", choices=("plan", "stage", "review", "apply"))
|
||||
parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope")
|
||||
parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE)
|
||||
parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--confirm-edition", help="Exact official definitive edition, for example 2026-v3")
|
||||
parser.add_argument("--confirm-plan-sha256", help="Exact staged-plan hash required for review/apply")
|
||||
parser.add_argument("--confirm-review-sha256", help="Exact review-evidence hash required for apply")
|
||||
parser.add_argument("--approve", action="store_true", help="Explicitly approve the staged evidence during review")
|
||||
parser.add_argument("--reviewer", help="Named human/operator approving the staged evidence")
|
||||
parser.add_argument("--review-note", default="", help="Optional bounded review note")
|
||||
parser.add_argument("--plan-path", type=Path, help="Override the governed staged-plan path")
|
||||
parser.add_argument("--review-path", type=Path, help="Override the governed review-evidence path")
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_AGRICULTURE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--evidence-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_ALZ_REFRESH_EVIDENCE_ROOT", DEFAULT_EVIDENCE_ROOT)),
|
||||
)
|
||||
parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short official-catalog cache")
|
||||
parser.add_argument("--request-timeout", type=int, default=900)
|
||||
parser.add_argument("--api-timeout", type=int, default=180)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
parser.add_argument("--max-features", type=int, default=250_000)
|
||||
parser.add_argument("--max-archive-mb", type=int, default=250)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]:
|
||||
endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}"
|
||||
request = Request(
|
||||
endpoint,
|
||||
headers={"Accept": "application/json", "User-Agent": "GeoIntel-ALZ-release/1.0"},
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=timeout) as response:
|
||||
payload = json.load(response)
|
||||
except HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
|
||||
raise RuntimeError("GeoIntel API response is not a canonical data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def validate_project_scope(args: argparse.Namespace) -> None:
|
||||
project = api_data(args.api_url, f"projects/{args.project_id}", args.api_timeout)
|
||||
expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name
|
||||
if project.get("name") != expected_name:
|
||||
raise RuntimeError(
|
||||
f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'"
|
||||
)
|
||||
|
||||
|
||||
def parse_catalog_datetime(value: Any) -> datetime:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise RuntimeError("The official ALZ catalog did not provide a publication date")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("The official ALZ catalog publication date is invalid") from exc
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def release_from_catalog_item(item: dict[str, Any]) -> AgriculturalReleaseConfig:
|
||||
match = REMOTE_VERSION_PATTERN.fullmatch(str(item.get("remote_version") or ""))
|
||||
if not match:
|
||||
raise RuntimeError("The official ALZ catalog did not provide one definitive YYYY-v3 edition")
|
||||
year = int(match.group(1))
|
||||
published_at = parse_catalog_datetime(item.get("remote_published_at"))
|
||||
archive_url = (
|
||||
"https://www.landbouwvlaanderen.be/bestanden/gis/"
|
||||
f"agpa_{year}_{published_at.date().isoformat()}_public.zip"
|
||||
)
|
||||
return resolve_release_config(year, archive_url=archive_url)
|
||||
|
||||
|
||||
def fetch_release_decision_from_item(args: argparse.Namespace, item: dict[str, Any]) -> dict[str, Any]:
|
||||
allowed_degraded = item.get("error_code") == "CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING"
|
||||
if (
|
||||
item.get("source_name") != SOURCE_NAME
|
||||
or item.get("reachable") is not True
|
||||
or item.get("status") not in {"available", "degraded"}
|
||||
or (item.get("status") == "degraded" and not allowed_degraded)
|
||||
or "definitive_archive" not in set(item.get("matched_layers") or [])
|
||||
or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or ""))
|
||||
or item.get("metadata_identifier") != "alz-agricultural-use-parcels"
|
||||
):
|
||||
raise RuntimeError("The official ALZ catalog is not safely available for release planning")
|
||||
release = release_from_catalog_item(item)
|
||||
local_version = str(item.get("local_source_version") or "")
|
||||
local_match = LOCAL_VERSION_PATTERN.fullmatch(local_version)
|
||||
if local_version and not local_match:
|
||||
status = "blocked_local_version"
|
||||
elif not local_version:
|
||||
status = "not_loaded"
|
||||
elif int(local_match.group(1)) == release.year:
|
||||
status = "current"
|
||||
elif int(local_match.group(1)) < release.year:
|
||||
status = "update_available"
|
||||
else:
|
||||
status = "blocked_remote_older"
|
||||
message = str(item.get("message") or "")
|
||||
provisional_match = PROVISIONAL_VERSION_PATTERN.search(message)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"status": status,
|
||||
"project_id": args.project_id,
|
||||
"scope": args.scope,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"catalog_checked_at": item.get("checked_at"),
|
||||
"local_source_version": local_version or None,
|
||||
"release": {
|
||||
"year": release.year,
|
||||
"edition": release.definitive_version,
|
||||
"archive_url": release.archive_url,
|
||||
},
|
||||
"catalog_identity": {
|
||||
"metadata_identifier": item["metadata_identifier"],
|
||||
"metadata_url": item.get("metadata_url"),
|
||||
"remote_version": release.definitive_version,
|
||||
"remote_published_at": item.get("remote_published_at"),
|
||||
"capabilities_sha256": item["capabilities_sha256"],
|
||||
},
|
||||
"provisional_release": provisional_match.group(1) if provisional_match else None,
|
||||
"provisional_release_importable": False,
|
||||
"automatic_download": False,
|
||||
"automatic_import": False,
|
||||
"destructive_replacement": False,
|
||||
"next_action": "stage" if status in ACTIONABLE_STATUSES else None,
|
||||
}
|
||||
|
||||
|
||||
def fetch_release_decision(args: argparse.Namespace, *, refresh: bool) -> dict[str, Any]:
|
||||
query = "true" if refresh else "false"
|
||||
report = api_data(
|
||||
args.api_url,
|
||||
f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={query}",
|
||||
args.api_timeout,
|
||||
)
|
||||
matches = [item for item in report.get("items") or [] if item.get("source_name") == SOURCE_NAME]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Source catalog report did not contain exactly one ALZ agriculture contract")
|
||||
return fetch_release_decision_from_item(args, matches[0])
|
||||
|
||||
|
||||
def require_release_confirmation(args: argparse.Namespace, release: AgriculturalReleaseConfig) -> None:
|
||||
if args.confirm_edition != release.definitive_version:
|
||||
raise RuntimeError(f"Explicit --confirm-edition {release.definitive_version} is required")
|
||||
|
||||
|
||||
def internal_base_url(api_url: str) -> str:
|
||||
value = api_url.rstrip("/")
|
||||
if value.endswith("/api/v1"):
|
||||
value = value[:-7]
|
||||
parsed = urlparse(value)
|
||||
if parsed.hostname not in {"127.0.0.1", "localhost", "::1"}:
|
||||
raise RuntimeError("Stage/apply must run inside GeoIntel against the local backend API")
|
||||
return value
|
||||
|
||||
|
||||
def default_plan_path(args: argparse.Namespace, year: int) -> Path:
|
||||
return args.evidence_root / args.scope / str(year) / "staged-plan.json"
|
||||
|
||||
|
||||
def default_review_path(args: argparse.Namespace, year: int) -> Path:
|
||||
return args.evidence_root / args.scope / str(year) / "review-evidence.json"
|
||||
|
||||
|
||||
def governed_evidence_path(args: argparse.Namespace, path: Path) -> Path:
|
||||
if not path.resolve().is_relative_to(args.evidence_root.resolve()):
|
||||
raise RuntimeError(f"Release evidence path is outside the governed evidence root: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def build_operator_command(
|
||||
args: argparse.Namespace,
|
||||
release: AgriculturalReleaseConfig,
|
||||
*,
|
||||
fetch_only: bool,
|
||||
) -> list[str]:
|
||||
command = [
|
||||
sys.executable,
|
||||
str(Path(__file__).resolve().parent / "provision_agricultural_parcel_history.py"),
|
||||
"--base-url",
|
||||
internal_base_url(args.api_url),
|
||||
"--scope",
|
||||
args.scope,
|
||||
"--years",
|
||||
str(release.year),
|
||||
"--archive-url",
|
||||
release.archive_url,
|
||||
"--output-root",
|
||||
str(args.output_root),
|
||||
"--request-timeout",
|
||||
str(args.request_timeout),
|
||||
"--import-timeout",
|
||||
str(args.import_timeout),
|
||||
"--max-features",
|
||||
str(args.max_features),
|
||||
"--max-archive-mb",
|
||||
str(args.max_archive_mb),
|
||||
]
|
||||
if fetch_only:
|
||||
command.extend(("--force", "--fetch-only"))
|
||||
return command
|
||||
|
||||
|
||||
def run_operator(command: list[str], *, action: str) -> dict[str, Any]:
|
||||
print(f"ALZ agriculture: {action}...", file=sys.stderr, flush=True)
|
||||
completed = subprocess.run(command, check=False, capture_output=True, text=True, encoding="utf-8")
|
||||
if completed.returncode != 0:
|
||||
detail = completed.stderr.strip() or completed.stdout.strip() or "operator returned no diagnostics"
|
||||
raise RuntimeError(f"ALZ agriculture {action} failed with exit {completed.returncode}: {detail[-3000:]}")
|
||||
try:
|
||||
payload = json.loads(completed.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"ALZ agriculture {action} returned invalid JSON") from exc
|
||||
if payload.get("status") != "ok":
|
||||
raise RuntimeError(f"ALZ agriculture {action} did not report success")
|
||||
return payload
|
||||
|
||||
|
||||
def canonical_sha256(payload: dict[str, Any], hash_field: str) -> str:
|
||||
content = {key: value for key, value in payload.items() if key != hash_field}
|
||||
encoded = json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".partial")
|
||||
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def _previous_manifest(output_root: Path, scope_key: str, year: int) -> tuple[Path, dict[str, Any]] | None:
|
||||
scope_root = output_root / scope_key
|
||||
if not scope_root.is_dir():
|
||||
return None
|
||||
previous_years = sorted(
|
||||
(int(path.name) for path in scope_root.iterdir() if path.is_dir() and path.name.isdigit() and int(path.name) < year),
|
||||
reverse=True,
|
||||
)
|
||||
for previous_year in previous_years:
|
||||
manifests = list((scope_root / str(previous_year)).glob("*.manifest.json"))
|
||||
if len(manifests) != 1:
|
||||
continue
|
||||
payload = json.loads(manifests[0].read_text(encoding="utf-8"))
|
||||
if payload.get("year") == previous_year and payload.get("scope_key") == scope_key:
|
||||
return manifests[0], payload
|
||||
return None
|
||||
|
||||
|
||||
def _relative_change(current: float, previous: float) -> float | None:
|
||||
if previous <= 0:
|
||||
return None
|
||||
return round((current - previous) / previous, 8)
|
||||
|
||||
|
||||
def validate_staged_release(args: argparse.Namespace, release: AgriculturalReleaseConfig) -> dict[str, Any]:
|
||||
paths = artifact_paths(args.output_root, args.scope, release.year, archive_url=release.archive_url)
|
||||
resolved_output = args.output_root.resolve()
|
||||
for key in ("archive", "artifact", "codelist", "manifest"):
|
||||
if not paths[key].resolve().is_relative_to(resolved_output):
|
||||
raise RuntimeError(f"Staged ALZ {key} is outside the governed output root: {paths[key]}")
|
||||
manifest = reusable_artifact(
|
||||
paths,
|
||||
year=release.year,
|
||||
scope_key=args.scope,
|
||||
archive_url=release.archive_url,
|
||||
)
|
||||
if manifest is None:
|
||||
raise RuntimeError("Staged ALZ artifacts are incomplete or no longer match their checksums")
|
||||
fields = set(manifest.get("source_fields") or [])
|
||||
if (
|
||||
manifest.get("source_url") != release.archive_url
|
||||
or manifest.get("source_crs") != SOURCE_CRS
|
||||
or manifest.get("output_crs") != OUTPUT_CRS
|
||||
or not STABLE_REQUIRED_FIELDS.issubset(fields)
|
||||
or int(manifest.get("feature_count") or 0) < 1
|
||||
or float(manifest.get("clipped_area_ha") or 0) <= 0
|
||||
or int(manifest.get("source_feature_count") or 0) < int(manifest.get("feature_count") or 0)
|
||||
):
|
||||
raise RuntimeError("Staged ALZ manifest does not satisfy the definitive release contract")
|
||||
max_archive_bytes = min(args.max_archive_mb * 1024 * 1024, MAX_ARCHIVE_BYTES)
|
||||
if paths["archive"].stat().st_size > max_archive_bytes:
|
||||
raise RuntimeError("Staged ALZ archive exceeds the configured release limit")
|
||||
codelist = json.loads(paths["codelist"].read_text(encoding="utf-8"))
|
||||
crop_entries = codelist.get("crop_entries") if isinstance(codelist, dict) else None
|
||||
if codelist.get("year") != release.year or not isinstance(crop_entries, list) or not crop_entries:
|
||||
raise RuntimeError("Staged ALZ crop-code evidence is incomplete")
|
||||
previous_result = _previous_manifest(args.output_root, args.scope, release.year)
|
||||
baseline = None
|
||||
if previous_result is not None:
|
||||
previous_path, previous = previous_result
|
||||
if not previous_path.resolve().is_relative_to(resolved_output):
|
||||
raise RuntimeError("Previous ALZ baseline manifest is outside the governed output root")
|
||||
current_count = int(manifest["feature_count"])
|
||||
previous_count = int(previous.get("feature_count") or 0)
|
||||
current_area = float(manifest["clipped_area_ha"])
|
||||
previous_area = float(previous.get("clipped_area_ha") or 0)
|
||||
baseline = {
|
||||
"year": int(previous["year"]),
|
||||
"manifest_path": str(previous_path),
|
||||
"manifest_sha256": sha256_file(previous_path),
|
||||
"feature_count": previous_count,
|
||||
"clipped_area_ha": previous_area,
|
||||
"feature_count_change_ratio": _relative_change(current_count, previous_count),
|
||||
"clipped_area_change_ratio": _relative_change(current_area, previous_area),
|
||||
}
|
||||
return {
|
||||
"manifest_path": str(paths["manifest"]),
|
||||
"manifest_sha256": sha256_file(paths["manifest"]),
|
||||
"archive_path": str(paths["archive"]),
|
||||
"archive_sha256": manifest["source_archive_sha256"],
|
||||
"archive_size_bytes": manifest["source_archive_size_bytes"],
|
||||
"artifact_path": str(paths["artifact"]),
|
||||
"artifact_sha256": manifest["artifact_sha256"],
|
||||
"crop_code_list_path": str(paths["codelist"]),
|
||||
"crop_code_list_sha256": manifest["crop_code_list_sha256"],
|
||||
"crop_entry_count": len(crop_entries),
|
||||
"crop_code_conflicts": codelist.get("code_title_conflicts") or {},
|
||||
"source_feature_count": manifest["source_feature_count"],
|
||||
"feature_count": manifest["feature_count"],
|
||||
"clipped_feature_count": manifest["clipped_feature_count"],
|
||||
"clipped_area_ha": manifest["clipped_area_ha"],
|
||||
"source_fields": sorted(fields),
|
||||
"member_nis_codes": manifest.get("member_nis_codes") or [],
|
||||
"baseline": baseline,
|
||||
}
|
||||
|
||||
|
||||
def build_staged_plan(
|
||||
args: argparse.Namespace,
|
||||
decision: dict[str, Any],
|
||||
release: AgriculturalReleaseConfig,
|
||||
operator_result: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"status": "staged",
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"project_id": args.project_id,
|
||||
"scope": args.scope,
|
||||
"local_source_version_before_apply": decision.get("local_source_version"),
|
||||
"release": decision["release"],
|
||||
"catalog_identity": decision["catalog_identity"],
|
||||
"evidence": validate_staged_release(args, release),
|
||||
"operator_result": operator_result,
|
||||
"review_required": True,
|
||||
"apply_requires_plan_sha256": True,
|
||||
"apply_requires_review_sha256": True,
|
||||
"automatic_import": False,
|
||||
"destructive_replacement": False,
|
||||
"existing_snapshots_retained": True,
|
||||
}
|
||||
payload["plan_sha256"] = canonical_sha256(payload, "plan_sha256")
|
||||
return payload
|
||||
|
||||
|
||||
def _release_from_payload(payload: dict[str, Any]) -> AgriculturalReleaseConfig:
|
||||
release = payload.get("release") or {}
|
||||
year = release.get("year") if isinstance(release, dict) else None
|
||||
edition = release.get("edition") if isinstance(release, dict) else None
|
||||
archive_url = release.get("archive_url") if isinstance(release, dict) else None
|
||||
if (
|
||||
not isinstance(year, int)
|
||||
or isinstance(year, bool)
|
||||
or edition != f"{year}-v3"
|
||||
or not isinstance(archive_url, str)
|
||||
):
|
||||
raise RuntimeError("Release evidence is missing required ALZ identity fields")
|
||||
return resolve_release_config(year, archive_url=archive_url)
|
||||
|
||||
|
||||
def load_staged_plan(
|
||||
args: argparse.Namespace,
|
||||
release: AgriculturalReleaseConfig,
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"Staged ALZ plan is missing: {path}")
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
actual_sha = canonical_sha256(payload, "plan_sha256")
|
||||
if payload.get("plan_sha256") != actual_sha:
|
||||
raise RuntimeError("Staged ALZ plan checksum is invalid")
|
||||
if args.confirm_plan_sha256 != actual_sha:
|
||||
raise RuntimeError(f"Explicit --confirm-plan-sha256 {actual_sha} is required")
|
||||
if (
|
||||
payload.get("status") != "staged"
|
||||
or payload.get("project_id") != args.project_id
|
||||
or payload.get("scope") != args.scope
|
||||
or _release_from_payload(payload) != release
|
||||
):
|
||||
raise RuntimeError("Staged ALZ plan identity is invalid")
|
||||
current_evidence = validate_staged_release(args, release)
|
||||
if current_evidence != payload.get("evidence"):
|
||||
raise RuntimeError("Staged ALZ artifacts no longer match the approved plan")
|
||||
return path, payload
|
||||
|
||||
|
||||
def require_catalog_unchanged(plan: dict[str, Any], decision: dict[str, Any]) -> None:
|
||||
if plan.get("release") != decision.get("release") or plan.get("catalog_identity") != decision.get("catalog_identity"):
|
||||
raise RuntimeError("The official ALZ release evidence changed; create and review a new staged plan")
|
||||
|
||||
|
||||
def build_review_evidence(args: argparse.Namespace, plan_path: Path, plan: dict[str, Any]) -> dict[str, Any]:
|
||||
reviewer = str(args.reviewer or "").strip()
|
||||
note = str(args.review_note or "").strip()
|
||||
if not args.approve or len(reviewer) < 2 or len(reviewer) > 120:
|
||||
raise RuntimeError("Review requires --approve and a named --reviewer between 2 and 120 characters")
|
||||
if len(note) > 1000:
|
||||
raise RuntimeError("Review note must not exceed 1000 characters")
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"status": "approved",
|
||||
"reviewed_at": datetime.now(timezone.utc).isoformat(),
|
||||
"reviewer": reviewer,
|
||||
"review_note": note or None,
|
||||
"project_id": args.project_id,
|
||||
"scope": args.scope,
|
||||
"release": plan["release"],
|
||||
"staged_plan_path": str(plan_path),
|
||||
"staged_plan_sha256": plan["plan_sha256"],
|
||||
"reviewed_checks": [
|
||||
"official_catalog_identity",
|
||||
"source_archive_checksum",
|
||||
"geopackage_schema_and_crs",
|
||||
"scope_feature_and_area_accounting",
|
||||
"crop_code_list_and_conflicts",
|
||||
"previous_definitive_edition_delta",
|
||||
"provisional_snapshot_exclusion",
|
||||
"immutable_dataset_apply",
|
||||
],
|
||||
}
|
||||
payload["review_sha256"] = canonical_sha256(payload, "review_sha256")
|
||||
return payload
|
||||
|
||||
|
||||
def load_review_evidence(
|
||||
args: argparse.Namespace,
|
||||
release: AgriculturalReleaseConfig,
|
||||
plan: dict[str, Any],
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"Approved ALZ review evidence is missing: {path}")
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
actual_sha = canonical_sha256(payload, "review_sha256")
|
||||
if payload.get("review_sha256") != actual_sha:
|
||||
raise RuntimeError("ALZ review evidence checksum is invalid")
|
||||
if args.confirm_review_sha256 != actual_sha:
|
||||
raise RuntimeError(f"Explicit --confirm-review-sha256 {actual_sha} is required")
|
||||
if (
|
||||
payload.get("status") != "approved"
|
||||
or payload.get("project_id") != args.project_id
|
||||
or payload.get("scope") != args.scope
|
||||
or payload.get("release") != plan.get("release")
|
||||
or payload.get("staged_plan_sha256") != plan.get("plan_sha256")
|
||||
or not str(payload.get("reviewer") or "").strip()
|
||||
):
|
||||
raise RuntimeError("ALZ review evidence does not authorize this staged plan")
|
||||
return path, payload
|
||||
|
||||
|
||||
def _release_result(operator_result: dict[str, Any], year: int) -> dict[str, Any]:
|
||||
matches = [item for item in operator_result.get("years") or [] if int(item.get("year") or 0) == year]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Agriculture operator did not return exactly one result for the approved edition")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
if min(args.request_timeout, args.api_timeout, args.import_timeout, args.max_features, args.max_archive_mb) <= 0:
|
||||
raise ValueError("All timeout and safety limits must be positive")
|
||||
validate_project_scope(args)
|
||||
refresh = args.refresh_catalog or args.action in {"stage", "review", "apply"}
|
||||
decision = fetch_release_decision(args, refresh=refresh)
|
||||
release = _release_from_payload(decision)
|
||||
|
||||
if args.action == "plan":
|
||||
print(json.dumps({"status": "ok", "action": "plan", "decision": decision}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
require_release_confirmation(args, release)
|
||||
if args.action == "stage":
|
||||
if decision.get("status") not in ACTIONABLE_STATUSES:
|
||||
raise RuntimeError(f"ALZ release is not safely stageable: {decision.get('status')}")
|
||||
operator_result = run_operator(build_operator_command(args, release, fetch_only=True), action="staging")
|
||||
staged_result = _release_result(operator_result, release.year)
|
||||
if staged_result.get("status") != "prepared":
|
||||
raise RuntimeError("ALZ staging did not produce prepared release evidence")
|
||||
plan = build_staged_plan(args, decision, release, operator_result)
|
||||
plan_path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
|
||||
write_json(plan_path, plan)
|
||||
print(json.dumps({"status": "staged", "plan_path": str(plan_path), **plan}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
plan_path, plan = load_staged_plan(args, release)
|
||||
require_catalog_unchanged(plan, decision)
|
||||
if args.action == "review":
|
||||
review = build_review_evidence(args, plan_path, plan)
|
||||
review_path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
|
||||
write_json(review_path, review)
|
||||
print(json.dumps({"status": "approved", "review_path": str(review_path), **review}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
review_path, review = load_review_evidence(args, release, plan)
|
||||
operator_result = run_operator(build_operator_command(args, release, fetch_only=False), action="applying")
|
||||
applied_result = _release_result(operator_result, release.year)
|
||||
if applied_result.get("status") not in {"imported", "existing"} or not applied_result.get("dataset_id"):
|
||||
raise RuntimeError("ALZ apply did not return one persisted immutable Dataset")
|
||||
final_decision = fetch_release_decision(args, refresh=True)
|
||||
if (
|
||||
final_decision.get("status") != "current"
|
||||
or final_decision.get("local_source_version") != f"{release.year}-definitive"
|
||||
):
|
||||
raise RuntimeError("Applied ALZ Dataset did not become the current local definitive edition")
|
||||
evidence = {
|
||||
"schema_version": 1,
|
||||
"status": "applied",
|
||||
"applied_at": datetime.now(timezone.utc).isoformat(),
|
||||
"project_id": args.project_id,
|
||||
"scope": args.scope,
|
||||
"release": plan["release"],
|
||||
"staged_plan_path": str(plan_path),
|
||||
"staged_plan_sha256": plan["plan_sha256"],
|
||||
"review_path": str(review_path),
|
||||
"review_sha256": review["review_sha256"],
|
||||
"reviewer": review["reviewer"],
|
||||
"dataset_id": applied_result["dataset_id"],
|
||||
"dataset_status": applied_result["status"],
|
||||
"feature_count": applied_result.get("feature_count"),
|
||||
"final_catalog_decision": final_decision,
|
||||
"existing_snapshots_retained": True,
|
||||
}
|
||||
evidence["applied_evidence_sha256"] = canonical_sha256(evidence, "applied_evidence_sha256")
|
||||
evidence_path = plan_path.with_name("applied-evidence.json")
|
||||
write_json(evidence_path, evidence)
|
||||
print(json.dumps({"status": "applied", "evidence_path": str(evidence_path), **evidence}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
except (OSError, RuntimeError, ValueError, KeyError, json.JSONDecodeError) as exc:
|
||||
print(
|
||||
json.dumps({"status": "error", "action": args.action, "message": str(exc)}, ensure_ascii=False),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -10,17 +10,19 @@ directly and never uses the provisional current-campaign snapshot.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import zipfile
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import requests
|
||||
from pyproj import Transformer
|
||||
@@ -51,6 +53,10 @@ SOURCE_CRS = "EPSG:31370"
|
||||
OUTPUT_CRS = "EPSG:4326"
|
||||
SCHEMA_VERSION = 1
|
||||
MAX_ARCHIVE_BYTES = 250 * 1024 * 1024
|
||||
MAX_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024
|
||||
MAX_ARCHIVE_MEMBERS = 32
|
||||
ARCHIVE_HOST = "www.landbouwvlaanderen.be"
|
||||
ARCHIVE_PATH_PATTERN = re.compile(r"^/bestanden/gis/agpa_(20[0-9]{2})_(20[0-9]{2}-[0-9]{2}-[0-9]{2})_public\.zip$")
|
||||
|
||||
ARCHIVE_URLS = {
|
||||
2008: "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2008_2022-03-23_public.zip",
|
||||
@@ -74,6 +80,16 @@ ARCHIVE_URLS = {
|
||||
}
|
||||
SUPPORTED_YEARS = tuple(ARCHIVE_URLS)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AgriculturalReleaseConfig:
|
||||
year: int
|
||||
archive_url: str
|
||||
|
||||
@property
|
||||
def definitive_version(self) -> str:
|
||||
return f"{self.year}-v3"
|
||||
|
||||
STABLE_REQUIRED_FIELDS = {
|
||||
"agpakey",
|
||||
"parcelnumber",
|
||||
@@ -131,6 +147,10 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
|
||||
parser.add_argument("--years", default=",".join(str(year) for year in SUPPORTED_YEARS))
|
||||
parser.add_argument(
|
||||
"--archive-url",
|
||||
help="Exact official archive URL for one explicitly confirmed future definitive edition.",
|
||||
)
|
||||
parser.add_argument("--output-root", type=Path, default=Path(os.environ.get("GEOINTEL_AGRICULTURE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)))
|
||||
parser.add_argument("--request-timeout", type=int, default=900)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
@@ -201,21 +221,29 @@ def response_data(response: requests.Response) -> Any:
|
||||
def api_items(session: requests.Session, url: str, timeout: int) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
while True:
|
||||
expected_total: int | None = None
|
||||
while expected_total is None or offset < expected_total:
|
||||
response = session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout)
|
||||
data = response_data(response)
|
||||
if isinstance(data, dict):
|
||||
page = data.get("items") or data.get("results") or []
|
||||
total = int(data.get("total") or len(page))
|
||||
page_total = int(data.get("total") if data.get("total") is not None else len(page))
|
||||
else:
|
||||
page = data
|
||||
total = len(page) if isinstance(page, list) else 0
|
||||
page_total = len(page) if isinstance(page, list) else 0
|
||||
if not isinstance(page, list):
|
||||
raise RuntimeError(f"Expected a list response from {url}")
|
||||
if expected_total is None:
|
||||
expected_total = page_total
|
||||
elif page_total != expected_total:
|
||||
raise RuntimeError("GeoIntel pagination total changed while reading the agricultural workspace")
|
||||
items.extend(item for item in page if isinstance(item, dict))
|
||||
if not page or len(items) >= total:
|
||||
return items
|
||||
if not page:
|
||||
break
|
||||
offset += len(page)
|
||||
if expected_total is not None and len(items) != expected_total:
|
||||
raise RuntimeError(f"GeoIntel list response for {url} returned {len(items)} of {expected_total} items")
|
||||
return items
|
||||
|
||||
|
||||
def locate_workspace(
|
||||
@@ -251,6 +279,53 @@ def parse_years(raw: str) -> list[int]:
|
||||
return years
|
||||
|
||||
|
||||
def validate_archive_url(url: str, *, expected_year: int) -> str:
|
||||
parsed = urlsplit(url)
|
||||
match = ARCHIVE_PATH_PATTERN.fullmatch(parsed.path)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname != ARCHIVE_HOST
|
||||
or parsed.port not in {None, 443}
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or not match
|
||||
or int(match.group(1)) != expected_year
|
||||
):
|
||||
raise ValueError("Agricultural release archive is outside the official ALZ URL contract")
|
||||
try:
|
||||
datetime.strptime(match.group(2), "%Y-%m-%d")
|
||||
except ValueError as exc:
|
||||
raise ValueError("Agricultural release archive contains an invalid publication date") from exc
|
||||
return url
|
||||
|
||||
|
||||
def resolve_release_config(year: int, *, archive_url: str | None = None) -> AgriculturalReleaseConfig:
|
||||
known_url = ARCHIVE_URLS.get(year)
|
||||
if known_url is not None:
|
||||
if archive_url is not None and archive_url != known_url:
|
||||
raise ValueError(f"The official retained archive identity for {year} may not be overridden")
|
||||
return AgriculturalReleaseConfig(year=year, archive_url=known_url)
|
||||
if year <= max(SUPPORTED_YEARS) or not archive_url:
|
||||
raise ValueError(
|
||||
f"One future definitive edition after {max(SUPPORTED_YEARS)} may be supplied with --archive-url"
|
||||
)
|
||||
return AgriculturalReleaseConfig(year=year, archive_url=validate_archive_url(archive_url, expected_year=year))
|
||||
|
||||
|
||||
def resolve_release_configs(raw_years: str, *, archive_url: str | None = None) -> list[AgriculturalReleaseConfig]:
|
||||
if archive_url is None:
|
||||
return [resolve_release_config(year) for year in parse_years(raw_years)]
|
||||
try:
|
||||
years = sorted({int(value.strip()) for value in raw_years.split(",") if value.strip()})
|
||||
except ValueError as exc:
|
||||
raise ValueError("Years must be a comma-separated list of integers") from exc
|
||||
if len(years) != 1:
|
||||
raise ValueError("--archive-url requires exactly one explicitly selected definitive year")
|
||||
return [resolve_release_config(years[0], archive_url=archive_url)]
|
||||
|
||||
|
||||
def polygonal_geometry(geometry):
|
||||
if geometry is None or geometry.is_empty:
|
||||
return None
|
||||
@@ -308,6 +383,10 @@ def download_archive(
|
||||
temporary.unlink(missing_ok=True)
|
||||
response = session.get(url, timeout=timeout, stream=True)
|
||||
response.raise_for_status()
|
||||
final_url = str(getattr(response, "url", "") or url)
|
||||
expected_year = int(ARCHIVE_PATH_PATTERN.fullmatch(urlsplit(url).path).group(1))
|
||||
if validate_archive_url(final_url, expected_year=expected_year) != url:
|
||||
raise RuntimeError("Official archive download redirected to a different release identity")
|
||||
content_length = int(response.headers.get("content-length") or 0)
|
||||
if content_length > max_bytes:
|
||||
raise RuntimeError(f"Official archive exceeds the configured {max_bytes // (1024 * 1024)} MiB limit")
|
||||
@@ -331,7 +410,12 @@ def download_archive(
|
||||
|
||||
def archive_geopackage_member(path: Path) -> str:
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
members = [item.filename for item in archive.infolist() if not item.is_dir() and item.filename.lower().endswith(".gpkg")]
|
||||
entries = archive.infolist()
|
||||
if len(entries) > MAX_ARCHIVE_MEMBERS:
|
||||
raise RuntimeError(f"Official archive contains more than {MAX_ARCHIVE_MEMBERS} members")
|
||||
if sum(item.file_size for item in entries if not item.is_dir()) > MAX_EXTRACTED_BYTES:
|
||||
raise RuntimeError("Official archive exceeds the extracted-size safety limit")
|
||||
members = [item.filename for item in entries if not item.is_dir() and item.filename.lower().endswith(".gpkg")]
|
||||
if len(members) != 1:
|
||||
raise RuntimeError(f"Official archive must contain exactly one GeoPackage; found {len(members)}")
|
||||
member = members[0]
|
||||
@@ -351,7 +435,12 @@ def extract_geopackage(archive_path: Path, destination_dir: Path) -> Path:
|
||||
member = validate_archive(archive_path)
|
||||
destination = destination_dir / Path(member).name
|
||||
with zipfile.ZipFile(archive_path) as archive, archive.open(member) as source, destination.open("wb") as target:
|
||||
shutil.copyfileobj(source, target, length=1024 * 1024)
|
||||
extracted_bytes = 0
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
extracted_bytes += len(chunk)
|
||||
if extracted_bytes > MAX_EXTRACTED_BYTES:
|
||||
raise RuntimeError("Official GeoPackage exceeded the extracted-size safety limit while streaming")
|
||||
target.write(chunk)
|
||||
if destination.stat().st_size == 0:
|
||||
raise RuntimeError("Extracted official GeoPackage is empty")
|
||||
return destination
|
||||
@@ -470,23 +559,39 @@ def normalize_frame(frame, *, year: int, boundary_lambert72, max_features: int)
|
||||
}
|
||||
|
||||
|
||||
def artifact_paths(output_root: Path, scope_key: str, year: int) -> dict[str, Path]:
|
||||
def artifact_paths(
|
||||
output_root: Path,
|
||||
scope_key: str,
|
||||
year: int,
|
||||
*,
|
||||
archive_url: str | None = None,
|
||||
) -> dict[str, Path]:
|
||||
release = resolve_release_config(year, archive_url=archive_url)
|
||||
directory = output_root / scope_key / str(year)
|
||||
return {
|
||||
"directory": directory,
|
||||
"archive": directory / Path(ARCHIVE_URLS[year]).name,
|
||||
"archive": directory / Path(urlsplit(release.archive_url).path).name,
|
||||
"artifact": directory / f"agricultural_use_parcels_{year}_{scope_key}.geojson",
|
||||
"codelist": directory / f"agricultural_use_parcels_{year}_crop_codes.json",
|
||||
"manifest": directory / f"agricultural_use_parcels_{year}_{scope_key}.manifest.json",
|
||||
}
|
||||
|
||||
|
||||
def reusable_artifact(paths: dict[str, Path], *, year: int, scope_key: str) -> dict[str, Any] | None:
|
||||
def reusable_artifact(
|
||||
paths: dict[str, Path],
|
||||
*,
|
||||
year: int,
|
||||
scope_key: str,
|
||||
archive_url: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if not all(paths[key].is_file() for key in ("archive", "artifact", "codelist", "manifest")):
|
||||
return None
|
||||
manifest = json.loads(paths["manifest"].read_text(encoding="utf-8"))
|
||||
if manifest.get("schema_version") != SCHEMA_VERSION or manifest.get("year") != year or manifest.get("scope_key") != scope_key:
|
||||
return None
|
||||
release = resolve_release_config(year, archive_url=archive_url)
|
||||
if manifest.get("source_url") != release.archive_url:
|
||||
return None
|
||||
if manifest.get("source_archive_sha256") != sha256_file(paths["archive"]):
|
||||
return None
|
||||
if manifest.get("artifact_sha256") != sha256_file(paths["artifact"]):
|
||||
@@ -508,16 +613,18 @@ def prepare_year(
|
||||
max_archive_bytes: int,
|
||||
max_features: int,
|
||||
force: bool,
|
||||
archive_url: str | None = None,
|
||||
) -> tuple[dict[str, Path], dict[str, Any]]:
|
||||
paths = artifact_paths(output_root, scope.key, year)
|
||||
release = resolve_release_config(year, archive_url=archive_url)
|
||||
paths = artifact_paths(output_root, scope.key, year, archive_url=release.archive_url)
|
||||
paths["directory"].mkdir(parents=True, exist_ok=True)
|
||||
if not force:
|
||||
reused = reusable_artifact(paths, year=year, scope_key=scope.key)
|
||||
reused = reusable_artifact(paths, year=year, scope_key=scope.key, archive_url=release.archive_url)
|
||||
if reused is not None:
|
||||
return paths, reused
|
||||
download = download_archive(
|
||||
session,
|
||||
ARCHIVE_URLS[year],
|
||||
release.archive_url,
|
||||
paths["archive"],
|
||||
timeout=request_timeout,
|
||||
max_bytes=max_archive_bytes,
|
||||
@@ -546,7 +653,7 @@ def prepare_year(
|
||||
"scope_name": scope.display_name,
|
||||
"scope_type": scope.scope_type,
|
||||
"member_nis_codes": list(scope.nis_codes),
|
||||
"source_url": ARCHIVE_URLS[year],
|
||||
"source_url": release.archive_url,
|
||||
"catalog_url": CATALOG_URL,
|
||||
"data_catalog_url": DATA_CATALOG_URL,
|
||||
"attribution": ATTRIBUTION,
|
||||
@@ -635,7 +742,7 @@ def upload_artifact(
|
||||
"operator_tool": "provision_agricultural_parcel_history.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"geometry_clipped_to_area": True,
|
||||
"source_archive_url": ARCHIVE_URLS[year],
|
||||
"source_archive_url": manifest["source_url"],
|
||||
"source_archive_path": str(paths["archive"]),
|
||||
"source_archive_sha256": manifest["source_archive_sha256"],
|
||||
"crop_code_list_path": str(paths["codelist"]),
|
||||
@@ -684,7 +791,7 @@ def existing_dataset_for_year(datasets: list[dict[str, Any]], *, area_id: str, y
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
years = parse_years(args.years)
|
||||
releases = resolve_release_configs(args.years, archive_url=args.archive_url)
|
||||
if args.max_features < 1 or args.max_archive_mb < 1:
|
||||
raise ValueError("Feature and archive safety limits must be positive")
|
||||
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||
@@ -693,7 +800,8 @@ def main() -> int:
|
||||
project_id, area_id, boundary, datasets = locate_workspace(api_session, base_url, scope, args.import_timeout)
|
||||
results: list[dict[str, Any]] = []
|
||||
with build_session() as official_session:
|
||||
for year in years:
|
||||
for release in releases:
|
||||
year = release.year
|
||||
paths, manifest = prepare_year(
|
||||
official_session,
|
||||
year=year,
|
||||
@@ -704,6 +812,7 @@ def main() -> int:
|
||||
max_archive_bytes=min(args.max_archive_mb * 1024 * 1024, MAX_ARCHIVE_BYTES),
|
||||
max_features=args.max_features,
|
||||
force=args.force,
|
||||
archive_url=release.archive_url,
|
||||
)
|
||||
existing = existing_dataset_for_year(datasets, area_id=area_id, year=year)
|
||||
if existing is not None:
|
||||
|
||||
@@ -54,6 +54,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/manage_alz_agriculture_release.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py
|
||||
|
||||
Reference in New Issue
Block a user