Add governed ALZ release promotion
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-17 00:53:34 +02:00
parent 904f8bd9d2
commit ad2c3481c4
15 changed files with 1357 additions and 19 deletions
+17
View File
@@ -7,6 +7,23 @@
# Changelog
## Sprint 229 Governed ALZ definitive release promotion (2026-07-17)
- Added an operator-only `plan -> stage -> review -> apply` coordinator for
future definitive ALZ agricultural-use parcel editions. A provisional v1/v2
campaign snapshot remains visible release evidence but is never importable.
- Derived the exact archive identity from the allowlisted publication year and
date reported by the existing source-catalog probe. Arbitrary provider URLs,
current/older releases, catalog drift and changed staged bytes fail closed.
- Extended the existing agricultural provisioner to accept exactly one
explicitly governed future edition, validate final download identity, keep
streamed size bounds and read every paginated workspace Dataset consistently.
- Bound archive, normalized GeoJSON, GeoPackage schema/CRS, crop-code list,
scope totals and previous-edition deltas into named human review evidence.
Apply still creates only an immutable Dataset through DatasetService.
- Added focused trust-boundary, tamper, drift, approval, apply and packaging
tests without adding an API route, migration, scheduler or frontend action.
## Sprint 228 Governed Statbel population release promotion (2026-07-17)
- Added an operator-only `plan -> stage -> review -> apply` coordinator for
+29
View File
@@ -1219,6 +1219,35 @@ transport region. Every annual source ZIP and crop code list remains under the
storage volume. PostGIS computes exact hectares for drawn rectangles and
persisted Areas; parcel identities are deliberately unavailable for lineage.
Future definitive editions use the separate four-phase release coordinator:
```bash
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py plan \
--project-id <KEMPEN_PROJECT_ID> --refresh-catalog
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py stage \
--project-id <KEMPEN_PROJECT_ID> \
--confirm-edition <YYYY-v3_FROM_PLAN>
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py review \
--project-id <KEMPEN_PROJECT_ID> \
--confirm-edition <YYYY-v3_FROM_PLAN> \
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
--approve --reviewer "<OPERATOR_NAME>"
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py apply \
--project-id <KEMPEN_PROJECT_ID> \
--confirm-edition <YYYY-v3_FROM_PLAN> \
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
--confirm-review-sha256 <SHA256_FROM_REVIEW>
```
The manager accepts only one catalog-confirmed definitive v3 release. `plan`
writes nothing; `stage` downloads and normalizes without PostGIS mutation;
`review` binds a named approval; `apply` revalidates catalog, hashes, schema,
crop codes, scope accounting and previous-edition deltas before delegating to
DatasetService. Provisional v1/v2 snapshots never enter the historical series.
## Buildings and Addresses Register snapshot
After the Mol Area and regional GRB buildings have been provisioned, prepare
@@ -0,0 +1,419 @@
from __future__ import annotations
import argparse
import importlib.util
import json
from pathlib import Path
import sys
import zipfile
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
def load_script(name: str):
path = SCRIPTS / name
module_name = f"test_{path.stem}_sprint229"
spec = importlib.util.spec_from_file_location(module_name, path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
OPERATOR = load_script("provision_agricultural_parcel_history.py")
MANAGER = load_script("manage_alz_agriculture_release.py")
def arguments(tmp_path: Path, **overrides) -> argparse.Namespace:
values = {
"action": "plan",
"project_id": "00000000-0000-0000-0000-000000000001",
"scope": "kempen-transport-region",
"api_url": "http://127.0.0.1:8000/api/v1",
"confirm_edition": None,
"confirm_plan_sha256": None,
"confirm_review_sha256": None,
"approve": False,
"reviewer": None,
"review_note": "",
"plan_path": None,
"review_path": None,
"output_root": tmp_path / "operator-evidence" / "agricultural-use-parcels",
"evidence_root": tmp_path / "operator-evidence" / "alz-agriculture-refresh",
"refresh_catalog": False,
"request_timeout": 900,
"api_timeout": 180,
"import_timeout": 3600,
"max_features": 250_000,
"max_archive_mb": 250,
}
values.update(overrides)
return argparse.Namespace(**values)
def catalog_item(
*,
remote: str = "2026-v3",
local: str | None = "2025-definitive",
catalog_hash: str = "a" * 64,
checked_at: str = "2027-03-16T08:00:00Z",
) -> dict:
year = int(remote[:4])
published = "2027-03-15T00:00:00Z" if year == 2026 else "2026-05-13T00:00:00Z"
return {
"source_name": MANAGER.SOURCE_NAME,
"status": "available",
"reachable": True,
"error_code": None,
"remote_version": remote,
"remote_published_at": published,
"local_source_version": local,
"message": f"Definitieve editie {remote}; actuele publicatie {year + 1}-v1 is voorlopig.",
"matched_layers": ["definitive_archive", "current_snapshot"],
"capabilities_sha256": catalog_hash,
"metadata_identifier": "alz-agricultural-use-parcels",
"metadata_url": "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen",
"checked_at": checked_at,
}
def decision(args: argparse.Namespace, *, remote: str = "2026-v3", local: str | None = "2025-definitive") -> dict:
return MANAGER.fetch_release_decision_from_item(args, catalog_item(remote=remote, local=local))
def release_2026():
return MANAGER.release_from_catalog_item(catalog_item())
def write_staged_artifacts(args: argparse.Namespace, release) -> dict:
paths = OPERATOR.artifact_paths(
args.output_root,
args.scope,
release.year,
archive_url=release.archive_url,
)
paths["directory"].mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(paths["archive"], "w") as archive:
archive.writestr(f"agpa_{release.year}.gpkg", b"official geopackage")
paths["artifact"].write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
codelist = {
"year": release.year,
"crop_entries": [{"code": "201", "title": "Mais", "group_title": "Mais"}],
"code_title_conflicts": {},
}
paths["codelist"].write_text(json.dumps(codelist), encoding="utf-8")
baseline_dir = args.output_root / args.scope / "2025"
baseline_dir.mkdir(parents=True, exist_ok=True)
(baseline_dir / "agricultural_use_parcels_2025_kempen-transport-region.manifest.json").write_text(
json.dumps(
{
"year": 2025,
"scope_key": args.scope,
"feature_count": 120_000,
"clipped_area_ha": 62_000.0,
}
),
encoding="utf-8",
)
manifest = {
"schema_version": 1,
"year": release.year,
"scope_key": args.scope,
"member_nis_codes": ["13025", "13003"],
"source_url": release.archive_url,
"source_crs": OPERATOR.SOURCE_CRS,
"output_crs": OPERATOR.OUTPUT_CRS,
"source_archive_sha256": OPERATOR.sha256_file(paths["archive"]),
"source_archive_size_bytes": paths["archive"].stat().st_size,
"source_feature_count": 180_000,
"source_fields": sorted(OPERATOR.STABLE_REQUIRED_FIELDS),
"crop_code_list_sha256": OPERATOR.sha256_file(paths["codelist"]),
"artifact_sha256": OPERATOR.sha256_file(paths["artifact"]),
"feature_count": 121_500,
"clipped_feature_count": 800,
"clipped_area_ha": 62_500.0,
}
paths["manifest"].write_text(json.dumps(manifest), encoding="utf-8")
return paths
def staged_plan(args: argparse.Namespace, release, release_decision: dict) -> tuple[Path, dict]:
write_staged_artifacts(args, release)
result = {
"status": "ok",
"scope": args.scope,
"years": [{"year": release.year, "status": "prepared", "feature_count": 121_500}],
}
plan = MANAGER.build_staged_plan(args, release_decision, release, result)
path = MANAGER.default_plan_path(args, release.year)
MANAGER.write_json(path, plan)
return path, plan
def test_future_release_config_accepts_only_one_exact_official_edition() -> None:
url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip"
release = OPERATOR.resolve_release_config(2026, archive_url=url)
assert release.definitive_version == "2026-v3"
assert release.archive_url == url
future = OPERATOR.resolve_release_config(2027, archive_url=url.replace("2026", "2027"))
assert future.definitive_version == "2027-v3"
with pytest.raises(ValueError, match="official ALZ URL"):
OPERATOR.resolve_release_config(2026, archive_url=url.replace("www.landbouwvlaanderen.be", "example.com"))
with pytest.raises(ValueError, match="may not be overridden"):
OPERATOR.resolve_release_config(2025, archive_url=url.replace("2026", "2025"))
def test_future_archive_requires_one_selected_year() -> None:
url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip"
with pytest.raises(ValueError, match="exactly one"):
OPERATOR.resolve_release_configs("2025,2026", archive_url=url)
def test_agriculture_workspace_pagination_is_complete_and_total_consistent() -> None:
rows = [{"id": index} for index in range(401)]
class Response:
ok = True
status_code = 200
text = ""
def __init__(self, payload: dict) -> None:
self.payload = payload
def json(self) -> dict:
return {"data": self.payload}
class Session:
drift = False
def get(self, _url: str, *, params: dict, timeout: int):
assert timeout == 30
offset = int(params["offset"])
total = len(rows) + (1 if self.drift and offset else 0)
return Response({"items": rows[offset : offset + int(params["limit"])], "total": total})
assert OPERATOR.api_items(Session(), "http://backend/datasets", 30) == rows
drifting = Session()
drifting.drift = True
with pytest.raises(RuntimeError, match="total changed"):
OPERATOR.api_items(drifting, "http://backend/datasets", 30)
def test_archive_download_rejects_oversize_before_streaming(tmp_path: Path) -> None:
url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip"
class Response:
headers = {"content-length": "101"}
def __init__(self) -> None:
self.url = url
def raise_for_status(self) -> None:
return None
def iter_content(self, *, chunk_size: int):
raise AssertionError(f"streaming should not start: {chunk_size}")
class Session:
def get(self, *_args, **_kwargs):
return Response()
with pytest.raises(RuntimeError, match="configured"):
OPERATOR.download_archive(Session(), url, tmp_path / "source.zip", timeout=30, max_bytes=100, force=True)
def test_archive_rejects_extracted_size_over_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
archive_path = tmp_path / "source.zip"
with zipfile.ZipFile(archive_path, "w") as archive:
archive.writestr("agpa_2026.gpkg", b"0123456789")
monkeypatch.setattr(OPERATOR, "MAX_EXTRACTED_BYTES", 9)
with pytest.raises(RuntimeError, match="extracted-size safety limit"):
OPERATOR.validate_archive(archive_path)
@pytest.mark.parametrize(
("remote", "local", "expected"),
[
("2026-v3", "2025-definitive", "update_available"),
("2025-v3", "2025-definitive", "current"),
("2026-v3", None, "not_loaded"),
("2025-v3", "2026-definitive", "blocked_remote_older"),
],
)
def test_catalog_decision_orders_only_definitive_editions(
tmp_path: Path,
remote: str,
local: str | None,
expected: str,
) -> None:
assert decision(arguments(tmp_path), remote=remote, local=local)["status"] == expected
def test_catalog_decision_keeps_provisional_snapshot_non_importable(tmp_path: Path) -> None:
release_decision = decision(arguments(tmp_path))
assert release_decision["release"]["edition"] == "2026-v3"
assert release_decision["provisional_release"] == "2027-v1"
assert release_decision["provisional_release_importable"] is False
with pytest.raises(RuntimeError, match="definitive YYYY-v3"):
MANAGER.fetch_release_decision_from_item(arguments(tmp_path), catalog_item(remote="2026-v1"))
def test_stage_and_apply_commands_are_separate_and_local_only(tmp_path: Path) -> None:
args = arguments(tmp_path)
release = release_2026()
stage = MANAGER.build_operator_command(args, release, fetch_only=True)
apply = MANAGER.build_operator_command(args, release, fetch_only=False)
assert "--force" in stage and "--fetch-only" in stage
assert "--force" not in apply and "--fetch-only" not in apply
assert stage[stage.index("--archive-url") + 1] == release.archive_url
with pytest.raises(RuntimeError, match="inside GeoIntel"):
MANAGER.internal_base_url("http://192.168.10.150:1202/api/v1")
def test_staged_plan_binds_source_schema_codelist_scope_and_baseline(tmp_path: Path) -> None:
args = arguments(tmp_path)
release = release_2026()
_, plan = staged_plan(args, release, decision(args))
evidence = plan["evidence"]
assert evidence["feature_count"] == 121_500
assert evidence["crop_entry_count"] == 1
assert evidence["member_nis_codes"] == ["13025", "13003"]
assert evidence["baseline"]["year"] == 2025
assert len(evidence["baseline"]["manifest_sha256"]) == 64
assert evidence["baseline"]["feature_count_change_ratio"] == 0.0125
assert plan["plan_sha256"] == MANAGER.canonical_sha256(plan, "plan_sha256")
def test_modified_staged_bytes_invalidate_plan(tmp_path: Path) -> None:
args = arguments(tmp_path)
release = release_2026()
path, plan = staged_plan(args, release, decision(args))
args.plan_path = path
args.confirm_plan_sha256 = plan["plan_sha256"]
Path(plan["evidence"]["artifact_path"]).write_text("tampered", encoding="utf-8")
with pytest.raises(RuntimeError, match="incomplete or no longer match"):
MANAGER.load_staged_plan(args, release)
def test_plan_and_review_paths_must_remain_governed(tmp_path: Path) -> None:
args = arguments(tmp_path, plan_path=tmp_path / "outside.json")
with pytest.raises(RuntimeError, match="outside the governed"):
MANAGER.governed_evidence_path(args, args.plan_path)
def test_review_requires_named_approval_and_exact_plan_hash(tmp_path: Path) -> None:
args = arguments(tmp_path)
release = release_2026()
path, plan = staged_plan(args, release, decision(args))
with pytest.raises(RuntimeError, match="--approve"):
MANAGER.build_review_evidence(args, path, plan)
args.approve = True
args.reviewer = "Jens"
review = MANAGER.build_review_evidence(args, path, plan)
assert review["staged_plan_sha256"] == plan["plan_sha256"]
assert review["review_sha256"] == MANAGER.canonical_sha256(review, "review_sha256")
def test_review_tampering_and_catalog_drift_fail_closed(tmp_path: Path) -> None:
args = arguments(tmp_path, approve=True, reviewer="Jens")
release = release_2026()
path, plan = staged_plan(args, release, decision(args))
review = MANAGER.build_review_evidence(args, path, plan)
review_path = MANAGER.default_review_path(args, release.year)
MANAGER.write_json(review_path, review)
payload = json.loads(review_path.read_text(encoding="utf-8"))
payload["reviewer"] = "changed"
MANAGER.write_json(review_path, payload)
args.review_path = review_path
args.confirm_review_sha256 = review["review_sha256"]
with pytest.raises(RuntimeError, match="checksum is invalid"):
MANAGER.load_review_evidence(args, release, plan)
changed = MANAGER.fetch_release_decision_from_item(args, catalog_item(catalog_hash="b" * 64))
with pytest.raises(RuntimeError, match="evidence changed"):
MANAGER.require_catalog_unchanged(plan, changed)
def test_catalog_check_timestamp_does_not_create_false_drift(tmp_path: Path) -> None:
args = arguments(tmp_path)
first = MANAGER.fetch_release_decision_from_item(args, catalog_item(checked_at="2027-03-16T08:00:00Z"))
second = MANAGER.fetch_release_decision_from_item(args, catalog_item(checked_at="2027-03-16T09:00:00Z"))
plan = {"release": first["release"], "catalog_identity": first["catalog_identity"]}
MANAGER.require_catalog_unchanged(plan, second)
assert first["catalog_checked_at"] != second["catalog_checked_at"]
def test_current_edition_cannot_stage_or_write_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
args = arguments(tmp_path, action="stage", confirm_edition="2025-v3")
current = decision(args, remote="2025-v3", local="2025-definitive")
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: current)
monkeypatch.setattr(
MANAGER,
"run_operator",
lambda *_args, **_kwargs: pytest.fail("operator must not run for current edition"),
)
assert MANAGER.main() == 1
assert not args.evidence_root.exists()
def test_full_apply_requires_review_and_verifies_final_dataset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
args = arguments(tmp_path, action="apply", confirm_edition="2026-v3", approve=True, reviewer="Jens")
release = release_2026()
update = decision(args)
path, plan = staged_plan(args, release, update)
review = MANAGER.build_review_evidence(args, path, plan)
review_path = MANAGER.default_review_path(args, release.year)
MANAGER.write_json(review_path, review)
args.confirm_plan_sha256 = plan["plan_sha256"]
args.confirm_review_sha256 = review["review_sha256"]
current = decision(args, remote="2026-v3", local="2026-definitive")
decisions = iter((update, current))
monkeypatch.setattr(MANAGER, "parse_args", lambda: args)
monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None)
monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: next(decisions))
monkeypatch.setattr(
MANAGER,
"run_operator",
lambda *_args, **_kwargs: {
"status": "ok",
"years": [{"year": 2026, "status": "imported", "dataset_id": "dataset-2026", "feature_count": 121_500}],
},
)
assert MANAGER.main() == 0
applied = json.loads(path.with_name("applied-evidence.json").read_text(encoding="utf-8"))
assert applied["dataset_id"] == "dataset-2026"
assert applied["review_sha256"] == review["review_sha256"]
assert applied["applied_evidence_sha256"] == MANAGER.canonical_sha256(applied, "applied_evidence_sha256")
def test_manager_is_packaged_and_never_writes_vector_features_directly() -> None:
manager = (SCRIPTS / "manage_alz_agriculture_release.py").read_text(encoding="utf-8")
operator = (SCRIPTS / "provision_agricultural_parcel_history.py").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
assert "INSERT INTO vector_features" not in manager
assert "INSERT INTO vector_features" not in operator
assert "/datasets/upload" in operator
assert "COPY scripts/manage_alz_agriculture_release.py" in dockerfile
assert "py_compile scripts/manage_alz_agriculture_release.py" in readiness
+1
View File
@@ -91,6 +91,7 @@ COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_water
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
COPY scripts/manage_alz_agriculture_release.py /app/scripts/manage_alz_agriculture_release.py
COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py
COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
+9
View File
@@ -457,6 +457,15 @@ human `review` and checksum-confirmed `apply`. No additional public endpoint
is introduced. Apply delegates to the existing dataset upload contract and
creates a new immutable annual snapshot only after all evidence is unchanged.
Future definitive ALZ execution likewise remains outside the HTTP request
cycle in `scripts/manage_alz_agriculture_release.py`. It derives the exact
`agpa_<campaign>_<publication-date>_public.zip` identity from the existing
catalog response and accepts only `YYYY-v3`. `stage` is filesystem-only,
`review` is a named approval and `apply` requires exact plan/review hashes,
revalidates the current catalog and delegates to the existing Dataset upload
contract. No ALZ release endpoint, background task or provider URL parameter
is added; v1/v2 campaign snapshots remain non-importable.
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
not fetch vector features, raster pixels or models, create jobs/datasets, write
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
+43
View File
@@ -1,3 +1,46 @@
## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17)
Implemented:
- Added `scripts/manage_alz_agriculture_release.py` with separate read-only
`plan`, filesystem-only `stage`, named `review` and exact-hash `apply`
actions for the approved Kempen regional scope.
- Derived the only accepted future archive from the allowlisted definitive
`YYYY-v3` campaign and publication date exposed by the existing source-
catalog probe. Provisional v1/v2 snapshots remain visible but non-importable;
arbitrary URLs, current/older releases and catalog drift fail closed.
- Extended the existing agricultural provisioner to accept exactly one
explicitly governed future edition per run. Final response URLs and streamed
archive size are bounded, retained editions cannot be overridden and all
workspace API collections are read with stable-total pagination.
- Bound archive, normalized GeoJSON, GeoPackage schema/CRS, complete crop-code
list, scope counts/area and previous-definitive-edition manifest/deltas into
the staged plan. ZIP member/extracted size is bounded. Review and apply
require exact SHA-256 values; apply still uses DatasetService and preserves
all historical snapshots.
- Packaged the manager in the Unraid image/readiness gate and updated source,
data, API, persistence, storage and operator documentation. No endpoint,
migration, scheduler, browser fetch or frontend behavior changed.
Validation so far:
- 43 tests passed across the original ALZ importer, official catalog probe and
new release manager; 58 passed together with the Statbel release suite.
- Target Python compilation and Ruff pass. Coverage includes future release
identity, v1/v2 exclusion, pagination drift, download bounds, release
ordering, stage/apply separation, source/schema/codelist/baseline evidence,
path confinement, byte tampering, named review, catalog drift, full mocked
apply, current-edition refusal and runtime packaging.
- Complete readiness passed with 850 backend tests, 110 documented routes,
one Alembic head `202607160001`, frontend typecheck and production build.
Static full-chain Alembic SQL and shell syntax checks also passed. Local
Docker is not installed in the Codex Windows environment; Compose and live
PostGIS validation are therefore deferred to the Tower deployment gate.
Boundary:
- The official page currently advertises definitive `2025-v3` plus
provisional `2026-v1`. No newer definitive edition exists, so live stage,
review and apply must remain blocked; only a read-only plan and deliberate
current-edition refusal may be exercised after deployment.
## Sprint 228 - Governed Statbel population release promotion (2026-07-17)
Implemented:
+8
View File
@@ -283,6 +283,14 @@ which creates the ordinary annual `datasets`, `dataset_versions` and
year remains idempotent and previous annual snapshots are never updated or
deleted.
Definitive ALZ release management uses the same persistence boundary and adds
no migration or release table. Catalog planning, staged archive/GeoJSON,
crop-code evidence and named review live on the filesystem. Only an approved
apply invokes the existing Dataset upload service, producing the ordinary
annual Dataset, DatasetVersion and vector_features records with
`source_version=<year>-definitive`. Earlier annual snapshots are retained and
provisional v1/v2 publications cannot create rows.
## Geometry normalization
- User-drawn polygons arrive as EPSG:4326.
+23 -1
View File
@@ -173,7 +173,7 @@ gebeurd.
| --- | --- | --- |
| GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen |
| Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks; officiële DCAT-releaseprobe | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen |
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete officiële publicatieprobe; metricvergelijking zonder objectlineage | een nieuwere definitieve v3-editie eerst handmatig beoordelen en daarna via de bestaande begrensde operatorflow toevoegen |
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete publicatieprobe en plan-stage-review-apply promotie; metricvergelijking zonder objectlineage | alleen een nieuwere definitieve v3-editie na gestagede schema-/codelijst-/scopecontrole en benoemde review toevoegen |
| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald |
| landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven |
| bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave |
@@ -190,6 +190,22 @@ actuele `2026-v1` zichtbaar maar niet updategerechtigd blijft. De probe leest
alleen de allowlisted HTML-pagina en linkidentiteiten; ZIP-archieven worden pas
door de afzonderlijke operator opgehaald na menselijke editiebevestiging.
`scripts/manage_alz_agriculture_release.py` beheert zo'n toekomstige
definitieve editie in vier afzonderlijke operatorstappen. `plan` vergelijkt
read-only de nieuwste v3 met de lokale `*-definitive` Dataset. `stage` vereist
de exacte `YYYY-v3`, leidt de archief-URL af uit de gecontroleerde campagne en
publicatiedatum en voert de bestaande provisioner uitsluitend met
`--force --fetch-only` uit. Het staged plan bindt bronarchief, GeoPackage-
schema/CRS, genormaliseerde GeoJSON, gewascodelijst, scope-aantallen en de
verschillen met de vorige definitieve editie aan SHA-256.
`review` vereist een benoemde menselijke goedkeuring van exact dat plan.
`apply` vereist de plan- en reviewhash, controleert catalogus en alle bestanden
opnieuw en maakt via de bestaande uploadservice hoogstens een nieuwe immutable
Dataset. Een v1/v2-snapshot, huidige of oudere editie, cataloguswijziging,
gewijzigde bronbyte of evidence buiten de beheerde opslagroot blokkeert de
flow. Er is geen scheduler, browserdownload of automatische vervanging.
De Statbel-probe leest uitsluitend de officiële DCAT Turtle-catalogus en
selecteert de nieuwste unieke Nederlandstalige publicatie `Bevolking per
statistische sector`. Voor 2025 vereist GeoIntel de nieuwe REDEGEO-indeling;
@@ -409,6 +425,12 @@ annual field set, read through the GIS optional dependencies and deleted after
the normalized GeoJSON has been built. Geometry is clipped exactly against the
persisted scope Area in Lambert 72 and imported only through DatasetService.
Future definitive editions are accepted only as one explicit release per
operator run with an exact official
`agpa_<campaign>_<publication-date>_public.zip` identity. Reaching that URL is
not sufficient for persistence: the separate release manager must first stage
and bind all evidence, receive named approval and revalidate it during apply.
The regional default creates the series
`alz:agricultural-use-parcels:kempen-transport-region`; `--scope mol` creates an
independent Mol series. Selection and evolution expose exact intersected
+6
View File
@@ -307,6 +307,12 @@ official ZIP archive containing a Belgian Lambert 72 GeoPackage. Queryable
geometry is clipped against the persisted Area in EPSG:31370 and normalized to
EPSG:4326 before canonical `vector_features` persistence.
A future campaign becomes historical input only when the official publication
contract labels it `v3`. Early `v1` and `v2` snapshots are provisional and may
not create annual GeoIntel Datasets. The governed release plan binds the exact
archive publication date, schema, CRS, crop-code list, scope totals and deltas
against the previous definitive edition before named review and apply.
Stable source fields include `agpakey`, parcel number, declared source area,
reference id, spring crop, main crop, official main-crop group, production
method and source municipality. All annual source properties remain available,
+18
View File
@@ -204,6 +204,24 @@ artifact; Dataset and vector_feature rows remain the queryable PostGIS state.
The manifest binds source, crop-code list and upload artifact checksums. A
checksum conflict with an existing annual Dataset fails closed.
Future definitive-release decision evidence is stored separately from those
source artifacts:
```text
storage/operator-evidence/alz-agriculture-refresh/{scope}/{year}/
staged-plan.json
review-evidence.json
applied-evidence.json
```
The staged plan binds the official publication-page SHA-256 and definitive
archive identity to the retained archive, normalized GeoJSON, schema/CRS,
crop-code list, scope accounting and previous-edition delta. Review evidence
binds a named decision to the exact plan hash. Applied evidence binds both to
the resulting immutable Dataset id. Evidence paths outside this root and
source artifacts outside `agricultural-use-parcels` are rejected; these files
authorize neither deletion nor in-place Dataset replacement.
Buildings and Addresses Register snapshot evidence lives under:
```text
+1
View File
@@ -45,6 +45,7 @@
- [x] Add a fail-closed Statbel DCAT publication probe that distinguishes population year, sector-geometry year and the 2025 REDEGEO transition without downloading distributions.
- [x] Add a fail-closed Statbel population import preflight with archive/schema/CRS/join/total/baseline checks, retained checksums and explicit ZZZZ accounting before any new Dataset import.
- [x] Add an explicit Statbel population plan -> stage -> named review -> checksum-confirmed apply workflow that preserves every prior annual snapshot.
- [x] Add an explicit definitive ALZ plan -> stage -> named review -> checksum-confirmed apply workflow while keeping v1/v2 campaign snapshots non-importable.
- [ ] Extend catalogue probes only to additional sources that publish a stable official edition contract; do not add background polling or infer releases from HTTP dates alone.
## Governed source expansion backlog
+39
View File
@@ -1498,6 +1498,45 @@ Datasets. `--force` refreshes retained evidence but cannot silently replace a
conflicting persisted annual checksum. Use `--scope mol` for an independent
municipal series.
### Governed future definitive ALZ release
Run the four phases only inside the GeoIntel container. The project id must
belong to `Kempen Regional Workbench`:
```bash
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py plan \
--project-id <KEMPEN_PROJECT_ID> --refresh-catalog
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py stage \
--project-id <KEMPEN_PROJECT_ID> \
--confirm-edition <YYYY-v3_FROM_PLAN>
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py review \
--project-id <KEMPEN_PROJECT_ID> \
--confirm-edition <YYYY-v3_FROM_PLAN> \
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
--approve --reviewer "<OPERATOR_NAME>" \
--review-note "Schema, gewascodes, scope en jaarverschillen nagekeken"
docker exec geointel python /app/scripts/manage_alz_agriculture_release.py apply \
--project-id <KEMPEN_PROJECT_ID> \
--confirm-edition <YYYY-v3_FROM_PLAN> \
--confirm-plan-sha256 <SHA256_FROM_STAGE> \
--confirm-review-sha256 <SHA256_FROM_REVIEW>
```
`plan` is read-only. `stage` derives the exact archive from the official
catalog campaign/publication date, downloads within the 250 MiB ceiling and
runs the existing provisioner with `--force --fetch-only`. The staged plan
binds archive, GeoJSON, schema, CRS, crop-code list, scope counts and previous
definitive-edition manifest hash/deltas. ZIP member count and extracted size
are bounded before the GeoPackage is read. `review` imports nothing. `apply` revalidates the
catalog and every byte before using the canonical Dataset upload route.
Only a definitive `YYYY-v3` is eligible. Current or older editions, v1/v2
snapshots, changed catalog/source evidence and paths outside the governed
roots fail closed. Existing annual Datasets remain immutable and queryable.
## Buildings and Addresses Register snapshot
Prepare and audit the current official Mol snapshot without persistence:
+616
View File
@@ -0,0 +1,616 @@
#!/usr/bin/env python3
"""Plan, stage, review and explicitly apply one definitive ALZ edition.
Planning reads only the canonical source-catalog report. Staging downloads and
prepares one official v3 archive without database persistence. Review records
a named approval. Apply requires the exact plan and review hashes and reuses
the canonical agricultural DatasetService upload path.
"""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
from hashlib import sha256
import json
import os
from pathlib import Path
import re
import subprocess
import sys
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urlparse
from urllib.request import Request, urlopen
from geographic_scopes import GEOGRAPHIC_SCOPES
from provision_agricultural_parcel_history import (
AgriculturalReleaseConfig,
MAX_ARCHIVE_BYTES,
OUTPUT_CRS,
SOURCE_CRS,
STABLE_REQUIRED_FIELDS,
artifact_paths,
resolve_release_config,
reusable_artifact,
sha256_file,
)
DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1"
DEFAULT_SCOPE = "kempen-transport-region"
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/agricultural-use-parcels")
DEFAULT_EVIDENCE_ROOT = Path("/app/storage/operator-evidence/alz-agriculture-refresh")
SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels"
REMOTE_VERSION_PATTERN = re.compile(r"^(20[0-9]{2})-v3$")
LOCAL_VERSION_PATTERN = re.compile(r"^(20[0-9]{2})-definitive$")
SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
PROVISIONAL_VERSION_PATTERN = re.compile(r"\b(20[0-9]{2}-v[12])\b")
ACTIONABLE_STATUSES = {"update_available", "not_loaded"}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Governed definitive ALZ release: plan, stage, review, then checksum-confirmed apply."
)
parser.add_argument("action", choices=("plan", "stage", "review", "apply"))
parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope")
parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE)
parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL))
parser.add_argument("--confirm-edition", help="Exact official definitive edition, for example 2026-v3")
parser.add_argument("--confirm-plan-sha256", help="Exact staged-plan hash required for review/apply")
parser.add_argument("--confirm-review-sha256", help="Exact review-evidence hash required for apply")
parser.add_argument("--approve", action="store_true", help="Explicitly approve the staged evidence during review")
parser.add_argument("--reviewer", help="Named human/operator approving the staged evidence")
parser.add_argument("--review-note", default="", help="Optional bounded review note")
parser.add_argument("--plan-path", type=Path, help="Override the governed staged-plan path")
parser.add_argument("--review-path", type=Path, help="Override the governed review-evidence path")
parser.add_argument(
"--output-root",
type=Path,
default=Path(os.environ.get("GEOINTEL_AGRICULTURE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
)
parser.add_argument(
"--evidence-root",
type=Path,
default=Path(os.environ.get("GEOINTEL_ALZ_REFRESH_EVIDENCE_ROOT", DEFAULT_EVIDENCE_ROOT)),
)
parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short official-catalog cache")
parser.add_argument("--request-timeout", type=int, default=900)
parser.add_argument("--api-timeout", type=int, default=180)
parser.add_argument("--import-timeout", type=int, default=3600)
parser.add_argument("--max-features", type=int, default=250_000)
parser.add_argument("--max-archive-mb", type=int, default=250)
return parser.parse_args()
def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]:
endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}"
request = Request(
endpoint,
headers={"Accept": "application/json", "User-Agent": "GeoIntel-ALZ-release/1.0"},
)
try:
with urlopen(request, timeout=timeout) as response:
payload = json.load(response)
except HTTPError as exc:
body = exc.read().decode("utf-8", errors="replace")
raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc
except URLError as exc:
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
raise RuntimeError("GeoIntel API response is not a canonical data envelope")
return payload["data"]
def validate_project_scope(args: argparse.Namespace) -> None:
project = api_data(args.api_url, f"projects/{args.project_id}", args.api_timeout)
expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name
if project.get("name") != expected_name:
raise RuntimeError(
f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'"
)
def parse_catalog_datetime(value: Any) -> datetime:
if not isinstance(value, str) or not value.strip():
raise RuntimeError("The official ALZ catalog did not provide a publication date")
try:
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
except ValueError as exc:
raise RuntimeError("The official ALZ catalog publication date is invalid") from exc
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def release_from_catalog_item(item: dict[str, Any]) -> AgriculturalReleaseConfig:
match = REMOTE_VERSION_PATTERN.fullmatch(str(item.get("remote_version") or ""))
if not match:
raise RuntimeError("The official ALZ catalog did not provide one definitive YYYY-v3 edition")
year = int(match.group(1))
published_at = parse_catalog_datetime(item.get("remote_published_at"))
archive_url = (
"https://www.landbouwvlaanderen.be/bestanden/gis/"
f"agpa_{year}_{published_at.date().isoformat()}_public.zip"
)
return resolve_release_config(year, archive_url=archive_url)
def fetch_release_decision_from_item(args: argparse.Namespace, item: dict[str, Any]) -> dict[str, Any]:
allowed_degraded = item.get("error_code") == "CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING"
if (
item.get("source_name") != SOURCE_NAME
or item.get("reachable") is not True
or item.get("status") not in {"available", "degraded"}
or (item.get("status") == "degraded" and not allowed_degraded)
or "definitive_archive" not in set(item.get("matched_layers") or [])
or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or ""))
or item.get("metadata_identifier") != "alz-agricultural-use-parcels"
):
raise RuntimeError("The official ALZ catalog is not safely available for release planning")
release = release_from_catalog_item(item)
local_version = str(item.get("local_source_version") or "")
local_match = LOCAL_VERSION_PATTERN.fullmatch(local_version)
if local_version and not local_match:
status = "blocked_local_version"
elif not local_version:
status = "not_loaded"
elif int(local_match.group(1)) == release.year:
status = "current"
elif int(local_match.group(1)) < release.year:
status = "update_available"
else:
status = "blocked_remote_older"
message = str(item.get("message") or "")
provisional_match = PROVISIONAL_VERSION_PATTERN.search(message)
return {
"schema_version": 1,
"status": status,
"project_id": args.project_id,
"scope": args.scope,
"generated_at": datetime.now(timezone.utc).isoformat(),
"catalog_checked_at": item.get("checked_at"),
"local_source_version": local_version or None,
"release": {
"year": release.year,
"edition": release.definitive_version,
"archive_url": release.archive_url,
},
"catalog_identity": {
"metadata_identifier": item["metadata_identifier"],
"metadata_url": item.get("metadata_url"),
"remote_version": release.definitive_version,
"remote_published_at": item.get("remote_published_at"),
"capabilities_sha256": item["capabilities_sha256"],
},
"provisional_release": provisional_match.group(1) if provisional_match else None,
"provisional_release_importable": False,
"automatic_download": False,
"automatic_import": False,
"destructive_replacement": False,
"next_action": "stage" if status in ACTIONABLE_STATUSES else None,
}
def fetch_release_decision(args: argparse.Namespace, *, refresh: bool) -> dict[str, Any]:
query = "true" if refresh else "false"
report = api_data(
args.api_url,
f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={query}",
args.api_timeout,
)
matches = [item for item in report.get("items") or [] if item.get("source_name") == SOURCE_NAME]
if len(matches) != 1:
raise RuntimeError("Source catalog report did not contain exactly one ALZ agriculture contract")
return fetch_release_decision_from_item(args, matches[0])
def require_release_confirmation(args: argparse.Namespace, release: AgriculturalReleaseConfig) -> None:
if args.confirm_edition != release.definitive_version:
raise RuntimeError(f"Explicit --confirm-edition {release.definitive_version} is required")
def internal_base_url(api_url: str) -> str:
value = api_url.rstrip("/")
if value.endswith("/api/v1"):
value = value[:-7]
parsed = urlparse(value)
if parsed.hostname not in {"127.0.0.1", "localhost", "::1"}:
raise RuntimeError("Stage/apply must run inside GeoIntel against the local backend API")
return value
def default_plan_path(args: argparse.Namespace, year: int) -> Path:
return args.evidence_root / args.scope / str(year) / "staged-plan.json"
def default_review_path(args: argparse.Namespace, year: int) -> Path:
return args.evidence_root / args.scope / str(year) / "review-evidence.json"
def governed_evidence_path(args: argparse.Namespace, path: Path) -> Path:
if not path.resolve().is_relative_to(args.evidence_root.resolve()):
raise RuntimeError(f"Release evidence path is outside the governed evidence root: {path}")
return path
def build_operator_command(
args: argparse.Namespace,
release: AgriculturalReleaseConfig,
*,
fetch_only: bool,
) -> list[str]:
command = [
sys.executable,
str(Path(__file__).resolve().parent / "provision_agricultural_parcel_history.py"),
"--base-url",
internal_base_url(args.api_url),
"--scope",
args.scope,
"--years",
str(release.year),
"--archive-url",
release.archive_url,
"--output-root",
str(args.output_root),
"--request-timeout",
str(args.request_timeout),
"--import-timeout",
str(args.import_timeout),
"--max-features",
str(args.max_features),
"--max-archive-mb",
str(args.max_archive_mb),
]
if fetch_only:
command.extend(("--force", "--fetch-only"))
return command
def run_operator(command: list[str], *, action: str) -> dict[str, Any]:
print(f"ALZ agriculture: {action}...", file=sys.stderr, flush=True)
completed = subprocess.run(command, check=False, capture_output=True, text=True, encoding="utf-8")
if completed.returncode != 0:
detail = completed.stderr.strip() or completed.stdout.strip() or "operator returned no diagnostics"
raise RuntimeError(f"ALZ agriculture {action} failed with exit {completed.returncode}: {detail[-3000:]}")
try:
payload = json.loads(completed.stdout)
except json.JSONDecodeError as exc:
raise RuntimeError(f"ALZ agriculture {action} returned invalid JSON") from exc
if payload.get("status") != "ok":
raise RuntimeError(f"ALZ agriculture {action} did not report success")
return payload
def canonical_sha256(payload: dict[str, Any], hash_field: str) -> str:
content = {key: value for key, value in payload.items() if key != hash_field}
encoded = json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
return sha256(encoded).hexdigest()
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".partial")
temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
def _previous_manifest(output_root: Path, scope_key: str, year: int) -> tuple[Path, dict[str, Any]] | None:
scope_root = output_root / scope_key
if not scope_root.is_dir():
return None
previous_years = sorted(
(int(path.name) for path in scope_root.iterdir() if path.is_dir() and path.name.isdigit() and int(path.name) < year),
reverse=True,
)
for previous_year in previous_years:
manifests = list((scope_root / str(previous_year)).glob("*.manifest.json"))
if len(manifests) != 1:
continue
payload = json.loads(manifests[0].read_text(encoding="utf-8"))
if payload.get("year") == previous_year and payload.get("scope_key") == scope_key:
return manifests[0], payload
return None
def _relative_change(current: float, previous: float) -> float | None:
if previous <= 0:
return None
return round((current - previous) / previous, 8)
def validate_staged_release(args: argparse.Namespace, release: AgriculturalReleaseConfig) -> dict[str, Any]:
paths = artifact_paths(args.output_root, args.scope, release.year, archive_url=release.archive_url)
resolved_output = args.output_root.resolve()
for key in ("archive", "artifact", "codelist", "manifest"):
if not paths[key].resolve().is_relative_to(resolved_output):
raise RuntimeError(f"Staged ALZ {key} is outside the governed output root: {paths[key]}")
manifest = reusable_artifact(
paths,
year=release.year,
scope_key=args.scope,
archive_url=release.archive_url,
)
if manifest is None:
raise RuntimeError("Staged ALZ artifacts are incomplete or no longer match their checksums")
fields = set(manifest.get("source_fields") or [])
if (
manifest.get("source_url") != release.archive_url
or manifest.get("source_crs") != SOURCE_CRS
or manifest.get("output_crs") != OUTPUT_CRS
or not STABLE_REQUIRED_FIELDS.issubset(fields)
or int(manifest.get("feature_count") or 0) < 1
or float(manifest.get("clipped_area_ha") or 0) <= 0
or int(manifest.get("source_feature_count") or 0) < int(manifest.get("feature_count") or 0)
):
raise RuntimeError("Staged ALZ manifest does not satisfy the definitive release contract")
max_archive_bytes = min(args.max_archive_mb * 1024 * 1024, MAX_ARCHIVE_BYTES)
if paths["archive"].stat().st_size > max_archive_bytes:
raise RuntimeError("Staged ALZ archive exceeds the configured release limit")
codelist = json.loads(paths["codelist"].read_text(encoding="utf-8"))
crop_entries = codelist.get("crop_entries") if isinstance(codelist, dict) else None
if codelist.get("year") != release.year or not isinstance(crop_entries, list) or not crop_entries:
raise RuntimeError("Staged ALZ crop-code evidence is incomplete")
previous_result = _previous_manifest(args.output_root, args.scope, release.year)
baseline = None
if previous_result is not None:
previous_path, previous = previous_result
if not previous_path.resolve().is_relative_to(resolved_output):
raise RuntimeError("Previous ALZ baseline manifest is outside the governed output root")
current_count = int(manifest["feature_count"])
previous_count = int(previous.get("feature_count") or 0)
current_area = float(manifest["clipped_area_ha"])
previous_area = float(previous.get("clipped_area_ha") or 0)
baseline = {
"year": int(previous["year"]),
"manifest_path": str(previous_path),
"manifest_sha256": sha256_file(previous_path),
"feature_count": previous_count,
"clipped_area_ha": previous_area,
"feature_count_change_ratio": _relative_change(current_count, previous_count),
"clipped_area_change_ratio": _relative_change(current_area, previous_area),
}
return {
"manifest_path": str(paths["manifest"]),
"manifest_sha256": sha256_file(paths["manifest"]),
"archive_path": str(paths["archive"]),
"archive_sha256": manifest["source_archive_sha256"],
"archive_size_bytes": manifest["source_archive_size_bytes"],
"artifact_path": str(paths["artifact"]),
"artifact_sha256": manifest["artifact_sha256"],
"crop_code_list_path": str(paths["codelist"]),
"crop_code_list_sha256": manifest["crop_code_list_sha256"],
"crop_entry_count": len(crop_entries),
"crop_code_conflicts": codelist.get("code_title_conflicts") or {},
"source_feature_count": manifest["source_feature_count"],
"feature_count": manifest["feature_count"],
"clipped_feature_count": manifest["clipped_feature_count"],
"clipped_area_ha": manifest["clipped_area_ha"],
"source_fields": sorted(fields),
"member_nis_codes": manifest.get("member_nis_codes") or [],
"baseline": baseline,
}
def build_staged_plan(
args: argparse.Namespace,
decision: dict[str, Any],
release: AgriculturalReleaseConfig,
operator_result: dict[str, Any],
) -> dict[str, Any]:
payload: dict[str, Any] = {
"schema_version": 1,
"status": "staged",
"created_at": datetime.now(timezone.utc).isoformat(),
"project_id": args.project_id,
"scope": args.scope,
"local_source_version_before_apply": decision.get("local_source_version"),
"release": decision["release"],
"catalog_identity": decision["catalog_identity"],
"evidence": validate_staged_release(args, release),
"operator_result": operator_result,
"review_required": True,
"apply_requires_plan_sha256": True,
"apply_requires_review_sha256": True,
"automatic_import": False,
"destructive_replacement": False,
"existing_snapshots_retained": True,
}
payload["plan_sha256"] = canonical_sha256(payload, "plan_sha256")
return payload
def _release_from_payload(payload: dict[str, Any]) -> AgriculturalReleaseConfig:
release = payload.get("release") or {}
year = release.get("year") if isinstance(release, dict) else None
edition = release.get("edition") if isinstance(release, dict) else None
archive_url = release.get("archive_url") if isinstance(release, dict) else None
if (
not isinstance(year, int)
or isinstance(year, bool)
or edition != f"{year}-v3"
or not isinstance(archive_url, str)
):
raise RuntimeError("Release evidence is missing required ALZ identity fields")
return resolve_release_config(year, archive_url=archive_url)
def load_staged_plan(
args: argparse.Namespace,
release: AgriculturalReleaseConfig,
) -> tuple[Path, dict[str, Any]]:
path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
if not path.is_file():
raise RuntimeError(f"Staged ALZ plan is missing: {path}")
payload = json.loads(path.read_text(encoding="utf-8"))
actual_sha = canonical_sha256(payload, "plan_sha256")
if payload.get("plan_sha256") != actual_sha:
raise RuntimeError("Staged ALZ plan checksum is invalid")
if args.confirm_plan_sha256 != actual_sha:
raise RuntimeError(f"Explicit --confirm-plan-sha256 {actual_sha} is required")
if (
payload.get("status") != "staged"
or payload.get("project_id") != args.project_id
or payload.get("scope") != args.scope
or _release_from_payload(payload) != release
):
raise RuntimeError("Staged ALZ plan identity is invalid")
current_evidence = validate_staged_release(args, release)
if current_evidence != payload.get("evidence"):
raise RuntimeError("Staged ALZ artifacts no longer match the approved plan")
return path, payload
def require_catalog_unchanged(plan: dict[str, Any], decision: dict[str, Any]) -> None:
if plan.get("release") != decision.get("release") or plan.get("catalog_identity") != decision.get("catalog_identity"):
raise RuntimeError("The official ALZ release evidence changed; create and review a new staged plan")
def build_review_evidence(args: argparse.Namespace, plan_path: Path, plan: dict[str, Any]) -> dict[str, Any]:
reviewer = str(args.reviewer or "").strip()
note = str(args.review_note or "").strip()
if not args.approve or len(reviewer) < 2 or len(reviewer) > 120:
raise RuntimeError("Review requires --approve and a named --reviewer between 2 and 120 characters")
if len(note) > 1000:
raise RuntimeError("Review note must not exceed 1000 characters")
payload: dict[str, Any] = {
"schema_version": 1,
"status": "approved",
"reviewed_at": datetime.now(timezone.utc).isoformat(),
"reviewer": reviewer,
"review_note": note or None,
"project_id": args.project_id,
"scope": args.scope,
"release": plan["release"],
"staged_plan_path": str(plan_path),
"staged_plan_sha256": plan["plan_sha256"],
"reviewed_checks": [
"official_catalog_identity",
"source_archive_checksum",
"geopackage_schema_and_crs",
"scope_feature_and_area_accounting",
"crop_code_list_and_conflicts",
"previous_definitive_edition_delta",
"provisional_snapshot_exclusion",
"immutable_dataset_apply",
],
}
payload["review_sha256"] = canonical_sha256(payload, "review_sha256")
return payload
def load_review_evidence(
args: argparse.Namespace,
release: AgriculturalReleaseConfig,
plan: dict[str, Any],
) -> tuple[Path, dict[str, Any]]:
path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
if not path.is_file():
raise RuntimeError(f"Approved ALZ review evidence is missing: {path}")
payload = json.loads(path.read_text(encoding="utf-8"))
actual_sha = canonical_sha256(payload, "review_sha256")
if payload.get("review_sha256") != actual_sha:
raise RuntimeError("ALZ review evidence checksum is invalid")
if args.confirm_review_sha256 != actual_sha:
raise RuntimeError(f"Explicit --confirm-review-sha256 {actual_sha} is required")
if (
payload.get("status") != "approved"
or payload.get("project_id") != args.project_id
or payload.get("scope") != args.scope
or payload.get("release") != plan.get("release")
or payload.get("staged_plan_sha256") != plan.get("plan_sha256")
or not str(payload.get("reviewer") or "").strip()
):
raise RuntimeError("ALZ review evidence does not authorize this staged plan")
return path, payload
def _release_result(operator_result: dict[str, Any], year: int) -> dict[str, Any]:
matches = [item for item in operator_result.get("years") or [] if int(item.get("year") or 0) == year]
if len(matches) != 1:
raise RuntimeError("Agriculture operator did not return exactly one result for the approved edition")
return matches[0]
def main() -> int:
args = parse_args()
try:
if min(args.request_timeout, args.api_timeout, args.import_timeout, args.max_features, args.max_archive_mb) <= 0:
raise ValueError("All timeout and safety limits must be positive")
validate_project_scope(args)
refresh = args.refresh_catalog or args.action in {"stage", "review", "apply"}
decision = fetch_release_decision(args, refresh=refresh)
release = _release_from_payload(decision)
if args.action == "plan":
print(json.dumps({"status": "ok", "action": "plan", "decision": decision}, ensure_ascii=False, indent=2))
return 0
require_release_confirmation(args, release)
if args.action == "stage":
if decision.get("status") not in ACTIONABLE_STATUSES:
raise RuntimeError(f"ALZ release is not safely stageable: {decision.get('status')}")
operator_result = run_operator(build_operator_command(args, release, fetch_only=True), action="staging")
staged_result = _release_result(operator_result, release.year)
if staged_result.get("status") != "prepared":
raise RuntimeError("ALZ staging did not produce prepared release evidence")
plan = build_staged_plan(args, decision, release, operator_result)
plan_path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year))
write_json(plan_path, plan)
print(json.dumps({"status": "staged", "plan_path": str(plan_path), **plan}, ensure_ascii=False, indent=2))
return 0
plan_path, plan = load_staged_plan(args, release)
require_catalog_unchanged(plan, decision)
if args.action == "review":
review = build_review_evidence(args, plan_path, plan)
review_path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year))
write_json(review_path, review)
print(json.dumps({"status": "approved", "review_path": str(review_path), **review}, ensure_ascii=False, indent=2))
return 0
review_path, review = load_review_evidence(args, release, plan)
operator_result = run_operator(build_operator_command(args, release, fetch_only=False), action="applying")
applied_result = _release_result(operator_result, release.year)
if applied_result.get("status") not in {"imported", "existing"} or not applied_result.get("dataset_id"):
raise RuntimeError("ALZ apply did not return one persisted immutable Dataset")
final_decision = fetch_release_decision(args, refresh=True)
if (
final_decision.get("status") != "current"
or final_decision.get("local_source_version") != f"{release.year}-definitive"
):
raise RuntimeError("Applied ALZ Dataset did not become the current local definitive edition")
evidence = {
"schema_version": 1,
"status": "applied",
"applied_at": datetime.now(timezone.utc).isoformat(),
"project_id": args.project_id,
"scope": args.scope,
"release": plan["release"],
"staged_plan_path": str(plan_path),
"staged_plan_sha256": plan["plan_sha256"],
"review_path": str(review_path),
"review_sha256": review["review_sha256"],
"reviewer": review["reviewer"],
"dataset_id": applied_result["dataset_id"],
"dataset_status": applied_result["status"],
"feature_count": applied_result.get("feature_count"),
"final_catalog_decision": final_decision,
"existing_snapshots_retained": True,
}
evidence["applied_evidence_sha256"] = canonical_sha256(evidence, "applied_evidence_sha256")
evidence_path = plan_path.with_name("applied-evidence.json")
write_json(evidence_path, evidence)
print(json.dumps({"status": "applied", "evidence_path": str(evidence_path), **evidence}, ensure_ascii=False, indent=2))
return 0
except (OSError, RuntimeError, ValueError, KeyError, json.JSONDecodeError) as exc:
print(
json.dumps({"status": "error", "action": args.action, "message": str(exc)}, ensure_ascii=False),
file=sys.stderr,
)
return 1
if __name__ == "__main__":
raise SystemExit(main())
+127 -18
View File
@@ -10,17 +10,19 @@ directly and never uses the provisional current-campaign snapshot.
from __future__ import annotations
import argparse
from dataclasses import dataclass
import hashlib
import json
import math
import os
import shutil
import re
import sys
import tempfile
import zipfile
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any, Iterable
from urllib.parse import urlsplit
import requests
from pyproj import Transformer
@@ -51,6 +53,10 @@ SOURCE_CRS = "EPSG:31370"
OUTPUT_CRS = "EPSG:4326"
SCHEMA_VERSION = 1
MAX_ARCHIVE_BYTES = 250 * 1024 * 1024
MAX_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024
MAX_ARCHIVE_MEMBERS = 32
ARCHIVE_HOST = "www.landbouwvlaanderen.be"
ARCHIVE_PATH_PATTERN = re.compile(r"^/bestanden/gis/agpa_(20[0-9]{2})_(20[0-9]{2}-[0-9]{2}-[0-9]{2})_public\.zip$")
ARCHIVE_URLS = {
2008: "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2008_2022-03-23_public.zip",
@@ -74,6 +80,16 @@ ARCHIVE_URLS = {
}
SUPPORTED_YEARS = tuple(ARCHIVE_URLS)
@dataclass(frozen=True)
class AgriculturalReleaseConfig:
year: int
archive_url: str
@property
def definitive_version(self) -> str:
return f"{self.year}-v3"
STABLE_REQUIRED_FIELDS = {
"agpakey",
"parcelnumber",
@@ -131,6 +147,10 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
parser.add_argument("--years", default=",".join(str(year) for year in SUPPORTED_YEARS))
parser.add_argument(
"--archive-url",
help="Exact official archive URL for one explicitly confirmed future definitive edition.",
)
parser.add_argument("--output-root", type=Path, default=Path(os.environ.get("GEOINTEL_AGRICULTURE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)))
parser.add_argument("--request-timeout", type=int, default=900)
parser.add_argument("--import-timeout", type=int, default=3600)
@@ -201,21 +221,29 @@ def response_data(response: requests.Response) -> Any:
def api_items(session: requests.Session, url: str, timeout: int) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
offset = 0
while True:
expected_total: int | None = None
while expected_total is None or offset < expected_total:
response = session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout)
data = response_data(response)
if isinstance(data, dict):
page = data.get("items") or data.get("results") or []
total = int(data.get("total") or len(page))
page_total = int(data.get("total") if data.get("total") is not None else len(page))
else:
page = data
total = len(page) if isinstance(page, list) else 0
page_total = len(page) if isinstance(page, list) else 0
if not isinstance(page, list):
raise RuntimeError(f"Expected a list response from {url}")
if expected_total is None:
expected_total = page_total
elif page_total != expected_total:
raise RuntimeError("GeoIntel pagination total changed while reading the agricultural workspace")
items.extend(item for item in page if isinstance(item, dict))
if not page or len(items) >= total:
return items
if not page:
break
offset += len(page)
if expected_total is not None and len(items) != expected_total:
raise RuntimeError(f"GeoIntel list response for {url} returned {len(items)} of {expected_total} items")
return items
def locate_workspace(
@@ -251,6 +279,53 @@ def parse_years(raw: str) -> list[int]:
return years
def validate_archive_url(url: str, *, expected_year: int) -> str:
parsed = urlsplit(url)
match = ARCHIVE_PATH_PATTERN.fullmatch(parsed.path)
if (
parsed.scheme != "https"
or parsed.hostname != ARCHIVE_HOST
or parsed.port not in {None, 443}
or parsed.username
or parsed.password
or parsed.query
or parsed.fragment
or not match
or int(match.group(1)) != expected_year
):
raise ValueError("Agricultural release archive is outside the official ALZ URL contract")
try:
datetime.strptime(match.group(2), "%Y-%m-%d")
except ValueError as exc:
raise ValueError("Agricultural release archive contains an invalid publication date") from exc
return url
def resolve_release_config(year: int, *, archive_url: str | None = None) -> AgriculturalReleaseConfig:
known_url = ARCHIVE_URLS.get(year)
if known_url is not None:
if archive_url is not None and archive_url != known_url:
raise ValueError(f"The official retained archive identity for {year} may not be overridden")
return AgriculturalReleaseConfig(year=year, archive_url=known_url)
if year <= max(SUPPORTED_YEARS) or not archive_url:
raise ValueError(
f"One future definitive edition after {max(SUPPORTED_YEARS)} may be supplied with --archive-url"
)
return AgriculturalReleaseConfig(year=year, archive_url=validate_archive_url(archive_url, expected_year=year))
def resolve_release_configs(raw_years: str, *, archive_url: str | None = None) -> list[AgriculturalReleaseConfig]:
if archive_url is None:
return [resolve_release_config(year) for year in parse_years(raw_years)]
try:
years = sorted({int(value.strip()) for value in raw_years.split(",") if value.strip()})
except ValueError as exc:
raise ValueError("Years must be a comma-separated list of integers") from exc
if len(years) != 1:
raise ValueError("--archive-url requires exactly one explicitly selected definitive year")
return [resolve_release_config(years[0], archive_url=archive_url)]
def polygonal_geometry(geometry):
if geometry is None or geometry.is_empty:
return None
@@ -308,6 +383,10 @@ def download_archive(
temporary.unlink(missing_ok=True)
response = session.get(url, timeout=timeout, stream=True)
response.raise_for_status()
final_url = str(getattr(response, "url", "") or url)
expected_year = int(ARCHIVE_PATH_PATTERN.fullmatch(urlsplit(url).path).group(1))
if validate_archive_url(final_url, expected_year=expected_year) != url:
raise RuntimeError("Official archive download redirected to a different release identity")
content_length = int(response.headers.get("content-length") or 0)
if content_length > max_bytes:
raise RuntimeError(f"Official archive exceeds the configured {max_bytes // (1024 * 1024)} MiB limit")
@@ -331,7 +410,12 @@ def download_archive(
def archive_geopackage_member(path: Path) -> str:
with zipfile.ZipFile(path) as archive:
members = [item.filename for item in archive.infolist() if not item.is_dir() and item.filename.lower().endswith(".gpkg")]
entries = archive.infolist()
if len(entries) > MAX_ARCHIVE_MEMBERS:
raise RuntimeError(f"Official archive contains more than {MAX_ARCHIVE_MEMBERS} members")
if sum(item.file_size for item in entries if not item.is_dir()) > MAX_EXTRACTED_BYTES:
raise RuntimeError("Official archive exceeds the extracted-size safety limit")
members = [item.filename for item in entries if not item.is_dir() and item.filename.lower().endswith(".gpkg")]
if len(members) != 1:
raise RuntimeError(f"Official archive must contain exactly one GeoPackage; found {len(members)}")
member = members[0]
@@ -351,7 +435,12 @@ def extract_geopackage(archive_path: Path, destination_dir: Path) -> Path:
member = validate_archive(archive_path)
destination = destination_dir / Path(member).name
with zipfile.ZipFile(archive_path) as archive, archive.open(member) as source, destination.open("wb") as target:
shutil.copyfileobj(source, target, length=1024 * 1024)
extracted_bytes = 0
for chunk in iter(lambda: source.read(1024 * 1024), b""):
extracted_bytes += len(chunk)
if extracted_bytes > MAX_EXTRACTED_BYTES:
raise RuntimeError("Official GeoPackage exceeded the extracted-size safety limit while streaming")
target.write(chunk)
if destination.stat().st_size == 0:
raise RuntimeError("Extracted official GeoPackage is empty")
return destination
@@ -470,23 +559,39 @@ def normalize_frame(frame, *, year: int, boundary_lambert72, max_features: int)
}
def artifact_paths(output_root: Path, scope_key: str, year: int) -> dict[str, Path]:
def artifact_paths(
output_root: Path,
scope_key: str,
year: int,
*,
archive_url: str | None = None,
) -> dict[str, Path]:
release = resolve_release_config(year, archive_url=archive_url)
directory = output_root / scope_key / str(year)
return {
"directory": directory,
"archive": directory / Path(ARCHIVE_URLS[year]).name,
"archive": directory / Path(urlsplit(release.archive_url).path).name,
"artifact": directory / f"agricultural_use_parcels_{year}_{scope_key}.geojson",
"codelist": directory / f"agricultural_use_parcels_{year}_crop_codes.json",
"manifest": directory / f"agricultural_use_parcels_{year}_{scope_key}.manifest.json",
}
def reusable_artifact(paths: dict[str, Path], *, year: int, scope_key: str) -> dict[str, Any] | None:
def reusable_artifact(
paths: dict[str, Path],
*,
year: int,
scope_key: str,
archive_url: str | None = None,
) -> dict[str, Any] | None:
if not all(paths[key].is_file() for key in ("archive", "artifact", "codelist", "manifest")):
return None
manifest = json.loads(paths["manifest"].read_text(encoding="utf-8"))
if manifest.get("schema_version") != SCHEMA_VERSION or manifest.get("year") != year or manifest.get("scope_key") != scope_key:
return None
release = resolve_release_config(year, archive_url=archive_url)
if manifest.get("source_url") != release.archive_url:
return None
if manifest.get("source_archive_sha256") != sha256_file(paths["archive"]):
return None
if manifest.get("artifact_sha256") != sha256_file(paths["artifact"]):
@@ -508,16 +613,18 @@ def prepare_year(
max_archive_bytes: int,
max_features: int,
force: bool,
archive_url: str | None = None,
) -> tuple[dict[str, Path], dict[str, Any]]:
paths = artifact_paths(output_root, scope.key, year)
release = resolve_release_config(year, archive_url=archive_url)
paths = artifact_paths(output_root, scope.key, year, archive_url=release.archive_url)
paths["directory"].mkdir(parents=True, exist_ok=True)
if not force:
reused = reusable_artifact(paths, year=year, scope_key=scope.key)
reused = reusable_artifact(paths, year=year, scope_key=scope.key, archive_url=release.archive_url)
if reused is not None:
return paths, reused
download = download_archive(
session,
ARCHIVE_URLS[year],
release.archive_url,
paths["archive"],
timeout=request_timeout,
max_bytes=max_archive_bytes,
@@ -546,7 +653,7 @@ def prepare_year(
"scope_name": scope.display_name,
"scope_type": scope.scope_type,
"member_nis_codes": list(scope.nis_codes),
"source_url": ARCHIVE_URLS[year],
"source_url": release.archive_url,
"catalog_url": CATALOG_URL,
"data_catalog_url": DATA_CATALOG_URL,
"attribution": ATTRIBUTION,
@@ -635,7 +742,7 @@ def upload_artifact(
"operator_tool": "provision_agricultural_parcel_history.py",
"operator_explicit_fetch": True,
"geometry_clipped_to_area": True,
"source_archive_url": ARCHIVE_URLS[year],
"source_archive_url": manifest["source_url"],
"source_archive_path": str(paths["archive"]),
"source_archive_sha256": manifest["source_archive_sha256"],
"crop_code_list_path": str(paths["codelist"]),
@@ -684,7 +791,7 @@ def existing_dataset_for_year(datasets: list[dict[str, Any]], *, area_id: str, y
def main() -> int:
args = parse_args()
try:
years = parse_years(args.years)
releases = resolve_release_configs(args.years, archive_url=args.archive_url)
if args.max_features < 1 or args.max_archive_mb < 1:
raise ValueError("Feature and archive safety limits must be positive")
scope = GEOGRAPHIC_SCOPES[args.scope]
@@ -693,7 +800,8 @@ def main() -> int:
project_id, area_id, boundary, datasets = locate_workspace(api_session, base_url, scope, args.import_timeout)
results: list[dict[str, Any]] = []
with build_session() as official_session:
for year in years:
for release in releases:
year = release.year
paths, manifest = prepare_year(
official_session,
year=year,
@@ -704,6 +812,7 @@ def main() -> int:
max_archive_bytes=min(args.max_archive_mb * 1024 * 1024, MAX_ARCHIVE_BYTES),
max_features=args.max_features,
force=args.force,
archive_url=release.archive_url,
)
existing = existing_dataset_for_year(datasets, area_id=area_id, year=year)
if existing is not None:
+1
View File
@@ -54,6 +54,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
${PYTHON_BIN} -m py_compile scripts/manage_alz_agriculture_release.py
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py