From ad2c3481c4817dc6fa1d010677b1ed8ce99f0cbf Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 17 Jul 2026 00:53:34 +0200 Subject: [PATCH] Add governed ALZ release promotion --- CHANGELOG.md | 17 + backend/README.md | 29 + .../test_sprint229_alz_release_management.py | 419 ++++++++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docs/API_CONTRACTS.md | 9 + docs/CODEX_EXECUTION_LOG.md | 43 ++ docs/DATABASE_IMPLEMENTATION_PLAN.md | 8 + docs/DATA_SOURCES.md | 24 +- docs/DATA_SPECIFICATION.md | 6 + docs/STORAGE_ARCHITECTURE.md | 18 + docs/TODO.md | 1 + scripts/README.md | 39 ++ scripts/manage_alz_agriculture_release.py | 616 ++++++++++++++++++ .../provision_agricultural_parcel_history.py | 145 ++++- scripts/run_readiness_check.sh | 1 + 15 files changed, 1357 insertions(+), 19 deletions(-) create mode 100644 backend/tests/test_sprint229_alz_release_management.py create mode 100644 scripts/manage_alz_agriculture_release.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 90a8b597..eb1b89a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ # Changelog +## Sprint 229 Governed ALZ definitive release promotion (2026-07-17) + +- Added an operator-only `plan -> stage -> review -> apply` coordinator for + future definitive ALZ agricultural-use parcel editions. A provisional v1/v2 + campaign snapshot remains visible release evidence but is never importable. +- Derived the exact archive identity from the allowlisted publication year and + date reported by the existing source-catalog probe. Arbitrary provider URLs, + current/older releases, catalog drift and changed staged bytes fail closed. +- Extended the existing agricultural provisioner to accept exactly one + explicitly governed future edition, validate final download identity, keep + streamed size bounds and read every paginated workspace Dataset consistently. +- Bound archive, normalized GeoJSON, GeoPackage schema/CRS, crop-code list, + scope totals and previous-edition deltas into named human review evidence. + Apply still creates only an immutable Dataset through DatasetService. +- Added focused trust-boundary, tamper, drift, approval, apply and packaging + tests without adding an API route, migration, scheduler or frontend action. + ## Sprint 228 Governed Statbel population release promotion (2026-07-17) - Added an operator-only `plan -> stage -> review -> apply` coordinator for diff --git a/backend/README.md b/backend/README.md index f3a64206..451e8c4c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1219,6 +1219,35 @@ transport region. Every annual source ZIP and crop code list remains under the storage volume. PostGIS computes exact hectares for drawn rectangles and persisted Areas; parcel identities are deliberately unavailable for lineage. +Future definitive editions use the separate four-phase release coordinator: + +```bash +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py plan \ + --project-id --refresh-catalog + +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py stage \ + --project-id \ + --confirm-edition + +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py review \ + --project-id \ + --confirm-edition \ + --confirm-plan-sha256 \ + --approve --reviewer "" + +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py apply \ + --project-id \ + --confirm-edition \ + --confirm-plan-sha256 \ + --confirm-review-sha256 +``` + +The manager accepts only one catalog-confirmed definitive v3 release. `plan` +writes nothing; `stage` downloads and normalizes without PostGIS mutation; +`review` binds a named approval; `apply` revalidates catalog, hashes, schema, +crop codes, scope accounting and previous-edition deltas before delegating to +DatasetService. Provisional v1/v2 snapshots never enter the historical series. + ## Buildings and Addresses Register snapshot After the Mol Area and regional GRB buildings have been provisioned, prepare diff --git a/backend/tests/test_sprint229_alz_release_management.py b/backend/tests/test_sprint229_alz_release_management.py new file mode 100644 index 00000000..edaf1657 --- /dev/null +++ b/backend/tests/test_sprint229_alz_release_management.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import argparse +import importlib.util +import json +from pathlib import Path +import sys +import zipfile + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + + +def load_script(name: str): + path = SCRIPTS / name + module_name = f"test_{path.stem}_sprint229" + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +OPERATOR = load_script("provision_agricultural_parcel_history.py") +MANAGER = load_script("manage_alz_agriculture_release.py") + + +def arguments(tmp_path: Path, **overrides) -> argparse.Namespace: + values = { + "action": "plan", + "project_id": "00000000-0000-0000-0000-000000000001", + "scope": "kempen-transport-region", + "api_url": "http://127.0.0.1:8000/api/v1", + "confirm_edition": None, + "confirm_plan_sha256": None, + "confirm_review_sha256": None, + "approve": False, + "reviewer": None, + "review_note": "", + "plan_path": None, + "review_path": None, + "output_root": tmp_path / "operator-evidence" / "agricultural-use-parcels", + "evidence_root": tmp_path / "operator-evidence" / "alz-agriculture-refresh", + "refresh_catalog": False, + "request_timeout": 900, + "api_timeout": 180, + "import_timeout": 3600, + "max_features": 250_000, + "max_archive_mb": 250, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def catalog_item( + *, + remote: str = "2026-v3", + local: str | None = "2025-definitive", + catalog_hash: str = "a" * 64, + checked_at: str = "2027-03-16T08:00:00Z", +) -> dict: + year = int(remote[:4]) + published = "2027-03-15T00:00:00Z" if year == 2026 else "2026-05-13T00:00:00Z" + return { + "source_name": MANAGER.SOURCE_NAME, + "status": "available", + "reachable": True, + "error_code": None, + "remote_version": remote, + "remote_published_at": published, + "local_source_version": local, + "message": f"Definitieve editie {remote}; actuele publicatie {year + 1}-v1 is voorlopig.", + "matched_layers": ["definitive_archive", "current_snapshot"], + "capabilities_sha256": catalog_hash, + "metadata_identifier": "alz-agricultural-use-parcels", + "metadata_url": "https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen", + "checked_at": checked_at, + } + + +def decision(args: argparse.Namespace, *, remote: str = "2026-v3", local: str | None = "2025-definitive") -> dict: + return MANAGER.fetch_release_decision_from_item(args, catalog_item(remote=remote, local=local)) + + +def release_2026(): + return MANAGER.release_from_catalog_item(catalog_item()) + + +def write_staged_artifacts(args: argparse.Namespace, release) -> dict: + paths = OPERATOR.artifact_paths( + args.output_root, + args.scope, + release.year, + archive_url=release.archive_url, + ) + paths["directory"].mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(paths["archive"], "w") as archive: + archive.writestr(f"agpa_{release.year}.gpkg", b"official geopackage") + paths["artifact"].write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8") + codelist = { + "year": release.year, + "crop_entries": [{"code": "201", "title": "Mais", "group_title": "Mais"}], + "code_title_conflicts": {}, + } + paths["codelist"].write_text(json.dumps(codelist), encoding="utf-8") + baseline_dir = args.output_root / args.scope / "2025" + baseline_dir.mkdir(parents=True, exist_ok=True) + (baseline_dir / "agricultural_use_parcels_2025_kempen-transport-region.manifest.json").write_text( + json.dumps( + { + "year": 2025, + "scope_key": args.scope, + "feature_count": 120_000, + "clipped_area_ha": 62_000.0, + } + ), + encoding="utf-8", + ) + manifest = { + "schema_version": 1, + "year": release.year, + "scope_key": args.scope, + "member_nis_codes": ["13025", "13003"], + "source_url": release.archive_url, + "source_crs": OPERATOR.SOURCE_CRS, + "output_crs": OPERATOR.OUTPUT_CRS, + "source_archive_sha256": OPERATOR.sha256_file(paths["archive"]), + "source_archive_size_bytes": paths["archive"].stat().st_size, + "source_feature_count": 180_000, + "source_fields": sorted(OPERATOR.STABLE_REQUIRED_FIELDS), + "crop_code_list_sha256": OPERATOR.sha256_file(paths["codelist"]), + "artifact_sha256": OPERATOR.sha256_file(paths["artifact"]), + "feature_count": 121_500, + "clipped_feature_count": 800, + "clipped_area_ha": 62_500.0, + } + paths["manifest"].write_text(json.dumps(manifest), encoding="utf-8") + return paths + + +def staged_plan(args: argparse.Namespace, release, release_decision: dict) -> tuple[Path, dict]: + write_staged_artifacts(args, release) + result = { + "status": "ok", + "scope": args.scope, + "years": [{"year": release.year, "status": "prepared", "feature_count": 121_500}], + } + plan = MANAGER.build_staged_plan(args, release_decision, release, result) + path = MANAGER.default_plan_path(args, release.year) + MANAGER.write_json(path, plan) + return path, plan + + +def test_future_release_config_accepts_only_one_exact_official_edition() -> None: + url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip" + release = OPERATOR.resolve_release_config(2026, archive_url=url) + + assert release.definitive_version == "2026-v3" + assert release.archive_url == url + future = OPERATOR.resolve_release_config(2027, archive_url=url.replace("2026", "2027")) + assert future.definitive_version == "2027-v3" + with pytest.raises(ValueError, match="official ALZ URL"): + OPERATOR.resolve_release_config(2026, archive_url=url.replace("www.landbouwvlaanderen.be", "example.com")) + with pytest.raises(ValueError, match="may not be overridden"): + OPERATOR.resolve_release_config(2025, archive_url=url.replace("2026", "2025")) + + +def test_future_archive_requires_one_selected_year() -> None: + url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip" + with pytest.raises(ValueError, match="exactly one"): + OPERATOR.resolve_release_configs("2025,2026", archive_url=url) + + +def test_agriculture_workspace_pagination_is_complete_and_total_consistent() -> None: + rows = [{"id": index} for index in range(401)] + + class Response: + ok = True + status_code = 200 + text = "" + + def __init__(self, payload: dict) -> None: + self.payload = payload + + def json(self) -> dict: + return {"data": self.payload} + + class Session: + drift = False + + def get(self, _url: str, *, params: dict, timeout: int): + assert timeout == 30 + offset = int(params["offset"]) + total = len(rows) + (1 if self.drift and offset else 0) + return Response({"items": rows[offset : offset + int(params["limit"])], "total": total}) + + assert OPERATOR.api_items(Session(), "http://backend/datasets", 30) == rows + drifting = Session() + drifting.drift = True + with pytest.raises(RuntimeError, match="total changed"): + OPERATOR.api_items(drifting, "http://backend/datasets", 30) + + +def test_archive_download_rejects_oversize_before_streaming(tmp_path: Path) -> None: + url = "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2026_2027-03-15_public.zip" + + class Response: + headers = {"content-length": "101"} + + def __init__(self) -> None: + self.url = url + + def raise_for_status(self) -> None: + return None + + def iter_content(self, *, chunk_size: int): + raise AssertionError(f"streaming should not start: {chunk_size}") + + class Session: + def get(self, *_args, **_kwargs): + return Response() + + with pytest.raises(RuntimeError, match="configured"): + OPERATOR.download_archive(Session(), url, tmp_path / "source.zip", timeout=30, max_bytes=100, force=True) + + +def test_archive_rejects_extracted_size_over_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + archive_path = tmp_path / "source.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("agpa_2026.gpkg", b"0123456789") + monkeypatch.setattr(OPERATOR, "MAX_EXTRACTED_BYTES", 9) + + with pytest.raises(RuntimeError, match="extracted-size safety limit"): + OPERATOR.validate_archive(archive_path) + + +@pytest.mark.parametrize( + ("remote", "local", "expected"), + [ + ("2026-v3", "2025-definitive", "update_available"), + ("2025-v3", "2025-definitive", "current"), + ("2026-v3", None, "not_loaded"), + ("2025-v3", "2026-definitive", "blocked_remote_older"), + ], +) +def test_catalog_decision_orders_only_definitive_editions( + tmp_path: Path, + remote: str, + local: str | None, + expected: str, +) -> None: + assert decision(arguments(tmp_path), remote=remote, local=local)["status"] == expected + + +def test_catalog_decision_keeps_provisional_snapshot_non_importable(tmp_path: Path) -> None: + release_decision = decision(arguments(tmp_path)) + + assert release_decision["release"]["edition"] == "2026-v3" + assert release_decision["provisional_release"] == "2027-v1" + assert release_decision["provisional_release_importable"] is False + with pytest.raises(RuntimeError, match="definitive YYYY-v3"): + MANAGER.fetch_release_decision_from_item(arguments(tmp_path), catalog_item(remote="2026-v1")) + + +def test_stage_and_apply_commands_are_separate_and_local_only(tmp_path: Path) -> None: + args = arguments(tmp_path) + release = release_2026() + stage = MANAGER.build_operator_command(args, release, fetch_only=True) + apply = MANAGER.build_operator_command(args, release, fetch_only=False) + + assert "--force" in stage and "--fetch-only" in stage + assert "--force" not in apply and "--fetch-only" not in apply + assert stage[stage.index("--archive-url") + 1] == release.archive_url + with pytest.raises(RuntimeError, match="inside GeoIntel"): + MANAGER.internal_base_url("http://192.168.10.150:1202/api/v1") + + +def test_staged_plan_binds_source_schema_codelist_scope_and_baseline(tmp_path: Path) -> None: + args = arguments(tmp_path) + release = release_2026() + _, plan = staged_plan(args, release, decision(args)) + + evidence = plan["evidence"] + assert evidence["feature_count"] == 121_500 + assert evidence["crop_entry_count"] == 1 + assert evidence["member_nis_codes"] == ["13025", "13003"] + assert evidence["baseline"]["year"] == 2025 + assert len(evidence["baseline"]["manifest_sha256"]) == 64 + assert evidence["baseline"]["feature_count_change_ratio"] == 0.0125 + assert plan["plan_sha256"] == MANAGER.canonical_sha256(plan, "plan_sha256") + + +def test_modified_staged_bytes_invalidate_plan(tmp_path: Path) -> None: + args = arguments(tmp_path) + release = release_2026() + path, plan = staged_plan(args, release, decision(args)) + args.plan_path = path + args.confirm_plan_sha256 = plan["plan_sha256"] + Path(plan["evidence"]["artifact_path"]).write_text("tampered", encoding="utf-8") + + with pytest.raises(RuntimeError, match="incomplete or no longer match"): + MANAGER.load_staged_plan(args, release) + + +def test_plan_and_review_paths_must_remain_governed(tmp_path: Path) -> None: + args = arguments(tmp_path, plan_path=tmp_path / "outside.json") + with pytest.raises(RuntimeError, match="outside the governed"): + MANAGER.governed_evidence_path(args, args.plan_path) + + +def test_review_requires_named_approval_and_exact_plan_hash(tmp_path: Path) -> None: + args = arguments(tmp_path) + release = release_2026() + path, plan = staged_plan(args, release, decision(args)) + + with pytest.raises(RuntimeError, match="--approve"): + MANAGER.build_review_evidence(args, path, plan) + args.approve = True + args.reviewer = "Jens" + review = MANAGER.build_review_evidence(args, path, plan) + assert review["staged_plan_sha256"] == plan["plan_sha256"] + assert review["review_sha256"] == MANAGER.canonical_sha256(review, "review_sha256") + + +def test_review_tampering_and_catalog_drift_fail_closed(tmp_path: Path) -> None: + args = arguments(tmp_path, approve=True, reviewer="Jens") + release = release_2026() + path, plan = staged_plan(args, release, decision(args)) + review = MANAGER.build_review_evidence(args, path, plan) + review_path = MANAGER.default_review_path(args, release.year) + MANAGER.write_json(review_path, review) + payload = json.loads(review_path.read_text(encoding="utf-8")) + payload["reviewer"] = "changed" + MANAGER.write_json(review_path, payload) + args.review_path = review_path + args.confirm_review_sha256 = review["review_sha256"] + + with pytest.raises(RuntimeError, match="checksum is invalid"): + MANAGER.load_review_evidence(args, release, plan) + changed = MANAGER.fetch_release_decision_from_item(args, catalog_item(catalog_hash="b" * 64)) + with pytest.raises(RuntimeError, match="evidence changed"): + MANAGER.require_catalog_unchanged(plan, changed) + + +def test_catalog_check_timestamp_does_not_create_false_drift(tmp_path: Path) -> None: + args = arguments(tmp_path) + first = MANAGER.fetch_release_decision_from_item(args, catalog_item(checked_at="2027-03-16T08:00:00Z")) + second = MANAGER.fetch_release_decision_from_item(args, catalog_item(checked_at="2027-03-16T09:00:00Z")) + plan = {"release": first["release"], "catalog_identity": first["catalog_identity"]} + + MANAGER.require_catalog_unchanged(plan, second) + assert first["catalog_checked_at"] != second["catalog_checked_at"] + + +def test_current_edition_cannot_stage_or_write_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + args = arguments(tmp_path, action="stage", confirm_edition="2025-v3") + current = decision(args, remote="2025-v3", local="2025-definitive") + monkeypatch.setattr(MANAGER, "parse_args", lambda: args) + monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None) + monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: current) + monkeypatch.setattr( + MANAGER, + "run_operator", + lambda *_args, **_kwargs: pytest.fail("operator must not run for current edition"), + ) + + assert MANAGER.main() == 1 + assert not args.evidence_root.exists() + + +def test_full_apply_requires_review_and_verifies_final_dataset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + args = arguments(tmp_path, action="apply", confirm_edition="2026-v3", approve=True, reviewer="Jens") + release = release_2026() + update = decision(args) + path, plan = staged_plan(args, release, update) + review = MANAGER.build_review_evidence(args, path, plan) + review_path = MANAGER.default_review_path(args, release.year) + MANAGER.write_json(review_path, review) + args.confirm_plan_sha256 = plan["plan_sha256"] + args.confirm_review_sha256 = review["review_sha256"] + current = decision(args, remote="2026-v3", local="2026-definitive") + decisions = iter((update, current)) + monkeypatch.setattr(MANAGER, "parse_args", lambda: args) + monkeypatch.setattr(MANAGER, "validate_project_scope", lambda _args: None) + monkeypatch.setattr(MANAGER, "fetch_release_decision", lambda *_args, **_kwargs: next(decisions)) + monkeypatch.setattr( + MANAGER, + "run_operator", + lambda *_args, **_kwargs: { + "status": "ok", + "years": [{"year": 2026, "status": "imported", "dataset_id": "dataset-2026", "feature_count": 121_500}], + }, + ) + + assert MANAGER.main() == 0 + applied = json.loads(path.with_name("applied-evidence.json").read_text(encoding="utf-8")) + assert applied["dataset_id"] == "dataset-2026" + assert applied["review_sha256"] == review["review_sha256"] + assert applied["applied_evidence_sha256"] == MANAGER.canonical_sha256(applied, "applied_evidence_sha256") + + +def test_manager_is_packaged_and_never_writes_vector_features_directly() -> None: + manager = (SCRIPTS / "manage_alz_agriculture_release.py").read_text(encoding="utf-8") + operator = (SCRIPTS / "provision_agricultural_parcel_history.py").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert "INSERT INTO vector_features" not in manager + assert "INSERT INTO vector_features" not in operator + assert "/datasets/upload" in operator + assert "COPY scripts/manage_alz_agriculture_release.py" in dockerfile + assert "py_compile scripts/manage_alz_agriculture_release.py" in readiness diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 2f789f98..19a9c351 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -91,6 +91,7 @@ COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_water COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py +COPY scripts/manage_alz_agriculture_release.py /app/scripts/manage_alz_agriculture_release.py COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index a2e3124a..078db80e 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -457,6 +457,15 @@ human `review` and checksum-confirmed `apply`. No additional public endpoint is introduced. Apply delegates to the existing dataset upload contract and creates a new immutable annual snapshot only after all evidence is unchanged. +Future definitive ALZ execution likewise remains outside the HTTP request +cycle in `scripts/manage_alz_agriculture_release.py`. It derives the exact +`agpa___public.zip` identity from the existing +catalog response and accepts only `YYYY-v3`. `stage` is filesystem-only, +`review` is a named approval and `apply` requires exact plan/review hashes, +revalidates the current catalog and delegates to the existing Dataset upload +contract. No ALZ release endpoint, background task or provider URL parameter +is added; v1/v2 campaign snapshots remain non-importable. + The endpoint accepts no arbitrary URL, feature query, area or layer. It does not fetch vector features, raster pixels or models, create jobs/datasets, write to PostGIS or trigger an import. The normal `source-freshness` endpoint remains diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index a418578f..03b07042 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,46 @@ +## Sprint 229 - Governed ALZ definitive release promotion (2026-07-17) + +Implemented: +- Added `scripts/manage_alz_agriculture_release.py` with separate read-only + `plan`, filesystem-only `stage`, named `review` and exact-hash `apply` + actions for the approved Kempen regional scope. +- Derived the only accepted future archive from the allowlisted definitive + `YYYY-v3` campaign and publication date exposed by the existing source- + catalog probe. Provisional v1/v2 snapshots remain visible but non-importable; + arbitrary URLs, current/older releases and catalog drift fail closed. +- Extended the existing agricultural provisioner to accept exactly one + explicitly governed future edition per run. Final response URLs and streamed + archive size are bounded, retained editions cannot be overridden and all + workspace API collections are read with stable-total pagination. +- Bound archive, normalized GeoJSON, GeoPackage schema/CRS, complete crop-code + list, scope counts/area and previous-definitive-edition manifest/deltas into + the staged plan. ZIP member/extracted size is bounded. Review and apply + require exact SHA-256 values; apply still uses DatasetService and preserves + all historical snapshots. +- Packaged the manager in the Unraid image/readiness gate and updated source, + data, API, persistence, storage and operator documentation. No endpoint, + migration, scheduler, browser fetch or frontend behavior changed. + +Validation so far: +- 43 tests passed across the original ALZ importer, official catalog probe and + new release manager; 58 passed together with the Statbel release suite. +- Target Python compilation and Ruff pass. Coverage includes future release + identity, v1/v2 exclusion, pagination drift, download bounds, release + ordering, stage/apply separation, source/schema/codelist/baseline evidence, + path confinement, byte tampering, named review, catalog drift, full mocked + apply, current-edition refusal and runtime packaging. +- Complete readiness passed with 850 backend tests, 110 documented routes, + one Alembic head `202607160001`, frontend typecheck and production build. + Static full-chain Alembic SQL and shell syntax checks also passed. Local + Docker is not installed in the Codex Windows environment; Compose and live + PostGIS validation are therefore deferred to the Tower deployment gate. + +Boundary: +- The official page currently advertises definitive `2025-v3` plus + provisional `2026-v1`. No newer definitive edition exists, so live stage, + review and apply must remain blocked; only a read-only plan and deliberate + current-edition refusal may be exercised after deployment. + ## Sprint 228 - Governed Statbel population release promotion (2026-07-17) Implemented: diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index e5b6d707..8d789113 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -283,6 +283,14 @@ which creates the ordinary annual `datasets`, `dataset_versions` and year remains idempotent and previous annual snapshots are never updated or deleted. +Definitive ALZ release management uses the same persistence boundary and adds +no migration or release table. Catalog planning, staged archive/GeoJSON, +crop-code evidence and named review live on the filesystem. Only an approved +apply invokes the existing Dataset upload service, producing the ordinary +annual Dataset, DatasetVersion and vector_features records with +`source_version=-definitive`. Earlier annual snapshots are retained and +provisional v1/v2 publications cannot create rows. + ## Geometry normalization - User-drawn polygons arrive as EPSG:4326. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 161ae11d..22a6e71b 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -173,7 +173,7 @@ gebeurd. | --- | --- | --- | | GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen | | Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks; officiële DCAT-releaseprobe | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen | -| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete officiële publicatieprobe; metricvergelijking zonder objectlineage | een nieuwere definitieve v3-editie eerst handmatig beoordelen en daarna via de bestaande begrensde operatorflow toevoegen | +| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete publicatieprobe en plan-stage-review-apply promotie; metricvergelijking zonder objectlineage | alleen een nieuwere definitieve v3-editie na gestagede schema-/codelijst-/scopecontrole en benoemde review toevoegen | | orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald | | landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven | | bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave | @@ -190,6 +190,22 @@ actuele `2026-v1` zichtbaar maar niet updategerechtigd blijft. De probe leest alleen de allowlisted HTML-pagina en linkidentiteiten; ZIP-archieven worden pas door de afzonderlijke operator opgehaald na menselijke editiebevestiging. +`scripts/manage_alz_agriculture_release.py` beheert zo'n toekomstige +definitieve editie in vier afzonderlijke operatorstappen. `plan` vergelijkt +read-only de nieuwste v3 met de lokale `*-definitive` Dataset. `stage` vereist +de exacte `YYYY-v3`, leidt de archief-URL af uit de gecontroleerde campagne en +publicatiedatum en voert de bestaande provisioner uitsluitend met +`--force --fetch-only` uit. Het staged plan bindt bronarchief, GeoPackage- +schema/CRS, genormaliseerde GeoJSON, gewascodelijst, scope-aantallen en de +verschillen met de vorige definitieve editie aan SHA-256. + +`review` vereist een benoemde menselijke goedkeuring van exact dat plan. +`apply` vereist de plan- en reviewhash, controleert catalogus en alle bestanden +opnieuw en maakt via de bestaande uploadservice hoogstens een nieuwe immutable +Dataset. Een v1/v2-snapshot, huidige of oudere editie, cataloguswijziging, +gewijzigde bronbyte of evidence buiten de beheerde opslagroot blokkeert de +flow. Er is geen scheduler, browserdownload of automatische vervanging. + De Statbel-probe leest uitsluitend de officiële DCAT Turtle-catalogus en selecteert de nieuwste unieke Nederlandstalige publicatie `Bevolking per statistische sector`. Voor 2025 vereist GeoIntel de nieuwe REDEGEO-indeling; @@ -409,6 +425,12 @@ annual field set, read through the GIS optional dependencies and deleted after the normalized GeoJSON has been built. Geometry is clipped exactly against the persisted scope Area in Lambert 72 and imported only through DatasetService. +Future definitive editions are accepted only as one explicit release per +operator run with an exact official +`agpa___public.zip` identity. Reaching that URL is +not sufficient for persistence: the separate release manager must first stage +and bind all evidence, receive named approval and revalidate it during apply. + The regional default creates the series `alz:agricultural-use-parcels:kempen-transport-region`; `--scope mol` creates an independent Mol series. Selection and evolution expose exact intersected diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index eef48407..8ddb2201 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -307,6 +307,12 @@ official ZIP archive containing a Belgian Lambert 72 GeoPackage. Queryable geometry is clipped against the persisted Area in EPSG:31370 and normalized to EPSG:4326 before canonical `vector_features` persistence. +A future campaign becomes historical input only when the official publication +contract labels it `v3`. Early `v1` and `v2` snapshots are provisional and may +not create annual GeoIntel Datasets. The governed release plan binds the exact +archive publication date, schema, CRS, crop-code list, scope totals and deltas +against the previous definitive edition before named review and apply. + Stable source fields include `agpakey`, parcel number, declared source area, reference id, spring crop, main crop, official main-crop group, production method and source municipality. All annual source properties remain available, diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 402371f5..32cb1804 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -204,6 +204,24 @@ artifact; Dataset and vector_feature rows remain the queryable PostGIS state. The manifest binds source, crop-code list and upload artifact checksums. A checksum conflict with an existing annual Dataset fails closed. +Future definitive-release decision evidence is stored separately from those +source artifacts: + +```text +storage/operator-evidence/alz-agriculture-refresh/{scope}/{year}/ + staged-plan.json + review-evidence.json + applied-evidence.json +``` + +The staged plan binds the official publication-page SHA-256 and definitive +archive identity to the retained archive, normalized GeoJSON, schema/CRS, +crop-code list, scope accounting and previous-edition delta. Review evidence +binds a named decision to the exact plan hash. Applied evidence binds both to +the resulting immutable Dataset id. Evidence paths outside this root and +source artifacts outside `agricultural-use-parcels` are rejected; these files +authorize neither deletion nor in-place Dataset replacement. + Buildings and Addresses Register snapshot evidence lives under: ```text diff --git a/docs/TODO.md b/docs/TODO.md index eecc0b72..267b241b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -45,6 +45,7 @@ - [x] Add a fail-closed Statbel DCAT publication probe that distinguishes population year, sector-geometry year and the 2025 REDEGEO transition without downloading distributions. - [x] Add a fail-closed Statbel population import preflight with archive/schema/CRS/join/total/baseline checks, retained checksums and explicit ZZZZ accounting before any new Dataset import. - [x] Add an explicit Statbel population plan -> stage -> named review -> checksum-confirmed apply workflow that preserves every prior annual snapshot. +- [x] Add an explicit definitive ALZ plan -> stage -> named review -> checksum-confirmed apply workflow while keeping v1/v2 campaign snapshots non-importable. - [ ] Extend catalogue probes only to additional sources that publish a stable official edition contract; do not add background polling or infer releases from HTTP dates alone. ## Governed source expansion backlog diff --git a/scripts/README.md b/scripts/README.md index 652c6150..9a5129fc 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1498,6 +1498,45 @@ Datasets. `--force` refreshes retained evidence but cannot silently replace a conflicting persisted annual checksum. Use `--scope mol` for an independent municipal series. +### Governed future definitive ALZ release + +Run the four phases only inside the GeoIntel container. The project id must +belong to `Kempen Regional Workbench`: + +```bash +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py plan \ + --project-id --refresh-catalog + +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py stage \ + --project-id \ + --confirm-edition + +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py review \ + --project-id \ + --confirm-edition \ + --confirm-plan-sha256 \ + --approve --reviewer "" \ + --review-note "Schema, gewascodes, scope en jaarverschillen nagekeken" + +docker exec geointel python /app/scripts/manage_alz_agriculture_release.py apply \ + --project-id \ + --confirm-edition \ + --confirm-plan-sha256 \ + --confirm-review-sha256 +``` + +`plan` is read-only. `stage` derives the exact archive from the official +catalog campaign/publication date, downloads within the 250 MiB ceiling and +runs the existing provisioner with `--force --fetch-only`. The staged plan +binds archive, GeoJSON, schema, CRS, crop-code list, scope counts and previous +definitive-edition manifest hash/deltas. ZIP member count and extracted size +are bounded before the GeoPackage is read. `review` imports nothing. `apply` revalidates the +catalog and every byte before using the canonical Dataset upload route. + +Only a definitive `YYYY-v3` is eligible. Current or older editions, v1/v2 +snapshots, changed catalog/source evidence and paths outside the governed +roots fail closed. Existing annual Datasets remain immutable and queryable. + ## Buildings and Addresses Register snapshot Prepare and audit the current official Mol snapshot without persistence: diff --git a/scripts/manage_alz_agriculture_release.py b/scripts/manage_alz_agriculture_release.py new file mode 100644 index 00000000..896a7aaa --- /dev/null +++ b/scripts/manage_alz_agriculture_release.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python3 +"""Plan, stage, review and explicitly apply one definitive ALZ edition. + +Planning reads only the canonical source-catalog report. Staging downloads and +prepares one official v3 archive without database persistence. Review records +a named approval. Apply requires the exact plan and review hashes and reuses +the canonical agricultural DatasetService upload path. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +from hashlib import sha256 +import json +import os +from pathlib import Path +import re +import subprocess +import sys +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from geographic_scopes import GEOGRAPHIC_SCOPES +from provision_agricultural_parcel_history import ( + AgriculturalReleaseConfig, + MAX_ARCHIVE_BYTES, + OUTPUT_CRS, + SOURCE_CRS, + STABLE_REQUIRED_FIELDS, + artifact_paths, + resolve_release_config, + reusable_artifact, + sha256_file, +) + + +DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1" +DEFAULT_SCOPE = "kempen-transport-region" +DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/agricultural-use-parcels") +DEFAULT_EVIDENCE_ROOT = Path("/app/storage/operator-evidence/alz-agriculture-refresh") +SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels" +REMOTE_VERSION_PATTERN = re.compile(r"^(20[0-9]{2})-v3$") +LOCAL_VERSION_PATTERN = re.compile(r"^(20[0-9]{2})-definitive$") +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +PROVISIONAL_VERSION_PATTERN = re.compile(r"\b(20[0-9]{2}-v[12])\b") +ACTIONABLE_STATUSES = {"update_available", "not_loaded"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Governed definitive ALZ release: plan, stage, review, then checksum-confirmed apply." + ) + parser.add_argument("action", choices=("plan", "stage", "review", "apply")) + parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope") + parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE) + parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL)) + parser.add_argument("--confirm-edition", help="Exact official definitive edition, for example 2026-v3") + parser.add_argument("--confirm-plan-sha256", help="Exact staged-plan hash required for review/apply") + parser.add_argument("--confirm-review-sha256", help="Exact review-evidence hash required for apply") + parser.add_argument("--approve", action="store_true", help="Explicitly approve the staged evidence during review") + parser.add_argument("--reviewer", help="Named human/operator approving the staged evidence") + parser.add_argument("--review-note", default="", help="Optional bounded review note") + parser.add_argument("--plan-path", type=Path, help="Override the governed staged-plan path") + parser.add_argument("--review-path", type=Path, help="Override the governed review-evidence path") + parser.add_argument( + "--output-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_AGRICULTURE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)), + ) + parser.add_argument( + "--evidence-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_ALZ_REFRESH_EVIDENCE_ROOT", DEFAULT_EVIDENCE_ROOT)), + ) + parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short official-catalog cache") + parser.add_argument("--request-timeout", type=int, default=900) + parser.add_argument("--api-timeout", type=int, default=180) + parser.add_argument("--import-timeout", type=int, default=3600) + parser.add_argument("--max-features", type=int, default=250_000) + parser.add_argument("--max-archive-mb", type=int, default=250) + return parser.parse_args() + + +def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]: + endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}" + request = Request( + endpoint, + headers={"Accept": "application/json", "User-Agent": "GeoIntel-ALZ-release/1.0"}, + ) + try: + with urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc + except URLError as exc: + raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): + raise RuntimeError("GeoIntel API response is not a canonical data envelope") + return payload["data"] + + +def validate_project_scope(args: argparse.Namespace) -> None: + project = api_data(args.api_url, f"projects/{args.project_id}", args.api_timeout) + expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name + if project.get("name") != expected_name: + raise RuntimeError( + f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'" + ) + + +def parse_catalog_datetime(value: Any) -> datetime: + if not isinstance(value, str) or not value.strip(): + raise RuntimeError("The official ALZ catalog did not provide a publication date") + try: + parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) + except ValueError as exc: + raise RuntimeError("The official ALZ catalog publication date is invalid") from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def release_from_catalog_item(item: dict[str, Any]) -> AgriculturalReleaseConfig: + match = REMOTE_VERSION_PATTERN.fullmatch(str(item.get("remote_version") or "")) + if not match: + raise RuntimeError("The official ALZ catalog did not provide one definitive YYYY-v3 edition") + year = int(match.group(1)) + published_at = parse_catalog_datetime(item.get("remote_published_at")) + archive_url = ( + "https://www.landbouwvlaanderen.be/bestanden/gis/" + f"agpa_{year}_{published_at.date().isoformat()}_public.zip" + ) + return resolve_release_config(year, archive_url=archive_url) + + +def fetch_release_decision_from_item(args: argparse.Namespace, item: dict[str, Any]) -> dict[str, Any]: + allowed_degraded = item.get("error_code") == "CATALOG_ALZ_CURRENT_SNAPSHOT_MISSING" + if ( + item.get("source_name") != SOURCE_NAME + or item.get("reachable") is not True + or item.get("status") not in {"available", "degraded"} + or (item.get("status") == "degraded" and not allowed_degraded) + or "definitive_archive" not in set(item.get("matched_layers") or []) + or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or "")) + or item.get("metadata_identifier") != "alz-agricultural-use-parcels" + ): + raise RuntimeError("The official ALZ catalog is not safely available for release planning") + release = release_from_catalog_item(item) + local_version = str(item.get("local_source_version") or "") + local_match = LOCAL_VERSION_PATTERN.fullmatch(local_version) + if local_version and not local_match: + status = "blocked_local_version" + elif not local_version: + status = "not_loaded" + elif int(local_match.group(1)) == release.year: + status = "current" + elif int(local_match.group(1)) < release.year: + status = "update_available" + else: + status = "blocked_remote_older" + message = str(item.get("message") or "") + provisional_match = PROVISIONAL_VERSION_PATTERN.search(message) + return { + "schema_version": 1, + "status": status, + "project_id": args.project_id, + "scope": args.scope, + "generated_at": datetime.now(timezone.utc).isoformat(), + "catalog_checked_at": item.get("checked_at"), + "local_source_version": local_version or None, + "release": { + "year": release.year, + "edition": release.definitive_version, + "archive_url": release.archive_url, + }, + "catalog_identity": { + "metadata_identifier": item["metadata_identifier"], + "metadata_url": item.get("metadata_url"), + "remote_version": release.definitive_version, + "remote_published_at": item.get("remote_published_at"), + "capabilities_sha256": item["capabilities_sha256"], + }, + "provisional_release": provisional_match.group(1) if provisional_match else None, + "provisional_release_importable": False, + "automatic_download": False, + "automatic_import": False, + "destructive_replacement": False, + "next_action": "stage" if status in ACTIONABLE_STATUSES else None, + } + + +def fetch_release_decision(args: argparse.Namespace, *, refresh: bool) -> dict[str, Any]: + query = "true" if refresh else "false" + report = api_data( + args.api_url, + f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={query}", + args.api_timeout, + ) + matches = [item for item in report.get("items") or [] if item.get("source_name") == SOURCE_NAME] + if len(matches) != 1: + raise RuntimeError("Source catalog report did not contain exactly one ALZ agriculture contract") + return fetch_release_decision_from_item(args, matches[0]) + + +def require_release_confirmation(args: argparse.Namespace, release: AgriculturalReleaseConfig) -> None: + if args.confirm_edition != release.definitive_version: + raise RuntimeError(f"Explicit --confirm-edition {release.definitive_version} is required") + + +def internal_base_url(api_url: str) -> str: + value = api_url.rstrip("/") + if value.endswith("/api/v1"): + value = value[:-7] + parsed = urlparse(value) + if parsed.hostname not in {"127.0.0.1", "localhost", "::1"}: + raise RuntimeError("Stage/apply must run inside GeoIntel against the local backend API") + return value + + +def default_plan_path(args: argparse.Namespace, year: int) -> Path: + return args.evidence_root / args.scope / str(year) / "staged-plan.json" + + +def default_review_path(args: argparse.Namespace, year: int) -> Path: + return args.evidence_root / args.scope / str(year) / "review-evidence.json" + + +def governed_evidence_path(args: argparse.Namespace, path: Path) -> Path: + if not path.resolve().is_relative_to(args.evidence_root.resolve()): + raise RuntimeError(f"Release evidence path is outside the governed evidence root: {path}") + return path + + +def build_operator_command( + args: argparse.Namespace, + release: AgriculturalReleaseConfig, + *, + fetch_only: bool, +) -> list[str]: + command = [ + sys.executable, + str(Path(__file__).resolve().parent / "provision_agricultural_parcel_history.py"), + "--base-url", + internal_base_url(args.api_url), + "--scope", + args.scope, + "--years", + str(release.year), + "--archive-url", + release.archive_url, + "--output-root", + str(args.output_root), + "--request-timeout", + str(args.request_timeout), + "--import-timeout", + str(args.import_timeout), + "--max-features", + str(args.max_features), + "--max-archive-mb", + str(args.max_archive_mb), + ] + if fetch_only: + command.extend(("--force", "--fetch-only")) + return command + + +def run_operator(command: list[str], *, action: str) -> dict[str, Any]: + print(f"ALZ agriculture: {action}...", file=sys.stderr, flush=True) + completed = subprocess.run(command, check=False, capture_output=True, text=True, encoding="utf-8") + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() or "operator returned no diagnostics" + raise RuntimeError(f"ALZ agriculture {action} failed with exit {completed.returncode}: {detail[-3000:]}") + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError(f"ALZ agriculture {action} returned invalid JSON") from exc + if payload.get("status") != "ok": + raise RuntimeError(f"ALZ agriculture {action} did not report success") + return payload + + +def canonical_sha256(payload: dict[str, Any], hash_field: str) -> str: + content = {key: value for key, value in payload.items() if key != hash_field} + encoded = json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return sha256(encoded).hexdigest() + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".partial") + temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + +def _previous_manifest(output_root: Path, scope_key: str, year: int) -> tuple[Path, dict[str, Any]] | None: + scope_root = output_root / scope_key + if not scope_root.is_dir(): + return None + previous_years = sorted( + (int(path.name) for path in scope_root.iterdir() if path.is_dir() and path.name.isdigit() and int(path.name) < year), + reverse=True, + ) + for previous_year in previous_years: + manifests = list((scope_root / str(previous_year)).glob("*.manifest.json")) + if len(manifests) != 1: + continue + payload = json.loads(manifests[0].read_text(encoding="utf-8")) + if payload.get("year") == previous_year and payload.get("scope_key") == scope_key: + return manifests[0], payload + return None + + +def _relative_change(current: float, previous: float) -> float | None: + if previous <= 0: + return None + return round((current - previous) / previous, 8) + + +def validate_staged_release(args: argparse.Namespace, release: AgriculturalReleaseConfig) -> dict[str, Any]: + paths = artifact_paths(args.output_root, args.scope, release.year, archive_url=release.archive_url) + resolved_output = args.output_root.resolve() + for key in ("archive", "artifact", "codelist", "manifest"): + if not paths[key].resolve().is_relative_to(resolved_output): + raise RuntimeError(f"Staged ALZ {key} is outside the governed output root: {paths[key]}") + manifest = reusable_artifact( + paths, + year=release.year, + scope_key=args.scope, + archive_url=release.archive_url, + ) + if manifest is None: + raise RuntimeError("Staged ALZ artifacts are incomplete or no longer match their checksums") + fields = set(manifest.get("source_fields") or []) + if ( + manifest.get("source_url") != release.archive_url + or manifest.get("source_crs") != SOURCE_CRS + or manifest.get("output_crs") != OUTPUT_CRS + or not STABLE_REQUIRED_FIELDS.issubset(fields) + or int(manifest.get("feature_count") or 0) < 1 + or float(manifest.get("clipped_area_ha") or 0) <= 0 + or int(manifest.get("source_feature_count") or 0) < int(manifest.get("feature_count") or 0) + ): + raise RuntimeError("Staged ALZ manifest does not satisfy the definitive release contract") + max_archive_bytes = min(args.max_archive_mb * 1024 * 1024, MAX_ARCHIVE_BYTES) + if paths["archive"].stat().st_size > max_archive_bytes: + raise RuntimeError("Staged ALZ archive exceeds the configured release limit") + codelist = json.loads(paths["codelist"].read_text(encoding="utf-8")) + crop_entries = codelist.get("crop_entries") if isinstance(codelist, dict) else None + if codelist.get("year") != release.year or not isinstance(crop_entries, list) or not crop_entries: + raise RuntimeError("Staged ALZ crop-code evidence is incomplete") + previous_result = _previous_manifest(args.output_root, args.scope, release.year) + baseline = None + if previous_result is not None: + previous_path, previous = previous_result + if not previous_path.resolve().is_relative_to(resolved_output): + raise RuntimeError("Previous ALZ baseline manifest is outside the governed output root") + current_count = int(manifest["feature_count"]) + previous_count = int(previous.get("feature_count") or 0) + current_area = float(manifest["clipped_area_ha"]) + previous_area = float(previous.get("clipped_area_ha") or 0) + baseline = { + "year": int(previous["year"]), + "manifest_path": str(previous_path), + "manifest_sha256": sha256_file(previous_path), + "feature_count": previous_count, + "clipped_area_ha": previous_area, + "feature_count_change_ratio": _relative_change(current_count, previous_count), + "clipped_area_change_ratio": _relative_change(current_area, previous_area), + } + return { + "manifest_path": str(paths["manifest"]), + "manifest_sha256": sha256_file(paths["manifest"]), + "archive_path": str(paths["archive"]), + "archive_sha256": manifest["source_archive_sha256"], + "archive_size_bytes": manifest["source_archive_size_bytes"], + "artifact_path": str(paths["artifact"]), + "artifact_sha256": manifest["artifact_sha256"], + "crop_code_list_path": str(paths["codelist"]), + "crop_code_list_sha256": manifest["crop_code_list_sha256"], + "crop_entry_count": len(crop_entries), + "crop_code_conflicts": codelist.get("code_title_conflicts") or {}, + "source_feature_count": manifest["source_feature_count"], + "feature_count": manifest["feature_count"], + "clipped_feature_count": manifest["clipped_feature_count"], + "clipped_area_ha": manifest["clipped_area_ha"], + "source_fields": sorted(fields), + "member_nis_codes": manifest.get("member_nis_codes") or [], + "baseline": baseline, + } + + +def build_staged_plan( + args: argparse.Namespace, + decision: dict[str, Any], + release: AgriculturalReleaseConfig, + operator_result: dict[str, Any], +) -> dict[str, Any]: + payload: dict[str, Any] = { + "schema_version": 1, + "status": "staged", + "created_at": datetime.now(timezone.utc).isoformat(), + "project_id": args.project_id, + "scope": args.scope, + "local_source_version_before_apply": decision.get("local_source_version"), + "release": decision["release"], + "catalog_identity": decision["catalog_identity"], + "evidence": validate_staged_release(args, release), + "operator_result": operator_result, + "review_required": True, + "apply_requires_plan_sha256": True, + "apply_requires_review_sha256": True, + "automatic_import": False, + "destructive_replacement": False, + "existing_snapshots_retained": True, + } + payload["plan_sha256"] = canonical_sha256(payload, "plan_sha256") + return payload + + +def _release_from_payload(payload: dict[str, Any]) -> AgriculturalReleaseConfig: + release = payload.get("release") or {} + year = release.get("year") if isinstance(release, dict) else None + edition = release.get("edition") if isinstance(release, dict) else None + archive_url = release.get("archive_url") if isinstance(release, dict) else None + if ( + not isinstance(year, int) + or isinstance(year, bool) + or edition != f"{year}-v3" + or not isinstance(archive_url, str) + ): + raise RuntimeError("Release evidence is missing required ALZ identity fields") + return resolve_release_config(year, archive_url=archive_url) + + +def load_staged_plan( + args: argparse.Namespace, + release: AgriculturalReleaseConfig, +) -> tuple[Path, dict[str, Any]]: + path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year)) + if not path.is_file(): + raise RuntimeError(f"Staged ALZ plan is missing: {path}") + payload = json.loads(path.read_text(encoding="utf-8")) + actual_sha = canonical_sha256(payload, "plan_sha256") + if payload.get("plan_sha256") != actual_sha: + raise RuntimeError("Staged ALZ plan checksum is invalid") + if args.confirm_plan_sha256 != actual_sha: + raise RuntimeError(f"Explicit --confirm-plan-sha256 {actual_sha} is required") + if ( + payload.get("status") != "staged" + or payload.get("project_id") != args.project_id + or payload.get("scope") != args.scope + or _release_from_payload(payload) != release + ): + raise RuntimeError("Staged ALZ plan identity is invalid") + current_evidence = validate_staged_release(args, release) + if current_evidence != payload.get("evidence"): + raise RuntimeError("Staged ALZ artifacts no longer match the approved plan") + return path, payload + + +def require_catalog_unchanged(plan: dict[str, Any], decision: dict[str, Any]) -> None: + if plan.get("release") != decision.get("release") or plan.get("catalog_identity") != decision.get("catalog_identity"): + raise RuntimeError("The official ALZ release evidence changed; create and review a new staged plan") + + +def build_review_evidence(args: argparse.Namespace, plan_path: Path, plan: dict[str, Any]) -> dict[str, Any]: + reviewer = str(args.reviewer or "").strip() + note = str(args.review_note or "").strip() + if not args.approve or len(reviewer) < 2 or len(reviewer) > 120: + raise RuntimeError("Review requires --approve and a named --reviewer between 2 and 120 characters") + if len(note) > 1000: + raise RuntimeError("Review note must not exceed 1000 characters") + payload: dict[str, Any] = { + "schema_version": 1, + "status": "approved", + "reviewed_at": datetime.now(timezone.utc).isoformat(), + "reviewer": reviewer, + "review_note": note or None, + "project_id": args.project_id, + "scope": args.scope, + "release": plan["release"], + "staged_plan_path": str(plan_path), + "staged_plan_sha256": plan["plan_sha256"], + "reviewed_checks": [ + "official_catalog_identity", + "source_archive_checksum", + "geopackage_schema_and_crs", + "scope_feature_and_area_accounting", + "crop_code_list_and_conflicts", + "previous_definitive_edition_delta", + "provisional_snapshot_exclusion", + "immutable_dataset_apply", + ], + } + payload["review_sha256"] = canonical_sha256(payload, "review_sha256") + return payload + + +def load_review_evidence( + args: argparse.Namespace, + release: AgriculturalReleaseConfig, + plan: dict[str, Any], +) -> tuple[Path, dict[str, Any]]: + path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year)) + if not path.is_file(): + raise RuntimeError(f"Approved ALZ review evidence is missing: {path}") + payload = json.loads(path.read_text(encoding="utf-8")) + actual_sha = canonical_sha256(payload, "review_sha256") + if payload.get("review_sha256") != actual_sha: + raise RuntimeError("ALZ review evidence checksum is invalid") + if args.confirm_review_sha256 != actual_sha: + raise RuntimeError(f"Explicit --confirm-review-sha256 {actual_sha} is required") + if ( + payload.get("status") != "approved" + or payload.get("project_id") != args.project_id + or payload.get("scope") != args.scope + or payload.get("release") != plan.get("release") + or payload.get("staged_plan_sha256") != plan.get("plan_sha256") + or not str(payload.get("reviewer") or "").strip() + ): + raise RuntimeError("ALZ review evidence does not authorize this staged plan") + return path, payload + + +def _release_result(operator_result: dict[str, Any], year: int) -> dict[str, Any]: + matches = [item for item in operator_result.get("years") or [] if int(item.get("year") or 0) == year] + if len(matches) != 1: + raise RuntimeError("Agriculture operator did not return exactly one result for the approved edition") + return matches[0] + + +def main() -> int: + args = parse_args() + try: + if min(args.request_timeout, args.api_timeout, args.import_timeout, args.max_features, args.max_archive_mb) <= 0: + raise ValueError("All timeout and safety limits must be positive") + validate_project_scope(args) + refresh = args.refresh_catalog or args.action in {"stage", "review", "apply"} + decision = fetch_release_decision(args, refresh=refresh) + release = _release_from_payload(decision) + + if args.action == "plan": + print(json.dumps({"status": "ok", "action": "plan", "decision": decision}, ensure_ascii=False, indent=2)) + return 0 + + require_release_confirmation(args, release) + if args.action == "stage": + if decision.get("status") not in ACTIONABLE_STATUSES: + raise RuntimeError(f"ALZ release is not safely stageable: {decision.get('status')}") + operator_result = run_operator(build_operator_command(args, release, fetch_only=True), action="staging") + staged_result = _release_result(operator_result, release.year) + if staged_result.get("status") != "prepared": + raise RuntimeError("ALZ staging did not produce prepared release evidence") + plan = build_staged_plan(args, decision, release, operator_result) + plan_path = governed_evidence_path(args, args.plan_path or default_plan_path(args, release.year)) + write_json(plan_path, plan) + print(json.dumps({"status": "staged", "plan_path": str(plan_path), **plan}, ensure_ascii=False, indent=2)) + return 0 + + plan_path, plan = load_staged_plan(args, release) + require_catalog_unchanged(plan, decision) + if args.action == "review": + review = build_review_evidence(args, plan_path, plan) + review_path = governed_evidence_path(args, args.review_path or default_review_path(args, release.year)) + write_json(review_path, review) + print(json.dumps({"status": "approved", "review_path": str(review_path), **review}, ensure_ascii=False, indent=2)) + return 0 + + review_path, review = load_review_evidence(args, release, plan) + operator_result = run_operator(build_operator_command(args, release, fetch_only=False), action="applying") + applied_result = _release_result(operator_result, release.year) + if applied_result.get("status") not in {"imported", "existing"} or not applied_result.get("dataset_id"): + raise RuntimeError("ALZ apply did not return one persisted immutable Dataset") + final_decision = fetch_release_decision(args, refresh=True) + if ( + final_decision.get("status") != "current" + or final_decision.get("local_source_version") != f"{release.year}-definitive" + ): + raise RuntimeError("Applied ALZ Dataset did not become the current local definitive edition") + evidence = { + "schema_version": 1, + "status": "applied", + "applied_at": datetime.now(timezone.utc).isoformat(), + "project_id": args.project_id, + "scope": args.scope, + "release": plan["release"], + "staged_plan_path": str(plan_path), + "staged_plan_sha256": plan["plan_sha256"], + "review_path": str(review_path), + "review_sha256": review["review_sha256"], + "reviewer": review["reviewer"], + "dataset_id": applied_result["dataset_id"], + "dataset_status": applied_result["status"], + "feature_count": applied_result.get("feature_count"), + "final_catalog_decision": final_decision, + "existing_snapshots_retained": True, + } + evidence["applied_evidence_sha256"] = canonical_sha256(evidence, "applied_evidence_sha256") + evidence_path = plan_path.with_name("applied-evidence.json") + write_json(evidence_path, evidence) + print(json.dumps({"status": "applied", "evidence_path": str(evidence_path), **evidence}, ensure_ascii=False, indent=2)) + return 0 + except (OSError, RuntimeError, ValueError, KeyError, json.JSONDecodeError) as exc: + print( + json.dumps({"status": "error", "action": args.action, "message": str(exc)}, ensure_ascii=False), + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/provision_agricultural_parcel_history.py b/scripts/provision_agricultural_parcel_history.py index 04548e64..11a64572 100644 --- a/scripts/provision_agricultural_parcel_history.py +++ b/scripts/provision_agricultural_parcel_history.py @@ -10,17 +10,19 @@ directly and never uses the provisional current-campaign snapshot. from __future__ import annotations import argparse +from dataclasses import dataclass import hashlib import json import math import os -import shutil +import re import sys import tempfile import zipfile from datetime import date, datetime, timezone from pathlib import Path from typing import Any, Iterable +from urllib.parse import urlsplit import requests from pyproj import Transformer @@ -51,6 +53,10 @@ SOURCE_CRS = "EPSG:31370" OUTPUT_CRS = "EPSG:4326" SCHEMA_VERSION = 1 MAX_ARCHIVE_BYTES = 250 * 1024 * 1024 +MAX_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024 +MAX_ARCHIVE_MEMBERS = 32 +ARCHIVE_HOST = "www.landbouwvlaanderen.be" +ARCHIVE_PATH_PATTERN = re.compile(r"^/bestanden/gis/agpa_(20[0-9]{2})_(20[0-9]{2}-[0-9]{2}-[0-9]{2})_public\.zip$") ARCHIVE_URLS = { 2008: "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2008_2022-03-23_public.zip", @@ -74,6 +80,16 @@ ARCHIVE_URLS = { } SUPPORTED_YEARS = tuple(ARCHIVE_URLS) + +@dataclass(frozen=True) +class AgriculturalReleaseConfig: + year: int + archive_url: str + + @property + def definitive_version(self) -> str: + return f"{self.year}-v3" + STABLE_REQUIRED_FIELDS = { "agpakey", "parcelnumber", @@ -131,6 +147,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL)) parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY) parser.add_argument("--years", default=",".join(str(year) for year in SUPPORTED_YEARS)) + parser.add_argument( + "--archive-url", + help="Exact official archive URL for one explicitly confirmed future definitive edition.", + ) parser.add_argument("--output-root", type=Path, default=Path(os.environ.get("GEOINTEL_AGRICULTURE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT))) parser.add_argument("--request-timeout", type=int, default=900) parser.add_argument("--import-timeout", type=int, default=3600) @@ -201,21 +221,29 @@ def response_data(response: requests.Response) -> Any: def api_items(session: requests.Session, url: str, timeout: int) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] offset = 0 - while True: + expected_total: int | None = None + while expected_total is None or offset < expected_total: response = session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout) data = response_data(response) if isinstance(data, dict): page = data.get("items") or data.get("results") or [] - total = int(data.get("total") or len(page)) + page_total = int(data.get("total") if data.get("total") is not None else len(page)) else: page = data - total = len(page) if isinstance(page, list) else 0 + page_total = len(page) if isinstance(page, list) else 0 if not isinstance(page, list): raise RuntimeError(f"Expected a list response from {url}") + if expected_total is None: + expected_total = page_total + elif page_total != expected_total: + raise RuntimeError("GeoIntel pagination total changed while reading the agricultural workspace") items.extend(item for item in page if isinstance(item, dict)) - if not page or len(items) >= total: - return items + if not page: + break offset += len(page) + if expected_total is not None and len(items) != expected_total: + raise RuntimeError(f"GeoIntel list response for {url} returned {len(items)} of {expected_total} items") + return items def locate_workspace( @@ -251,6 +279,53 @@ def parse_years(raw: str) -> list[int]: return years +def validate_archive_url(url: str, *, expected_year: int) -> str: + parsed = urlsplit(url) + match = ARCHIVE_PATH_PATTERN.fullmatch(parsed.path) + if ( + parsed.scheme != "https" + or parsed.hostname != ARCHIVE_HOST + or parsed.port not in {None, 443} + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or not match + or int(match.group(1)) != expected_year + ): + raise ValueError("Agricultural release archive is outside the official ALZ URL contract") + try: + datetime.strptime(match.group(2), "%Y-%m-%d") + except ValueError as exc: + raise ValueError("Agricultural release archive contains an invalid publication date") from exc + return url + + +def resolve_release_config(year: int, *, archive_url: str | None = None) -> AgriculturalReleaseConfig: + known_url = ARCHIVE_URLS.get(year) + if known_url is not None: + if archive_url is not None and archive_url != known_url: + raise ValueError(f"The official retained archive identity for {year} may not be overridden") + return AgriculturalReleaseConfig(year=year, archive_url=known_url) + if year <= max(SUPPORTED_YEARS) or not archive_url: + raise ValueError( + f"One future definitive edition after {max(SUPPORTED_YEARS)} may be supplied with --archive-url" + ) + return AgriculturalReleaseConfig(year=year, archive_url=validate_archive_url(archive_url, expected_year=year)) + + +def resolve_release_configs(raw_years: str, *, archive_url: str | None = None) -> list[AgriculturalReleaseConfig]: + if archive_url is None: + return [resolve_release_config(year) for year in parse_years(raw_years)] + try: + years = sorted({int(value.strip()) for value in raw_years.split(",") if value.strip()}) + except ValueError as exc: + raise ValueError("Years must be a comma-separated list of integers") from exc + if len(years) != 1: + raise ValueError("--archive-url requires exactly one explicitly selected definitive year") + return [resolve_release_config(years[0], archive_url=archive_url)] + + def polygonal_geometry(geometry): if geometry is None or geometry.is_empty: return None @@ -308,6 +383,10 @@ def download_archive( temporary.unlink(missing_ok=True) response = session.get(url, timeout=timeout, stream=True) response.raise_for_status() + final_url = str(getattr(response, "url", "") or url) + expected_year = int(ARCHIVE_PATH_PATTERN.fullmatch(urlsplit(url).path).group(1)) + if validate_archive_url(final_url, expected_year=expected_year) != url: + raise RuntimeError("Official archive download redirected to a different release identity") content_length = int(response.headers.get("content-length") or 0) if content_length > max_bytes: raise RuntimeError(f"Official archive exceeds the configured {max_bytes // (1024 * 1024)} MiB limit") @@ -331,7 +410,12 @@ def download_archive( def archive_geopackage_member(path: Path) -> str: with zipfile.ZipFile(path) as archive: - members = [item.filename for item in archive.infolist() if not item.is_dir() and item.filename.lower().endswith(".gpkg")] + entries = archive.infolist() + if len(entries) > MAX_ARCHIVE_MEMBERS: + raise RuntimeError(f"Official archive contains more than {MAX_ARCHIVE_MEMBERS} members") + if sum(item.file_size for item in entries if not item.is_dir()) > MAX_EXTRACTED_BYTES: + raise RuntimeError("Official archive exceeds the extracted-size safety limit") + members = [item.filename for item in entries if not item.is_dir() and item.filename.lower().endswith(".gpkg")] if len(members) != 1: raise RuntimeError(f"Official archive must contain exactly one GeoPackage; found {len(members)}") member = members[0] @@ -351,7 +435,12 @@ def extract_geopackage(archive_path: Path, destination_dir: Path) -> Path: member = validate_archive(archive_path) destination = destination_dir / Path(member).name with zipfile.ZipFile(archive_path) as archive, archive.open(member) as source, destination.open("wb") as target: - shutil.copyfileobj(source, target, length=1024 * 1024) + extracted_bytes = 0 + for chunk in iter(lambda: source.read(1024 * 1024), b""): + extracted_bytes += len(chunk) + if extracted_bytes > MAX_EXTRACTED_BYTES: + raise RuntimeError("Official GeoPackage exceeded the extracted-size safety limit while streaming") + target.write(chunk) if destination.stat().st_size == 0: raise RuntimeError("Extracted official GeoPackage is empty") return destination @@ -470,23 +559,39 @@ def normalize_frame(frame, *, year: int, boundary_lambert72, max_features: int) } -def artifact_paths(output_root: Path, scope_key: str, year: int) -> dict[str, Path]: +def artifact_paths( + output_root: Path, + scope_key: str, + year: int, + *, + archive_url: str | None = None, +) -> dict[str, Path]: + release = resolve_release_config(year, archive_url=archive_url) directory = output_root / scope_key / str(year) return { "directory": directory, - "archive": directory / Path(ARCHIVE_URLS[year]).name, + "archive": directory / Path(urlsplit(release.archive_url).path).name, "artifact": directory / f"agricultural_use_parcels_{year}_{scope_key}.geojson", "codelist": directory / f"agricultural_use_parcels_{year}_crop_codes.json", "manifest": directory / f"agricultural_use_parcels_{year}_{scope_key}.manifest.json", } -def reusable_artifact(paths: dict[str, Path], *, year: int, scope_key: str) -> dict[str, Any] | None: +def reusable_artifact( + paths: dict[str, Path], + *, + year: int, + scope_key: str, + archive_url: str | None = None, +) -> dict[str, Any] | None: if not all(paths[key].is_file() for key in ("archive", "artifact", "codelist", "manifest")): return None manifest = json.loads(paths["manifest"].read_text(encoding="utf-8")) if manifest.get("schema_version") != SCHEMA_VERSION or manifest.get("year") != year or manifest.get("scope_key") != scope_key: return None + release = resolve_release_config(year, archive_url=archive_url) + if manifest.get("source_url") != release.archive_url: + return None if manifest.get("source_archive_sha256") != sha256_file(paths["archive"]): return None if manifest.get("artifact_sha256") != sha256_file(paths["artifact"]): @@ -508,16 +613,18 @@ def prepare_year( max_archive_bytes: int, max_features: int, force: bool, + archive_url: str | None = None, ) -> tuple[dict[str, Path], dict[str, Any]]: - paths = artifact_paths(output_root, scope.key, year) + release = resolve_release_config(year, archive_url=archive_url) + paths = artifact_paths(output_root, scope.key, year, archive_url=release.archive_url) paths["directory"].mkdir(parents=True, exist_ok=True) if not force: - reused = reusable_artifact(paths, year=year, scope_key=scope.key) + reused = reusable_artifact(paths, year=year, scope_key=scope.key, archive_url=release.archive_url) if reused is not None: return paths, reused download = download_archive( session, - ARCHIVE_URLS[year], + release.archive_url, paths["archive"], timeout=request_timeout, max_bytes=max_archive_bytes, @@ -546,7 +653,7 @@ def prepare_year( "scope_name": scope.display_name, "scope_type": scope.scope_type, "member_nis_codes": list(scope.nis_codes), - "source_url": ARCHIVE_URLS[year], + "source_url": release.archive_url, "catalog_url": CATALOG_URL, "data_catalog_url": DATA_CATALOG_URL, "attribution": ATTRIBUTION, @@ -635,7 +742,7 @@ def upload_artifact( "operator_tool": "provision_agricultural_parcel_history.py", "operator_explicit_fetch": True, "geometry_clipped_to_area": True, - "source_archive_url": ARCHIVE_URLS[year], + "source_archive_url": manifest["source_url"], "source_archive_path": str(paths["archive"]), "source_archive_sha256": manifest["source_archive_sha256"], "crop_code_list_path": str(paths["codelist"]), @@ -684,7 +791,7 @@ def existing_dataset_for_year(datasets: list[dict[str, Any]], *, area_id: str, y def main() -> int: args = parse_args() try: - years = parse_years(args.years) + releases = resolve_release_configs(args.years, archive_url=args.archive_url) if args.max_features < 1 or args.max_archive_mb < 1: raise ValueError("Feature and archive safety limits must be positive") scope = GEOGRAPHIC_SCOPES[args.scope] @@ -693,7 +800,8 @@ def main() -> int: project_id, area_id, boundary, datasets = locate_workspace(api_session, base_url, scope, args.import_timeout) results: list[dict[str, Any]] = [] with build_session() as official_session: - for year in years: + for release in releases: + year = release.year paths, manifest = prepare_year( official_session, year=year, @@ -704,6 +812,7 @@ def main() -> int: max_archive_bytes=min(args.max_archive_mb * 1024 * 1024, MAX_ARCHIVE_BYTES), max_features=args.max_features, force=args.force, + archive_url=release.archive_url, ) existing = existing_dataset_for_year(datasets, area_id=area_id, year=year) if existing is not None: diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index c91c3e28..df3920d5 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -54,6 +54,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py +${PYTHON_BIN} -m py_compile scripts/manage_alz_agriculture_release.py ${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py