diff --git a/CHANGELOG.md b/CHANGELOG.md index b5d44d57..15457199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ # Changelog +## Sprint 210 Regional BWK/Natura 2000 expansion (2026-07-16) + +- Added an explicit 28-municipality operator for the official INBO BWK/Natura + 2000 state-2025 WFS, reusing the governed Mol normalization and metrics. +- Added exact EPSG:31370 municipality clipping, partition-unique source ids, + deterministic gzip source evidence, checksum-bound cache reuse and one + canonical regional Dataset upload. +- Made map dataset selection prefer an exact Area snapshot over a broader + compatible layer and changed the source inventory to report overlapping + area snapshots without summing duplicate coverage. +- Added focused GIS, persistence-contract, packaging and UI regression tests. + No API contract, database migration, AI model or product scope changed. + ## Sprint 209 Regional historical land-use expansion (2026-07-15) - Added an explicit operator for the official 1778, 1873 and 1969 historical diff --git a/backend/README.md b/backend/README.md index e61e1925..344a8bd1 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1289,3 +1289,17 @@ evidence without persistence. A conflicting checksum for an already persisted state-2025 Mol Dataset fails closed instead of creating a silent replacement. PostGIS selection summaries keep BWK value classes separate and label PHAB-derived habitat hectares as estimates. + +For the complete approved Kempen transport region, run the partitioned +operator after `provision_geographic_scope.py --scope kempen-transport-region`: + +```bash +docker exec geointel python /app/scripts/provision_regional_bwk_natura2000.py +``` + +Use `--fetch-only` to build and validate all 28 municipality partitions without +database persistence. A normal rerun validates and reuses the immutable source +evidence and existing Dataset. `--force` explicitly refetches the WFS but still +fails closed if a different state-2025 checksum is already persisted. The +regional output uses the same selection-summary API as Mol; no new endpoint or +direct PostGIS write is introduced. diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index 626f5e6d..04158e07 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -26,6 +26,7 @@ FULL_AREA_CLIPPED_OPERATOR_TOOLS = { "provision_regional_historical_landuse.py", "provision_waterinfo_station_history.py", "provision_mol_bwk_natura2000.py", + "provision_regional_bwk_natura2000.py", "provision_agricultural_parcel_history.py", "provision_buildings_addresses_register.py", } diff --git a/backend/tests/test_sprint210_regional_bwk_natura2000.py b/backend/tests/test_sprint210_regional_bwk_natura2000.py new file mode 100644 index 00000000..45bcec34 --- /dev/null +++ b/backend/tests/test_sprint210_regional_bwk_natura2000.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import gzip +import importlib.util +import json +from pathlib import Path +import sys + +import pytest +from shapely.geometry import box, mapping, shape + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + + +def load_script(): + path = SCRIPTS / "provision_regional_bwk_natura2000.py" + spec = importlib.util.spec_from_file_location("test_provision_regional_bwk_natura2000", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class FakeResponse: + status_code = 200 + ok = True + text = "" + + def __init__(self, payload): + self.payload = payload + self.content = json.dumps(payload, separators=(",", ":")).encode("utf-8") + + def json(self): + return self.payload + + +class FakeApiSession: + def __init__(self, payload): + self.payload = payload + self.calls = [] + + def post(self, url, **kwargs): + self.calls.append((url, kwargs)) + return FakeResponse({"data": self.payload}) + + +def source_feature(feature_id: str, geometry): + return { + "type": "Feature", + "id": feature_id, + "geometry": mapping(geometry), + "properties": { + "UIDN": feature_id, + "EVAL": "z", + "HAB1": "9190", + "PHAB1": 50, + "HABLEGENDE": "hab", + }, + } + + +def test_partition_retains_gzipped_source_and_applies_member_context(tmp_path: Path, monkeypatch) -> None: + module = load_script() + scope = module.GeographicScope( + key="test-region", + display_name="Test region", + project_name="Test", + project_region="Test", + area_name="Test area", + authority_name="Test", + authority_url="https://example.test", + scope_type="test", + limitation_message="Test", + members=(module.ScopeMember("Mol", "13025"),), + ) + boundary = box(5.0, 51.0, 5.1, 51.1) + payload = { + "type": "FeatureCollection", + "features": [source_feature("Bwkhab.1", box(4.98, 51.02, 5.05, 51.08))], + } + raw_bytes = json.dumps(payload, separators=(",", ":")).encode("utf-8") + monkeypatch.setattr( + module.bwk, + "iter_wfs_pages", + lambda *_args, **_kwargs: iter([(payload, "https://example.test/page", raw_bytes)]), + ) + + manifest = module.prepare_partition( + object(), + output_root=tmp_path, + scope=scope, + member=scope.members[0], + boundary_wgs84=boundary, + page_limit=100, + max_features=1000, + timeout=30, + force=False, + ) + output = json.loads(Path(manifest["output_path"]).read_text(encoding="utf-8")) + feature = output["features"][0] + raw_path = Path(manifest["manifest_path"]).parent / manifest["raw_pages"][0]["artifact_path"] + + assert manifest["feature_count"] == 1 + assert feature["id"] == "BWK:Bwkhab:Bwkhab.1:13025" + assert feature["properties"]["municipality"] == "Mol" + assert feature["properties"]["coverage_scope"] == "test-region" + assert shape(feature["geometry"]).bounds == pytest.approx((5.0, 51.02, 5.05, 51.08), abs=1e-5) + assert gzip.decompress(raw_path.read_bytes()) == raw_bytes + + cached = module.prepare_partition( + object(), + output_root=tmp_path, + scope=scope, + member=scope.members[0], + boundary_wgs84=boundary, + page_limit=100, + max_features=1000, + timeout=30, + force=False, + ) + assert cached["output_sha256"] == manifest["output_sha256"] + + +def test_snapshot_assembles_unique_partitions_and_metrics(tmp_path: Path) -> None: + module = load_script() + scope = module.GeographicScope( + key="test-region", + display_name="Test region", + project_name="Test", + project_region="Test", + area_name="Test area", + authority_name="Test", + authority_url="https://example.test", + scope_type="test", + limitation_message="Test", + members=(module.ScopeMember("Left", "10001"), module.ScopeMember("Right", "10002")), + ) + partitions = [] + for index, member in enumerate(scope.members): + output_path, manifest_path, _raw_dir = module.partition_paths(tmp_path / scope.key, member.nis_code) + feature = source_feature(f"Bwkhab.{index}", box(index, 0, index + 0.5, 0.5)) + feature["id"] = f"BWK:Bwkhab:Bwkhab.{index}:{member.nis_code}" + feature["properties"].update( + { + "clipped_area_ha": 1.0 + index, + "bwk_evaluation_code": "z", + "habitat_status_code": "hab", + "natura2000_area_ha": 0.5, + "regional_biotope_area_ha": 0.25, + "uncertain_habitat_area_ha": 0.0, + } + ) + module.bwk.write_json_atomic(output_path, {"type": "FeatureCollection", "features": [feature]}) + partitions.append( + { + "municipality": member.name, + "nis_code": member.nis_code, + "feature_count": 1, + "raw_source_feature_count": 1, + "page_count": 1, + "output_path": str(output_path), + "output_sha256": module.bwk.sha256_file(output_path), + "manifest_path": str(manifest_path), + } + ) + + output_path, _manifest_path, manifest = module.assemble_snapshot( + output_root=tmp_path, + scope=scope, + partitions=partitions, + member_boundaries_sha256="boundaries-hash", + max_total_features=10, + ) + output = json.loads(output_path.read_text(encoding="utf-8")) + + assert manifest["coverage_complete"] is True + assert manifest["feature_count"] == 2 + assert manifest["evaluation_area_ha"]["z"] == 3.0 + assert manifest["natura2000_area_ha"] == 1.0 + assert len({feature["id"] for feature in output["features"]}) == 2 + + +def test_upload_contract_is_regional_partitioned_and_canonical(tmp_path: Path) -> None: + module = load_script() + scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] + path = tmp_path / "bwk.geojson" + path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8") + manifest_path = tmp_path / "manifest.json" + manifest = { + "coverage_complete": True, + "feature_count": 42, + "output_sha256": "output-hash", + "partition_identity_sha256": "partition-hash", + "partitions": [{} for _ in scope.members], + "generated_at": "2026-07-16T00:00:00+00:00", + "limitations": ["test"], + } + session = FakeApiSession({"id": "dataset-id", "feature_count": 42}) + + result = module.upload_snapshot( + session, + base_url="http://backend:8000", + project_id="project-id", + area_id="area-id", + scope=scope, + path=path, + manifest_path=manifest_path, + manifest=manifest, + timeout=30, + ) + data = session.calls[0][1]["data"] + source_metadata = json.loads(data["source_metadata_json"]) + provenance = json.loads(data["provenance_metadata_json"]) + + assert result["id"] == "dataset-id" + assert data["area_id"] == "area-id" + assert data["temporal_series_key"] == "inbo-bwk-natura2000:kempen-transport-region" + assert source_metadata["coverage_scope"] == "kempen-transport-region" + assert source_metadata["member_count"] == 28 + assert source_metadata["partitioned_source_audit"] is True + assert source_metadata["selection_metrics"] == module.bwk.selection_metrics() + assert provenance["operator_tool"] == "provision_regional_bwk_natura2000.py" + assert provenance["raw_source_responses_retained"] is True + + +def test_regional_operator_is_packaged_release_checked_and_exact_area_is_preferred() -> None: + dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8") + service = (ROOT / "backend/app/services/vector_feature_service.py").read_text(encoding="utf-8") + workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8") + catalog = (ROOT / "frontend/src/components/datasets/SourceCatalogPanel.tsx").read_text(encoding="utf-8") + + assert "COPY scripts/provision_regional_bwk_natura2000.py" in dockerfile + assert "py_compile scripts/provision_regional_bwk_natura2000.py" in readiness + assert '"provision_regional_bwk_natura2000.py"' in service + assert "dataset.area_id === selectedAreaId ? 10_000_000" in workspace + assert "largestBwkSnapshot" in catalog diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 55a064b5..6e4bafc3 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -82,6 +82,7 @@ COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_reg COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py +COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 2e12948a..ac30b657 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8880,3 +8880,34 @@ Next: next implementation pass can focus on current-only nature/terrain/flood regional expansion or on a benchmarked data-refresh scheduler; it must not reinterpret these historical surfaces as modern object lineage. + +## Sprint 210 - Regional BWK/Natura 2000 expansion (2026-07-16) + +Implemented: +- Added `provision_regional_bwk_natura2000.py` for the approved 28-municipality + Kempen transport-region scope, using the existing official INBO state-2025 + source contract and governed Mol feature normalization. +- Added per-municipality WFS pagination, exact EPSG:31370 clipping, + municipality-suffixed feature identities, deterministic gzip retention of + every exact source response and checksum-bound partition/snapshot reuse. +- Kept one canonical regional Dataset upload behind DatasetService. No direct + `vector_features` write, migration, endpoint or fabricated time series was + introduced. +- Made map theme selection prefer an exact selected-Area Dataset over a broader + compatible regional Dataset. Updated the source inventory to present + overlapping BWK snapshots as area coverages without double-counting their + feature totals. +- Added focused tests for source evidence, GIS clipping, context fields, + regional assembly, semantic metrics, upload metadata, packaging and UI + selection behavior. + +Validation so far: +- Focused BWK suites pass 11 tests. +- Full local readiness passes backend compilation, 689 backend tests, API + contract audit, one Alembic head (`202607150001`), frontend typecheck/build + and script syntax gates. + +Open for this pass: +- Deploy the validated operator to Tower, fetch and persist all 28 official + partitions, verify idempotency and PostGIS row/geometry/checksum consistency, + then validate complete-region and Mol map/metric behavior in the browser. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index e4fbab62..c27a1eb2 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -445,3 +445,25 @@ Sprint 7B exposes provider metadata only. It does not perform GRB WFS calls, OSM | `fixture` | fixture | true | demo/test fixture layers | `dataset_role=reference`, `source_name=fixture` | Future provider output must flow through `DatasetService` and `VectorFeatureService`; providers must not write directly to `vector_features`. + +## Regional BWK/Natura 2000 state 2025 + +`scripts/provision_regional_bwk_natura2000.py` expands the governed Mol +operator to the approved 28-municipality Kempen transport-region scope. It +queries the same official INBO `BWK:Bwkhab` WFS collection separately for each +official VRBG municipality boundary. Every response is retained as a +deterministic gzip artifact with both compressed-artifact and original-response +SHA256 checksums. + +Geometry is intersected with each municipality in EPSG:31370 before it is +transformed to EPSG:4326. Source polygons crossing a municipality boundary are +therefore split into auditable pieces and receive a NIS suffix; those pieces +must not be interpreted as independent source observations. The 28 validated +partitions are assembled into one regional snapshot and persisted through the +ordinary Dataset upload API with `source_name=inbo_bwk_natura2000`, +`reference_layer_name=nature_value` and +`temporal_series_key=inbo-bwk-natura2000:kempen-transport-region`. + +The state remains a single 2025 map edition, not an annual time series. BWK +valuation, Natura 2000 shares, regional biotopes and uncertain habitat remain +separate fields and metrics. PHAB-derived hectares remain explicitly estimated. diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 4a457399..3640db48 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -238,3 +238,24 @@ parameters_json ## Local development default Use local filesystem paths. Keep MinIO/object storage as future extension. + +## Regional BWK/Natura 2000 evidence + +The regional state-2025 operator stores immutable evidence under: + +```text +storage/operator-evidence/bwk-natura2000-2025/regional/ + kempen-transport-region/partitions/{nis_code}/ + raw/*.json.gz + bwk_natura2000_2025.geojson + manifest.json + snapshots/kempen-transport-region/ + bwk_natura2000_2025.geojson + manifest.json +``` + +Partition reuse requires the same source version, municipality identity, +boundary checksum, normalized output checksum and all raw evidence checksums. +Snapshot reuse additionally binds the ordered set of 28 partition output +checksums. Only the assembled snapshot enters DatasetService/PostGIS; raw WFS +responses and partition files remain operator evidence on persistent storage. diff --git a/docs/TODO.md b/docs/TODO.md index f4c110b9..b86d162f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -25,6 +25,7 @@ - [x] Integrate governed Waterinfo/VMM annual station series without presenting point measurements as area-wide water volume. - [x] Add bounded historical orthophoto acquisition for official 1971-2025 products with a map overlay and no current-GRB QA on old imagery. - [x] Add BWK/Natura 2000 through an explicit provider/operator contract. +- [x] Expand BWK/Natura 2000 state 2025 from Mol to all 28 approved Kempen municipalities with partitioned source evidence. - [x] Add annual agricultural-use parcels through an explicit provider/operator contract. - [x] Extend the official 1778/1873/1969 historical buildings, water and roads series from Mol to the approved regional scope with partitioned source audits. - [x] Connect a drawn rectangle to bounded official orthophoto acquisition, local configured-YOLO detection and persisted GRB QA. diff --git a/frontend/README.md b/frontend/README.md index 99004faf..9eb27297 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -529,3 +529,13 @@ integral as current, permanent or concurrent water volume. - `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md` - `docs/API_CONTRACTS.md` - `docs/REPOSITORY_CONVENTIONS.md` + +## Area-aware map layer selection + +Map themes can have both a complete regional Dataset and a smaller exact-Area +snapshot from the same official source. The workbench prefers the Dataset whose +`area_id` exactly matches the selected Area; when no exact snapshot exists, the +regional Dataset remains available and the backend clips metrics and map output +to the selected municipality or drawn rectangle. The Sources workspace reports +overlapping BWK area snapshots as separate coverages and does not sum their +feature counts as if they were disjoint observations. diff --git a/frontend/src/components/datasets/SourceCatalogPanel.tsx b/frontend/src/components/datasets/SourceCatalogPanel.tsx index efcbbc39..cdf61c5b 100644 --- a/frontend/src/components/datasets/SourceCatalogPanel.tsx +++ b/frontend/src/components/datasets/SourceCatalogPanel.tsx @@ -117,6 +117,10 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E String(dataset.source_metadata?.['product_key'] ?? 'most_recent') !== 'most_recent', ) const bwkDatasets = ready.filter((dataset) => dataset.source_name === 'inbo_bwk_natura2000') + const largestBwkSnapshot = bwkDatasets.reduce( + (largest, dataset) => !largest || (dataset.feature_count ?? 0) > (largest.feature_count ?? 0) ? dataset : largest, + null, + ) const agricultureDatasets = ready.filter((dataset) => dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels') const buildingsRegisterDatasets = ready.filter( (dataset) => dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register', @@ -214,7 +218,10 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E {bwkDatasets.length > 0 ? (
BWK / Natura 2000 - Toestand 2025 · {bwkDatasets.reduce((total, dataset) => total + (dataset.feature_count ?? 0), 0).toLocaleString('nl-BE')} kaartvlakken + + Toestand 2025 · {bwkDatasets.length} gebiedsdekking{bwkDatasets.length === 1 ? '' : 'en'} ·{' '} + {(largestBwkSnapshot?.feature_count ?? 0).toLocaleString('nl-BE')} kaartvlakken in de grootste dekking +

Biologische waardering en habitataandelen blijven afzonderlijke, brongetrouwe metingen met herkomstinformatie.

) : null} diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index a181c14d..95266b09 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -178,6 +178,7 @@ function pickThemeDataset( ) candidates.sort((left, right) => { const score = (dataset: DatasetCreateResponse) => + (dataset.area_id && dataset.area_id === selectedAreaId ? 10_000_000 : 0) + (dataset.reference_layer_name && theme.tokens.includes(dataset.reference_layer_name.toLowerCase()) ? 1_000_000 : 0) + (dataset.source_name === 'grb' ? 100_000 : 0) + (dataset.source_name === 'department_omgeving_land_use' ? 90_000 : 0) + diff --git a/scripts/README.md b/scripts/README.md index c368fd75..c755e9d2 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1612,3 +1612,24 @@ REMOTE_PATH=/mnt/user/appdata/geointel REMOTE_REPO=gitea-widefrog:NuklearRabbit/geointel.git FRONTEND_URL=http://192.168.10.150:1202 ``` + +## Regional BWK/Natura 2000 state 2025 + +Prepare all official municipality partitions and inspect the combined manifest +without persisting a Dataset: + +```bash +docker exec geointel python /app/scripts/provision_regional_bwk_natura2000.py --fetch-only +``` + +Import the checksum-bound regional snapshot through DatasetService: + +```bash +docker exec geointel python /app/scripts/provision_regional_bwk_natura2000.py +``` + +The command requires the canonical geographic-scope manifest and 28-member +boundary artifact. Defaults cap each municipality at 30,000 source features and +the assembled snapshot at 300,000 features. It never truncates silently, never +writes `vector_features` directly and never turns the single 2025 state into a +fabricated historical series. diff --git a/scripts/provision_mol_bwk_natura2000.py b/scripts/provision_mol_bwk_natura2000.py index 4d3ada52..54becdff 100644 --- a/scripts/provision_mol_bwk_natura2000.py +++ b/scripts/provision_mol_bwk_natura2000.py @@ -319,7 +319,15 @@ def habitat_breakdown(properties: dict[str, Any]) -> tuple[list[dict[str, Any]], return entries, min(natura_share, 100.0), min(regional_share, 100.0), min(uncertain_share, 100.0) -def normalize_feature(feature: dict[str, Any], boundary_lambert72) -> tuple[dict[str, Any] | None, bool]: +def normalize_feature( + feature: dict[str, Any], + boundary_lambert72, + *, + municipality: str = "Mol", + nis_code: str = "13025", + coverage_scope: str = "municipality", + feature_id_suffix: str | None = None, +) -> tuple[dict[str, Any] | None, bool]: geometry_payload = feature.get("geometry") if not geometry_payload: return None, False @@ -342,6 +350,8 @@ def normalize_feature(feature: dict[str, Any], boundary_lambert72) -> tuple[dict if not source_id: source_id = sha256_bytes(json.dumps(geometry_payload, sort_keys=True).encode("utf-8")) stable_id = f"BWK:Bwkhab:{raw_properties.get('UIDN') or source_id}" + if feature_id_suffix: + stable_id = f"{stable_id}:{feature_id_suffix}" evaluation_code = str(raw_properties.get("EVAL") or "").strip().lower() habitat_status = str(raw_properties.get("HABLEGENDE") or "").strip().lower() habitats, natura_share, regional_share, uncertain_share = habitat_breakdown(raw_properties) @@ -357,9 +367,9 @@ def normalize_feature(feature: dict[str, Any], boundary_lambert72) -> tuple[dict "reference_layer_name": "nature_value", "theme": "nature_value", "authority_level": "authoritative", - "coverage_scope": "municipality", - "municipality": "Mol", - "nis_code": "13025", + "coverage_scope": coverage_scope, + "municipality": municipality, + "nis_code": nis_code, "source_version": SOURCE_VERSION, "attribution": ATTRIBUTION, "bwk_evaluation_code": evaluation_code or "unknown", diff --git a/scripts/provision_regional_bwk_natura2000.py b/scripts/provision_regional_bwk_natura2000.py new file mode 100644 index 00000000..9e93e3f3 --- /dev/null +++ b/scripts/provision_regional_bwk_natura2000.py @@ -0,0 +1,587 @@ +"""Provision an audited BWK/Natura 2000 snapshot for an approved region. + +The official WFS is queried per municipality. Every source response is retained +as deterministic gzip evidence, geometry is clipped in EPSG:31370, and one +regional GeoJSON snapshot is persisted through the canonical DatasetService API. +""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +import gzip +import json +import os +from pathlib import Path +import sys +from typing import Any + +import requests +from shapely.geometry import mapping, shape +from shapely.ops import transform as transform_geometry + +from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope, ScopeMember +import provision_mol_bwk_natura2000 as bwk + + +DEFAULT_SCOPE_KEY = "kempen-transport-region" +DEFAULT_SCOPE_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes") +DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/bwk-natura2000-2025/regional") +SCHEMA_VERSION = 1 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Provision regional official BWK/Natura 2000 reference data.") + parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY) + parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", bwk.DEFAULT_API_URL)) + parser.add_argument( + "--scope-output-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_SCOPE_OUTPUT_ROOT)), + ) + parser.add_argument( + "--output-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_REGIONAL_BWK_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)), + ) + parser.add_argument("--page-limit", type=int, default=1000) + parser.add_argument("--max-features-per-partition", type=int, default=30_000) + parser.add_argument("--max-total-features", type=int, default=300_000) + parser.add_argument("--request-timeout", type=int, default=300) + parser.add_argument("--import-timeout", type=int, default=3600) + parser.add_argument("--fetch-only", action="store_true") + parser.add_argument("--force", action="store_true") + return parser.parse_args() + + +def boundary_sha256(geometry) -> str: + return bwk.sha256_bytes(json.dumps(mapping(geometry), sort_keys=True, separators=(",", ":")).encode("utf-8")) + + +def resolve_member_boundaries(scope: GeographicScope, scope_output_root: Path) -> tuple[Path, str]: + scope_dir = scope_output_root / scope.key + manifest_path = scope_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json" + if not manifest_path.is_file(): + raise RuntimeError(f"Official scope manifest is missing at {manifest_path}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if ( + manifest.get("status") != "complete" + or manifest.get("scope_key") != scope.key + or int(manifest.get("member_count") or 0) != len(scope.members) + ): + raise RuntimeError(f"Official scope manifest at {manifest_path} is incomplete or inconsistent") + members_path = scope_dir / str(manifest.get("municipalities_filename") or "") + expected_sha256 = str(manifest.get("municipalities_sha256") or "") + if not members_path.is_file() or not expected_sha256 or bwk.sha256_file(members_path) != expected_sha256: + raise RuntimeError("Official municipality-boundary artifact is missing or fails its scope checksum") + return members_path, expected_sha256 + + +def load_member_boundaries(path: Path, scope: GeographicScope) -> dict[str, tuple[ScopeMember, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(features, list): + raise RuntimeError("Municipality-boundary artifact is not a GeoJSON FeatureCollection") + expected = {member.nis_code: member for member in scope.members} + selected: dict[str, tuple[ScopeMember, Any]] = {} + for feature in features: + properties = feature.get("properties") or {} + nis_code = str(properties.get("nis_code") or properties.get("NISCODE") or "") + if nis_code not in expected: + continue + if nis_code in selected: + raise RuntimeError(f"Municipality-boundary artifact contains duplicate NIS code {nis_code}") + geometry = bwk.polygonal_geometry(shape(feature.get("geometry"))) + if geometry is None: + raise RuntimeError(f"Municipality boundary for {expected[nis_code].name} is invalid") + selected[nis_code] = (expected[nis_code], geometry) + missing = [member.name for member in scope.members if member.nis_code not in selected] + if missing: + raise RuntimeError(f"Municipality-boundary artifact is missing: {', '.join(missing)}") + return {member.nis_code: selected[member.nis_code] for member in scope.members} + + +def partition_paths(output_root: Path, nis_code: str) -> tuple[Path, Path, Path]: + partition_dir = output_root / "partitions" / nis_code + return partition_dir / "bwk_natura2000_2025.geojson", partition_dir / "manifest.json", partition_dir / "raw" + + +def cached_partition( + output_root: Path, + member: ScopeMember, + boundary_hash: str, +) -> dict[str, Any] | None: + output_path, manifest_path, _raw_dir = partition_paths(output_root, member.nis_code) + if not output_path.is_file() or not manifest_path.is_file(): + return None + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if ( + manifest.get("schema_version") != SCHEMA_VERSION + or manifest.get("source_version") != bwk.SOURCE_VERSION + or manifest.get("nis_code") != member.nis_code + or manifest.get("boundary_sha256") != boundary_hash + or manifest.get("output_sha256") != bwk.sha256_file(output_path) + ): + return None + for page in manifest.get("raw_pages") or []: + page_path = manifest_path.parent / str(page.get("artifact_path") or "") + if not page_path.is_file() or page.get("artifact_sha256") != bwk.sha256_file(page_path): + return None + try: + raw_bytes = gzip.decompress(page_path.read_bytes()) + except (OSError, EOFError): + return None + if page.get("response_sha256") != bwk.sha256_bytes(raw_bytes): + return None + manifest["output_path"] = str(output_path) + manifest["manifest_path"] = str(manifest_path) + return manifest + + +def prepare_partition( + session, + *, + output_root: Path, + scope: GeographicScope, + member: ScopeMember, + boundary_wgs84, + page_limit: int, + max_features: int, + timeout: int, + force: bool, +) -> dict[str, Any]: + output_path, manifest_path, raw_dir = partition_paths(output_root, member.nis_code) + current_boundary_hash = boundary_sha256(boundary_wgs84) + if not force: + reusable = cached_partition(output_root, member, current_boundary_hash) + if reusable: + return reusable + + raw_dir.mkdir(parents=True, exist_ok=True) + boundary_lambert72 = bwk.polygonal_geometry(transform_geometry(bwk.TO_LAMBERT72.transform, boundary_wgs84)) + if boundary_lambert72 is None: + raise RuntimeError(f"Boundary for {member.name} could not be transformed to EPSG:31370") + + retained: list[dict[str, Any]] = [] + raw_pages: list[dict[str, Any]] = [] + seen_source_ids: set[str] = set() + raw_feature_count = 0 + duplicate_count = 0 + rejected_count = 0 + clipped_count = 0 + for page_number, (payload, source_url, raw_bytes) in enumerate( + bwk.iter_wfs_pages(session, boundary_wgs84.bounds, page_limit=page_limit, timeout=timeout), + start=1, + ): + page_features = list(payload.get("features") or []) + raw_feature_count += len(page_features) + if raw_feature_count > max_features: + raise RuntimeError( + f"BWK WFS returned more than {max_features} features for {member.name}; refusing a truncated partition" + ) + compressed = gzip.compress(raw_bytes, mtime=0) + raw_path = raw_dir / f"bwk_bwkhab_page_{page_number:05d}.json.gz" + bwk.write_bytes_atomic(raw_path, compressed) + raw_pages.append( + { + "artifact_path": str(raw_path.relative_to(manifest_path.parent)), + "artifact_sha256": bwk.sha256_bytes(compressed), + "response_sha256": bwk.sha256_bytes(raw_bytes), + "response_size_bytes": len(raw_bytes), + "feature_count": len(page_features), + "source_url": source_url, + } + ) + for source_feature in page_features: + source_id = str(source_feature.get("id") or (source_feature.get("properties") or {}).get("UIDN") or "") + if source_id and source_id in seen_source_ids: + duplicate_count += 1 + continue + if source_id: + seen_source_ids.add(source_id) + normalized, was_clipped = bwk.normalize_feature( + source_feature, + boundary_lambert72, + municipality=member.name, + nis_code=member.nis_code, + coverage_scope=scope.key, + feature_id_suffix=member.nis_code, + ) + if normalized is None: + rejected_count += 1 + continue + clipped_count += int(was_clipped) + retained.append(normalized) + + artifact = { + "type": "FeatureCollection", + "name": f"BWK en Natura 2000 - toestand 2025 - {member.name}", + "features": retained, + "source": "INBO BWK/Natura 2000 WFS", + "source_version": bwk.SOURCE_VERSION, + "coverage_scope": scope.key, + "municipality": member.name, + "nis_code": member.nis_code, + "reference_truncated": False, + } + bwk.write_json_atomic(output_path, artifact) + manifest = { + "schema_version": SCHEMA_VERSION, + "source_version": bwk.SOURCE_VERSION, + "scope_key": scope.key, + "municipality": member.name, + "nis_code": member.nis_code, + "boundary_sha256": current_boundary_hash, + "boundary_bbox_wgs84": list(boundary_wgs84.bounds), + "page_limit": page_limit, + "page_count": len(raw_pages), + "raw_source_feature_count": raw_feature_count, + "feature_count": len(retained), + "duplicate_count": duplicate_count, + "rejected_or_outside_count": rejected_count, + "clipped_feature_count": clipped_count, + "reference_truncated": False, + "raw_pages": raw_pages, + "output_path": str(output_path), + "output_sha256": bwk.sha256_file(output_path), + "generated_at": bwk.utc_now(), + } + bwk.write_json_atomic(manifest_path, manifest, pretty=True) + manifest["manifest_path"] = str(manifest_path) + return manifest + + +def snapshot_paths(output_root: Path, scope: GeographicScope) -> tuple[Path, Path]: + snapshot_dir = output_root / "snapshots" / scope.key + return snapshot_dir / "bwk_natura2000_2025.geojson", snapshot_dir / "manifest.json" + + +def assemble_snapshot( + *, + output_root: Path, + scope: GeographicScope, + partitions: list[dict[str, Any]], + member_boundaries_sha256: str, + max_total_features: int, +) -> tuple[Path, Path, dict[str, Any]]: + if [item.get("nis_code") for item in partitions] != list(scope.nis_codes): + raise RuntimeError("BWK partition order/completeness does not match the approved geographic scope") + output_path, manifest_path = snapshot_paths(output_root, scope) + partition_identity = bwk.sha256_bytes( + json.dumps( + [(item["nis_code"], item["output_sha256"]) for item in partitions], + separators=(",", ":"), + ).encode("utf-8") + ) + if output_path.is_file() and manifest_path.is_file(): + try: + existing = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + existing = {} + if ( + existing.get("partition_identity_sha256") == partition_identity + and existing.get("output_sha256") == bwk.sha256_file(output_path) + ): + return output_path, manifest_path, existing + + features: list[dict[str, Any]] = [] + feature_ids: set[str] = set() + evaluation_area: dict[str, float] = defaultdict(float) + habitat_status_area: dict[str, float] = defaultdict(float) + natura_area = 0.0 + regional_area = 0.0 + uncertain_area = 0.0 + for partition in partitions: + payload = json.loads(Path(str(partition["output_path"])).read_text(encoding="utf-8")) + partition_features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(partition_features, list) or len(partition_features) != int(partition["feature_count"]): + raise RuntimeError(f"BWK partition feature-count drift for NIS {partition['nis_code']}") + for feature in partition_features: + feature_id = str(feature.get("id") or "") + if not feature_id or feature_id in feature_ids: + raise RuntimeError(f"Duplicate or missing regional BWK feature id {feature_id!r}") + feature_ids.add(feature_id) + properties = feature.get("properties") or {} + area = float(properties.get("clipped_area_ha") or 0.0) + evaluation_area[str(properties.get("bwk_evaluation_code") or "unknown")] += area + habitat_status_area[str(properties.get("habitat_status_code") or "unknown")] += area + natura_area += float(properties.get("natura2000_area_ha") or 0.0) + regional_area += float(properties.get("regional_biotope_area_ha") or 0.0) + uncertain_area += float(properties.get("uncertain_habitat_area_ha") or 0.0) + features.append(feature) + if len(features) > max_total_features: + raise RuntimeError( + f"Regional BWK snapshot exceeds the {max_total_features} feature safety limit; refusing truncation" + ) + + generated_at = bwk.utc_now() + artifact = { + "type": "FeatureCollection", + "name": f"BWK en Natura 2000 - toestand 2025 - {scope.display_name}", + "features": features, + "source": "INBO BWK/Natura 2000 WFS", + "source_version": bwk.SOURCE_VERSION, + "attribution": bwk.ATTRIBUTION, + "catalog_url": bwk.CATALOG_URL, + "report_url": bwk.REPORT_URL, + "coverage_scope": scope.key, + "member_count": len(scope.members), + "reference_truncated": False, + "generated_at": generated_at, + } + bwk.write_json_atomic(output_path, artifact) + empty_partitions = [item["nis_code"] for item in partitions if int(item["feature_count"]) == 0] + manifest = { + "schema_version": SCHEMA_VERSION, + "source_version": bwk.SOURCE_VERSION, + "scope_key": scope.key, + "scope_display_name": scope.display_name, + "member_count": len(scope.members), + "member_nis_codes": list(scope.nis_codes), + "member_boundaries_sha256": member_boundaries_sha256, + "coverage_complete": len(partitions) == len(scope.members), + "empty_partitions": empty_partitions, + "feature_count": len(features), + "raw_source_feature_count": sum(int(item["raw_source_feature_count"]) for item in partitions), + "raw_response_count": sum(int(item["page_count"]) for item in partitions), + "partition_identity_sha256": partition_identity, + "partitions": [ + { + "municipality": item["municipality"], + "nis_code": item["nis_code"], + "feature_count": item["feature_count"], + "raw_source_feature_count": item["raw_source_feature_count"], + "output_sha256": item["output_sha256"], + "manifest_path": item["manifest_path"], + } + for item in partitions + ], + "evaluation_area_ha": {key: round(value, 6) for key, value in sorted(evaluation_area.items())}, + "habitat_status_area_ha": {key: round(value, 6) for key, value in sorted(habitat_status_area.items())}, + "natura2000_area_ha": round(natura_area, 6), + "regional_biotope_area_ha": round(regional_area, 6), + "uncertain_habitat_area_ha": round(uncertain_area, 6), + "reference_truncated": False, + "output_sha256": bwk.sha256_file(output_path), + "output_size_bytes": output_path.stat().st_size, + "generated_at": generated_at, + "limitations": [ + "The 2025 edition is the best available map state, not one uniform 2025 field survey.", + "PHAB percentages may be theoretical shares; partial selections scale shares by intersected polygon area.", + "Municipality partitions split source polygons at administrative boundaries; source ids carry a NIS suffix.", + "The real field situation remains authoritative for policy and legal use.", + ], + } + bwk.write_json_atomic(manifest_path, manifest, pretty=True) + return output_path, manifest_path, manifest + + +def locate_workspace(session, base_url: str, scope: GeographicScope, timeout: int): + projects = bwk.paginated_items(session, f"{base_url}/api/v1/projects", timeout=timeout) + project = next((item for item in projects if item.get("name") == scope.project_name), None) + if not project: + raise RuntimeError(f"Project {scope.project_name!r} is missing") + project_id = str(project["id"]) + areas = bwk.paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/areas", timeout=timeout) + area = next((item for item in areas if item.get("name") == scope.area_name), None) + if not area: + raise RuntimeError(f"Official scope Area {scope.area_name!r} is missing") + datasets = bwk.paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/datasets", timeout=timeout) + return project_id, str(area["id"]), datasets + + +def upload_snapshot( + session, + *, + base_url: str, + project_id: str, + area_id: str, + scope: GeographicScope, + path: Path, + manifest_path: Path, + manifest: dict[str, Any], + timeout: int, +) -> dict[str, Any]: + source_metadata = { + "provider": "Instituut voor Natuur- en Bosonderzoek", + "theme": "nature_value", + "layer_name": "BWK/Natura 2000 toestand 2025", + "authority_level": "authoritative", + "coverage_scope": scope.key, + "scope_type": scope.scope_type, + "scope_display_name": scope.display_name, + "member_count": len(scope.members), + "member_nis_codes": list(scope.nis_codes), + "partitioned_source_audit": True, + "coverage_complete": bool(manifest["coverage_complete"]), + "feature_count": manifest["feature_count"], + "geometry_clipped_to_area": True, + "semantic_metrics": False, + "attribution": bwk.ATTRIBUTION, + "catalog_url": bwk.CATALOG_URL, + "report_url": bwk.REPORT_URL, + "selection_aggregation": { + "metric_key": "bwk_mapped_area", + "method": "intersection_area", + "label": "BWK-gekarteerde oppervlakte", + "unit": "ha", + "geometry_dimension": 2, + "warning": "Uitgave 2025 is de best beschikbare kaarttoestand, maar niet elk kaartvlak is in 2025 op terrein gekarteerd.", + }, + "selection_metrics": bwk.selection_metrics(), + } + provenance_metadata = { + "operator_tool": "provision_regional_bwk_natura2000.py", + "operator_explicit_fetch": True, + "geometry_clipped_to_area": True, + "source_type_name": bwk.TYPE_NAME, + "wfs_url": bwk.WFS_URL, + "catalog_url": bwk.CATALOG_URL, + "report_url": bwk.REPORT_URL, + "manifest_path": str(manifest_path), + "combined_output_sha256": manifest["output_sha256"], + "partition_count": len(manifest["partitions"]), + "partition_identity_sha256": manifest["partition_identity_sha256"], + "raw_source_responses_retained": True, + "reference_truncated": False, + "generated_at": manifest["generated_at"], + "limitations": manifest["limitations"], + } + with path.open("rb") as handle: + response = session.post( + f"{base_url}/api/v1/projects/{project_id}/datasets/upload", + data={ + "dataset_type": "vector", + "source": "operator_official_import", + "dataset_role": "reference", + "source_name": "inbo_bwk_natura2000", + "reference_layer_name": "nature_value", + "source_metadata_json": json.dumps(source_metadata, ensure_ascii=False), + "provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False), + "area_id": area_id, + "temporal_series_key": f"inbo-bwk-natura2000:{scope.key}", + "observed_at": bwk.PUBLICATION_DATE, + "temporal_granularity": "snapshot", + "source_version": bwk.SOURCE_VERSION, + }, + files={"file": (path.name, handle, "application/geo+json")}, + timeout=timeout, + ) + return bwk.response_data(response) + + +def main() -> int: + args = parse_args() + try: + if ( + args.page_limit < 1 + or args.page_limit > 5000 + or args.max_features_per_partition < args.page_limit + or args.max_total_features < args.max_features_per_partition + ): + raise ValueError("Invalid BWK page or feature safety limits") + scope = GEOGRAPHIC_SCOPES[args.scope] + members_path, members_sha256 = resolve_member_boundaries(scope, args.scope_output_root) + boundaries = load_member_boundaries(members_path, scope) + source_session = bwk.source_session() + source_session.headers.update({"User-Agent": "GeoIntel-BWK-Natura2000-Regional-Operator/1.0"}) + with source_session: + partitions = [ + prepare_partition( + source_session, + output_root=args.output_root / scope.key, + scope=scope, + member=member, + boundary_wgs84=boundary, + page_limit=args.page_limit, + max_features=args.max_features_per_partition, + timeout=args.request_timeout, + force=args.force, + ) + for member, boundary in boundaries.values() + ] + path, manifest_path, manifest = assemble_snapshot( + output_root=args.output_root, + scope=scope, + partitions=partitions, + member_boundaries_sha256=members_sha256, + max_total_features=args.max_total_features, + ) + result: dict[str, Any] + if args.fetch_only: + result = {"status": "prepared", "artifact_path": str(path), "feature_count": manifest["feature_count"]} + else: + with requests.Session() as api_session: + project_id, area_id, datasets = locate_workspace( + api_session, args.base_url.rstrip("/"), scope, args.import_timeout + ) + existing = next( + ( + item + for item in datasets + if item.get("source_name") == "inbo_bwk_natura2000" + and item.get("source_version") == bwk.SOURCE_VERSION + and str(item.get("area_id") or "") == area_id + ), + None, + ) + if existing: + if str(existing.get("checksum_sha256") or "") != manifest["output_sha256"]: + raise RuntimeError("A different regional BWK 2025 artifact already exists; refusing replacement") + result = { + "status": "existing", + "dataset_id": existing["id"], + "feature_count": existing.get("feature_count"), + } + else: + dataset = upload_snapshot( + api_session, + base_url=args.base_url.rstrip("/"), + project_id=project_id, + area_id=area_id, + scope=scope, + path=path, + manifest_path=manifest_path, + manifest=manifest, + timeout=args.import_timeout, + ) + result = { + "status": "imported", + "dataset_id": dataset["id"], + "feature_count": dataset.get("feature_count"), + } + except (KeyError, OSError, RuntimeError, ValueError, requests.RequestException) as exc: + print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 1 + + print( + json.dumps( + { + "status": "ok", + "scope": scope.key, + "member_count": len(scope.members), + "partition_count": len(partitions), + "feature_count": manifest["feature_count"], + "raw_source_feature_count": manifest["raw_source_feature_count"], + "raw_response_count": manifest["raw_response_count"], + "manifest_path": str(manifest_path), + "metrics": { + "evaluation_area_ha": manifest["evaluation_area_ha"], + "natura2000_area_ha": manifest["natura2000_area_ha"], + "regional_biotope_area_ha": manifest["regional_biotope_area_ha"], + "uncertain_habitat_area_ha": manifest["uncertain_habitat_area_ha"], + }, + "result": result, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index b32fdc82..ca85c429 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -50,6 +50,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py ${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py +${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py ${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py