Expand BWK nature data across Kempen
This commit is contained in:
@@ -7,6 +7,19 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 210 Regional BWK/Natura 2000 expansion (2026-07-16)
|
||||
|
||||
- Added an explicit 28-municipality operator for the official INBO BWK/Natura
|
||||
2000 state-2025 WFS, reusing the governed Mol normalization and metrics.
|
||||
- Added exact EPSG:31370 municipality clipping, partition-unique source ids,
|
||||
deterministic gzip source evidence, checksum-bound cache reuse and one
|
||||
canonical regional Dataset upload.
|
||||
- Made map dataset selection prefer an exact Area snapshot over a broader
|
||||
compatible layer and changed the source inventory to report overlapping
|
||||
area snapshots without summing duplicate coverage.
|
||||
- Added focused GIS, persistence-contract, packaging and UI regression tests.
|
||||
No API contract, database migration, AI model or product scope changed.
|
||||
|
||||
## Sprint 209 Regional historical land-use expansion (2026-07-15)
|
||||
|
||||
- Added an explicit operator for the official 1778, 1873 and 1969 historical
|
||||
|
||||
@@ -1289,3 +1289,17 @@ evidence without persistence. A conflicting checksum for an already persisted
|
||||
state-2025 Mol Dataset fails closed instead of creating a silent replacement.
|
||||
PostGIS selection summaries keep BWK value classes separate and label
|
||||
PHAB-derived habitat hectares as estimates.
|
||||
|
||||
For the complete approved Kempen transport region, run the partitioned
|
||||
operator after `provision_geographic_scope.py --scope kempen-transport-region`:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_bwk_natura2000.py
|
||||
```
|
||||
|
||||
Use `--fetch-only` to build and validate all 28 municipality partitions without
|
||||
database persistence. A normal rerun validates and reuses the immutable source
|
||||
evidence and existing Dataset. `--force` explicitly refetches the WFS but still
|
||||
fails closed if a different state-2025 checksum is already persisted. The
|
||||
regional output uses the same selection-summary API as Mol; no new endpoint or
|
||||
direct PostGIS write is introduced.
|
||||
|
||||
@@ -26,6 +26,7 @@ FULL_AREA_CLIPPED_OPERATOR_TOOLS = {
|
||||
"provision_regional_historical_landuse.py",
|
||||
"provision_waterinfo_station_history.py",
|
||||
"provision_mol_bwk_natura2000.py",
|
||||
"provision_regional_bwk_natura2000.py",
|
||||
"provision_agricultural_parcel_history.py",
|
||||
"provision_buildings_addresses_register.py",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box, mapping, shape
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
|
||||
def load_script():
|
||||
path = SCRIPTS / "provision_regional_bwk_natura2000.py"
|
||||
spec = importlib.util.spec_from_file_location("test_provision_regional_bwk_natura2000", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
ok = True
|
||||
text = ""
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.content = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakeApiSession:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.calls = []
|
||||
|
||||
def post(self, url, **kwargs):
|
||||
self.calls.append((url, kwargs))
|
||||
return FakeResponse({"data": self.payload})
|
||||
|
||||
|
||||
def source_feature(feature_id: str, geometry):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": mapping(geometry),
|
||||
"properties": {
|
||||
"UIDN": feature_id,
|
||||
"EVAL": "z",
|
||||
"HAB1": "9190",
|
||||
"PHAB1": 50,
|
||||
"HABLEGENDE": "hab",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_partition_retains_gzipped_source_and_applies_member_context(tmp_path: Path, monkeypatch) -> None:
|
||||
module = load_script()
|
||||
scope = module.GeographicScope(
|
||||
key="test-region",
|
||||
display_name="Test region",
|
||||
project_name="Test",
|
||||
project_region="Test",
|
||||
area_name="Test area",
|
||||
authority_name="Test",
|
||||
authority_url="https://example.test",
|
||||
scope_type="test",
|
||||
limitation_message="Test",
|
||||
members=(module.ScopeMember("Mol", "13025"),),
|
||||
)
|
||||
boundary = box(5.0, 51.0, 5.1, 51.1)
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [source_feature("Bwkhab.1", box(4.98, 51.02, 5.05, 51.08))],
|
||||
}
|
||||
raw_bytes = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
monkeypatch.setattr(
|
||||
module.bwk,
|
||||
"iter_wfs_pages",
|
||||
lambda *_args, **_kwargs: iter([(payload, "https://example.test/page", raw_bytes)]),
|
||||
)
|
||||
|
||||
manifest = module.prepare_partition(
|
||||
object(),
|
||||
output_root=tmp_path,
|
||||
scope=scope,
|
||||
member=scope.members[0],
|
||||
boundary_wgs84=boundary,
|
||||
page_limit=100,
|
||||
max_features=1000,
|
||||
timeout=30,
|
||||
force=False,
|
||||
)
|
||||
output = json.loads(Path(manifest["output_path"]).read_text(encoding="utf-8"))
|
||||
feature = output["features"][0]
|
||||
raw_path = Path(manifest["manifest_path"]).parent / manifest["raw_pages"][0]["artifact_path"]
|
||||
|
||||
assert manifest["feature_count"] == 1
|
||||
assert feature["id"] == "BWK:Bwkhab:Bwkhab.1:13025"
|
||||
assert feature["properties"]["municipality"] == "Mol"
|
||||
assert feature["properties"]["coverage_scope"] == "test-region"
|
||||
assert shape(feature["geometry"]).bounds == pytest.approx((5.0, 51.02, 5.05, 51.08), abs=1e-5)
|
||||
assert gzip.decompress(raw_path.read_bytes()) == raw_bytes
|
||||
|
||||
cached = module.prepare_partition(
|
||||
object(),
|
||||
output_root=tmp_path,
|
||||
scope=scope,
|
||||
member=scope.members[0],
|
||||
boundary_wgs84=boundary,
|
||||
page_limit=100,
|
||||
max_features=1000,
|
||||
timeout=30,
|
||||
force=False,
|
||||
)
|
||||
assert cached["output_sha256"] == manifest["output_sha256"]
|
||||
|
||||
|
||||
def test_snapshot_assembles_unique_partitions_and_metrics(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
scope = module.GeographicScope(
|
||||
key="test-region",
|
||||
display_name="Test region",
|
||||
project_name="Test",
|
||||
project_region="Test",
|
||||
area_name="Test area",
|
||||
authority_name="Test",
|
||||
authority_url="https://example.test",
|
||||
scope_type="test",
|
||||
limitation_message="Test",
|
||||
members=(module.ScopeMember("Left", "10001"), module.ScopeMember("Right", "10002")),
|
||||
)
|
||||
partitions = []
|
||||
for index, member in enumerate(scope.members):
|
||||
output_path, manifest_path, _raw_dir = module.partition_paths(tmp_path / scope.key, member.nis_code)
|
||||
feature = source_feature(f"Bwkhab.{index}", box(index, 0, index + 0.5, 0.5))
|
||||
feature["id"] = f"BWK:Bwkhab:Bwkhab.{index}:{member.nis_code}"
|
||||
feature["properties"].update(
|
||||
{
|
||||
"clipped_area_ha": 1.0 + index,
|
||||
"bwk_evaluation_code": "z",
|
||||
"habitat_status_code": "hab",
|
||||
"natura2000_area_ha": 0.5,
|
||||
"regional_biotope_area_ha": 0.25,
|
||||
"uncertain_habitat_area_ha": 0.0,
|
||||
}
|
||||
)
|
||||
module.bwk.write_json_atomic(output_path, {"type": "FeatureCollection", "features": [feature]})
|
||||
partitions.append(
|
||||
{
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"feature_count": 1,
|
||||
"raw_source_feature_count": 1,
|
||||
"page_count": 1,
|
||||
"output_path": str(output_path),
|
||||
"output_sha256": module.bwk.sha256_file(output_path),
|
||||
"manifest_path": str(manifest_path),
|
||||
}
|
||||
)
|
||||
|
||||
output_path, _manifest_path, manifest = module.assemble_snapshot(
|
||||
output_root=tmp_path,
|
||||
scope=scope,
|
||||
partitions=partitions,
|
||||
member_boundaries_sha256="boundaries-hash",
|
||||
max_total_features=10,
|
||||
)
|
||||
output = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
|
||||
assert manifest["coverage_complete"] is True
|
||||
assert manifest["feature_count"] == 2
|
||||
assert manifest["evaluation_area_ha"]["z"] == 3.0
|
||||
assert manifest["natura2000_area_ha"] == 1.0
|
||||
assert len({feature["id"] for feature in output["features"]}) == 2
|
||||
|
||||
|
||||
def test_upload_contract_is_regional_partitioned_and_canonical(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
|
||||
path = tmp_path / "bwk.geojson"
|
||||
path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest = {
|
||||
"coverage_complete": True,
|
||||
"feature_count": 42,
|
||||
"output_sha256": "output-hash",
|
||||
"partition_identity_sha256": "partition-hash",
|
||||
"partitions": [{} for _ in scope.members],
|
||||
"generated_at": "2026-07-16T00:00:00+00:00",
|
||||
"limitations": ["test"],
|
||||
}
|
||||
session = FakeApiSession({"id": "dataset-id", "feature_count": 42})
|
||||
|
||||
result = module.upload_snapshot(
|
||||
session,
|
||||
base_url="http://backend:8000",
|
||||
project_id="project-id",
|
||||
area_id="area-id",
|
||||
scope=scope,
|
||||
path=path,
|
||||
manifest_path=manifest_path,
|
||||
manifest=manifest,
|
||||
timeout=30,
|
||||
)
|
||||
data = session.calls[0][1]["data"]
|
||||
source_metadata = json.loads(data["source_metadata_json"])
|
||||
provenance = json.loads(data["provenance_metadata_json"])
|
||||
|
||||
assert result["id"] == "dataset-id"
|
||||
assert data["area_id"] == "area-id"
|
||||
assert data["temporal_series_key"] == "inbo-bwk-natura2000:kempen-transport-region"
|
||||
assert source_metadata["coverage_scope"] == "kempen-transport-region"
|
||||
assert source_metadata["member_count"] == 28
|
||||
assert source_metadata["partitioned_source_audit"] is True
|
||||
assert source_metadata["selection_metrics"] == module.bwk.selection_metrics()
|
||||
assert provenance["operator_tool"] == "provision_regional_bwk_natura2000.py"
|
||||
assert provenance["raw_source_responses_retained"] is True
|
||||
|
||||
|
||||
def test_regional_operator_is_packaged_release_checked_and_exact_area_is_preferred() -> None:
|
||||
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
service = (ROOT / "backend/app/services/vector_feature_service.py").read_text(encoding="utf-8")
|
||||
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
catalog = (ROOT / "frontend/src/components/datasets/SourceCatalogPanel.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "COPY scripts/provision_regional_bwk_natura2000.py" in dockerfile
|
||||
assert "py_compile scripts/provision_regional_bwk_natura2000.py" in readiness
|
||||
assert '"provision_regional_bwk_natura2000.py"' in service
|
||||
assert "dataset.area_id === selectedAreaId ? 10_000_000" in workspace
|
||||
assert "largestBwkSnapshot" in catalog
|
||||
@@ -82,6 +82,7 @@ COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_reg
|
||||
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
||||
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
|
||||
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
||||
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
||||
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
||||
COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py
|
||||
COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py
|
||||
|
||||
@@ -8880,3 +8880,34 @@ Next:
|
||||
next implementation pass can focus on current-only nature/terrain/flood
|
||||
regional expansion or on a benchmarked data-refresh scheduler; it must not
|
||||
reinterpret these historical surfaces as modern object lineage.
|
||||
|
||||
## Sprint 210 - Regional BWK/Natura 2000 expansion (2026-07-16)
|
||||
|
||||
Implemented:
|
||||
- Added `provision_regional_bwk_natura2000.py` for the approved 28-municipality
|
||||
Kempen transport-region scope, using the existing official INBO state-2025
|
||||
source contract and governed Mol feature normalization.
|
||||
- Added per-municipality WFS pagination, exact EPSG:31370 clipping,
|
||||
municipality-suffixed feature identities, deterministic gzip retention of
|
||||
every exact source response and checksum-bound partition/snapshot reuse.
|
||||
- Kept one canonical regional Dataset upload behind DatasetService. No direct
|
||||
`vector_features` write, migration, endpoint or fabricated time series was
|
||||
introduced.
|
||||
- Made map theme selection prefer an exact selected-Area Dataset over a broader
|
||||
compatible regional Dataset. Updated the source inventory to present
|
||||
overlapping BWK snapshots as area coverages without double-counting their
|
||||
feature totals.
|
||||
- Added focused tests for source evidence, GIS clipping, context fields,
|
||||
regional assembly, semantic metrics, upload metadata, packaging and UI
|
||||
selection behavior.
|
||||
|
||||
Validation so far:
|
||||
- Focused BWK suites pass 11 tests.
|
||||
- Full local readiness passes backend compilation, 689 backend tests, API
|
||||
contract audit, one Alembic head (`202607150001`), frontend typecheck/build
|
||||
and script syntax gates.
|
||||
|
||||
Open for this pass:
|
||||
- Deploy the validated operator to Tower, fetch and persist all 28 official
|
||||
partitions, verify idempotency and PostGIS row/geometry/checksum consistency,
|
||||
then validate complete-region and Mol map/metric behavior in the browser.
|
||||
|
||||
@@ -445,3 +445,25 @@ Sprint 7B exposes provider metadata only. It does not perform GRB WFS calls, OSM
|
||||
| `fixture` | fixture | true | demo/test fixture layers | `dataset_role=reference`, `source_name=fixture` |
|
||||
|
||||
Future provider output must flow through `DatasetService` and `VectorFeatureService`; providers must not write directly to `vector_features`.
|
||||
|
||||
## Regional BWK/Natura 2000 state 2025
|
||||
|
||||
`scripts/provision_regional_bwk_natura2000.py` expands the governed Mol
|
||||
operator to the approved 28-municipality Kempen transport-region scope. It
|
||||
queries the same official INBO `BWK:Bwkhab` WFS collection separately for each
|
||||
official VRBG municipality boundary. Every response is retained as a
|
||||
deterministic gzip artifact with both compressed-artifact and original-response
|
||||
SHA256 checksums.
|
||||
|
||||
Geometry is intersected with each municipality in EPSG:31370 before it is
|
||||
transformed to EPSG:4326. Source polygons crossing a municipality boundary are
|
||||
therefore split into auditable pieces and receive a NIS suffix; those pieces
|
||||
must not be interpreted as independent source observations. The 28 validated
|
||||
partitions are assembled into one regional snapshot and persisted through the
|
||||
ordinary Dataset upload API with `source_name=inbo_bwk_natura2000`,
|
||||
`reference_layer_name=nature_value` and
|
||||
`temporal_series_key=inbo-bwk-natura2000:kempen-transport-region`.
|
||||
|
||||
The state remains a single 2025 map edition, not an annual time series. BWK
|
||||
valuation, Natura 2000 shares, regional biotopes and uncertain habitat remain
|
||||
separate fields and metrics. PHAB-derived hectares remain explicitly estimated.
|
||||
|
||||
@@ -238,3 +238,24 @@ parameters_json
|
||||
## Local development default
|
||||
|
||||
Use local filesystem paths. Keep MinIO/object storage as future extension.
|
||||
|
||||
## Regional BWK/Natura 2000 evidence
|
||||
|
||||
The regional state-2025 operator stores immutable evidence under:
|
||||
|
||||
```text
|
||||
storage/operator-evidence/bwk-natura2000-2025/regional/
|
||||
kempen-transport-region/partitions/{nis_code}/
|
||||
raw/*.json.gz
|
||||
bwk_natura2000_2025.geojson
|
||||
manifest.json
|
||||
snapshots/kempen-transport-region/
|
||||
bwk_natura2000_2025.geojson
|
||||
manifest.json
|
||||
```
|
||||
|
||||
Partition reuse requires the same source version, municipality identity,
|
||||
boundary checksum, normalized output checksum and all raw evidence checksums.
|
||||
Snapshot reuse additionally binds the ordered set of 28 partition output
|
||||
checksums. Only the assembled snapshot enters DatasetService/PostGIS; raw WFS
|
||||
responses and partition files remain operator evidence on persistent storage.
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
- [x] Integrate governed Waterinfo/VMM annual station series without presenting point measurements as area-wide water volume.
|
||||
- [x] Add bounded historical orthophoto acquisition for official 1971-2025 products with a map overlay and no current-GRB QA on old imagery.
|
||||
- [x] Add BWK/Natura 2000 through an explicit provider/operator contract.
|
||||
- [x] Expand BWK/Natura 2000 state 2025 from Mol to all 28 approved Kempen municipalities with partitioned source evidence.
|
||||
- [x] Add annual agricultural-use parcels through an explicit provider/operator contract.
|
||||
- [x] Extend the official 1778/1873/1969 historical buildings, water and roads series from Mol to the approved regional scope with partitioned source audits.
|
||||
- [x] Connect a drawn rectangle to bounded official orthophoto acquisition, local configured-YOLO detection and persisted GRB QA.
|
||||
|
||||
@@ -529,3 +529,13 @@ integral as current, permanent or concurrent water volume.
|
||||
- `docs/40-build-launch/SPRINT_1_SCOPE_FREEZE.md`
|
||||
- `docs/API_CONTRACTS.md`
|
||||
- `docs/REPOSITORY_CONVENTIONS.md`
|
||||
|
||||
## Area-aware map layer selection
|
||||
|
||||
Map themes can have both a complete regional Dataset and a smaller exact-Area
|
||||
snapshot from the same official source. The workbench prefers the Dataset whose
|
||||
`area_id` exactly matches the selected Area; when no exact snapshot exists, the
|
||||
regional Dataset remains available and the backend clips metrics and map output
|
||||
to the selected municipality or drawn rectangle. The Sources workspace reports
|
||||
overlapping BWK area snapshots as separate coverages and does not sum their
|
||||
feature counts as if they were disjoint observations.
|
||||
|
||||
@@ -117,6 +117,10 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
||||
String(dataset.source_metadata?.['product_key'] ?? 'most_recent') !== 'most_recent',
|
||||
)
|
||||
const bwkDatasets = ready.filter((dataset) => dataset.source_name === 'inbo_bwk_natura2000')
|
||||
const largestBwkSnapshot = bwkDatasets.reduce<DatasetCreateResponse | null>(
|
||||
(largest, dataset) => !largest || (dataset.feature_count ?? 0) > (largest.feature_count ?? 0) ? dataset : largest,
|
||||
null,
|
||||
)
|
||||
const agricultureDatasets = ready.filter((dataset) => dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels')
|
||||
const buildingsRegisterDatasets = ready.filter(
|
||||
(dataset) => dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register',
|
||||
@@ -214,7 +218,10 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
||||
{bwkDatasets.length > 0 ? (
|
||||
<article>
|
||||
<strong>BWK / Natura 2000</strong>
|
||||
<span>Toestand 2025 · {bwkDatasets.reduce((total, dataset) => total + (dataset.feature_count ?? 0), 0).toLocaleString('nl-BE')} kaartvlakken</span>
|
||||
<span>
|
||||
Toestand 2025 · {bwkDatasets.length} gebiedsdekking{bwkDatasets.length === 1 ? '' : 'en'} ·{' '}
|
||||
{(largestBwkSnapshot?.feature_count ?? 0).toLocaleString('nl-BE')} kaartvlakken in de grootste dekking
|
||||
</span>
|
||||
<p>Biologische waardering en habitataandelen blijven afzonderlijke, brongetrouwe metingen met herkomstinformatie.</p>
|
||||
</article>
|
||||
) : null}
|
||||
|
||||
@@ -178,6 +178,7 @@ function pickThemeDataset(
|
||||
)
|
||||
candidates.sort((left, right) => {
|
||||
const score = (dataset: DatasetCreateResponse) =>
|
||||
(dataset.area_id && dataset.area_id === selectedAreaId ? 10_000_000 : 0) +
|
||||
(dataset.reference_layer_name && theme.tokens.includes(dataset.reference_layer_name.toLowerCase()) ? 1_000_000 : 0) +
|
||||
(dataset.source_name === 'grb' ? 100_000 : 0) +
|
||||
(dataset.source_name === 'department_omgeving_land_use' ? 90_000 : 0) +
|
||||
|
||||
@@ -1612,3 +1612,24 @@ REMOTE_PATH=/mnt/user/appdata/geointel
|
||||
REMOTE_REPO=gitea-widefrog:NuklearRabbit/geointel.git
|
||||
FRONTEND_URL=http://192.168.10.150:1202
|
||||
```
|
||||
|
||||
## Regional BWK/Natura 2000 state 2025
|
||||
|
||||
Prepare all official municipality partitions and inspect the combined manifest
|
||||
without persisting a Dataset:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_bwk_natura2000.py --fetch-only
|
||||
```
|
||||
|
||||
Import the checksum-bound regional snapshot through DatasetService:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_bwk_natura2000.py
|
||||
```
|
||||
|
||||
The command requires the canonical geographic-scope manifest and 28-member
|
||||
boundary artifact. Defaults cap each municipality at 30,000 source features and
|
||||
the assembled snapshot at 300,000 features. It never truncates silently, never
|
||||
writes `vector_features` directly and never turns the single 2025 state into a
|
||||
fabricated historical series.
|
||||
|
||||
@@ -319,7 +319,15 @@ def habitat_breakdown(properties: dict[str, Any]) -> tuple[list[dict[str, Any]],
|
||||
return entries, min(natura_share, 100.0), min(regional_share, 100.0), min(uncertain_share, 100.0)
|
||||
|
||||
|
||||
def normalize_feature(feature: dict[str, Any], boundary_lambert72) -> tuple[dict[str, Any] | None, bool]:
|
||||
def normalize_feature(
|
||||
feature: dict[str, Any],
|
||||
boundary_lambert72,
|
||||
*,
|
||||
municipality: str = "Mol",
|
||||
nis_code: str = "13025",
|
||||
coverage_scope: str = "municipality",
|
||||
feature_id_suffix: str | None = None,
|
||||
) -> tuple[dict[str, Any] | None, bool]:
|
||||
geometry_payload = feature.get("geometry")
|
||||
if not geometry_payload:
|
||||
return None, False
|
||||
@@ -342,6 +350,8 @@ def normalize_feature(feature: dict[str, Any], boundary_lambert72) -> tuple[dict
|
||||
if not source_id:
|
||||
source_id = sha256_bytes(json.dumps(geometry_payload, sort_keys=True).encode("utf-8"))
|
||||
stable_id = f"BWK:Bwkhab:{raw_properties.get('UIDN') or source_id}"
|
||||
if feature_id_suffix:
|
||||
stable_id = f"{stable_id}:{feature_id_suffix}"
|
||||
evaluation_code = str(raw_properties.get("EVAL") or "").strip().lower()
|
||||
habitat_status = str(raw_properties.get("HABLEGENDE") or "").strip().lower()
|
||||
habitats, natura_share, regional_share, uncertain_share = habitat_breakdown(raw_properties)
|
||||
@@ -357,9 +367,9 @@ def normalize_feature(feature: dict[str, Any], boundary_lambert72) -> tuple[dict
|
||||
"reference_layer_name": "nature_value",
|
||||
"theme": "nature_value",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": "municipality",
|
||||
"municipality": "Mol",
|
||||
"nis_code": "13025",
|
||||
"coverage_scope": coverage_scope,
|
||||
"municipality": municipality,
|
||||
"nis_code": nis_code,
|
||||
"source_version": SOURCE_VERSION,
|
||||
"attribution": ATTRIBUTION,
|
||||
"bwk_evaluation_code": evaluation_code or "unknown",
|
||||
|
||||
@@ -0,0 +1,587 @@
|
||||
"""Provision an audited BWK/Natura 2000 snapshot for an approved region.
|
||||
|
||||
The official WFS is queried per municipality. Every source response is retained
|
||||
as deterministic gzip evidence, geometry is clipped in EPSG:31370, and one
|
||||
regional GeoJSON snapshot is persisted through the canonical DatasetService API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from shapely.geometry import mapping, shape
|
||||
from shapely.ops import transform as transform_geometry
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope, ScopeMember
|
||||
import provision_mol_bwk_natura2000 as bwk
|
||||
|
||||
|
||||
DEFAULT_SCOPE_KEY = "kempen-transport-region"
|
||||
DEFAULT_SCOPE_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/bwk-natura2000-2025/regional")
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision regional official BWK/Natura 2000 reference data.")
|
||||
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", bwk.DEFAULT_API_URL))
|
||||
parser.add_argument(
|
||||
"--scope-output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_SCOPE_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_REGIONAL_BWK_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument("--page-limit", type=int, default=1000)
|
||||
parser.add_argument("--max-features-per-partition", type=int, default=30_000)
|
||||
parser.add_argument("--max-total-features", type=int, default=300_000)
|
||||
parser.add_argument("--request-timeout", type=int, default=300)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
parser.add_argument("--force", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def boundary_sha256(geometry) -> str:
|
||||
return bwk.sha256_bytes(json.dumps(mapping(geometry), sort_keys=True, separators=(",", ":")).encode("utf-8"))
|
||||
|
||||
|
||||
def resolve_member_boundaries(scope: GeographicScope, scope_output_root: Path) -> tuple[Path, str]:
|
||||
scope_dir = scope_output_root / scope.key
|
||||
manifest_path = scope_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
raise RuntimeError(f"Official scope manifest is missing at {manifest_path}")
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if (
|
||||
manifest.get("status") != "complete"
|
||||
or manifest.get("scope_key") != scope.key
|
||||
or int(manifest.get("member_count") or 0) != len(scope.members)
|
||||
):
|
||||
raise RuntimeError(f"Official scope manifest at {manifest_path} is incomplete or inconsistent")
|
||||
members_path = scope_dir / str(manifest.get("municipalities_filename") or "")
|
||||
expected_sha256 = str(manifest.get("municipalities_sha256") or "")
|
||||
if not members_path.is_file() or not expected_sha256 or bwk.sha256_file(members_path) != expected_sha256:
|
||||
raise RuntimeError("Official municipality-boundary artifact is missing or fails its scope checksum")
|
||||
return members_path, expected_sha256
|
||||
|
||||
|
||||
def load_member_boundaries(path: Path, scope: GeographicScope) -> dict[str, tuple[ScopeMember, Any]]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
if not isinstance(features, list):
|
||||
raise RuntimeError("Municipality-boundary artifact is not a GeoJSON FeatureCollection")
|
||||
expected = {member.nis_code: member for member in scope.members}
|
||||
selected: dict[str, tuple[ScopeMember, Any]] = {}
|
||||
for feature in features:
|
||||
properties = feature.get("properties") or {}
|
||||
nis_code = str(properties.get("nis_code") or properties.get("NISCODE") or "")
|
||||
if nis_code not in expected:
|
||||
continue
|
||||
if nis_code in selected:
|
||||
raise RuntimeError(f"Municipality-boundary artifact contains duplicate NIS code {nis_code}")
|
||||
geometry = bwk.polygonal_geometry(shape(feature.get("geometry")))
|
||||
if geometry is None:
|
||||
raise RuntimeError(f"Municipality boundary for {expected[nis_code].name} is invalid")
|
||||
selected[nis_code] = (expected[nis_code], geometry)
|
||||
missing = [member.name for member in scope.members if member.nis_code not in selected]
|
||||
if missing:
|
||||
raise RuntimeError(f"Municipality-boundary artifact is missing: {', '.join(missing)}")
|
||||
return {member.nis_code: selected[member.nis_code] for member in scope.members}
|
||||
|
||||
|
||||
def partition_paths(output_root: Path, nis_code: str) -> tuple[Path, Path, Path]:
|
||||
partition_dir = output_root / "partitions" / nis_code
|
||||
return partition_dir / "bwk_natura2000_2025.geojson", partition_dir / "manifest.json", partition_dir / "raw"
|
||||
|
||||
|
||||
def cached_partition(
|
||||
output_root: Path,
|
||||
member: ScopeMember,
|
||||
boundary_hash: str,
|
||||
) -> dict[str, Any] | None:
|
||||
output_path, manifest_path, _raw_dir = partition_paths(output_root, member.nis_code)
|
||||
if not output_path.is_file() or not manifest_path.is_file():
|
||||
return None
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if (
|
||||
manifest.get("schema_version") != SCHEMA_VERSION
|
||||
or manifest.get("source_version") != bwk.SOURCE_VERSION
|
||||
or manifest.get("nis_code") != member.nis_code
|
||||
or manifest.get("boundary_sha256") != boundary_hash
|
||||
or manifest.get("output_sha256") != bwk.sha256_file(output_path)
|
||||
):
|
||||
return None
|
||||
for page in manifest.get("raw_pages") or []:
|
||||
page_path = manifest_path.parent / str(page.get("artifact_path") or "")
|
||||
if not page_path.is_file() or page.get("artifact_sha256") != bwk.sha256_file(page_path):
|
||||
return None
|
||||
try:
|
||||
raw_bytes = gzip.decompress(page_path.read_bytes())
|
||||
except (OSError, EOFError):
|
||||
return None
|
||||
if page.get("response_sha256") != bwk.sha256_bytes(raw_bytes):
|
||||
return None
|
||||
manifest["output_path"] = str(output_path)
|
||||
manifest["manifest_path"] = str(manifest_path)
|
||||
return manifest
|
||||
|
||||
|
||||
def prepare_partition(
|
||||
session,
|
||||
*,
|
||||
output_root: Path,
|
||||
scope: GeographicScope,
|
||||
member: ScopeMember,
|
||||
boundary_wgs84,
|
||||
page_limit: int,
|
||||
max_features: int,
|
||||
timeout: int,
|
||||
force: bool,
|
||||
) -> dict[str, Any]:
|
||||
output_path, manifest_path, raw_dir = partition_paths(output_root, member.nis_code)
|
||||
current_boundary_hash = boundary_sha256(boundary_wgs84)
|
||||
if not force:
|
||||
reusable = cached_partition(output_root, member, current_boundary_hash)
|
||||
if reusable:
|
||||
return reusable
|
||||
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
boundary_lambert72 = bwk.polygonal_geometry(transform_geometry(bwk.TO_LAMBERT72.transform, boundary_wgs84))
|
||||
if boundary_lambert72 is None:
|
||||
raise RuntimeError(f"Boundary for {member.name} could not be transformed to EPSG:31370")
|
||||
|
||||
retained: list[dict[str, Any]] = []
|
||||
raw_pages: list[dict[str, Any]] = []
|
||||
seen_source_ids: set[str] = set()
|
||||
raw_feature_count = 0
|
||||
duplicate_count = 0
|
||||
rejected_count = 0
|
||||
clipped_count = 0
|
||||
for page_number, (payload, source_url, raw_bytes) in enumerate(
|
||||
bwk.iter_wfs_pages(session, boundary_wgs84.bounds, page_limit=page_limit, timeout=timeout),
|
||||
start=1,
|
||||
):
|
||||
page_features = list(payload.get("features") or [])
|
||||
raw_feature_count += len(page_features)
|
||||
if raw_feature_count > max_features:
|
||||
raise RuntimeError(
|
||||
f"BWK WFS returned more than {max_features} features for {member.name}; refusing a truncated partition"
|
||||
)
|
||||
compressed = gzip.compress(raw_bytes, mtime=0)
|
||||
raw_path = raw_dir / f"bwk_bwkhab_page_{page_number:05d}.json.gz"
|
||||
bwk.write_bytes_atomic(raw_path, compressed)
|
||||
raw_pages.append(
|
||||
{
|
||||
"artifact_path": str(raw_path.relative_to(manifest_path.parent)),
|
||||
"artifact_sha256": bwk.sha256_bytes(compressed),
|
||||
"response_sha256": bwk.sha256_bytes(raw_bytes),
|
||||
"response_size_bytes": len(raw_bytes),
|
||||
"feature_count": len(page_features),
|
||||
"source_url": source_url,
|
||||
}
|
||||
)
|
||||
for source_feature in page_features:
|
||||
source_id = str(source_feature.get("id") or (source_feature.get("properties") or {}).get("UIDN") or "")
|
||||
if source_id and source_id in seen_source_ids:
|
||||
duplicate_count += 1
|
||||
continue
|
||||
if source_id:
|
||||
seen_source_ids.add(source_id)
|
||||
normalized, was_clipped = bwk.normalize_feature(
|
||||
source_feature,
|
||||
boundary_lambert72,
|
||||
municipality=member.name,
|
||||
nis_code=member.nis_code,
|
||||
coverage_scope=scope.key,
|
||||
feature_id_suffix=member.nis_code,
|
||||
)
|
||||
if normalized is None:
|
||||
rejected_count += 1
|
||||
continue
|
||||
clipped_count += int(was_clipped)
|
||||
retained.append(normalized)
|
||||
|
||||
artifact = {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"BWK en Natura 2000 - toestand 2025 - {member.name}",
|
||||
"features": retained,
|
||||
"source": "INBO BWK/Natura 2000 WFS",
|
||||
"source_version": bwk.SOURCE_VERSION,
|
||||
"coverage_scope": scope.key,
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"reference_truncated": False,
|
||||
}
|
||||
bwk.write_json_atomic(output_path, artifact)
|
||||
manifest = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"source_version": bwk.SOURCE_VERSION,
|
||||
"scope_key": scope.key,
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"boundary_sha256": current_boundary_hash,
|
||||
"boundary_bbox_wgs84": list(boundary_wgs84.bounds),
|
||||
"page_limit": page_limit,
|
||||
"page_count": len(raw_pages),
|
||||
"raw_source_feature_count": raw_feature_count,
|
||||
"feature_count": len(retained),
|
||||
"duplicate_count": duplicate_count,
|
||||
"rejected_or_outside_count": rejected_count,
|
||||
"clipped_feature_count": clipped_count,
|
||||
"reference_truncated": False,
|
||||
"raw_pages": raw_pages,
|
||||
"output_path": str(output_path),
|
||||
"output_sha256": bwk.sha256_file(output_path),
|
||||
"generated_at": bwk.utc_now(),
|
||||
}
|
||||
bwk.write_json_atomic(manifest_path, manifest, pretty=True)
|
||||
manifest["manifest_path"] = str(manifest_path)
|
||||
return manifest
|
||||
|
||||
|
||||
def snapshot_paths(output_root: Path, scope: GeographicScope) -> tuple[Path, Path]:
|
||||
snapshot_dir = output_root / "snapshots" / scope.key
|
||||
return snapshot_dir / "bwk_natura2000_2025.geojson", snapshot_dir / "manifest.json"
|
||||
|
||||
|
||||
def assemble_snapshot(
|
||||
*,
|
||||
output_root: Path,
|
||||
scope: GeographicScope,
|
||||
partitions: list[dict[str, Any]],
|
||||
member_boundaries_sha256: str,
|
||||
max_total_features: int,
|
||||
) -> tuple[Path, Path, dict[str, Any]]:
|
||||
if [item.get("nis_code") for item in partitions] != list(scope.nis_codes):
|
||||
raise RuntimeError("BWK partition order/completeness does not match the approved geographic scope")
|
||||
output_path, manifest_path = snapshot_paths(output_root, scope)
|
||||
partition_identity = bwk.sha256_bytes(
|
||||
json.dumps(
|
||||
[(item["nis_code"], item["output_sha256"]) for item in partitions],
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
)
|
||||
if output_path.is_file() and manifest_path.is_file():
|
||||
try:
|
||||
existing = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
existing = {}
|
||||
if (
|
||||
existing.get("partition_identity_sha256") == partition_identity
|
||||
and existing.get("output_sha256") == bwk.sha256_file(output_path)
|
||||
):
|
||||
return output_path, manifest_path, existing
|
||||
|
||||
features: list[dict[str, Any]] = []
|
||||
feature_ids: set[str] = set()
|
||||
evaluation_area: dict[str, float] = defaultdict(float)
|
||||
habitat_status_area: dict[str, float] = defaultdict(float)
|
||||
natura_area = 0.0
|
||||
regional_area = 0.0
|
||||
uncertain_area = 0.0
|
||||
for partition in partitions:
|
||||
payload = json.loads(Path(str(partition["output_path"])).read_text(encoding="utf-8"))
|
||||
partition_features = payload.get("features") if isinstance(payload, dict) else None
|
||||
if not isinstance(partition_features, list) or len(partition_features) != int(partition["feature_count"]):
|
||||
raise RuntimeError(f"BWK partition feature-count drift for NIS {partition['nis_code']}")
|
||||
for feature in partition_features:
|
||||
feature_id = str(feature.get("id") or "")
|
||||
if not feature_id or feature_id in feature_ids:
|
||||
raise RuntimeError(f"Duplicate or missing regional BWK feature id {feature_id!r}")
|
||||
feature_ids.add(feature_id)
|
||||
properties = feature.get("properties") or {}
|
||||
area = float(properties.get("clipped_area_ha") or 0.0)
|
||||
evaluation_area[str(properties.get("bwk_evaluation_code") or "unknown")] += area
|
||||
habitat_status_area[str(properties.get("habitat_status_code") or "unknown")] += area
|
||||
natura_area += float(properties.get("natura2000_area_ha") or 0.0)
|
||||
regional_area += float(properties.get("regional_biotope_area_ha") or 0.0)
|
||||
uncertain_area += float(properties.get("uncertain_habitat_area_ha") or 0.0)
|
||||
features.append(feature)
|
||||
if len(features) > max_total_features:
|
||||
raise RuntimeError(
|
||||
f"Regional BWK snapshot exceeds the {max_total_features} feature safety limit; refusing truncation"
|
||||
)
|
||||
|
||||
generated_at = bwk.utc_now()
|
||||
artifact = {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"BWK en Natura 2000 - toestand 2025 - {scope.display_name}",
|
||||
"features": features,
|
||||
"source": "INBO BWK/Natura 2000 WFS",
|
||||
"source_version": bwk.SOURCE_VERSION,
|
||||
"attribution": bwk.ATTRIBUTION,
|
||||
"catalog_url": bwk.CATALOG_URL,
|
||||
"report_url": bwk.REPORT_URL,
|
||||
"coverage_scope": scope.key,
|
||||
"member_count": len(scope.members),
|
||||
"reference_truncated": False,
|
||||
"generated_at": generated_at,
|
||||
}
|
||||
bwk.write_json_atomic(output_path, artifact)
|
||||
empty_partitions = [item["nis_code"] for item in partitions if int(item["feature_count"]) == 0]
|
||||
manifest = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"source_version": bwk.SOURCE_VERSION,
|
||||
"scope_key": scope.key,
|
||||
"scope_display_name": scope.display_name,
|
||||
"member_count": len(scope.members),
|
||||
"member_nis_codes": list(scope.nis_codes),
|
||||
"member_boundaries_sha256": member_boundaries_sha256,
|
||||
"coverage_complete": len(partitions) == len(scope.members),
|
||||
"empty_partitions": empty_partitions,
|
||||
"feature_count": len(features),
|
||||
"raw_source_feature_count": sum(int(item["raw_source_feature_count"]) for item in partitions),
|
||||
"raw_response_count": sum(int(item["page_count"]) for item in partitions),
|
||||
"partition_identity_sha256": partition_identity,
|
||||
"partitions": [
|
||||
{
|
||||
"municipality": item["municipality"],
|
||||
"nis_code": item["nis_code"],
|
||||
"feature_count": item["feature_count"],
|
||||
"raw_source_feature_count": item["raw_source_feature_count"],
|
||||
"output_sha256": item["output_sha256"],
|
||||
"manifest_path": item["manifest_path"],
|
||||
}
|
||||
for item in partitions
|
||||
],
|
||||
"evaluation_area_ha": {key: round(value, 6) for key, value in sorted(evaluation_area.items())},
|
||||
"habitat_status_area_ha": {key: round(value, 6) for key, value in sorted(habitat_status_area.items())},
|
||||
"natura2000_area_ha": round(natura_area, 6),
|
||||
"regional_biotope_area_ha": round(regional_area, 6),
|
||||
"uncertain_habitat_area_ha": round(uncertain_area, 6),
|
||||
"reference_truncated": False,
|
||||
"output_sha256": bwk.sha256_file(output_path),
|
||||
"output_size_bytes": output_path.stat().st_size,
|
||||
"generated_at": generated_at,
|
||||
"limitations": [
|
||||
"The 2025 edition is the best available map state, not one uniform 2025 field survey.",
|
||||
"PHAB percentages may be theoretical shares; partial selections scale shares by intersected polygon area.",
|
||||
"Municipality partitions split source polygons at administrative boundaries; source ids carry a NIS suffix.",
|
||||
"The real field situation remains authoritative for policy and legal use.",
|
||||
],
|
||||
}
|
||||
bwk.write_json_atomic(manifest_path, manifest, pretty=True)
|
||||
return output_path, manifest_path, manifest
|
||||
|
||||
|
||||
def locate_workspace(session, base_url: str, scope: GeographicScope, timeout: int):
|
||||
projects = bwk.paginated_items(session, f"{base_url}/api/v1/projects", timeout=timeout)
|
||||
project = next((item for item in projects if item.get("name") == scope.project_name), None)
|
||||
if not project:
|
||||
raise RuntimeError(f"Project {scope.project_name!r} is missing")
|
||||
project_id = str(project["id"])
|
||||
areas = bwk.paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/areas", timeout=timeout)
|
||||
area = next((item for item in areas if item.get("name") == scope.area_name), None)
|
||||
if not area:
|
||||
raise RuntimeError(f"Official scope Area {scope.area_name!r} is missing")
|
||||
datasets = bwk.paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/datasets", timeout=timeout)
|
||||
return project_id, str(area["id"]), datasets
|
||||
|
||||
|
||||
def upload_snapshot(
|
||||
session,
|
||||
*,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area_id: str,
|
||||
scope: GeographicScope,
|
||||
path: Path,
|
||||
manifest_path: Path,
|
||||
manifest: dict[str, Any],
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
source_metadata = {
|
||||
"provider": "Instituut voor Natuur- en Bosonderzoek",
|
||||
"theme": "nature_value",
|
||||
"layer_name": "BWK/Natura 2000 toestand 2025",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": scope.key,
|
||||
"scope_type": scope.scope_type,
|
||||
"scope_display_name": scope.display_name,
|
||||
"member_count": len(scope.members),
|
||||
"member_nis_codes": list(scope.nis_codes),
|
||||
"partitioned_source_audit": True,
|
||||
"coverage_complete": bool(manifest["coverage_complete"]),
|
||||
"feature_count": manifest["feature_count"],
|
||||
"geometry_clipped_to_area": True,
|
||||
"semantic_metrics": False,
|
||||
"attribution": bwk.ATTRIBUTION,
|
||||
"catalog_url": bwk.CATALOG_URL,
|
||||
"report_url": bwk.REPORT_URL,
|
||||
"selection_aggregation": {
|
||||
"metric_key": "bwk_mapped_area",
|
||||
"method": "intersection_area",
|
||||
"label": "BWK-gekarteerde oppervlakte",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"warning": "Uitgave 2025 is de best beschikbare kaarttoestand, maar niet elk kaartvlak is in 2025 op terrein gekarteerd.",
|
||||
},
|
||||
"selection_metrics": bwk.selection_metrics(),
|
||||
}
|
||||
provenance_metadata = {
|
||||
"operator_tool": "provision_regional_bwk_natura2000.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"geometry_clipped_to_area": True,
|
||||
"source_type_name": bwk.TYPE_NAME,
|
||||
"wfs_url": bwk.WFS_URL,
|
||||
"catalog_url": bwk.CATALOG_URL,
|
||||
"report_url": bwk.REPORT_URL,
|
||||
"manifest_path": str(manifest_path),
|
||||
"combined_output_sha256": manifest["output_sha256"],
|
||||
"partition_count": len(manifest["partitions"]),
|
||||
"partition_identity_sha256": manifest["partition_identity_sha256"],
|
||||
"raw_source_responses_retained": True,
|
||||
"reference_truncated": False,
|
||||
"generated_at": manifest["generated_at"],
|
||||
"limitations": manifest["limitations"],
|
||||
}
|
||||
with path.open("rb") as handle:
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||
data={
|
||||
"dataset_type": "vector",
|
||||
"source": "operator_official_import",
|
||||
"dataset_role": "reference",
|
||||
"source_name": "inbo_bwk_natura2000",
|
||||
"reference_layer_name": "nature_value",
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": f"inbo-bwk-natura2000:{scope.key}",
|
||||
"observed_at": bwk.PUBLICATION_DATE,
|
||||
"temporal_granularity": "snapshot",
|
||||
"source_version": bwk.SOURCE_VERSION,
|
||||
},
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
)
|
||||
return bwk.response_data(response)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
if (
|
||||
args.page_limit < 1
|
||||
or args.page_limit > 5000
|
||||
or args.max_features_per_partition < args.page_limit
|
||||
or args.max_total_features < args.max_features_per_partition
|
||||
):
|
||||
raise ValueError("Invalid BWK page or feature safety limits")
|
||||
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||
members_path, members_sha256 = resolve_member_boundaries(scope, args.scope_output_root)
|
||||
boundaries = load_member_boundaries(members_path, scope)
|
||||
source_session = bwk.source_session()
|
||||
source_session.headers.update({"User-Agent": "GeoIntel-BWK-Natura2000-Regional-Operator/1.0"})
|
||||
with source_session:
|
||||
partitions = [
|
||||
prepare_partition(
|
||||
source_session,
|
||||
output_root=args.output_root / scope.key,
|
||||
scope=scope,
|
||||
member=member,
|
||||
boundary_wgs84=boundary,
|
||||
page_limit=args.page_limit,
|
||||
max_features=args.max_features_per_partition,
|
||||
timeout=args.request_timeout,
|
||||
force=args.force,
|
||||
)
|
||||
for member, boundary in boundaries.values()
|
||||
]
|
||||
path, manifest_path, manifest = assemble_snapshot(
|
||||
output_root=args.output_root,
|
||||
scope=scope,
|
||||
partitions=partitions,
|
||||
member_boundaries_sha256=members_sha256,
|
||||
max_total_features=args.max_total_features,
|
||||
)
|
||||
result: dict[str, Any]
|
||||
if args.fetch_only:
|
||||
result = {"status": "prepared", "artifact_path": str(path), "feature_count": manifest["feature_count"]}
|
||||
else:
|
||||
with requests.Session() as api_session:
|
||||
project_id, area_id, datasets = locate_workspace(
|
||||
api_session, args.base_url.rstrip("/"), scope, args.import_timeout
|
||||
)
|
||||
existing = next(
|
||||
(
|
||||
item
|
||||
for item in datasets
|
||||
if item.get("source_name") == "inbo_bwk_natura2000"
|
||||
and item.get("source_version") == bwk.SOURCE_VERSION
|
||||
and str(item.get("area_id") or "") == area_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing:
|
||||
if str(existing.get("checksum_sha256") or "") != manifest["output_sha256"]:
|
||||
raise RuntimeError("A different regional BWK 2025 artifact already exists; refusing replacement")
|
||||
result = {
|
||||
"status": "existing",
|
||||
"dataset_id": existing["id"],
|
||||
"feature_count": existing.get("feature_count"),
|
||||
}
|
||||
else:
|
||||
dataset = upload_snapshot(
|
||||
api_session,
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
scope=scope,
|
||||
path=path,
|
||||
manifest_path=manifest_path,
|
||||
manifest=manifest,
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
result = {
|
||||
"status": "imported",
|
||||
"dataset_id": dataset["id"],
|
||||
"feature_count": dataset.get("feature_count"),
|
||||
}
|
||||
except (KeyError, OSError, RuntimeError, ValueError, requests.RequestException) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"scope": scope.key,
|
||||
"member_count": len(scope.members),
|
||||
"partition_count": len(partitions),
|
||||
"feature_count": manifest["feature_count"],
|
||||
"raw_source_feature_count": manifest["raw_source_feature_count"],
|
||||
"raw_response_count": manifest["raw_response_count"],
|
||||
"manifest_path": str(manifest_path),
|
||||
"metrics": {
|
||||
"evaluation_area_ha": manifest["evaluation_area_ha"],
|
||||
"natura2000_area_ha": manifest["natura2000_area_ha"],
|
||||
"regional_biotope_area_ha": manifest["regional_biotope_area_ha"],
|
||||
"uncertain_habitat_area_ha": manifest["uncertain_habitat_area_ha"],
|
||||
},
|
||||
"result": result,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -50,6 +50,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
|
||||
|
||||
Reference in New Issue
Block a user