feat: add official Kempen operating scope
This commit is contained in:
@@ -7,6 +7,14 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 189 Official Kempen operational scope (2026-07-14)
|
||||||
|
|
||||||
|
- Defined `Kempen` operationally as the official 28-municipality Vlaamse vervoerregio, with an explicit warning that this policy boundary is not the wider cultural or landscape Kempen.
|
||||||
|
- Added a central operator scope registry with current municipality names and NIS codes, including Mol `13025` and Nijlen `12026`.
|
||||||
|
- Added an explicit, idempotent VRBG scope provisioner that creates one regional boundary, 28 member boundaries, one regional Area, 28 municipality Areas and canonical source datasets through the existing API.
|
||||||
|
- Added a compact Mol/Kempen region selector to the map-first explorer and made its heading and full-area action scope-neutral.
|
||||||
|
- Kept thematic regional ingestion separate from the boundary pass so large GRB/WCS sources can be partitioned and validated without hidden startup fetches or truncated datasets.
|
||||||
|
|
||||||
## Sprint 188 Official modern Mol land-use series (2026-07-14)
|
## Sprint 188 Official modern Mol land-use series (2026-07-14)
|
||||||
|
|
||||||
- Added an explicit, reusable MercatorNet WCS operator for official Departement Omgeving land-use snapshots in 2013, 2016, 2019, 2022 and 2025.
|
- Added an explicit, reusable MercatorNet WCS operator for official Departement Omgeving land-use snapshots in 2013, 2016, 2019, 2022 and 2025.
|
||||||
|
|||||||
@@ -935,6 +935,27 @@ the same bbox-selected FeatureCollection as a normal export record with
|
|||||||
`export_type="vector_selection_geojson"`. This creates a handoff artifact only;
|
`export_type="vector_selection_geojson"`. This creates a handoff artifact only;
|
||||||
it does not create a derived dataset.
|
it does not create a derived dataset.
|
||||||
|
|
||||||
|
## Geographic scope provisioning
|
||||||
|
|
||||||
|
The explicit operator command below provisions the official 28-municipality
|
||||||
|
Vlaamse vervoerregio Kempen boundary foundation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/provision_geographic_scope.py \
|
||||||
|
--scope kempen-transport-region
|
||||||
|
```
|
||||||
|
|
||||||
|
It reads current `VRBG/Refgem` boundaries, validates every registered name and
|
||||||
|
NIS code, unions the regional geometry and creates one project, one regional
|
||||||
|
Area, 28 municipality Areas and two source datasets through the public API.
|
||||||
|
It never writes directly to PostGIS and does not run on startup. The persisted
|
||||||
|
scope limitation explicitly distinguishes the transport-policy region from a
|
||||||
|
cultural or landscape definition of Kempen.
|
||||||
|
|
||||||
|
Use `--fetch-only` for a source/geometry/checksum audit. The scope pass does
|
||||||
|
not fetch thematic GRB, population or land-use data; those remain separate,
|
||||||
|
bounded operator jobs.
|
||||||
|
|
||||||
## Temporal Mol data and evolution
|
## Temporal Mol data and evolution
|
||||||
|
|
||||||
Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`,
|
Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`,
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ def test_map_first_explorer_is_the_default_product_flow() -> None:
|
|||||||
assert "Wat bevindt zich in dit gebied?" in workspace
|
assert "Wat bevindt zich in dit gebied?" in workspace
|
||||||
assert "Kies een datathema" in workspace
|
assert "Kies een datathema" in workspace
|
||||||
assert "Teken rechthoek" in workspace
|
assert "Teken rechthoek" in workspace
|
||||||
assert "Volledige gemeente" in workspace
|
assert "Volledig werkgebied" in workspace
|
||||||
assert "Alle beschikbare thema" in workspace
|
assert "Alle beschikbare thema" in workspace
|
||||||
assert "Bron nog niet ingeladen" in workspace
|
assert "Bron nog niet ingeladen" in workspace
|
||||||
assert "useMapThemeSelectionInsights" in workspace
|
assert "useMapThemeSelectionInsights" in workspace
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from shapely.geometry import Polygon, shape
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPTS = ROOT / "scripts"
|
||||||
|
|
||||||
|
|
||||||
|
def load_module(name: str, filename: str):
|
||||||
|
scripts_path = str(SCRIPTS)
|
||||||
|
if scripts_path not in sys.path:
|
||||||
|
sys.path.insert(0, scripts_path)
|
||||||
|
spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename)
|
||||||
|
assert spec is not None
|
||||||
|
assert spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_kempen_scope_matches_official_28_municipality_policy_region() -> None:
|
||||||
|
scopes = load_module("geographic_scopes_test", "geographic_scopes.py")
|
||||||
|
scope = scopes.KEMPEN_TRANSPORT_REGION_SCOPE
|
||||||
|
|
||||||
|
assert scope.key == "kempen-transport-region"
|
||||||
|
assert scope.project_name == "Kempen Regional Workbench"
|
||||||
|
assert scope.scope_type == "transport_region"
|
||||||
|
assert len(scope.members) == 28
|
||||||
|
assert len(set(scope.nis_codes)) == 28
|
||||||
|
assert ("Mol", "13025") in {(member.name, member.nis_code) for member in scope.members}
|
||||||
|
assert ("Nijlen", "12026") in {(member.name, member.nis_code) for member in scope.members}
|
||||||
|
assert "vervoerregio-kempen" in scope.authority_url
|
||||||
|
assert "geen claim" in scope.limitation_message
|
||||||
|
|
||||||
|
|
||||||
|
def test_scope_union_preserves_member_identity_and_policy_limitation() -> None:
|
||||||
|
scopes = load_module("geographic_scopes_union_test", "geographic_scopes.py")
|
||||||
|
provisioner = load_module("provision_geographic_scope_test", "provision_geographic_scope.py")
|
||||||
|
scope = scopes.GeographicScope(
|
||||||
|
key="test-region",
|
||||||
|
display_name="Testregio",
|
||||||
|
project_name="Test Regional Workbench",
|
||||||
|
project_region="Test",
|
||||||
|
area_name="Testregio - operationele grens",
|
||||||
|
authority_name="Test authority",
|
||||||
|
authority_url="https://example.test/scope",
|
||||||
|
scope_type="policy_region",
|
||||||
|
limitation_message="Operationele testgrens; geen landschappelijke claim.",
|
||||||
|
members=(scopes.ScopeMember("Alpha", "10001"), scopes.ScopeMember("Beta", "10002")),
|
||||||
|
)
|
||||||
|
source_features = [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": "alpha",
|
||||||
|
"geometry": Polygon([(4.0, 51.0), (4.1, 51.0), (4.1, 51.1), (4.0, 51.1)]).__geo_interface__,
|
||||||
|
"properties": {"NAAM": "Alpha", "NISCODE": "10001"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": "beta",
|
||||||
|
"geometry": Polygon([(4.1, 51.0), (4.2, 51.0), (4.2, 51.1), (4.1, 51.1)]).__geo_interface__,
|
||||||
|
"properties": {"NAAM": "Beta", "NISCODE": "10002"},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
boundary, members, summary = provisioner.build_scope_payloads(
|
||||||
|
scope,
|
||||||
|
source_features,
|
||||||
|
source_url="https://example.test/vrbg",
|
||||||
|
generated_at="2026-07-14T00:00:00+00:00",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(boundary["features"]) == 1
|
||||||
|
assert len(members["features"]) == 2
|
||||||
|
assert shape(boundary["features"][0]["geometry"]).is_valid
|
||||||
|
assert boundary["features"][0]["properties"]["member_nis_codes"] == ["10001", "10002"]
|
||||||
|
assert boundary["features"][0]["properties"]["scope_limitation"] == scope.limitation_message
|
||||||
|
assert [feature["properties"]["municipality"] for feature in members["features"]] == ["Alpha", "Beta"]
|
||||||
|
assert summary["member_count"] == 2
|
||||||
|
assert summary["area_km2"] > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_scope_api_pagination_respects_canonical_limit() -> None:
|
||||||
|
provisioner = load_module("provision_geographic_scope_paging_test", "provision_geographic_scope.py")
|
||||||
|
|
||||||
|
class Response:
|
||||||
|
ok = True
|
||||||
|
status_code = 200
|
||||||
|
text = ""
|
||||||
|
|
||||||
|
def __init__(self, payload):
|
||||||
|
self.payload = payload
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return {"data": self.payload}
|
||||||
|
|
||||||
|
class Session:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.offsets = []
|
||||||
|
|
||||||
|
def get(self, url, *, params, timeout):
|
||||||
|
del url, timeout
|
||||||
|
self.offsets.append(params["offset"])
|
||||||
|
offset = params["offset"]
|
||||||
|
page = [{"id": index} for index in range(offset, min(offset + 200, 401))]
|
||||||
|
return Response({"items": page, "total": 401})
|
||||||
|
|
||||||
|
session = Session()
|
||||||
|
items = provisioner.list_paginated_items(session, "http://backend/api/v1/projects", timeout=30)
|
||||||
|
|
||||||
|
assert len(items) == 401
|
||||||
|
assert session.offsets == [0, 200, 400]
|
||||||
|
|
||||||
|
|
||||||
|
def test_kempen_scope_operator_is_packaged_and_exposed_in_map_flow() -> None:
|
||||||
|
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||||
|
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
|
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||||
|
app = (ROOT / "frontend/src/App.tsx").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "COPY scripts/geographic_scopes.py" in dockerfile
|
||||||
|
assert "COPY scripts/provision_geographic_scope.py" in dockerfile
|
||||||
|
assert "py_compile scripts/provision_geographic_scope.py" in readiness
|
||||||
|
assert "Immutable scope dataset" in (ROOT / "scripts/provision_geographic_scope.py").read_text(encoding="utf-8")
|
||||||
|
assert "Kempen (28 gemeenten)" in workspace
|
||||||
|
assert 'aria-label="Regio"' in workspace
|
||||||
|
assert "projects={projects}" in app
|
||||||
|
assert "onSelectProject={selectProject}" in app
|
||||||
@@ -77,6 +77,8 @@ COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_
|
|||||||
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
|
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
|
||||||
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
|
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
|
||||||
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
||||||
|
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
|
||||||
|
COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py
|
||||||
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
|
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
|
||||||
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
|
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
|
||||||
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
|
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
|
||||||
|
|||||||
@@ -7850,3 +7850,25 @@ Validation:
|
|||||||
|
|
||||||
Next:
|
Next:
|
||||||
- Define the exact municipality list or administrative boundary that GeoIntel will call `Kempen`, then reuse the proven source operators and temporal query path for that scope without changing Mol semantics.
|
- Define the exact municipality list or administrative boundary that GeoIntel will call `Kempen`, then reuse the proven source operators and temporal query path for that scope without changing Mol semantics.
|
||||||
|
|
||||||
|
## Sprint 189 Official Kempen operational scope (2026-07-14)
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
- Chose the official Vlaamse `Vervoerregio Kempen` as GeoIntel's reproducible operational regional scope. It has 28 municipalities and explicitly includes both Mol and Nijlen.
|
||||||
|
- Added `geographic_scopes.py` as the operator registry for scope identity, current municipality names/NIS codes, policy authority and limitation text.
|
||||||
|
- Added `provision_geographic_scope.py` to fetch current authoritative VRBG boundaries, validate all members, create a valid regional union and retain boundary/member GeoJSON plus checksums and manifest.
|
||||||
|
- The operator creates or reuses `Kempen Regional Workbench`, one regional Area, 28 municipality Areas and two source datasets exclusively through canonical API envelopes.
|
||||||
|
- Added a compact Mol/Kempen selector to the primary map flow, scope-aware heading text and the neutral `Volledig werkgebied` action.
|
||||||
|
- Kept thematic regional fetching outside the scope pass. No startup fetch, direct PostGIS write, fake metric or unbounded regional theme download was introduced.
|
||||||
|
|
||||||
|
Source proof:
|
||||||
|
- A live read-only VRBG run matched all 28 registered municipalities and current NIS codes.
|
||||||
|
- The valid union measured 1,399.2505 km2 in EPSG:31370 with WGS84 bbox `[4.59723873, 51.01047967, 5.26224853, 51.50511313]`.
|
||||||
|
- The scope is labelled as the transport-policy region and never as the complete cultural, landscape or historical Kempen.
|
||||||
|
|
||||||
|
Validation before deployment:
|
||||||
|
- New focused scope and map-flow suite passed 9 tests.
|
||||||
|
- Python scope/operator compilation and frontend TypeScript typecheck passed.
|
||||||
|
|
||||||
|
Next:
|
||||||
|
- Deploy and run the scope operator on Tower, verify 29 Areas and both boundary datasets, then design bounded regional theme partitions before importing high-volume GRB or WCS data.
|
||||||
|
|||||||
@@ -88,6 +88,24 @@ Kempen reuses the same operator with an explicit approved boundary, project,
|
|||||||
area and scope key; an ambiguous regional label is never converted into an
|
area and scope key; an ambiguous regional label is never converted into an
|
||||||
invented boundary.
|
invented boundary.
|
||||||
|
|
||||||
|
## Operational Kempen scope
|
||||||
|
|
||||||
|
The canonical regional workspace uses the official 28-municipality Vlaamse
|
||||||
|
`Vervoerregio Kempen` policy boundary. `scripts/geographic_scopes.py` is the
|
||||||
|
operator source of truth for its members and current NIS codes. The resulting
|
||||||
|
regional geometry is a union of current authoritative VRBG municipality
|
||||||
|
boundaries.
|
||||||
|
|
||||||
|
- Project: `Kempen Regional Workbench`.
|
||||||
|
- Regional Area: `Vervoerregio Kempen - officiële operationele grens`.
|
||||||
|
- Member Areas: one current official boundary for every municipality.
|
||||||
|
- Scope key: `kempen-transport-region`.
|
||||||
|
- Provenance: authority URL, VRBG request URL, member codes, fetch timestamp,
|
||||||
|
artifact checksums and the policy-boundary limitation.
|
||||||
|
- Scale policy: thematic providers are partitioned and validated separately;
|
||||||
|
no interactive request or application startup may download the entire
|
||||||
|
region or silently substitute missing data.
|
||||||
|
|
||||||
## User-uploaded raster strategy
|
## User-uploaded raster strategy
|
||||||
V1 must support controlled local datasets because public raster access and model compatibility can be difficult.
|
V1 must support controlled local datasets because public raster access and model compatibility can be difficult.
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,29 @@ start geen verborgen providerfetch. De resulterende GeoJSON-artefacten,
|
|||||||
checksums en bron-URL's worden onder persistent operator storage bewaard en
|
checksums en bron-URL's worden onder persistent operator storage bewaard en
|
||||||
via de bestaande DatasetService/vectorfeature-flow geïmporteerd.
|
via de bestaande DatasetService/vectorfeature-flow geïmporteerd.
|
||||||
|
|
||||||
|
### Operationele regio Kempen
|
||||||
|
|
||||||
|
Voor regionale analyse gebruikt GeoIntel de officiële `Vervoerregio Kempen`
|
||||||
|
van de Vlaamse overheid. Deze beleidsregio bestaat uit 28 gemeenten. Ze is
|
||||||
|
expliciet gekozen omdat ze actueel, reproduceerbaar en bestuurlijk
|
||||||
|
gedocumenteerd is. GeoIntel beweert niet dat deze grens samenvalt met de
|
||||||
|
ruimere culturele, landschappelijke of historische Kempen.
|
||||||
|
|
||||||
|
`scripts/geographic_scopes.py` registreert de 28 actuele namen en NIS-codes.
|
||||||
|
`scripts/provision_geographic_scope.py` controleert die tegen `VRBG/Refgem`,
|
||||||
|
maakt de regionale union en bewaart zowel de union als alle afzonderlijke
|
||||||
|
gemeentegrenzen met bron-URL, checksum, autoriteit en beperking. De gemeten
|
||||||
|
operationele grens omvat circa `1.399,25 km²`.
|
||||||
|
|
||||||
|
Bronnen:
|
||||||
|
|
||||||
|
- https://www.vlaanderen.be/mobiliteitsprofessionals/personenvervoer/basisbereikbaarheid/mobiliteitsuitdagingen-regionaal-aanpakken/vervoerregios/over-de-vervoerregio-kempen
|
||||||
|
- https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items
|
||||||
|
|
||||||
|
De scope-operator haalt geen thematische gegevens op. GRB, bevolking en
|
||||||
|
landgebruik volgen als afzonderlijke begrensde imports; ontbrekende regionale
|
||||||
|
thema's blijven zichtbaar onbeschikbaar.
|
||||||
|
|
||||||
### Mol population history
|
### Mol population history
|
||||||
|
|
||||||
`scripts/provision_mol_population_history.py` imports official Statbel
|
`scripts/provision_mol_population_history.py` imports official Statbel
|
||||||
|
|||||||
+2
-1
@@ -8,7 +8,8 @@
|
|||||||
- [x] Add official Mol GRB roads, water and parcels provisioning support.
|
- [x] Add official Mol GRB roads, water and parcels provisioning support.
|
||||||
- [x] Select, validate and provision official Statbel 2021-2025 population/statistical-sector snapshots for Mol.
|
- [x] Select, validate and provision official Statbel 2021-2025 population/statistical-sector snapshots for Mol.
|
||||||
- [x] Select, validate and provision official Departement Omgeving 2013-2025 forest snapshots from the 10 m land-use map.
|
- [x] Select, validate and provision official Departement Omgeving 2013-2025 forest snapshots from the 10 m land-use map.
|
||||||
- [ ] Define the exact administrative Kempen scope before provisioning municipality or regional copies of the proven source series.
|
- [x] Define and provision the official 28-municipality Vervoerregio Kempen as the explicit operational regional scope.
|
||||||
|
- [ ] Partition and provision regional theme datasets in bounded batches, starting with current buildings/context and then population/land-use time series.
|
||||||
|
|
||||||
This file now starts with the current implementation status. Older preparation/backlog sections are preserved below as historical planning context and should not be treated as the live sprint board without checking `docs/CODEX_EXECUTION_LOG.md`.
|
This file now starts with the current implementation status. Older preparation/backlog sections are preserved below as historical planning context and should not be treated as the live sprint board without checking `docs/CODEX_EXECUTION_LOG.md`.
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
React + TypeScript + MapLibre foundation for project/area/dataset workflow.
|
React + TypeScript + MapLibre foundation for project/area/dataset workflow.
|
||||||
|
|
||||||
Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available.
|
Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. A compact `Regio` selector switches between this workspace and the explicitly provisioned `Kempen Regional Workbench`, whose scope is the official 28-municipality Vlaamse vervoerregio. Explicit project and dataset selections remain authoritative.
|
||||||
|
|
||||||
The map-first explorer has two deliberate modes. `Latest state` selects the
|
The map-first explorer has two deliberate modes. `Latest state` selects the
|
||||||
latest explicitly dated source snapshot without claiming an old edition is
|
latest explicitly dated source snapshot without claiming an old edition is
|
||||||
@@ -21,7 +21,7 @@ estimates clearly marked and land-cover sources show intersected hectares. The
|
|||||||
advanced workbench remains available but is not required for the primary
|
advanced workbench remains available but is not required for the primary
|
||||||
choose-theme, draw-area, read-result flow.
|
choose-theme, draw-area, read-result flow.
|
||||||
|
|
||||||
The primary workflow is deliberately short: choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count.
|
The primary workflow is deliberately short: choose Mol or Kempen, choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count.
|
||||||
|
|
||||||
The theme catalog currently recognizes buildings, population, forest/green, water, roads and parcels from dataset names and canonical `reference_layer_name` metadata. A theme is enabled only when a ready persisted vector dataset exists; otherwise it states `Bron nog niet ingeladen`. This prevents missing population or land-cover sources from appearing as zero-valued observations. The previous technical Map workspace remains available through `Geavanceerde werkbank` for derived datasets, QA/QC evidence and export operations.
|
The theme catalog currently recognizes buildings, population, forest/green, water, roads and parcels from dataset names and canonical `reference_layer_name` metadata. A theme is enabled only when a ready persisted vector dataset exists; otherwise it states `Bron nog niet ingeladen`. This prevents missing population or land-cover sources from appearing as zero-valued observations. The previous technical Map workspace remains available through `Geavanceerde werkbank` for derived datasets, QA/QC evidence and export operations.
|
||||||
|
|
||||||
@@ -35,7 +35,7 @@ Wide and ultrawide screens keep a readable sidebar and centered work area, expan
|
|||||||
|
|
||||||
The Map workspace defaults to an OpenStreetMap road basemap with visible attribution so uploaded vectors, AOIs and QA overlays appear on a real street context. Set `VITE_MAP_STYLE_URL` to a managed MapLibre style URL to override this for production or high-volume deployments.
|
The Map workspace defaults to an OpenStreetMap road basemap with visible attribution so uploaded vectors, AOIs and QA overlays appear on a real street context. Set `VITE_MAP_STYLE_URL` to a managed MapLibre style URL to override this for production or high-volume deployments.
|
||||||
|
|
||||||
Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, GRB buildings use cyan/teal and existing detection/change-result colors remain distinct.
|
Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, while active source layers and their legends use consistent theme colors and existing detection/change-result colors remain distinct.
|
||||||
|
|
||||||
When the public OpenStreetMap fallback is active, the Map workspace shows a basemap usage notice. This keeps the local/demo default honest and reminds operators to configure a managed style URL before production or heavier tile traffic.
|
When the public OpenStreetMap fallback is active, the Map workspace shows a basemap usage notice. This keeps the local/demo default honest and reminds operators to configure a managed style URL before production or heavier tile traffic.
|
||||||
|
|
||||||
|
|||||||
@@ -911,6 +911,7 @@ function App(): JSX.Element {
|
|||||||
{activeWorkspace === 'map' ? (
|
{activeWorkspace === 'map' ? (
|
||||||
<MapWorkspace
|
<MapWorkspace
|
||||||
selectedProjectId={selectedProjectId}
|
selectedProjectId={selectedProjectId}
|
||||||
|
projects={projects}
|
||||||
areas={areas}
|
areas={areas}
|
||||||
selectedMapAreaId={selectedMapAreaId}
|
selectedMapAreaId={selectedMapAreaId}
|
||||||
areaFeatureCollection={areaFeatureCollection}
|
areaFeatureCollection={areaFeatureCollection}
|
||||||
@@ -957,6 +958,7 @@ function App(): JSX.Element {
|
|||||||
selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''}
|
selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''}
|
||||||
selectedFeature={selectedMapFeature}
|
selectedFeature={selectedMapFeature}
|
||||||
onSelectMapArea={setSelectedMapAreaId}
|
onSelectMapArea={setSelectedMapAreaId}
|
||||||
|
onSelectProject={selectProject}
|
||||||
onOpenDatasetInMap={openDatasetInMap}
|
onOpenDatasetInMap={openDatasetInMap}
|
||||||
onSetAreaLayerVisible={setAreaLayerVisible}
|
onSetAreaLayerVisible={setAreaLayerVisible}
|
||||||
onSetAreaLayerOpacity={setAreaLayerOpacity}
|
onSetAreaLayerOpacity={setAreaLayerOpacity}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useEffect, useMemo, useState } from 'react'
|
import { useEffect, useMemo, useState } from 'react'
|
||||||
import GeoMap from '../GeoMap'
|
import GeoMap from '../GeoMap'
|
||||||
import type { AreaRead, DatasetCreateResponse, MapViewportState, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
import type { AreaRead, DatasetCreateResponse, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||||
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
||||||
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
|
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
|
||||||
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
||||||
@@ -8,6 +8,8 @@ import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
|||||||
const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
|
const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
|
||||||
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
|
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
|
||||||
const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = []
|
const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = []
|
||||||
|
const MOL_PROJECT_NAME = 'Mol Municipality Workbench'
|
||||||
|
const KEMPEN_PROJECT_NAME = 'Kempen Regional Workbench'
|
||||||
|
|
||||||
type DataThemeId = 'buildings' | 'population' | 'forest' | 'water' | 'roads' | 'parcels'
|
type DataThemeId = 'buildings' | 'population' | 'forest' | 'water' | 'roads' | 'parcels'
|
||||||
|
|
||||||
@@ -163,6 +165,16 @@ function formatObservationDate(value: string | null | undefined): string {
|
|||||||
return new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short', day: 'numeric' }).format(new Date(value))
|
return new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short', day: 'numeric' }).format(new Date(value))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function operationalScopeProjectLabel(project: ProjectRead): string {
|
||||||
|
if (project.name === MOL_PROJECT_NAME) {
|
||||||
|
return 'Mol'
|
||||||
|
}
|
||||||
|
if (project.name === KEMPEN_PROJECT_NAME) {
|
||||||
|
return 'Kempen (28 gemeenten)'
|
||||||
|
}
|
||||||
|
return project.name
|
||||||
|
}
|
||||||
|
|
||||||
function selectionAreaSquareMetres(bbox: VectorSelectionBBox | null): number | null {
|
function selectionAreaSquareMetres(bbox: VectorSelectionBBox | null): number | null {
|
||||||
if (!bbox) {
|
if (!bbox) {
|
||||||
return null
|
return null
|
||||||
@@ -384,6 +396,7 @@ function downloadJsonFile(filename: string, payload: unknown): void {
|
|||||||
|
|
||||||
interface MapWorkspaceProps {
|
interface MapWorkspaceProps {
|
||||||
selectedProjectId: string | null
|
selectedProjectId: string | null
|
||||||
|
projects: ProjectRead[]
|
||||||
areas: AreaRead[]
|
areas: AreaRead[]
|
||||||
selectedMapAreaId: string
|
selectedMapAreaId: string
|
||||||
areaFeatureCollection: GeoJSON.FeatureCollection | null
|
areaFeatureCollection: GeoJSON.FeatureCollection | null
|
||||||
@@ -430,6 +443,7 @@ interface MapWorkspaceProps {
|
|||||||
availableMapDatasets: DatasetCreateResponse[]
|
availableMapDatasets: DatasetCreateResponse[]
|
||||||
selectedMapDatasetId: string
|
selectedMapDatasetId: string
|
||||||
onSelectMapArea: (areaId: string) => void
|
onSelectMapArea: (areaId: string) => void
|
||||||
|
onSelectProject: (projectId: string) => void
|
||||||
onOpenDatasetInMap: (dataset: DatasetCreateResponse) => void
|
onOpenDatasetInMap: (dataset: DatasetCreateResponse) => void
|
||||||
onSetAreaLayerVisible: (visible: boolean) => void
|
onSetAreaLayerVisible: (visible: boolean) => void
|
||||||
onSetAreaLayerOpacity: (opacity: number) => void
|
onSetAreaLayerOpacity: (opacity: number) => void
|
||||||
@@ -451,6 +465,7 @@ interface MapWorkspaceProps {
|
|||||||
|
|
||||||
export function MapWorkspace({
|
export function MapWorkspace({
|
||||||
selectedProjectId,
|
selectedProjectId,
|
||||||
|
projects,
|
||||||
areas,
|
areas,
|
||||||
selectedMapAreaId,
|
selectedMapAreaId,
|
||||||
areaFeatureCollection,
|
areaFeatureCollection,
|
||||||
@@ -497,6 +512,7 @@ export function MapWorkspace({
|
|||||||
availableMapDatasets,
|
availableMapDatasets,
|
||||||
selectedMapDatasetId,
|
selectedMapDatasetId,
|
||||||
onSelectMapArea,
|
onSelectMapArea,
|
||||||
|
onSelectProject,
|
||||||
onOpenDatasetInMap,
|
onOpenDatasetInMap,
|
||||||
onSetAreaLayerVisible,
|
onSetAreaLayerVisible,
|
||||||
onSetAreaLayerOpacity,
|
onSetAreaLayerOpacity,
|
||||||
@@ -574,6 +590,16 @@ export function MapWorkspace({
|
|||||||
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
|
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
|
||||||
const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id]
|
const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id]
|
||||||
const activeThemeDataset = themeDatasetMap[activeTheme.id]
|
const activeThemeDataset = themeDatasetMap[activeTheme.id]
|
||||||
|
const operationalScopeProjects = useMemo(() => {
|
||||||
|
const scoped = projects.filter((project) => project.name === MOL_PROJECT_NAME || project.name === KEMPEN_PROJECT_NAME)
|
||||||
|
const selected = projects.find((project) => project.id === selectedProjectId)
|
||||||
|
if (selected && !scoped.some((project) => project.id === selected.id)) {
|
||||||
|
return [selected, ...scoped]
|
||||||
|
}
|
||||||
|
return scoped
|
||||||
|
}, [projects, selectedProjectId])
|
||||||
|
const activeScopeProject = projects.find((project) => project.id === selectedProjectId) ?? null
|
||||||
|
const activeScopeLabel = activeScopeProject ? operationalScopeProjectLabel(activeScopeProject) : 'Werkgebied'
|
||||||
const activeTemporalSeriesGroups = useMemo(
|
const activeTemporalSeriesGroups = useMemo(
|
||||||
() => listThemeTemporalSeries(availableMapDatasets, activeTheme),
|
() => listThemeTemporalSeries(availableMapDatasets, activeTheme),
|
||||||
[activeTheme, availableMapDatasets],
|
[activeTheme, availableMapDatasets],
|
||||||
@@ -892,10 +918,10 @@ export function MapWorkspace({
|
|||||||
|
|
||||||
if (!advancedMode) {
|
if (!advancedMode) {
|
||||||
return (
|
return (
|
||||||
<section className="geo-explorer" data-testid="map-workspace" aria-label="Gebiedsverkenner Mol">
|
<section className="geo-explorer" data-testid="map-workspace" aria-label={`Gebiedsverkenner ${activeScopeLabel}`}>
|
||||||
<header className="geo-explorer-header">
|
<header className="geo-explorer-header">
|
||||||
<div>
|
<div>
|
||||||
<p className="eyebrow">Mol · geografische verkenner</p>
|
<p className="eyebrow">{activeScopeLabel} · geografische verkenner</p>
|
||||||
<h2>Wat bevindt zich in dit gebied?</h2>
|
<h2>Wat bevindt zich in dit gebied?</h2>
|
||||||
<p>Kies een datathema, teken een rechthoek en lees de beschikbare gegevens meteen uit.</p>
|
<p>Kies een datathema, teken een rechthoek en lees de beschikbare gegevens meteen uit.</p>
|
||||||
</div>
|
</div>
|
||||||
@@ -933,6 +959,19 @@ export function MapWorkspace({
|
|||||||
<p>Dit zijn databronnen, geen AI-modellen.</p>
|
<p>Dit zijn databronnen, geen AI-modellen.</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<label className="geo-project-scope-select">
|
||||||
|
Regio
|
||||||
|
<select
|
||||||
|
aria-label="Regio"
|
||||||
|
value={selectedProjectId ?? ''}
|
||||||
|
onChange={(event) => onSelectProject(event.target.value)}
|
||||||
|
disabled={operationalScopeProjects.length < 2}
|
||||||
|
>
|
||||||
|
{operationalScopeProjects.map((project) => (
|
||||||
|
<option key={project.id} value={project.id}>{operationalScopeProjectLabel(project)}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
<div className="geo-theme-list">
|
<div className="geo-theme-list">
|
||||||
{DATA_THEMES.map((theme) => {
|
{DATA_THEMES.map((theme) => {
|
||||||
const dataset = themeDatasetMap[theme.id]
|
const dataset = themeDatasetMap[theme.id]
|
||||||
@@ -1055,7 +1094,7 @@ export function MapWorkspace({
|
|||||||
type="button"
|
type="button"
|
||||||
onClick={() => selectedAreaBbox && void analyzeSelection(selectedAreaBbox)}
|
onClick={() => selectedAreaBbox && void analyzeSelection(selectedAreaBbox)}
|
||||||
>
|
>
|
||||||
Volledige gemeente
|
Volledig werkgebied
|
||||||
</button>
|
</button>
|
||||||
<button className="secondary-action" disabled={!mapSelectionBbox} type="button" onClick={clearAreaSelection}>
|
<button className="secondary-action" disabled={!mapSelectionBbox} type="button" onClick={clearAreaSelection}>
|
||||||
Wis selectie
|
Wis selectie
|
||||||
|
|||||||
@@ -5542,6 +5542,20 @@ section {
|
|||||||
gap: 0.36rem;
|
gap: 0.36rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.geo-project-scope-select {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.24rem;
|
||||||
|
color: #4c5b56;
|
||||||
|
font-size: 0.66rem;
|
||||||
|
font-weight: 800;
|
||||||
|
}
|
||||||
|
|
||||||
|
.geo-project-scope-select select {
|
||||||
|
min-height: 2.25rem;
|
||||||
|
padding: 0.35rem 0.45rem;
|
||||||
|
font-size: 0.72rem;
|
||||||
|
}
|
||||||
|
|
||||||
.geo-theme-option {
|
.geo-theme-option {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: 0.72rem minmax(0, 1fr) auto;
|
grid-template-columns: 0.72rem minmax(0, 1fr) auto;
|
||||||
|
|||||||
@@ -1254,6 +1254,38 @@ The 2013-2025 series is methodologically separate from the historical
|
|||||||
when both exist; it never calculates one continuous trend across those source
|
when both exist; it never calculates one continuous trend across those source
|
||||||
families.
|
families.
|
||||||
|
|
||||||
|
## Official Kempen operational scope
|
||||||
|
|
||||||
|
GeoIntel defines its regional `Kempen` workspace as the official Vlaamse
|
||||||
|
`Vervoerregio Kempen`: 28 explicitly registered municipalities. This is a
|
||||||
|
reproducible policy boundary, not a claim about the wider cultural,
|
||||||
|
landscape or historical Kempen.
|
||||||
|
|
||||||
|
Prepare and inspect the current VRBG union and all member boundaries without
|
||||||
|
changing persistence:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python3 /app/scripts/provision_geographic_scope.py \
|
||||||
|
--scope kempen-transport-region --fetch-only
|
||||||
|
```
|
||||||
|
|
||||||
|
Persist the complete scope foundation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python3 /app/scripts/provision_geographic_scope.py \
|
||||||
|
--scope kempen-transport-region
|
||||||
|
```
|
||||||
|
|
||||||
|
The command creates or reuses `Kempen Regional Workbench`, the regional Area,
|
||||||
|
28 municipality Areas and two VRBG source datasets through the canonical API.
|
||||||
|
Artifacts and checksums remain below
|
||||||
|
`/app/storage/operator-data/geographic-scopes/kempen-transport-region`.
|
||||||
|
Repeat runs are idempotent; `--force` refreshes today's source snapshot.
|
||||||
|
|
||||||
|
This command provisions boundaries only. Regional buildings, population,
|
||||||
|
land use, roads, water and parcels must be added by bounded source operators;
|
||||||
|
missing themes remain unavailable and are never filled with synthetic values.
|
||||||
|
|
||||||
## Tower deployment
|
## Tower deployment
|
||||||
|
|
||||||
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
|
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
"""Canonical operator scope definitions for GeoIntel.
|
||||||
|
|
||||||
|
These definitions describe administrative/policy scopes only. They do not
|
||||||
|
claim that a policy boundary equals a cultural or physical landscape region.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ScopeMember:
|
||||||
|
name: str
|
||||||
|
nis_code: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GeographicScope:
|
||||||
|
key: str
|
||||||
|
display_name: str
|
||||||
|
project_name: str
|
||||||
|
project_region: str
|
||||||
|
area_name: str
|
||||||
|
authority_name: str
|
||||||
|
authority_url: str
|
||||||
|
scope_type: str
|
||||||
|
limitation_message: str
|
||||||
|
members: tuple[ScopeMember, ...]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def nis_codes(self) -> tuple[str, ...]:
|
||||||
|
return tuple(member.nis_code for member in self.members)
|
||||||
|
|
||||||
|
|
||||||
|
MOL_SCOPE = GeographicScope(
|
||||||
|
key="mol",
|
||||||
|
display_name="Mol",
|
||||||
|
project_name="Mol Municipality Workbench",
|
||||||
|
project_region="Mol, Kempen",
|
||||||
|
area_name="Gemeente Mol - officiële grens",
|
||||||
|
authority_name="Digitaal Vlaanderen VRBG",
|
||||||
|
authority_url="https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items",
|
||||||
|
scope_type="municipality",
|
||||||
|
limitation_message="Officiële gemeentegrens; dit is geen perceelsgrens.",
|
||||||
|
members=(ScopeMember("Mol", "13025"),),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
KEMPEN_TRANSPORT_REGION_SCOPE = GeographicScope(
|
||||||
|
key="kempen-transport-region",
|
||||||
|
display_name="Kempen (28 gemeenten)",
|
||||||
|
project_name="Kempen Regional Workbench",
|
||||||
|
project_region="Vervoerregio Kempen, Vlaanderen",
|
||||||
|
area_name="Vervoerregio Kempen - officiële operationele grens",
|
||||||
|
authority_name="Vlaamse overheid - Vervoerregio Kempen",
|
||||||
|
authority_url=(
|
||||||
|
"https://www.vlaanderen.be/mobiliteitsprofessionals/personenvervoer/"
|
||||||
|
"basisbereikbaarheid/mobiliteitsuitdagingen-regionaal-aanpakken/"
|
||||||
|
"vervoerregios/over-de-vervoerregio-kempen"
|
||||||
|
),
|
||||||
|
scope_type="transport_region",
|
||||||
|
limitation_message=(
|
||||||
|
"Operationele beleidsgrens van de Vlaamse vervoerregio Kempen; "
|
||||||
|
"geen claim over de ruimere culturele, landschappelijke of historische Kempen."
|
||||||
|
),
|
||||||
|
members=(
|
||||||
|
ScopeMember("Arendonk", "13001"),
|
||||||
|
ScopeMember("Baarle-Hertog", "13002"),
|
||||||
|
ScopeMember("Balen", "13003"),
|
||||||
|
ScopeMember("Beerse", "13004"),
|
||||||
|
ScopeMember("Dessel", "13006"),
|
||||||
|
ScopeMember("Geel", "13008"),
|
||||||
|
ScopeMember("Grobbendonk", "13010"),
|
||||||
|
ScopeMember("Herentals", "13011"),
|
||||||
|
ScopeMember("Herenthout", "13012"),
|
||||||
|
ScopeMember("Herselt", "13013"),
|
||||||
|
ScopeMember("Hoogstraten", "13014"),
|
||||||
|
ScopeMember("Hulshout", "13016"),
|
||||||
|
ScopeMember("Kasterlee", "13017"),
|
||||||
|
ScopeMember("Laakdal", "13053"),
|
||||||
|
ScopeMember("Lille", "13019"),
|
||||||
|
ScopeMember("Meerhout", "13021"),
|
||||||
|
ScopeMember("Merksplas", "13023"),
|
||||||
|
ScopeMember("Mol", "13025"),
|
||||||
|
ScopeMember("Nijlen", "12026"),
|
||||||
|
ScopeMember("Olen", "13029"),
|
||||||
|
ScopeMember("Oud-Turnhout", "13031"),
|
||||||
|
ScopeMember("Ravels", "13035"),
|
||||||
|
ScopeMember("Retie", "13036"),
|
||||||
|
ScopeMember("Rijkevorsel", "13037"),
|
||||||
|
ScopeMember("Turnhout", "13040"),
|
||||||
|
ScopeMember("Vorselaar", "13044"),
|
||||||
|
ScopeMember("Vosselaar", "13046"),
|
||||||
|
ScopeMember("Westerlo", "13049"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
GEOGRAPHIC_SCOPES = {
|
||||||
|
scope.key: scope
|
||||||
|
for scope in (MOL_SCOPE, KEMPEN_TRANSPORT_REGION_SCOPE)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def validate_scope(scope: GeographicScope) -> None:
|
||||||
|
if not scope.key or not scope.project_name or not scope.area_name:
|
||||||
|
raise ValueError("Geographic scope identity fields must not be empty")
|
||||||
|
if len(scope.members) == 0:
|
||||||
|
raise ValueError(f"Geographic scope {scope.key} has no members")
|
||||||
|
names = [member.name.casefold() for member in scope.members]
|
||||||
|
codes = [member.nis_code for member in scope.members]
|
||||||
|
if len(names) != len(set(names)):
|
||||||
|
raise ValueError(f"Geographic scope {scope.key} contains duplicate municipality names")
|
||||||
|
if len(codes) != len(set(codes)):
|
||||||
|
raise ValueError(f"Geographic scope {scope.key} contains duplicate NIS codes")
|
||||||
|
if any(len(code) != 5 or not code.isdigit() for code in codes):
|
||||||
|
raise ValueError(f"Geographic scope {scope.key} contains an invalid NIS code")
|
||||||
|
|
||||||
|
|
||||||
|
for _scope in GEOGRAPHIC_SCOPES.values():
|
||||||
|
validate_scope(_scope)
|
||||||
@@ -0,0 +1,571 @@
|
|||||||
|
"""Provision an official geographic scope through the canonical GeoIntel API.
|
||||||
|
|
||||||
|
The operator fetches current VRBG municipality boundaries, creates one union
|
||||||
|
scope boundary plus a member-boundary artifact, and persists a project, the
|
||||||
|
regional area, all municipality areas and both datasets. It never runs during
|
||||||
|
application startup and never writes directly to PostGIS.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from pyproj import Transformer
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
|
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
||||||
|
from shapely.ops import transform, unary_union
|
||||||
|
from shapely.validation import make_valid
|
||||||
|
from urllib3.util.retry import Retry
|
||||||
|
|
||||||
|
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope
|
||||||
|
|
||||||
|
|
||||||
|
VRBG_ITEMS_URL = "https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items"
|
||||||
|
VRBG_ATTRIBUTION = "Bron: Voorlopig referentiebestand gemeentegrenzen, Digitaal Vlaanderen"
|
||||||
|
DEFAULT_SCOPE_KEY = "kempen-transport-region"
|
||||||
|
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
||||||
|
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||||
|
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description="Provision an official GeoIntel geographic scope.")
|
||||||
|
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
|
||||||
|
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-root",
|
||||||
|
type=Path,
|
||||||
|
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||||
|
)
|
||||||
|
parser.add_argument("--request-timeout", type=int, default=180)
|
||||||
|
parser.add_argument("--import-timeout", type=int, default=1800)
|
||||||
|
parser.add_argument("--force", action="store_true", help="Refresh official source artifacts for today's snapshot.")
|
||||||
|
parser.add_argument("--fetch-only", action="store_true", help="Validate and write artifacts without API persistence.")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def utc_now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def write_json_atomic(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
temporary = path.with_suffix(f"{path.suffix}.partial")
|
||||||
|
temporary.write_text(
|
||||||
|
json.dumps(
|
||||||
|
payload,
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2 if pretty else None,
|
||||||
|
separators=None if pretty else (",", ":"),
|
||||||
|
sort_keys=pretty,
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_polygonal(geometry):
|
||||||
|
if geometry is None or geometry.is_empty:
|
||||||
|
return None
|
||||||
|
if not geometry.is_valid:
|
||||||
|
geometry = make_valid(geometry)
|
||||||
|
if isinstance(geometry, (Polygon, MultiPolygon)):
|
||||||
|
return geometry
|
||||||
|
if isinstance(geometry, GeometryCollection):
|
||||||
|
polygons = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty]
|
||||||
|
if polygons:
|
||||||
|
merged = unary_union(polygons)
|
||||||
|
return merged if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty else None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def metric_area_km2(geometry) -> float:
|
||||||
|
transformer = Transformer.from_crs(4326, 31370, always_xy=True)
|
||||||
|
return float(transform(transformer.transform, geometry).area / 1_000_000)
|
||||||
|
|
||||||
|
|
||||||
|
def build_source_session() -> requests.Session:
|
||||||
|
retry = Retry(
|
||||||
|
total=5,
|
||||||
|
connect=5,
|
||||||
|
read=5,
|
||||||
|
status=5,
|
||||||
|
backoff_factor=1.0,
|
||||||
|
status_forcelist=(429, 500, 502, 503, 504),
|
||||||
|
allowed_methods=frozenset({"GET"}),
|
||||||
|
raise_on_status=True,
|
||||||
|
)
|
||||||
|
adapter = HTTPAdapter(max_retries=retry)
|
||||||
|
session = requests.Session()
|
||||||
|
session.headers.update({"User-Agent": "GeoIntel-Geographic-Scope-Operator/1.0"})
|
||||||
|
session.mount("https://", adapter)
|
||||||
|
session.mount("http://", adapter)
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_scope_members(session: requests.Session, scope: GeographicScope, timeout: int) -> tuple[list[dict[str, Any]], str]:
|
||||||
|
response = session.get(
|
||||||
|
VRBG_ITEMS_URL,
|
||||||
|
params={"f": "application/geo+json", "limit": "1000"},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
expected = {member.nis_code: member.name for member in scope.members}
|
||||||
|
selected: dict[str, dict[str, Any]] = {}
|
||||||
|
for feature in response.json().get("features") or []:
|
||||||
|
properties = feature.get("properties") or {}
|
||||||
|
nis_code = str(properties.get("NISCODE") or "")
|
||||||
|
if nis_code not in expected:
|
||||||
|
continue
|
||||||
|
if nis_code in selected:
|
||||||
|
raise RuntimeError(f"Official VRBG returned duplicate NIS code {nis_code}")
|
||||||
|
actual_name = str(properties.get("NAAM") or "")
|
||||||
|
if actual_name.casefold() != expected[nis_code].casefold():
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Official VRBG name drift for {nis_code}: expected {expected[nis_code]!r}, received {actual_name!r}"
|
||||||
|
)
|
||||||
|
selected[nis_code] = feature
|
||||||
|
missing = [f"{name} ({code})" for code, name in expected.items() if code not in selected]
|
||||||
|
if missing:
|
||||||
|
raise RuntimeError(f"Official VRBG is missing scope members: {', '.join(missing)}")
|
||||||
|
return [selected[member.nis_code] for member in scope.members], response.url
|
||||||
|
|
||||||
|
|
||||||
|
def build_scope_payloads(
|
||||||
|
scope: GeographicScope,
|
||||||
|
source_features: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
source_url: str,
|
||||||
|
generated_at: str,
|
||||||
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
||||||
|
if len(source_features) != len(scope.members):
|
||||||
|
raise RuntimeError(f"Expected {len(scope.members)} source boundaries, received {len(source_features)}")
|
||||||
|
member_features: list[dict[str, Any]] = []
|
||||||
|
member_geometries = []
|
||||||
|
for member, source_feature in zip(scope.members, source_features, strict=True):
|
||||||
|
properties = dict(source_feature.get("properties") or {})
|
||||||
|
if str(properties.get("NISCODE") or "") != member.nis_code:
|
||||||
|
raise RuntimeError(f"Scope member order/code mismatch for {member.name}")
|
||||||
|
geometry = normalize_polygonal(shape(source_feature.get("geometry")))
|
||||||
|
if geometry is None:
|
||||||
|
raise RuntimeError(f"Official boundary for {member.name} is empty, invalid or non-polygonal")
|
||||||
|
member_geometries.append(geometry)
|
||||||
|
properties.update(
|
||||||
|
{
|
||||||
|
"source_name": "vrbg",
|
||||||
|
"source_feature_id": str(source_feature.get("id") or member.nis_code),
|
||||||
|
"layer_type": "municipality_boundary",
|
||||||
|
"authority_level": "authoritative",
|
||||||
|
"coverage_scope": scope.key,
|
||||||
|
"scope_type": scope.scope_type,
|
||||||
|
"municipality": member.name,
|
||||||
|
"nis_code": member.nis_code,
|
||||||
|
"attribution": VRBG_ATTRIBUTION,
|
||||||
|
"source_url": source_url,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
member_features.append(
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": str(source_feature.get("id") or f"Refgem.{member.nis_code}"),
|
||||||
|
"geometry": mapping(geometry),
|
||||||
|
"properties": properties,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
boundary = normalize_polygonal(unary_union(member_geometries))
|
||||||
|
if boundary is None:
|
||||||
|
raise RuntimeError("Union of official scope member boundaries is invalid")
|
||||||
|
member_codes = list(scope.nis_codes)
|
||||||
|
boundary_payload = {
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"name": f"Official operation boundary - {scope.display_name}",
|
||||||
|
"crs": GEOJSON_CRS,
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": f"scope:{scope.key}",
|
||||||
|
"geometry": mapping(boundary),
|
||||||
|
"properties": {
|
||||||
|
"name": scope.area_name,
|
||||||
|
"source_name": "vrbg",
|
||||||
|
"source_feature_id": f"scope:{scope.key}",
|
||||||
|
"layer_type": "regional_boundary",
|
||||||
|
"authority_level": "authoritative",
|
||||||
|
"coverage_scope": scope.key,
|
||||||
|
"scope_type": scope.scope_type,
|
||||||
|
"member_count": len(scope.members),
|
||||||
|
"member_nis_codes": member_codes,
|
||||||
|
"scope_authority": scope.authority_name,
|
||||||
|
"scope_authority_url": scope.authority_url,
|
||||||
|
"scope_limitation": scope.limitation_message,
|
||||||
|
"attribution": VRBG_ATTRIBUTION,
|
||||||
|
"source_url": source_url,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source_url": source_url,
|
||||||
|
"scope_authority_url": scope.authority_url,
|
||||||
|
"scope_limitation": scope.limitation_message,
|
||||||
|
"generated_at": generated_at,
|
||||||
|
}
|
||||||
|
members_payload = {
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"name": f"Official municipality boundaries - {scope.display_name}",
|
||||||
|
"crs": GEOJSON_CRS,
|
||||||
|
"features": member_features,
|
||||||
|
"source_url": source_url,
|
||||||
|
"scope_authority_url": scope.authority_url,
|
||||||
|
"scope_limitation": scope.limitation_message,
|
||||||
|
"generated_at": generated_at,
|
||||||
|
}
|
||||||
|
summary = {
|
||||||
|
"scope_key": scope.key,
|
||||||
|
"scope_type": scope.scope_type,
|
||||||
|
"display_name": scope.display_name,
|
||||||
|
"member_count": len(scope.members),
|
||||||
|
"member_names": [member.name for member in scope.members],
|
||||||
|
"member_nis_codes": member_codes,
|
||||||
|
"area_km2": metric_area_km2(boundary),
|
||||||
|
"wgs84_bbox": list(boundary.bounds),
|
||||||
|
}
|
||||||
|
return boundary_payload, members_payload, summary
|
||||||
|
|
||||||
|
|
||||||
|
def artifact_paths(output_dir: Path, scope: GeographicScope, snapshot_date: str) -> tuple[Path, Path, Path]:
|
||||||
|
stem = scope.key.replace("-", "_")
|
||||||
|
return (
|
||||||
|
output_dir / f"{stem}_boundary_{snapshot_date}.geojson",
|
||||||
|
output_dir / f"{stem}_municipalities_{snapshot_date}.geojson",
|
||||||
|
output_dir / f"{stem}_scope_manifest.json",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def cached_artifacts(output_dir: Path, scope: GeographicScope) -> tuple[Path, Path, dict[str, Any]] | None:
|
||||||
|
manifest_path = output_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json"
|
||||||
|
if not manifest_path.exists():
|
||||||
|
return None
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
boundary_path = output_dir / str(manifest.get("boundary_filename") or "")
|
||||||
|
members_path = output_dir / str(manifest.get("municipalities_filename") or "")
|
||||||
|
if (
|
||||||
|
manifest.get("status") == "complete"
|
||||||
|
and manifest.get("scope_key") == scope.key
|
||||||
|
and manifest.get("member_count") == len(scope.members)
|
||||||
|
and boundary_path.is_file()
|
||||||
|
and members_path.is_file()
|
||||||
|
and sha256_file(boundary_path) == manifest.get("boundary_sha256")
|
||||||
|
and sha256_file(members_path) == manifest.get("municipalities_sha256")
|
||||||
|
):
|
||||||
|
return boundary_path, members_path, manifest
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_artifacts(args: argparse.Namespace, scope: GeographicScope) -> tuple[Path, Path, dict[str, Any]]:
|
||||||
|
output_dir = args.output_root / scope.key
|
||||||
|
if not args.force:
|
||||||
|
cached = cached_artifacts(output_dir, scope)
|
||||||
|
if cached:
|
||||||
|
return cached
|
||||||
|
generated_at = utc_now()
|
||||||
|
snapshot_date = generated_at[:10]
|
||||||
|
boundary_path, members_path, manifest_path = artifact_paths(output_dir, scope, snapshot_date)
|
||||||
|
with build_source_session() as session:
|
||||||
|
source_features, source_url = fetch_scope_members(session, scope, args.request_timeout)
|
||||||
|
boundary_payload, members_payload, summary = build_scope_payloads(
|
||||||
|
scope,
|
||||||
|
source_features,
|
||||||
|
source_url=source_url,
|
||||||
|
generated_at=generated_at,
|
||||||
|
)
|
||||||
|
write_json_atomic(boundary_path, boundary_payload)
|
||||||
|
write_json_atomic(members_path, members_payload)
|
||||||
|
manifest = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"status": "complete",
|
||||||
|
"generated_at": generated_at,
|
||||||
|
"observed_at": f"{snapshot_date}T00:00:00Z",
|
||||||
|
"boundary_filename": boundary_path.name,
|
||||||
|
"boundary_sha256": sha256_file(boundary_path),
|
||||||
|
"municipalities_filename": members_path.name,
|
||||||
|
"municipalities_sha256": sha256_file(members_path),
|
||||||
|
"vrbg_source_url": source_url,
|
||||||
|
"vrbg_attribution": VRBG_ATTRIBUTION,
|
||||||
|
"scope_authority_name": scope.authority_name,
|
||||||
|
"scope_authority_url": scope.authority_url,
|
||||||
|
"scope_limitation": scope.limitation_message,
|
||||||
|
**summary,
|
||||||
|
}
|
||||||
|
write_json_atomic(manifest_path, manifest, pretty=True)
|
||||||
|
return boundary_path, members_path, manifest
|
||||||
|
|
||||||
|
|
||||||
|
def response_data(response: requests.Response) -> Any:
|
||||||
|
try:
|
||||||
|
payload = response.json()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:500]}") from exc
|
||||||
|
if not response.ok:
|
||||||
|
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:1000]}")
|
||||||
|
if not isinstance(payload, dict) or "data" not in payload:
|
||||||
|
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
|
||||||
|
return payload["data"]
|
||||||
|
|
||||||
|
|
||||||
|
def list_paginated_items(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]:
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
offset = 0
|
||||||
|
total: int | None = None
|
||||||
|
while True:
|
||||||
|
page = response_data(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
|
||||||
|
page_items = list(page.get("items") or [])
|
||||||
|
items.extend(page_items)
|
||||||
|
if total is None and page.get("total") is not None:
|
||||||
|
total = int(page["total"])
|
||||||
|
if not page_items or (total is not None and len(items) >= total) or len(page_items) < 200:
|
||||||
|
break
|
||||||
|
offset += len(page_items)
|
||||||
|
if total is not None and len(items) != total:
|
||||||
|
raise RuntimeError(f"GeoIntel list response returned {len(items)} of {total} records for {url}")
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def find_or_create_project(session: requests.Session, base_url: str, scope: GeographicScope, timeout: int) -> dict[str, Any]:
|
||||||
|
projects = list_paginated_items(session, f"{base_url}/api/v1/projects", timeout=timeout)
|
||||||
|
existing = next((item for item in projects if item.get("name") == scope.project_name), None)
|
||||||
|
if existing:
|
||||||
|
return existing
|
||||||
|
return response_data(
|
||||||
|
session.post(
|
||||||
|
f"{base_url}/api/v1/projects",
|
||||||
|
json={
|
||||||
|
"name": scope.project_name,
|
||||||
|
"description": (
|
||||||
|
f"Operational GeoIntel scope for {scope.display_name}, composed from {len(scope.members)} current "
|
||||||
|
f"VRBG municipality boundaries. {scope.limitation_message}"
|
||||||
|
),
|
||||||
|
"region": scope.project_region,
|
||||||
|
},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def find_or_create_areas(
|
||||||
|
session: requests.Session,
|
||||||
|
base_url: str,
|
||||||
|
project_id: str,
|
||||||
|
scope: GeographicScope,
|
||||||
|
boundary_payload: dict[str, Any],
|
||||||
|
members_payload: dict[str, Any],
|
||||||
|
timeout: int,
|
||||||
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||||
|
areas = list_paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/areas", timeout=timeout)
|
||||||
|
by_name = {str(item.get("name")): item for item in areas}
|
||||||
|
|
||||||
|
def ensure(name: str, geometry: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
existing = by_name.get(name)
|
||||||
|
if existing:
|
||||||
|
return existing
|
||||||
|
created = response_data(
|
||||||
|
session.post(
|
||||||
|
f"{base_url}/api/v1/projects/{project_id}/areas",
|
||||||
|
json={"name": name, "crs": "EPSG:4326", "geometry": geometry},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
by_name[name] = created
|
||||||
|
return created
|
||||||
|
|
||||||
|
region_area = ensure(scope.area_name, boundary_payload["features"][0]["geometry"])
|
||||||
|
member_areas = [
|
||||||
|
ensure(f"Gemeente {member.name} - officiële grens", feature["geometry"])
|
||||||
|
for member, feature in zip(scope.members, members_payload["features"], strict=True)
|
||||||
|
]
|
||||||
|
return region_area, member_areas
|
||||||
|
|
||||||
|
|
||||||
|
def upload_dataset(
|
||||||
|
session: requests.Session,
|
||||||
|
*,
|
||||||
|
base_url: str,
|
||||||
|
project_id: str,
|
||||||
|
area_id: str,
|
||||||
|
path: Path,
|
||||||
|
source_metadata: dict[str, Any],
|
||||||
|
provenance_metadata: dict[str, Any],
|
||||||
|
temporal_series_key: str,
|
||||||
|
observed_at: str,
|
||||||
|
timeout: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
response = session.post(
|
||||||
|
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||||
|
data={
|
||||||
|
"dataset_type": "vector",
|
||||||
|
"source": "operator_official_import",
|
||||||
|
"dataset_role": "source",
|
||||||
|
"source_name": "vrbg",
|
||||||
|
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||||
|
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||||
|
"area_id": area_id,
|
||||||
|
"temporal_series_key": temporal_series_key,
|
||||||
|
"observed_at": observed_at,
|
||||||
|
"valid_from": observed_at,
|
||||||
|
"temporal_granularity": "snapshot",
|
||||||
|
"source_version": observed_at[:10],
|
||||||
|
},
|
||||||
|
files={"file": (path.name, handle, "application/geo+json")},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
return response_data(response)
|
||||||
|
|
||||||
|
|
||||||
|
def provision_scope(
|
||||||
|
args: argparse.Namespace,
|
||||||
|
scope: GeographicScope,
|
||||||
|
boundary_path: Path,
|
||||||
|
members_path: Path,
|
||||||
|
manifest: dict[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
base_url = args.base_url.rstrip("/")
|
||||||
|
boundary_payload = json.loads(boundary_path.read_text(encoding="utf-8"))
|
||||||
|
members_payload = json.loads(members_path.read_text(encoding="utf-8"))
|
||||||
|
with requests.Session() as session:
|
||||||
|
project = find_or_create_project(session, base_url, scope, args.import_timeout)
|
||||||
|
project_id = str(project["id"])
|
||||||
|
region_area, member_areas = find_or_create_areas(
|
||||||
|
session,
|
||||||
|
base_url,
|
||||||
|
project_id,
|
||||||
|
scope,
|
||||||
|
boundary_payload,
|
||||||
|
members_payload,
|
||||||
|
args.import_timeout,
|
||||||
|
)
|
||||||
|
datasets = list_paginated_items(
|
||||||
|
session,
|
||||||
|
f"{base_url}/api/v1/projects/{project_id}/datasets",
|
||||||
|
timeout=args.import_timeout,
|
||||||
|
)
|
||||||
|
source_metadata = {
|
||||||
|
"provider": "Digitaal Vlaanderen",
|
||||||
|
"collection": "VRBG/Refgem",
|
||||||
|
"authority_level": "authoritative",
|
||||||
|
"coverage_scope": scope.key,
|
||||||
|
"scope_type": scope.scope_type,
|
||||||
|
"scope_authority": scope.authority_name,
|
||||||
|
"scope_authority_url": scope.authority_url,
|
||||||
|
"scope_limitation": scope.limitation_message,
|
||||||
|
"member_count": len(scope.members),
|
||||||
|
"member_nis_codes": list(scope.nis_codes),
|
||||||
|
"attribution": VRBG_ATTRIBUTION,
|
||||||
|
}
|
||||||
|
common_provenance = {
|
||||||
|
"operator_tool": "provision_geographic_scope.py",
|
||||||
|
"operator_explicit_fetch": True,
|
||||||
|
"manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"),
|
||||||
|
"source_url": manifest["vrbg_source_url"],
|
||||||
|
"scope_authority_url": scope.authority_url,
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset_specs = (
|
||||||
|
(
|
||||||
|
boundary_path,
|
||||||
|
"regional_boundary",
|
||||||
|
f"vrbg:scope-boundary:{scope.key}",
|
||||||
|
manifest["boundary_sha256"],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
members_path,
|
||||||
|
"municipality_boundaries",
|
||||||
|
f"vrbg:scope-members:{scope.key}",
|
||||||
|
manifest["municipalities_sha256"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
persisted = []
|
||||||
|
for path, layer_type, series_key, checksum in dataset_specs:
|
||||||
|
existing = next((item for item in datasets if item.get("original_filename") == path.name), None)
|
||||||
|
if existing:
|
||||||
|
persisted_checksum = (existing.get("provenance_metadata") or {}).get("artifact_sha256")
|
||||||
|
if persisted_checksum and persisted_checksum != checksum:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Immutable scope dataset {path.name} has checksum {persisted_checksum}, "
|
||||||
|
f"but the refreshed source produced {checksum}; use a new observation date instead of overwriting it"
|
||||||
|
)
|
||||||
|
persisted.append(existing)
|
||||||
|
continue
|
||||||
|
created = upload_dataset(
|
||||||
|
session,
|
||||||
|
base_url=base_url,
|
||||||
|
project_id=project_id,
|
||||||
|
area_id=str(region_area["id"]),
|
||||||
|
path=path,
|
||||||
|
source_metadata={**source_metadata, "layer_type": layer_type},
|
||||||
|
provenance_metadata={**common_provenance, "artifact_sha256": checksum},
|
||||||
|
temporal_series_key=series_key,
|
||||||
|
observed_at=manifest["observed_at"],
|
||||||
|
timeout=args.import_timeout,
|
||||||
|
)
|
||||||
|
persisted.append(created)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"project_id": project_id,
|
||||||
|
"project_name": project.get("name"),
|
||||||
|
"region_area_id": str(region_area["id"]),
|
||||||
|
"region_area_name": region_area.get("name"),
|
||||||
|
"municipality_area_count": len(member_areas),
|
||||||
|
"boundary_dataset_id": str(persisted[0]["id"]),
|
||||||
|
"municipality_dataset_id": str(persisted[1]["id"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||||
|
try:
|
||||||
|
boundary_path, members_path, manifest = prepare_artifacts(args, scope)
|
||||||
|
workspace = None if args.fetch_only else provision_scope(args, scope, boundary_path, members_path, manifest)
|
||||||
|
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||||
|
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"mode": "fetch_only" if args.fetch_only else "provisioned",
|
||||||
|
"scope": scope.key,
|
||||||
|
"display_name": scope.display_name,
|
||||||
|
"member_count": manifest["member_count"],
|
||||||
|
"area_km2": manifest["area_km2"],
|
||||||
|
"wgs84_bbox": manifest["wgs84_bbox"],
|
||||||
|
"boundary_path": str(boundary_path),
|
||||||
|
"municipalities_path": str(members_path),
|
||||||
|
"workspace": workspace,
|
||||||
|
"limitation": scope.limitation_message,
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -47,6 +47,8 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_context_layers.py
|
|||||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py
|
${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||||
|
|||||||
Reference in New Issue
Block a user