feat: add official Kempen operating scope
This commit is contained in:
@@ -7,6 +7,14 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 189 Official Kempen operational scope (2026-07-14)
|
||||
|
||||
- Defined `Kempen` operationally as the official 28-municipality Vlaamse vervoerregio, with an explicit warning that this policy boundary is not the wider cultural or landscape Kempen.
|
||||
- Added a central operator scope registry with current municipality names and NIS codes, including Mol `13025` and Nijlen `12026`.
|
||||
- Added an explicit, idempotent VRBG scope provisioner that creates one regional boundary, 28 member boundaries, one regional Area, 28 municipality Areas and canonical source datasets through the existing API.
|
||||
- Added a compact Mol/Kempen region selector to the map-first explorer and made its heading and full-area action scope-neutral.
|
||||
- Kept thematic regional ingestion separate from the boundary pass so large GRB/WCS sources can be partitioned and validated without hidden startup fetches or truncated datasets.
|
||||
|
||||
## Sprint 188 Official modern Mol land-use series (2026-07-14)
|
||||
|
||||
- Added an explicit, reusable MercatorNet WCS operator for official Departement Omgeving land-use snapshots in 2013, 2016, 2019, 2022 and 2025.
|
||||
|
||||
@@ -935,6 +935,27 @@ the same bbox-selected FeatureCollection as a normal export record with
|
||||
`export_type="vector_selection_geojson"`. This creates a handoff artifact only;
|
||||
it does not create a derived dataset.
|
||||
|
||||
## Geographic scope provisioning
|
||||
|
||||
The explicit operator command below provisions the official 28-municipality
|
||||
Vlaamse vervoerregio Kempen boundary foundation:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_geographic_scope.py \
|
||||
--scope kempen-transport-region
|
||||
```
|
||||
|
||||
It reads current `VRBG/Refgem` boundaries, validates every registered name and
|
||||
NIS code, unions the regional geometry and creates one project, one regional
|
||||
Area, 28 municipality Areas and two source datasets through the public API.
|
||||
It never writes directly to PostGIS and does not run on startup. The persisted
|
||||
scope limitation explicitly distinguishes the transport-policy region from a
|
||||
cultural or landscape definition of Kempen.
|
||||
|
||||
Use `--fetch-only` for a source/geometry/checksum audit. The scope pass does
|
||||
not fetch thematic GRB, population or land-use data; those remain separate,
|
||||
bounded operator jobs.
|
||||
|
||||
## Temporal Mol data and evolution
|
||||
|
||||
Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`,
|
||||
|
||||
@@ -18,7 +18,7 @@ def test_map_first_explorer_is_the_default_product_flow() -> None:
|
||||
assert "Wat bevindt zich in dit gebied?" in workspace
|
||||
assert "Kies een datathema" in workspace
|
||||
assert "Teken rechthoek" in workspace
|
||||
assert "Volledige gemeente" in workspace
|
||||
assert "Volledig werkgebied" in workspace
|
||||
assert "Alle beschikbare thema" in workspace
|
||||
assert "Bron nog niet ingeladen" in workspace
|
||||
assert "useMapThemeSelectionInsights" in workspace
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from shapely.geometry import Polygon, shape
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
|
||||
|
||||
def load_module(name: str, filename: str):
|
||||
scripts_path = str(SCRIPTS)
|
||||
if scripts_path not in sys.path:
|
||||
sys.path.insert(0, scripts_path)
|
||||
spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_kempen_scope_matches_official_28_municipality_policy_region() -> None:
|
||||
scopes = load_module("geographic_scopes_test", "geographic_scopes.py")
|
||||
scope = scopes.KEMPEN_TRANSPORT_REGION_SCOPE
|
||||
|
||||
assert scope.key == "kempen-transport-region"
|
||||
assert scope.project_name == "Kempen Regional Workbench"
|
||||
assert scope.scope_type == "transport_region"
|
||||
assert len(scope.members) == 28
|
||||
assert len(set(scope.nis_codes)) == 28
|
||||
assert ("Mol", "13025") in {(member.name, member.nis_code) for member in scope.members}
|
||||
assert ("Nijlen", "12026") in {(member.name, member.nis_code) for member in scope.members}
|
||||
assert "vervoerregio-kempen" in scope.authority_url
|
||||
assert "geen claim" in scope.limitation_message
|
||||
|
||||
|
||||
def test_scope_union_preserves_member_identity_and_policy_limitation() -> None:
|
||||
scopes = load_module("geographic_scopes_union_test", "geographic_scopes.py")
|
||||
provisioner = load_module("provision_geographic_scope_test", "provision_geographic_scope.py")
|
||||
scope = scopes.GeographicScope(
|
||||
key="test-region",
|
||||
display_name="Testregio",
|
||||
project_name="Test Regional Workbench",
|
||||
project_region="Test",
|
||||
area_name="Testregio - operationele grens",
|
||||
authority_name="Test authority",
|
||||
authority_url="https://example.test/scope",
|
||||
scope_type="policy_region",
|
||||
limitation_message="Operationele testgrens; geen landschappelijke claim.",
|
||||
members=(scopes.ScopeMember("Alpha", "10001"), scopes.ScopeMember("Beta", "10002")),
|
||||
)
|
||||
source_features = [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "alpha",
|
||||
"geometry": Polygon([(4.0, 51.0), (4.1, 51.0), (4.1, 51.1), (4.0, 51.1)]).__geo_interface__,
|
||||
"properties": {"NAAM": "Alpha", "NISCODE": "10001"},
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "beta",
|
||||
"geometry": Polygon([(4.1, 51.0), (4.2, 51.0), (4.2, 51.1), (4.1, 51.1)]).__geo_interface__,
|
||||
"properties": {"NAAM": "Beta", "NISCODE": "10002"},
|
||||
},
|
||||
]
|
||||
|
||||
boundary, members, summary = provisioner.build_scope_payloads(
|
||||
scope,
|
||||
source_features,
|
||||
source_url="https://example.test/vrbg",
|
||||
generated_at="2026-07-14T00:00:00+00:00",
|
||||
)
|
||||
|
||||
assert len(boundary["features"]) == 1
|
||||
assert len(members["features"]) == 2
|
||||
assert shape(boundary["features"][0]["geometry"]).is_valid
|
||||
assert boundary["features"][0]["properties"]["member_nis_codes"] == ["10001", "10002"]
|
||||
assert boundary["features"][0]["properties"]["scope_limitation"] == scope.limitation_message
|
||||
assert [feature["properties"]["municipality"] for feature in members["features"]] == ["Alpha", "Beta"]
|
||||
assert summary["member_count"] == 2
|
||||
assert summary["area_km2"] > 0
|
||||
|
||||
|
||||
def test_scope_api_pagination_respects_canonical_limit() -> None:
|
||||
provisioner = load_module("provision_geographic_scope_paging_test", "provision_geographic_scope.py")
|
||||
|
||||
class Response:
|
||||
ok = True
|
||||
status_code = 200
|
||||
text = ""
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def json(self):
|
||||
return {"data": self.payload}
|
||||
|
||||
class Session:
|
||||
def __init__(self) -> None:
|
||||
self.offsets = []
|
||||
|
||||
def get(self, url, *, params, timeout):
|
||||
del url, timeout
|
||||
self.offsets.append(params["offset"])
|
||||
offset = params["offset"]
|
||||
page = [{"id": index} for index in range(offset, min(offset + 200, 401))]
|
||||
return Response({"items": page, "total": 401})
|
||||
|
||||
session = Session()
|
||||
items = provisioner.list_paginated_items(session, "http://backend/api/v1/projects", timeout=30)
|
||||
|
||||
assert len(items) == 401
|
||||
assert session.offsets == [0, 200, 400]
|
||||
|
||||
|
||||
def test_kempen_scope_operator_is_packaged_and_exposed_in_map_flow() -> None:
|
||||
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
app = (ROOT / "frontend/src/App.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "COPY scripts/geographic_scopes.py" in dockerfile
|
||||
assert "COPY scripts/provision_geographic_scope.py" in dockerfile
|
||||
assert "py_compile scripts/provision_geographic_scope.py" in readiness
|
||||
assert "Immutable scope dataset" in (ROOT / "scripts/provision_geographic_scope.py").read_text(encoding="utf-8")
|
||||
assert "Kempen (28 gemeenten)" in workspace
|
||||
assert 'aria-label="Regio"' in workspace
|
||||
assert "projects={projects}" in app
|
||||
assert "onSelectProject={selectProject}" in app
|
||||
@@ -77,6 +77,8 @@ COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_
|
||||
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
|
||||
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
|
||||
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
||||
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
|
||||
COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py
|
||||
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
|
||||
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
|
||||
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
|
||||
|
||||
@@ -7850,3 +7850,25 @@ Validation:
|
||||
|
||||
Next:
|
||||
- Define the exact municipality list or administrative boundary that GeoIntel will call `Kempen`, then reuse the proven source operators and temporal query path for that scope without changing Mol semantics.
|
||||
|
||||
## Sprint 189 Official Kempen operational scope (2026-07-14)
|
||||
|
||||
Implemented:
|
||||
- Chose the official Vlaamse `Vervoerregio Kempen` as GeoIntel's reproducible operational regional scope. It has 28 municipalities and explicitly includes both Mol and Nijlen.
|
||||
- Added `geographic_scopes.py` as the operator registry for scope identity, current municipality names/NIS codes, policy authority and limitation text.
|
||||
- Added `provision_geographic_scope.py` to fetch current authoritative VRBG boundaries, validate all members, create a valid regional union and retain boundary/member GeoJSON plus checksums and manifest.
|
||||
- The operator creates or reuses `Kempen Regional Workbench`, one regional Area, 28 municipality Areas and two source datasets exclusively through canonical API envelopes.
|
||||
- Added a compact Mol/Kempen selector to the primary map flow, scope-aware heading text and the neutral `Volledig werkgebied` action.
|
||||
- Kept thematic regional fetching outside the scope pass. No startup fetch, direct PostGIS write, fake metric or unbounded regional theme download was introduced.
|
||||
|
||||
Source proof:
|
||||
- A live read-only VRBG run matched all 28 registered municipalities and current NIS codes.
|
||||
- The valid union measured 1,399.2505 km2 in EPSG:31370 with WGS84 bbox `[4.59723873, 51.01047967, 5.26224853, 51.50511313]`.
|
||||
- The scope is labelled as the transport-policy region and never as the complete cultural, landscape or historical Kempen.
|
||||
|
||||
Validation before deployment:
|
||||
- New focused scope and map-flow suite passed 9 tests.
|
||||
- Python scope/operator compilation and frontend TypeScript typecheck passed.
|
||||
|
||||
Next:
|
||||
- Deploy and run the scope operator on Tower, verify 29 Areas and both boundary datasets, then design bounded regional theme partitions before importing high-volume GRB or WCS data.
|
||||
|
||||
@@ -88,6 +88,24 @@ Kempen reuses the same operator with an explicit approved boundary, project,
|
||||
area and scope key; an ambiguous regional label is never converted into an
|
||||
invented boundary.
|
||||
|
||||
## Operational Kempen scope
|
||||
|
||||
The canonical regional workspace uses the official 28-municipality Vlaamse
|
||||
`Vervoerregio Kempen` policy boundary. `scripts/geographic_scopes.py` is the
|
||||
operator source of truth for its members and current NIS codes. The resulting
|
||||
regional geometry is a union of current authoritative VRBG municipality
|
||||
boundaries.
|
||||
|
||||
- Project: `Kempen Regional Workbench`.
|
||||
- Regional Area: `Vervoerregio Kempen - officiële operationele grens`.
|
||||
- Member Areas: one current official boundary for every municipality.
|
||||
- Scope key: `kempen-transport-region`.
|
||||
- Provenance: authority URL, VRBG request URL, member codes, fetch timestamp,
|
||||
artifact checksums and the policy-boundary limitation.
|
||||
- Scale policy: thematic providers are partitioned and validated separately;
|
||||
no interactive request or application startup may download the entire
|
||||
region or silently substitute missing data.
|
||||
|
||||
## User-uploaded raster strategy
|
||||
V1 must support controlled local datasets because public raster access and model compatibility can be difficult.
|
||||
|
||||
|
||||
@@ -29,6 +29,29 @@ start geen verborgen providerfetch. De resulterende GeoJSON-artefacten,
|
||||
checksums en bron-URL's worden onder persistent operator storage bewaard en
|
||||
via de bestaande DatasetService/vectorfeature-flow geïmporteerd.
|
||||
|
||||
### Operationele regio Kempen
|
||||
|
||||
Voor regionale analyse gebruikt GeoIntel de officiële `Vervoerregio Kempen`
|
||||
van de Vlaamse overheid. Deze beleidsregio bestaat uit 28 gemeenten. Ze is
|
||||
expliciet gekozen omdat ze actueel, reproduceerbaar en bestuurlijk
|
||||
gedocumenteerd is. GeoIntel beweert niet dat deze grens samenvalt met de
|
||||
ruimere culturele, landschappelijke of historische Kempen.
|
||||
|
||||
`scripts/geographic_scopes.py` registreert de 28 actuele namen en NIS-codes.
|
||||
`scripts/provision_geographic_scope.py` controleert die tegen `VRBG/Refgem`,
|
||||
maakt de regionale union en bewaart zowel de union als alle afzonderlijke
|
||||
gemeentegrenzen met bron-URL, checksum, autoriteit en beperking. De gemeten
|
||||
operationele grens omvat circa `1.399,25 km²`.
|
||||
|
||||
Bronnen:
|
||||
|
||||
- https://www.vlaanderen.be/mobiliteitsprofessionals/personenvervoer/basisbereikbaarheid/mobiliteitsuitdagingen-regionaal-aanpakken/vervoerregios/over-de-vervoerregio-kempen
|
||||
- https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items
|
||||
|
||||
De scope-operator haalt geen thematische gegevens op. GRB, bevolking en
|
||||
landgebruik volgen als afzonderlijke begrensde imports; ontbrekende regionale
|
||||
thema's blijven zichtbaar onbeschikbaar.
|
||||
|
||||
### Mol population history
|
||||
|
||||
`scripts/provision_mol_population_history.py` imports official Statbel
|
||||
|
||||
+2
-1
@@ -8,7 +8,8 @@
|
||||
- [x] Add official Mol GRB roads, water and parcels provisioning support.
|
||||
- [x] Select, validate and provision official Statbel 2021-2025 population/statistical-sector snapshots for Mol.
|
||||
- [x] Select, validate and provision official Departement Omgeving 2013-2025 forest snapshots from the 10 m land-use map.
|
||||
- [ ] Define the exact administrative Kempen scope before provisioning municipality or regional copies of the proven source series.
|
||||
- [x] Define and provision the official 28-municipality Vervoerregio Kempen as the explicit operational regional scope.
|
||||
- [ ] Partition and provision regional theme datasets in bounded batches, starting with current buildings/context and then population/land-use time series.
|
||||
|
||||
This file now starts with the current implementation status. Older preparation/backlog sections are preserved below as historical planning context and should not be treated as the live sprint board without checking `docs/CODEX_EXECUTION_LOG.md`.
|
||||
|
||||
|
||||
+3
-3
@@ -2,7 +2,7 @@
|
||||
|
||||
React + TypeScript + MapLibre foundation for project/area/dataset workflow.
|
||||
|
||||
Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available.
|
||||
Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. A compact `Regio` selector switches between this workspace and the explicitly provisioned `Kempen Regional Workbench`, whose scope is the official 28-municipality Vlaamse vervoerregio. Explicit project and dataset selections remain authoritative.
|
||||
|
||||
The map-first explorer has two deliberate modes. `Latest state` selects the
|
||||
latest explicitly dated source snapshot without claiming an old edition is
|
||||
@@ -21,7 +21,7 @@ estimates clearly marked and land-cover sources show intersected hectares. The
|
||||
advanced workbench remains available but is not required for the primary
|
||||
choose-theme, draw-area, read-result flow.
|
||||
|
||||
The primary workflow is deliberately short: choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count.
|
||||
The primary workflow is deliberately short: choose Mol or Kempen, choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count.
|
||||
|
||||
The theme catalog currently recognizes buildings, population, forest/green, water, roads and parcels from dataset names and canonical `reference_layer_name` metadata. A theme is enabled only when a ready persisted vector dataset exists; otherwise it states `Bron nog niet ingeladen`. This prevents missing population or land-cover sources from appearing as zero-valued observations. The previous technical Map workspace remains available through `Geavanceerde werkbank` for derived datasets, QA/QC evidence and export operations.
|
||||
|
||||
@@ -35,7 +35,7 @@ Wide and ultrawide screens keep a readable sidebar and centered work area, expan
|
||||
|
||||
The Map workspace defaults to an OpenStreetMap road basemap with visible attribution so uploaded vectors, AOIs and QA overlays appear on a real street context. Set `VITE_MAP_STYLE_URL` to a managed MapLibre style URL to override this for production or high-volume deployments.
|
||||
|
||||
Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, GRB buildings use cyan/teal and existing detection/change-result colors remain distinct.
|
||||
Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, while active source layers and their legends use consistent theme colors and existing detection/change-result colors remain distinct.
|
||||
|
||||
When the public OpenStreetMap fallback is active, the Map workspace shows a basemap usage notice. This keeps the local/demo default honest and reminds operators to configure a managed style URL before production or heavier tile traffic.
|
||||
|
||||
|
||||
@@ -911,6 +911,7 @@ function App(): JSX.Element {
|
||||
{activeWorkspace === 'map' ? (
|
||||
<MapWorkspace
|
||||
selectedProjectId={selectedProjectId}
|
||||
projects={projects}
|
||||
areas={areas}
|
||||
selectedMapAreaId={selectedMapAreaId}
|
||||
areaFeatureCollection={areaFeatureCollection}
|
||||
@@ -957,6 +958,7 @@ function App(): JSX.Element {
|
||||
selectedMapDatasetId={selectedDataset && isVectorDatasetType(selectedDataset.dataset_type) ? selectedDataset.id : ''}
|
||||
selectedFeature={selectedMapFeature}
|
||||
onSelectMapArea={setSelectedMapAreaId}
|
||||
onSelectProject={selectProject}
|
||||
onOpenDatasetInMap={openDatasetInMap}
|
||||
onSetAreaLayerVisible={setAreaLayerVisible}
|
||||
onSetAreaLayerOpacity={setAreaLayerOpacity}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import GeoMap from '../GeoMap'
|
||||
import type { AreaRead, DatasetCreateResponse, MapViewportState, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||
import type { AreaRead, DatasetCreateResponse, MapViewportState, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
||||
import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights'
|
||||
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
||||
@@ -8,6 +8,8 @@ import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
||||
const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
|
||||
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
|
||||
const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = []
|
||||
const MOL_PROJECT_NAME = 'Mol Municipality Workbench'
|
||||
const KEMPEN_PROJECT_NAME = 'Kempen Regional Workbench'
|
||||
|
||||
type DataThemeId = 'buildings' | 'population' | 'forest' | 'water' | 'roads' | 'parcels'
|
||||
|
||||
@@ -163,6 +165,16 @@ function formatObservationDate(value: string | null | undefined): string {
|
||||
return new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short', day: 'numeric' }).format(new Date(value))
|
||||
}
|
||||
|
||||
function operationalScopeProjectLabel(project: ProjectRead): string {
|
||||
if (project.name === MOL_PROJECT_NAME) {
|
||||
return 'Mol'
|
||||
}
|
||||
if (project.name === KEMPEN_PROJECT_NAME) {
|
||||
return 'Kempen (28 gemeenten)'
|
||||
}
|
||||
return project.name
|
||||
}
|
||||
|
||||
function selectionAreaSquareMetres(bbox: VectorSelectionBBox | null): number | null {
|
||||
if (!bbox) {
|
||||
return null
|
||||
@@ -384,6 +396,7 @@ function downloadJsonFile(filename: string, payload: unknown): void {
|
||||
|
||||
interface MapWorkspaceProps {
|
||||
selectedProjectId: string | null
|
||||
projects: ProjectRead[]
|
||||
areas: AreaRead[]
|
||||
selectedMapAreaId: string
|
||||
areaFeatureCollection: GeoJSON.FeatureCollection | null
|
||||
@@ -430,6 +443,7 @@ interface MapWorkspaceProps {
|
||||
availableMapDatasets: DatasetCreateResponse[]
|
||||
selectedMapDatasetId: string
|
||||
onSelectMapArea: (areaId: string) => void
|
||||
onSelectProject: (projectId: string) => void
|
||||
onOpenDatasetInMap: (dataset: DatasetCreateResponse) => void
|
||||
onSetAreaLayerVisible: (visible: boolean) => void
|
||||
onSetAreaLayerOpacity: (opacity: number) => void
|
||||
@@ -451,6 +465,7 @@ interface MapWorkspaceProps {
|
||||
|
||||
export function MapWorkspace({
|
||||
selectedProjectId,
|
||||
projects,
|
||||
areas,
|
||||
selectedMapAreaId,
|
||||
areaFeatureCollection,
|
||||
@@ -497,6 +512,7 @@ export function MapWorkspace({
|
||||
availableMapDatasets,
|
||||
selectedMapDatasetId,
|
||||
onSelectMapArea,
|
||||
onSelectProject,
|
||||
onOpenDatasetInMap,
|
||||
onSetAreaLayerVisible,
|
||||
onSetAreaLayerOpacity,
|
||||
@@ -574,6 +590,16 @@ export function MapWorkspace({
|
||||
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
|
||||
const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id]
|
||||
const activeThemeDataset = themeDatasetMap[activeTheme.id]
|
||||
const operationalScopeProjects = useMemo(() => {
|
||||
const scoped = projects.filter((project) => project.name === MOL_PROJECT_NAME || project.name === KEMPEN_PROJECT_NAME)
|
||||
const selected = projects.find((project) => project.id === selectedProjectId)
|
||||
if (selected && !scoped.some((project) => project.id === selected.id)) {
|
||||
return [selected, ...scoped]
|
||||
}
|
||||
return scoped
|
||||
}, [projects, selectedProjectId])
|
||||
const activeScopeProject = projects.find((project) => project.id === selectedProjectId) ?? null
|
||||
const activeScopeLabel = activeScopeProject ? operationalScopeProjectLabel(activeScopeProject) : 'Werkgebied'
|
||||
const activeTemporalSeriesGroups = useMemo(
|
||||
() => listThemeTemporalSeries(availableMapDatasets, activeTheme),
|
||||
[activeTheme, availableMapDatasets],
|
||||
@@ -892,10 +918,10 @@ export function MapWorkspace({
|
||||
|
||||
if (!advancedMode) {
|
||||
return (
|
||||
<section className="geo-explorer" data-testid="map-workspace" aria-label="Gebiedsverkenner Mol">
|
||||
<section className="geo-explorer" data-testid="map-workspace" aria-label={`Gebiedsverkenner ${activeScopeLabel}`}>
|
||||
<header className="geo-explorer-header">
|
||||
<div>
|
||||
<p className="eyebrow">Mol · geografische verkenner</p>
|
||||
<p className="eyebrow">{activeScopeLabel} · geografische verkenner</p>
|
||||
<h2>Wat bevindt zich in dit gebied?</h2>
|
||||
<p>Kies een datathema, teken een rechthoek en lees de beschikbare gegevens meteen uit.</p>
|
||||
</div>
|
||||
@@ -933,6 +959,19 @@ export function MapWorkspace({
|
||||
<p>Dit zijn databronnen, geen AI-modellen.</p>
|
||||
</div>
|
||||
</div>
|
||||
<label className="geo-project-scope-select">
|
||||
Regio
|
||||
<select
|
||||
aria-label="Regio"
|
||||
value={selectedProjectId ?? ''}
|
||||
onChange={(event) => onSelectProject(event.target.value)}
|
||||
disabled={operationalScopeProjects.length < 2}
|
||||
>
|
||||
{operationalScopeProjects.map((project) => (
|
||||
<option key={project.id} value={project.id}>{operationalScopeProjectLabel(project)}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<div className="geo-theme-list">
|
||||
{DATA_THEMES.map((theme) => {
|
||||
const dataset = themeDatasetMap[theme.id]
|
||||
@@ -1055,7 +1094,7 @@ export function MapWorkspace({
|
||||
type="button"
|
||||
onClick={() => selectedAreaBbox && void analyzeSelection(selectedAreaBbox)}
|
||||
>
|
||||
Volledige gemeente
|
||||
Volledig werkgebied
|
||||
</button>
|
||||
<button className="secondary-action" disabled={!mapSelectionBbox} type="button" onClick={clearAreaSelection}>
|
||||
Wis selectie
|
||||
|
||||
@@ -5542,6 +5542,20 @@ section {
|
||||
gap: 0.36rem;
|
||||
}
|
||||
|
||||
.geo-project-scope-select {
|
||||
display: grid;
|
||||
gap: 0.24rem;
|
||||
color: #4c5b56;
|
||||
font-size: 0.66rem;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.geo-project-scope-select select {
|
||||
min-height: 2.25rem;
|
||||
padding: 0.35rem 0.45rem;
|
||||
font-size: 0.72rem;
|
||||
}
|
||||
|
||||
.geo-theme-option {
|
||||
display: grid;
|
||||
grid-template-columns: 0.72rem minmax(0, 1fr) auto;
|
||||
|
||||
@@ -1254,6 +1254,38 @@ The 2013-2025 series is methodologically separate from the historical
|
||||
when both exist; it never calculates one continuous trend across those source
|
||||
families.
|
||||
|
||||
## Official Kempen operational scope
|
||||
|
||||
GeoIntel defines its regional `Kempen` workspace as the official Vlaamse
|
||||
`Vervoerregio Kempen`: 28 explicitly registered municipalities. This is a
|
||||
reproducible policy boundary, not a claim about the wider cultural,
|
||||
landscape or historical Kempen.
|
||||
|
||||
Prepare and inspect the current VRBG union and all member boundaries without
|
||||
changing persistence:
|
||||
|
||||
```bash
|
||||
docker exec -it geointel python3 /app/scripts/provision_geographic_scope.py \
|
||||
--scope kempen-transport-region --fetch-only
|
||||
```
|
||||
|
||||
Persist the complete scope foundation:
|
||||
|
||||
```bash
|
||||
docker exec -it geointel python3 /app/scripts/provision_geographic_scope.py \
|
||||
--scope kempen-transport-region
|
||||
```
|
||||
|
||||
The command creates or reuses `Kempen Regional Workbench`, the regional Area,
|
||||
28 municipality Areas and two VRBG source datasets through the canonical API.
|
||||
Artifacts and checksums remain below
|
||||
`/app/storage/operator-data/geographic-scopes/kempen-transport-region`.
|
||||
Repeat runs are idempotent; `--force` refreshes today's source snapshot.
|
||||
|
||||
This command provisions boundaries only. Regional buildings, population,
|
||||
land use, roads, water and parcels must be added by bounded source operators;
|
||||
missing themes remain unavailable and are never filled with synthetic values.
|
||||
|
||||
## Tower deployment
|
||||
|
||||
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Canonical operator scope definitions for GeoIntel.
|
||||
|
||||
These definitions describe administrative/policy scopes only. They do not
|
||||
claim that a policy boundary equals a cultural or physical landscape region.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScopeMember:
|
||||
name: str
|
||||
nis_code: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GeographicScope:
|
||||
key: str
|
||||
display_name: str
|
||||
project_name: str
|
||||
project_region: str
|
||||
area_name: str
|
||||
authority_name: str
|
||||
authority_url: str
|
||||
scope_type: str
|
||||
limitation_message: str
|
||||
members: tuple[ScopeMember, ...]
|
||||
|
||||
@property
|
||||
def nis_codes(self) -> tuple[str, ...]:
|
||||
return tuple(member.nis_code for member in self.members)
|
||||
|
||||
|
||||
MOL_SCOPE = GeographicScope(
|
||||
key="mol",
|
||||
display_name="Mol",
|
||||
project_name="Mol Municipality Workbench",
|
||||
project_region="Mol, Kempen",
|
||||
area_name="Gemeente Mol - officiële grens",
|
||||
authority_name="Digitaal Vlaanderen VRBG",
|
||||
authority_url="https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items",
|
||||
scope_type="municipality",
|
||||
limitation_message="Officiële gemeentegrens; dit is geen perceelsgrens.",
|
||||
members=(ScopeMember("Mol", "13025"),),
|
||||
)
|
||||
|
||||
|
||||
KEMPEN_TRANSPORT_REGION_SCOPE = GeographicScope(
|
||||
key="kempen-transport-region",
|
||||
display_name="Kempen (28 gemeenten)",
|
||||
project_name="Kempen Regional Workbench",
|
||||
project_region="Vervoerregio Kempen, Vlaanderen",
|
||||
area_name="Vervoerregio Kempen - officiële operationele grens",
|
||||
authority_name="Vlaamse overheid - Vervoerregio Kempen",
|
||||
authority_url=(
|
||||
"https://www.vlaanderen.be/mobiliteitsprofessionals/personenvervoer/"
|
||||
"basisbereikbaarheid/mobiliteitsuitdagingen-regionaal-aanpakken/"
|
||||
"vervoerregios/over-de-vervoerregio-kempen"
|
||||
),
|
||||
scope_type="transport_region",
|
||||
limitation_message=(
|
||||
"Operationele beleidsgrens van de Vlaamse vervoerregio Kempen; "
|
||||
"geen claim over de ruimere culturele, landschappelijke of historische Kempen."
|
||||
),
|
||||
members=(
|
||||
ScopeMember("Arendonk", "13001"),
|
||||
ScopeMember("Baarle-Hertog", "13002"),
|
||||
ScopeMember("Balen", "13003"),
|
||||
ScopeMember("Beerse", "13004"),
|
||||
ScopeMember("Dessel", "13006"),
|
||||
ScopeMember("Geel", "13008"),
|
||||
ScopeMember("Grobbendonk", "13010"),
|
||||
ScopeMember("Herentals", "13011"),
|
||||
ScopeMember("Herenthout", "13012"),
|
||||
ScopeMember("Herselt", "13013"),
|
||||
ScopeMember("Hoogstraten", "13014"),
|
||||
ScopeMember("Hulshout", "13016"),
|
||||
ScopeMember("Kasterlee", "13017"),
|
||||
ScopeMember("Laakdal", "13053"),
|
||||
ScopeMember("Lille", "13019"),
|
||||
ScopeMember("Meerhout", "13021"),
|
||||
ScopeMember("Merksplas", "13023"),
|
||||
ScopeMember("Mol", "13025"),
|
||||
ScopeMember("Nijlen", "12026"),
|
||||
ScopeMember("Olen", "13029"),
|
||||
ScopeMember("Oud-Turnhout", "13031"),
|
||||
ScopeMember("Ravels", "13035"),
|
||||
ScopeMember("Retie", "13036"),
|
||||
ScopeMember("Rijkevorsel", "13037"),
|
||||
ScopeMember("Turnhout", "13040"),
|
||||
ScopeMember("Vorselaar", "13044"),
|
||||
ScopeMember("Vosselaar", "13046"),
|
||||
ScopeMember("Westerlo", "13049"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
GEOGRAPHIC_SCOPES = {
|
||||
scope.key: scope
|
||||
for scope in (MOL_SCOPE, KEMPEN_TRANSPORT_REGION_SCOPE)
|
||||
}
|
||||
|
||||
|
||||
def validate_scope(scope: GeographicScope) -> None:
|
||||
if not scope.key or not scope.project_name or not scope.area_name:
|
||||
raise ValueError("Geographic scope identity fields must not be empty")
|
||||
if len(scope.members) == 0:
|
||||
raise ValueError(f"Geographic scope {scope.key} has no members")
|
||||
names = [member.name.casefold() for member in scope.members]
|
||||
codes = [member.nis_code for member in scope.members]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError(f"Geographic scope {scope.key} contains duplicate municipality names")
|
||||
if len(codes) != len(set(codes)):
|
||||
raise ValueError(f"Geographic scope {scope.key} contains duplicate NIS codes")
|
||||
if any(len(code) != 5 or not code.isdigit() for code in codes):
|
||||
raise ValueError(f"Geographic scope {scope.key} contains an invalid NIS code")
|
||||
|
||||
|
||||
for _scope in GEOGRAPHIC_SCOPES.values():
|
||||
validate_scope(_scope)
|
||||
@@ -0,0 +1,571 @@
|
||||
"""Provision an official geographic scope through the canonical GeoIntel API.
|
||||
|
||||
The operator fetches current VRBG municipality boundaries, creates one union
|
||||
scope boundary plus a member-boundary artifact, and persists a project, the
|
||||
regional area, all municipality areas and both datasets. It never runs during
|
||||
application startup and never writes directly to PostGIS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from pyproj import Transformer
|
||||
from requests.adapters import HTTPAdapter
|
||||
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
||||
from shapely.ops import transform, unary_union
|
||||
from shapely.validation import make_valid
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope
|
||||
|
||||
|
||||
VRBG_ITEMS_URL = "https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items"
|
||||
VRBG_ATTRIBUTION = "Bron: Voorlopig referentiebestand gemeentegrenzen, Digitaal Vlaanderen"
|
||||
DEFAULT_SCOPE_KEY = "kempen-transport-region"
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision an official GeoIntel geographic scope.")
|
||||
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument("--request-timeout", type=int, default=180)
|
||||
parser.add_argument("--import-timeout", type=int, default=1800)
|
||||
parser.add_argument("--force", action="store_true", help="Refresh official source artifacts for today's snapshot.")
|
||||
parser.add_argument("--fetch-only", action="store_true", help="Validate and write artifacts without API persistence.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json_atomic(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(f"{path.suffix}.partial")
|
||||
temporary.write_text(
|
||||
json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
indent=2 if pretty else None,
|
||||
separators=None if pretty else (",", ":"),
|
||||
sort_keys=pretty,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def normalize_polygonal(geometry):
|
||||
if geometry is None or geometry.is_empty:
|
||||
return None
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if isinstance(geometry, (Polygon, MultiPolygon)):
|
||||
return geometry
|
||||
if isinstance(geometry, GeometryCollection):
|
||||
polygons = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty]
|
||||
if polygons:
|
||||
merged = unary_union(polygons)
|
||||
return merged if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty else None
|
||||
return None
|
||||
|
||||
|
||||
def metric_area_km2(geometry) -> float:
|
||||
transformer = Transformer.from_crs(4326, 31370, always_xy=True)
|
||||
return float(transform(transformer.transform, geometry).area / 1_000_000)
|
||||
|
||||
|
||||
def build_source_session() -> requests.Session:
|
||||
retry = Retry(
|
||||
total=5,
|
||||
connect=5,
|
||||
read=5,
|
||||
status=5,
|
||||
backoff_factor=1.0,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset({"GET"}),
|
||||
raise_on_status=True,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-Geographic-Scope-Operator/1.0"})
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
|
||||
def fetch_scope_members(session: requests.Session, scope: GeographicScope, timeout: int) -> tuple[list[dict[str, Any]], str]:
|
||||
response = session.get(
|
||||
VRBG_ITEMS_URL,
|
||||
params={"f": "application/geo+json", "limit": "1000"},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
expected = {member.nis_code: member.name for member in scope.members}
|
||||
selected: dict[str, dict[str, Any]] = {}
|
||||
for feature in response.json().get("features") or []:
|
||||
properties = feature.get("properties") or {}
|
||||
nis_code = str(properties.get("NISCODE") or "")
|
||||
if nis_code not in expected:
|
||||
continue
|
||||
if nis_code in selected:
|
||||
raise RuntimeError(f"Official VRBG returned duplicate NIS code {nis_code}")
|
||||
actual_name = str(properties.get("NAAM") or "")
|
||||
if actual_name.casefold() != expected[nis_code].casefold():
|
||||
raise RuntimeError(
|
||||
f"Official VRBG name drift for {nis_code}: expected {expected[nis_code]!r}, received {actual_name!r}"
|
||||
)
|
||||
selected[nis_code] = feature
|
||||
missing = [f"{name} ({code})" for code, name in expected.items() if code not in selected]
|
||||
if missing:
|
||||
raise RuntimeError(f"Official VRBG is missing scope members: {', '.join(missing)}")
|
||||
return [selected[member.nis_code] for member in scope.members], response.url
|
||||
|
||||
|
||||
def build_scope_payloads(
|
||||
scope: GeographicScope,
|
||||
source_features: list[dict[str, Any]],
|
||||
*,
|
||||
source_url: str,
|
||||
generated_at: str,
|
||||
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
||||
if len(source_features) != len(scope.members):
|
||||
raise RuntimeError(f"Expected {len(scope.members)} source boundaries, received {len(source_features)}")
|
||||
member_features: list[dict[str, Any]] = []
|
||||
member_geometries = []
|
||||
for member, source_feature in zip(scope.members, source_features, strict=True):
|
||||
properties = dict(source_feature.get("properties") or {})
|
||||
if str(properties.get("NISCODE") or "") != member.nis_code:
|
||||
raise RuntimeError(f"Scope member order/code mismatch for {member.name}")
|
||||
geometry = normalize_polygonal(shape(source_feature.get("geometry")))
|
||||
if geometry is None:
|
||||
raise RuntimeError(f"Official boundary for {member.name} is empty, invalid or non-polygonal")
|
||||
member_geometries.append(geometry)
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "vrbg",
|
||||
"source_feature_id": str(source_feature.get("id") or member.nis_code),
|
||||
"layer_type": "municipality_boundary",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": scope.key,
|
||||
"scope_type": scope.scope_type,
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
"source_url": source_url,
|
||||
}
|
||||
)
|
||||
member_features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": str(source_feature.get("id") or f"Refgem.{member.nis_code}"),
|
||||
"geometry": mapping(geometry),
|
||||
"properties": properties,
|
||||
}
|
||||
)
|
||||
|
||||
boundary = normalize_polygonal(unary_union(member_geometries))
|
||||
if boundary is None:
|
||||
raise RuntimeError("Union of official scope member boundaries is invalid")
|
||||
member_codes = list(scope.nis_codes)
|
||||
boundary_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"Official operation boundary - {scope.display_name}",
|
||||
"crs": GEOJSON_CRS,
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"scope:{scope.key}",
|
||||
"geometry": mapping(boundary),
|
||||
"properties": {
|
||||
"name": scope.area_name,
|
||||
"source_name": "vrbg",
|
||||
"source_feature_id": f"scope:{scope.key}",
|
||||
"layer_type": "regional_boundary",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": scope.key,
|
||||
"scope_type": scope.scope_type,
|
||||
"member_count": len(scope.members),
|
||||
"member_nis_codes": member_codes,
|
||||
"scope_authority": scope.authority_name,
|
||||
"scope_authority_url": scope.authority_url,
|
||||
"scope_limitation": scope.limitation_message,
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
"source_url": source_url,
|
||||
},
|
||||
}
|
||||
],
|
||||
"source_url": source_url,
|
||||
"scope_authority_url": scope.authority_url,
|
||||
"scope_limitation": scope.limitation_message,
|
||||
"generated_at": generated_at,
|
||||
}
|
||||
members_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"Official municipality boundaries - {scope.display_name}",
|
||||
"crs": GEOJSON_CRS,
|
||||
"features": member_features,
|
||||
"source_url": source_url,
|
||||
"scope_authority_url": scope.authority_url,
|
||||
"scope_limitation": scope.limitation_message,
|
||||
"generated_at": generated_at,
|
||||
}
|
||||
summary = {
|
||||
"scope_key": scope.key,
|
||||
"scope_type": scope.scope_type,
|
||||
"display_name": scope.display_name,
|
||||
"member_count": len(scope.members),
|
||||
"member_names": [member.name for member in scope.members],
|
||||
"member_nis_codes": member_codes,
|
||||
"area_km2": metric_area_km2(boundary),
|
||||
"wgs84_bbox": list(boundary.bounds),
|
||||
}
|
||||
return boundary_payload, members_payload, summary
|
||||
|
||||
|
||||
def artifact_paths(output_dir: Path, scope: GeographicScope, snapshot_date: str) -> tuple[Path, Path, Path]:
|
||||
stem = scope.key.replace("-", "_")
|
||||
return (
|
||||
output_dir / f"{stem}_boundary_{snapshot_date}.geojson",
|
||||
output_dir / f"{stem}_municipalities_{snapshot_date}.geojson",
|
||||
output_dir / f"{stem}_scope_manifest.json",
|
||||
)
|
||||
|
||||
|
||||
def cached_artifacts(output_dir: Path, scope: GeographicScope) -> tuple[Path, Path, dict[str, Any]] | None:
|
||||
manifest_path = output_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json"
|
||||
if not manifest_path.exists():
|
||||
return None
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
boundary_path = output_dir / str(manifest.get("boundary_filename") or "")
|
||||
members_path = output_dir / str(manifest.get("municipalities_filename") or "")
|
||||
if (
|
||||
manifest.get("status") == "complete"
|
||||
and manifest.get("scope_key") == scope.key
|
||||
and manifest.get("member_count") == len(scope.members)
|
||||
and boundary_path.is_file()
|
||||
and members_path.is_file()
|
||||
and sha256_file(boundary_path) == manifest.get("boundary_sha256")
|
||||
and sha256_file(members_path) == manifest.get("municipalities_sha256")
|
||||
):
|
||||
return boundary_path, members_path, manifest
|
||||
return None
|
||||
|
||||
|
||||
def prepare_artifacts(args: argparse.Namespace, scope: GeographicScope) -> tuple[Path, Path, dict[str, Any]]:
|
||||
output_dir = args.output_root / scope.key
|
||||
if not args.force:
|
||||
cached = cached_artifacts(output_dir, scope)
|
||||
if cached:
|
||||
return cached
|
||||
generated_at = utc_now()
|
||||
snapshot_date = generated_at[:10]
|
||||
boundary_path, members_path, manifest_path = artifact_paths(output_dir, scope, snapshot_date)
|
||||
with build_source_session() as session:
|
||||
source_features, source_url = fetch_scope_members(session, scope, args.request_timeout)
|
||||
boundary_payload, members_payload, summary = build_scope_payloads(
|
||||
scope,
|
||||
source_features,
|
||||
source_url=source_url,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
write_json_atomic(boundary_path, boundary_payload)
|
||||
write_json_atomic(members_path, members_payload)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "complete",
|
||||
"generated_at": generated_at,
|
||||
"observed_at": f"{snapshot_date}T00:00:00Z",
|
||||
"boundary_filename": boundary_path.name,
|
||||
"boundary_sha256": sha256_file(boundary_path),
|
||||
"municipalities_filename": members_path.name,
|
||||
"municipalities_sha256": sha256_file(members_path),
|
||||
"vrbg_source_url": source_url,
|
||||
"vrbg_attribution": VRBG_ATTRIBUTION,
|
||||
"scope_authority_name": scope.authority_name,
|
||||
"scope_authority_url": scope.authority_url,
|
||||
"scope_limitation": scope.limitation_message,
|
||||
**summary,
|
||||
}
|
||||
write_json_atomic(manifest_path, manifest, pretty=True)
|
||||
return boundary_path, members_path, manifest
|
||||
|
||||
|
||||
def response_data(response: requests.Response) -> Any:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:500]}") from exc
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:1000]}")
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def list_paginated_items(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
total: int | None = None
|
||||
while True:
|
||||
page = response_data(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
|
||||
page_items = list(page.get("items") or [])
|
||||
items.extend(page_items)
|
||||
if total is None and page.get("total") is not None:
|
||||
total = int(page["total"])
|
||||
if not page_items or (total is not None and len(items) >= total) or len(page_items) < 200:
|
||||
break
|
||||
offset += len(page_items)
|
||||
if total is not None and len(items) != total:
|
||||
raise RuntimeError(f"GeoIntel list response returned {len(items)} of {total} records for {url}")
|
||||
return items
|
||||
|
||||
|
||||
def find_or_create_project(session: requests.Session, base_url: str, scope: GeographicScope, timeout: int) -> dict[str, Any]:
|
||||
projects = list_paginated_items(session, f"{base_url}/api/v1/projects", timeout=timeout)
|
||||
existing = next((item for item in projects if item.get("name") == scope.project_name), None)
|
||||
if existing:
|
||||
return existing
|
||||
return response_data(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects",
|
||||
json={
|
||||
"name": scope.project_name,
|
||||
"description": (
|
||||
f"Operational GeoIntel scope for {scope.display_name}, composed from {len(scope.members)} current "
|
||||
f"VRBG municipality boundaries. {scope.limitation_message}"
|
||||
),
|
||||
"region": scope.project_region,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def find_or_create_areas(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
scope: GeographicScope,
|
||||
boundary_payload: dict[str, Any],
|
||||
members_payload: dict[str, Any],
|
||||
timeout: int,
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
areas = list_paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/areas", timeout=timeout)
|
||||
by_name = {str(item.get("name")): item for item in areas}
|
||||
|
||||
def ensure(name: str, geometry: dict[str, Any]) -> dict[str, Any]:
|
||||
existing = by_name.get(name)
|
||||
if existing:
|
||||
return existing
|
||||
created = response_data(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/areas",
|
||||
json={"name": name, "crs": "EPSG:4326", "geometry": geometry},
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
by_name[name] = created
|
||||
return created
|
||||
|
||||
region_area = ensure(scope.area_name, boundary_payload["features"][0]["geometry"])
|
||||
member_areas = [
|
||||
ensure(f"Gemeente {member.name} - officiële grens", feature["geometry"])
|
||||
for member, feature in zip(scope.members, members_payload["features"], strict=True)
|
||||
]
|
||||
return region_area, member_areas
|
||||
|
||||
|
||||
def upload_dataset(
|
||||
session: requests.Session,
|
||||
*,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area_id: str,
|
||||
path: Path,
|
||||
source_metadata: dict[str, Any],
|
||||
provenance_metadata: dict[str, Any],
|
||||
temporal_series_key: str,
|
||||
observed_at: str,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
with path.open("rb") as handle:
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||
data={
|
||||
"dataset_type": "vector",
|
||||
"source": "operator_official_import",
|
||||
"dataset_role": "source",
|
||||
"source_name": "vrbg",
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": temporal_series_key,
|
||||
"observed_at": observed_at,
|
||||
"valid_from": observed_at,
|
||||
"temporal_granularity": "snapshot",
|
||||
"source_version": observed_at[:10],
|
||||
},
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
)
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def provision_scope(
|
||||
args: argparse.Namespace,
|
||||
scope: GeographicScope,
|
||||
boundary_path: Path,
|
||||
members_path: Path,
|
||||
manifest: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
base_url = args.base_url.rstrip("/")
|
||||
boundary_payload = json.loads(boundary_path.read_text(encoding="utf-8"))
|
||||
members_payload = json.loads(members_path.read_text(encoding="utf-8"))
|
||||
with requests.Session() as session:
|
||||
project = find_or_create_project(session, base_url, scope, args.import_timeout)
|
||||
project_id = str(project["id"])
|
||||
region_area, member_areas = find_or_create_areas(
|
||||
session,
|
||||
base_url,
|
||||
project_id,
|
||||
scope,
|
||||
boundary_payload,
|
||||
members_payload,
|
||||
args.import_timeout,
|
||||
)
|
||||
datasets = list_paginated_items(
|
||||
session,
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets",
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
source_metadata = {
|
||||
"provider": "Digitaal Vlaanderen",
|
||||
"collection": "VRBG/Refgem",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": scope.key,
|
||||
"scope_type": scope.scope_type,
|
||||
"scope_authority": scope.authority_name,
|
||||
"scope_authority_url": scope.authority_url,
|
||||
"scope_limitation": scope.limitation_message,
|
||||
"member_count": len(scope.members),
|
||||
"member_nis_codes": list(scope.nis_codes),
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
}
|
||||
common_provenance = {
|
||||
"operator_tool": "provision_geographic_scope.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"),
|
||||
"source_url": manifest["vrbg_source_url"],
|
||||
"scope_authority_url": scope.authority_url,
|
||||
}
|
||||
|
||||
dataset_specs = (
|
||||
(
|
||||
boundary_path,
|
||||
"regional_boundary",
|
||||
f"vrbg:scope-boundary:{scope.key}",
|
||||
manifest["boundary_sha256"],
|
||||
),
|
||||
(
|
||||
members_path,
|
||||
"municipality_boundaries",
|
||||
f"vrbg:scope-members:{scope.key}",
|
||||
manifest["municipalities_sha256"],
|
||||
),
|
||||
)
|
||||
persisted = []
|
||||
for path, layer_type, series_key, checksum in dataset_specs:
|
||||
existing = next((item for item in datasets if item.get("original_filename") == path.name), None)
|
||||
if existing:
|
||||
persisted_checksum = (existing.get("provenance_metadata") or {}).get("artifact_sha256")
|
||||
if persisted_checksum and persisted_checksum != checksum:
|
||||
raise RuntimeError(
|
||||
f"Immutable scope dataset {path.name} has checksum {persisted_checksum}, "
|
||||
f"but the refreshed source produced {checksum}; use a new observation date instead of overwriting it"
|
||||
)
|
||||
persisted.append(existing)
|
||||
continue
|
||||
created = upload_dataset(
|
||||
session,
|
||||
base_url=base_url,
|
||||
project_id=project_id,
|
||||
area_id=str(region_area["id"]),
|
||||
path=path,
|
||||
source_metadata={**source_metadata, "layer_type": layer_type},
|
||||
provenance_metadata={**common_provenance, "artifact_sha256": checksum},
|
||||
temporal_series_key=series_key,
|
||||
observed_at=manifest["observed_at"],
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
persisted.append(created)
|
||||
|
||||
return {
|
||||
"project_id": project_id,
|
||||
"project_name": project.get("name"),
|
||||
"region_area_id": str(region_area["id"]),
|
||||
"region_area_name": region_area.get("name"),
|
||||
"municipality_area_count": len(member_areas),
|
||||
"boundary_dataset_id": str(persisted[0]["id"]),
|
||||
"municipality_dataset_id": str(persisted[1]["id"]),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||
try:
|
||||
boundary_path, members_path, manifest = prepare_artifacts(args, scope)
|
||||
workspace = None if args.fetch_only else provision_scope(args, scope, boundary_path, members_path, manifest)
|
||||
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"mode": "fetch_only" if args.fetch_only else "provisioned",
|
||||
"scope": scope.key,
|
||||
"display_name": scope.display_name,
|
||||
"member_count": manifest["member_count"],
|
||||
"area_km2": manifest["area_km2"],
|
||||
"wgs84_bbox": manifest["wgs84_bbox"],
|
||||
"boundary_path": str(boundary_path),
|
||||
"municipalities_path": str(members_path),
|
||||
"workspace": workspace,
|
||||
"limitation": scope.limitation_message,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -47,6 +47,8 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_context_layers.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||
${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py
|
||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||
|
||||
Reference in New Issue
Block a user