From bb7310e01577e14c0c0a4cae100124b175d743cb Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 18:32:32 +0200 Subject: [PATCH] feat: add official Kempen operating scope --- CHANGELOG.md | 8 + backend/README.md | 21 + ...sprint186_map_first_geographic_explorer.py | 2 +- backend/tests/test_sprint189_kempen_scope.py | 134 ++++ deploy/unraid/Dockerfile.all-in-one | 2 + docs/CODEX_EXECUTION_LOG.md | 22 + docs/DATASET_STRATEGY.md | 18 + docs/DATA_SOURCES.md | 23 + docs/TODO.md | 3 +- frontend/README.md | 6 +- frontend/src/App.tsx | 2 + frontend/src/components/map/MapWorkspace.tsx | 47 +- frontend/src/styles/app.css | 14 + scripts/README.md | 32 + scripts/geographic_scopes.py | 122 ++++ scripts/provision_geographic_scope.py | 571 ++++++++++++++++++ scripts/run_readiness_check.sh | 2 + 17 files changed, 1020 insertions(+), 9 deletions(-) create mode 100644 backend/tests/test_sprint189_kempen_scope.py create mode 100644 scripts/geographic_scopes.py create mode 100644 scripts/provision_geographic_scope.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 52954136..05145e09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 189 Official Kempen operational scope (2026-07-14) + +- Defined `Kempen` operationally as the official 28-municipality Vlaamse vervoerregio, with an explicit warning that this policy boundary is not the wider cultural or landscape Kempen. +- Added a central operator scope registry with current municipality names and NIS codes, including Mol `13025` and Nijlen `12026`. +- Added an explicit, idempotent VRBG scope provisioner that creates one regional boundary, 28 member boundaries, one regional Area, 28 municipality Areas and canonical source datasets through the existing API. +- Added a compact Mol/Kempen region selector to the map-first explorer and made its heading and full-area action scope-neutral. +- Kept thematic regional ingestion separate from the boundary pass so large GRB/WCS sources can be partitioned and validated without hidden startup fetches or truncated datasets. + ## Sprint 188 Official modern Mol land-use series (2026-07-14) - Added an explicit, reusable MercatorNet WCS operator for official Departement Omgeving land-use snapshots in 2013, 2016, 2019, 2022 and 2025. diff --git a/backend/README.md b/backend/README.md index 2d30546c..46a6fca6 100644 --- a/backend/README.md +++ b/backend/README.md @@ -935,6 +935,27 @@ the same bbox-selected FeatureCollection as a normal export record with `export_type="vector_selection_geojson"`. This creates a handoff artifact only; it does not create a derived dataset. +## Geographic scope provisioning + +The explicit operator command below provisions the official 28-municipality +Vlaamse vervoerregio Kempen boundary foundation: + +```bash +docker exec geointel python /app/scripts/provision_geographic_scope.py \ + --scope kempen-transport-region +``` + +It reads current `VRBG/Refgem` boundaries, validates every registered name and +NIS code, unions the regional geometry and creates one project, one regional +Area, 28 municipality Areas and two source datasets through the public API. +It never writes directly to PostGIS and does not run on startup. The persisted +scope limitation explicitly distinguishes the transport-policy region from a +cultural or landscape definition of Kempen. + +Use `--fetch-only` for a source/geometry/checksum audit. The scope pass does +not fetch thematic GRB, population or land-use data; those remain separate, +bounded operator jobs. + ## Temporal Mol data and evolution Dataset uploads accept `temporal_series_key`, `observed_at`, `valid_from`, diff --git a/backend/tests/test_sprint186_map_first_geographic_explorer.py b/backend/tests/test_sprint186_map_first_geographic_explorer.py index 438a6569..fc6a8bd9 100644 --- a/backend/tests/test_sprint186_map_first_geographic_explorer.py +++ b/backend/tests/test_sprint186_map_first_geographic_explorer.py @@ -18,7 +18,7 @@ def test_map_first_explorer_is_the_default_product_flow() -> None: assert "Wat bevindt zich in dit gebied?" in workspace assert "Kies een datathema" in workspace assert "Teken rechthoek" in workspace - assert "Volledige gemeente" in workspace + assert "Volledig werkgebied" in workspace assert "Alle beschikbare thema" in workspace assert "Bron nog niet ingeladen" in workspace assert "useMapThemeSelectionInsights" in workspace diff --git a/backend/tests/test_sprint189_kempen_scope.py b/backend/tests/test_sprint189_kempen_scope.py new file mode 100644 index 00000000..0dc94458 --- /dev/null +++ b/backend/tests/test_sprint189_kempen_scope.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys + +from shapely.geometry import Polygon, shape + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" + + +def load_module(name: str, filename: str): + scripts_path = str(SCRIPTS) + if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_kempen_scope_matches_official_28_municipality_policy_region() -> None: + scopes = load_module("geographic_scopes_test", "geographic_scopes.py") + scope = scopes.KEMPEN_TRANSPORT_REGION_SCOPE + + assert scope.key == "kempen-transport-region" + assert scope.project_name == "Kempen Regional Workbench" + assert scope.scope_type == "transport_region" + assert len(scope.members) == 28 + assert len(set(scope.nis_codes)) == 28 + assert ("Mol", "13025") in {(member.name, member.nis_code) for member in scope.members} + assert ("Nijlen", "12026") in {(member.name, member.nis_code) for member in scope.members} + assert "vervoerregio-kempen" in scope.authority_url + assert "geen claim" in scope.limitation_message + + +def test_scope_union_preserves_member_identity_and_policy_limitation() -> None: + scopes = load_module("geographic_scopes_union_test", "geographic_scopes.py") + provisioner = load_module("provision_geographic_scope_test", "provision_geographic_scope.py") + scope = scopes.GeographicScope( + key="test-region", + display_name="Testregio", + project_name="Test Regional Workbench", + project_region="Test", + area_name="Testregio - operationele grens", + authority_name="Test authority", + authority_url="https://example.test/scope", + scope_type="policy_region", + limitation_message="Operationele testgrens; geen landschappelijke claim.", + members=(scopes.ScopeMember("Alpha", "10001"), scopes.ScopeMember("Beta", "10002")), + ) + source_features = [ + { + "type": "Feature", + "id": "alpha", + "geometry": Polygon([(4.0, 51.0), (4.1, 51.0), (4.1, 51.1), (4.0, 51.1)]).__geo_interface__, + "properties": {"NAAM": "Alpha", "NISCODE": "10001"}, + }, + { + "type": "Feature", + "id": "beta", + "geometry": Polygon([(4.1, 51.0), (4.2, 51.0), (4.2, 51.1), (4.1, 51.1)]).__geo_interface__, + "properties": {"NAAM": "Beta", "NISCODE": "10002"}, + }, + ] + + boundary, members, summary = provisioner.build_scope_payloads( + scope, + source_features, + source_url="https://example.test/vrbg", + generated_at="2026-07-14T00:00:00+00:00", + ) + + assert len(boundary["features"]) == 1 + assert len(members["features"]) == 2 + assert shape(boundary["features"][0]["geometry"]).is_valid + assert boundary["features"][0]["properties"]["member_nis_codes"] == ["10001", "10002"] + assert boundary["features"][0]["properties"]["scope_limitation"] == scope.limitation_message + assert [feature["properties"]["municipality"] for feature in members["features"]] == ["Alpha", "Beta"] + assert summary["member_count"] == 2 + assert summary["area_km2"] > 0 + + +def test_scope_api_pagination_respects_canonical_limit() -> None: + provisioner = load_module("provision_geographic_scope_paging_test", "provision_geographic_scope.py") + + class Response: + ok = True + status_code = 200 + text = "" + + def __init__(self, payload): + self.payload = payload + + def json(self): + return {"data": self.payload} + + class Session: + def __init__(self) -> None: + self.offsets = [] + + def get(self, url, *, params, timeout): + del url, timeout + self.offsets.append(params["offset"]) + offset = params["offset"] + page = [{"id": index} for index in range(offset, min(offset + 200, 401))] + return Response({"items": page, "total": 401}) + + session = Session() + items = provisioner.list_paginated_items(session, "http://backend/api/v1/projects", timeout=30) + + assert len(items) == 401 + assert session.offsets == [0, 200, 400] + + +def test_kempen_scope_operator_is_packaged_and_exposed_in_map_flow() -> None: + dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8") + workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8") + app = (ROOT / "frontend/src/App.tsx").read_text(encoding="utf-8") + + assert "COPY scripts/geographic_scopes.py" in dockerfile + assert "COPY scripts/provision_geographic_scope.py" in dockerfile + assert "py_compile scripts/provision_geographic_scope.py" in readiness + assert "Immutable scope dataset" in (ROOT / "scripts/provision_geographic_scope.py").read_text(encoding="utf-8") + assert "Kempen (28 gemeenten)" in workspace + assert 'aria-label="Regio"' in workspace + assert "projects={projects}" in app + assert "onSelectProject={selectProject}" in app diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 8b2b6079..c35db4ee 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -77,6 +77,8 @@ COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_ COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py +COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py +COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index a1e7c0bd..e6af1183 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7850,3 +7850,25 @@ Validation: Next: - Define the exact municipality list or administrative boundary that GeoIntel will call `Kempen`, then reuse the proven source operators and temporal query path for that scope without changing Mol semantics. + +## Sprint 189 Official Kempen operational scope (2026-07-14) + +Implemented: +- Chose the official Vlaamse `Vervoerregio Kempen` as GeoIntel's reproducible operational regional scope. It has 28 municipalities and explicitly includes both Mol and Nijlen. +- Added `geographic_scopes.py` as the operator registry for scope identity, current municipality names/NIS codes, policy authority and limitation text. +- Added `provision_geographic_scope.py` to fetch current authoritative VRBG boundaries, validate all members, create a valid regional union and retain boundary/member GeoJSON plus checksums and manifest. +- The operator creates or reuses `Kempen Regional Workbench`, one regional Area, 28 municipality Areas and two source datasets exclusively through canonical API envelopes. +- Added a compact Mol/Kempen selector to the primary map flow, scope-aware heading text and the neutral `Volledig werkgebied` action. +- Kept thematic regional fetching outside the scope pass. No startup fetch, direct PostGIS write, fake metric or unbounded regional theme download was introduced. + +Source proof: +- A live read-only VRBG run matched all 28 registered municipalities and current NIS codes. +- The valid union measured 1,399.2505 km2 in EPSG:31370 with WGS84 bbox `[4.59723873, 51.01047967, 5.26224853, 51.50511313]`. +- The scope is labelled as the transport-policy region and never as the complete cultural, landscape or historical Kempen. + +Validation before deployment: +- New focused scope and map-flow suite passed 9 tests. +- Python scope/operator compilation and frontend TypeScript typecheck passed. + +Next: +- Deploy and run the scope operator on Tower, verify 29 Areas and both boundary datasets, then design bounded regional theme partitions before importing high-volume GRB or WCS data. diff --git a/docs/DATASET_STRATEGY.md b/docs/DATASET_STRATEGY.md index c3825e2d..30351904 100644 --- a/docs/DATASET_STRATEGY.md +++ b/docs/DATASET_STRATEGY.md @@ -88,6 +88,24 @@ Kempen reuses the same operator with an explicit approved boundary, project, area and scope key; an ambiguous regional label is never converted into an invented boundary. +## Operational Kempen scope + +The canonical regional workspace uses the official 28-municipality Vlaamse +`Vervoerregio Kempen` policy boundary. `scripts/geographic_scopes.py` is the +operator source of truth for its members and current NIS codes. The resulting +regional geometry is a union of current authoritative VRBG municipality +boundaries. + +- Project: `Kempen Regional Workbench`. +- Regional Area: `Vervoerregio Kempen - officiële operationele grens`. +- Member Areas: one current official boundary for every municipality. +- Scope key: `kempen-transport-region`. +- Provenance: authority URL, VRBG request URL, member codes, fetch timestamp, + artifact checksums and the policy-boundary limitation. +- Scale policy: thematic providers are partitioned and validated separately; + no interactive request or application startup may download the entire + region or silently substitute missing data. + ## User-uploaded raster strategy V1 must support controlled local datasets because public raster access and model compatibility can be difficult. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 31cd1e6e..0e17bb33 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -29,6 +29,29 @@ start geen verborgen providerfetch. De resulterende GeoJSON-artefacten, checksums en bron-URL's worden onder persistent operator storage bewaard en via de bestaande DatasetService/vectorfeature-flow geïmporteerd. +### Operationele regio Kempen + +Voor regionale analyse gebruikt GeoIntel de officiële `Vervoerregio Kempen` +van de Vlaamse overheid. Deze beleidsregio bestaat uit 28 gemeenten. Ze is +expliciet gekozen omdat ze actueel, reproduceerbaar en bestuurlijk +gedocumenteerd is. GeoIntel beweert niet dat deze grens samenvalt met de +ruimere culturele, landschappelijke of historische Kempen. + +`scripts/geographic_scopes.py` registreert de 28 actuele namen en NIS-codes. +`scripts/provision_geographic_scope.py` controleert die tegen `VRBG/Refgem`, +maakt de regionale union en bewaart zowel de union als alle afzonderlijke +gemeentegrenzen met bron-URL, checksum, autoriteit en beperking. De gemeten +operationele grens omvat circa `1.399,25 km²`. + +Bronnen: + +- https://www.vlaanderen.be/mobiliteitsprofessionals/personenvervoer/basisbereikbaarheid/mobiliteitsuitdagingen-regionaal-aanpakken/vervoerregios/over-de-vervoerregio-kempen +- https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items + +De scope-operator haalt geen thematische gegevens op. GRB, bevolking en +landgebruik volgen als afzonderlijke begrensde imports; ontbrekende regionale +thema's blijven zichtbaar onbeschikbaar. + ### Mol population history `scripts/provision_mol_population_history.py` imports official Statbel diff --git a/docs/TODO.md b/docs/TODO.md index 300eefdd..f6b0cfdb 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -8,7 +8,8 @@ - [x] Add official Mol GRB roads, water and parcels provisioning support. - [x] Select, validate and provision official Statbel 2021-2025 population/statistical-sector snapshots for Mol. - [x] Select, validate and provision official Departement Omgeving 2013-2025 forest snapshots from the 10 m land-use map. -- [ ] Define the exact administrative Kempen scope before provisioning municipality or regional copies of the proven source series. +- [x] Define and provision the official 28-municipality Vervoerregio Kempen as the explicit operational regional scope. +- [ ] Partition and provision regional theme datasets in bounded batches, starting with current buildings/context and then population/land-use time series. This file now starts with the current implementation status. Older preparation/backlog sections are preserved below as historical planning context and should not be treated as the live sprint board without checking `docs/CODEX_EXECUTION_LOG.md`. diff --git a/frontend/README.md b/frontend/README.md index 59b05d94..8a029bec 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -2,7 +2,7 @@ React + TypeScript + MapLibre foundation for project/area/dataset workflow. -Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available. +Mol is the primary operating context. On a fresh session the application opens the map-first geographic explorer, prefers the persisted `Mol Municipality Workbench`, selects the official NIS `13025` municipality boundary and activates the largest available authoritative building layer. A compact `Regio` selector switches between this workspace and the explicitly provisioned `Kempen Regional Workbench`, whose scope is the official 28-municipality Vlaamse vervoerregio. Explicit project and dataset selections remain authoritative. The map-first explorer has two deliberate modes. `Latest state` selects the latest explicitly dated source snapshot without claiming an old edition is @@ -21,7 +21,7 @@ estimates clearly marked and land-cover sources show intersected hectares. The advanced workbench remains available but is not required for the primary choose-theme, draw-area, read-result flow. -The primary workflow is deliberately short: choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count. +The primary workflow is deliberately short: choose Mol or Kempen, choose a data theme, drag a rectangle on the MapLibre map and read the resulting PostGIS evidence. Releasing the drag runs the active theme query and every other available theme query for the same EPSG:4326 bbox. The result panel shows selection area, exact intersection totals, active-theme density, source identity and bounded feature properties. Map rendering remains capped at 1,000 features while `total_feature_count` reports the exact database count. The theme catalog currently recognizes buildings, population, forest/green, water, roads and parcels from dataset names and canonical `reference_layer_name` metadata. A theme is enabled only when a ready persisted vector dataset exists; otherwise it states `Bron nog niet ingeladen`. This prevents missing population or land-cover sources from appearing as zero-valued observations. The previous technical Map workspace remains available through `Geavanceerde werkbank` for derived datasets, QA/QC evidence and export operations. @@ -35,7 +35,7 @@ Wide and ultrawide screens keep a readable sidebar and centered work area, expan The Map workspace defaults to an OpenStreetMap road basemap with visible attribution so uploaded vectors, AOIs and QA overlays appear on a real street context. Set `VITE_MAP_STYLE_URL` to a managed MapLibre style URL to override this for production or high-volume deployments. -Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, GRB buildings use cyan/teal and existing detection/change-result colors remain distinct. +Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, while active source layers and their legends use consistent theme colors and existing detection/change-result colors remain distinct. When the public OpenStreetMap fallback is active, the Map workspace shows a basemap usage notice. This keeps the local/demo default honest and reminds operators to configure a managed style URL before production or heavier tile traffic. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 38e5ea58..63a2aa00 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -911,6 +911,7 @@ function App(): JSX.Element { {activeWorkspace === 'map' ? ( void + onSelectProject: (projectId: string) => void onOpenDatasetInMap: (dataset: DatasetCreateResponse) => void onSetAreaLayerVisible: (visible: boolean) => void onSetAreaLayerOpacity: (opacity: number) => void @@ -451,6 +465,7 @@ interface MapWorkspaceProps { export function MapWorkspace({ selectedProjectId, + projects, areas, selectedMapAreaId, areaFeatureCollection, @@ -497,6 +512,7 @@ export function MapWorkspace({ availableMapDatasets, selectedMapDatasetId, onSelectMapArea, + onSelectProject, onOpenDatasetInMap, onSetAreaLayerVisible, onSetAreaLayerOpacity, @@ -574,6 +590,16 @@ export function MapWorkspace({ const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0] const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id] const activeThemeDataset = themeDatasetMap[activeTheme.id] + const operationalScopeProjects = useMemo(() => { + const scoped = projects.filter((project) => project.name === MOL_PROJECT_NAME || project.name === KEMPEN_PROJECT_NAME) + const selected = projects.find((project) => project.id === selectedProjectId) + if (selected && !scoped.some((project) => project.id === selected.id)) { + return [selected, ...scoped] + } + return scoped + }, [projects, selectedProjectId]) + const activeScopeProject = projects.find((project) => project.id === selectedProjectId) ?? null + const activeScopeLabel = activeScopeProject ? operationalScopeProjectLabel(activeScopeProject) : 'Werkgebied' const activeTemporalSeriesGroups = useMemo( () => listThemeTemporalSeries(availableMapDatasets, activeTheme), [activeTheme, availableMapDatasets], @@ -892,10 +918,10 @@ export function MapWorkspace({ if (!advancedMode) { return ( -
+
-

Mol · geografische verkenner

+

{activeScopeLabel} · geografische verkenner

Wat bevindt zich in dit gebied?

Kies een datathema, teken een rechthoek en lees de beschikbare gegevens meteen uit.

@@ -933,6 +959,19 @@ export function MapWorkspace({

Dit zijn databronnen, geen AI-modellen.

+
{DATA_THEMES.map((theme) => { const dataset = themeDatasetMap[theme.id] @@ -1055,7 +1094,7 @@ export function MapWorkspace({ type="button" onClick={() => selectedAreaBbox && void analyzeSelection(selectedAreaBbox)} > - Volledige gemeente + Volledig werkgebied