diff --git a/CHANGELOG.md b/CHANGELOG.md index 04b529d0..ef366a49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ # Changelog +## Sprint 181 Complete Mol municipality workspace (2026-07-14) + +- Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary. +- Added auditable source artifacts and a manifest with page count, checksums, exact WGS84 bounds, municipality area, feature totals and truncation state; incomplete pagination now fails closed. +- Provisioning remains explicit and imports Project, Area and Dataset records through existing canonical API routes and DatasetService/VectorFeatureService persistence rather than writing directly to PostGIS. +- Made the complete `Mol Municipality Workbench` the preferred fresh-session context and the official municipality boundary its lightweight default layer, ahead of historical Postel validation projects. +- Replaced large coordinate arrays and spread-based bounds calculations with streaming, memoized GeoJSON bounds so municipality-scale vector layers remain safe in MapLibre. +- Removed per-feature ORM refreshes after vector import while retaining one flush and commit, avoiding tens of thousands of redundant queries for full-municipality datasets. +- Added focused municipality clipping, truncation, persistence-scaling, runtime wiring and frontend-priority regression coverage. No migration or API contract changed. + ## Sprint 180 Premium workbench UX hardening (2026-07-14) - Rebuilt the workbench presentation hierarchy around grouped task navigation, a compact Mol context header and an optional selection-detail drawer instead of a permanent third column. diff --git a/README.md b/README.md index 3784f9e3..b2004496 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ GeoIntel Kempen is a GeoAI Workbench for the Belgian Kempen. It is designed as a portfolio-grade project combining GIS, remote sensing, raster/vector processing, computer vision, QA/QC and geospatial exports. -The primary operating focus is Mol. New workbench contexts, AOIs and operator samples start there, while the broader Kempen remains fully supported for cross-area validation and regional interoperability. +The primary operating focus is Mol. New workbench contexts, AOIs and operator samples start there, while the broader Kempen remains fully supported for cross-area validation and regional interoperability. The live operator environment can provision a complete `Mol Municipality Workbench` with the official NIS `13025` municipality boundary and every intersecting GRB GBG building; smaller Mol zones remain analysis and validation contexts instead of the default municipal map. GeoIntel is not a generic dashboard or chatbot. The core product is: diff --git a/backend/README.md b/backend/README.md index 5d263c9d..100e6ee2 100644 --- a/backend/README.md +++ b/backend/README.md @@ -490,6 +490,14 @@ background control. Prepare and execute that pack with the documented `prepare_operator_real_data_samples.py` and `run_mol_operational_validation.sh` commands in `scripts/README.md`. +For municipality-wide navigation, run +`/app/scripts/provision_mol_municipality_workspace.py` inside the all-in-one +container. It verifies the official Mol boundary (NIS `13025`), pages and clips +all GRB GBG buildings, records checksums/provenance under persistent operator +storage and imports both datasets through the existing HTTP service boundary. +The command is explicit and idempotent; it is never executed during backend +startup. See `scripts/README.md` for exact usage and refresh controls. + The current recommended local building model is `geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt` with tile size `512`, overlap `64` and confidence threshold `0.15`. Its SHA256 is diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index f7818bc4..6cb3a598 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -173,7 +173,6 @@ class VectorFeatureService: persisted.append(row) if commit: + db.flush() db.commit() - for row in persisted: - db.refresh(row) return persisted diff --git a/backend/tests/test_sprint181_mol_municipality_workspace.py b/backend/tests/test_sprint181_mol_municipality_workspace.py new file mode 100644 index 00000000..2ab55f37 --- /dev/null +++ b/backend/tests/test_sprint181_mol_municipality_workspace.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys +from uuid import uuid4 + +import pytest +from shapely.geometry import Polygon, shape + +from app.models import VectorFeature +from app.services.vector_feature_service import VectorFeatureService + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_provisioner(): + script_path = ROOT / "scripts" / "provision_mol_municipality_workspace.py" + spec = importlib.util.spec_from_file_location("mol_municipality_provisioner", script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def feature(feature_id: str, coordinates: list[list[list[float]]]): + return { + "type": "Feature", + "id": feature_id, + "geometry": {"type": "Polygon", "coordinates": coordinates}, + "properties": {"UIDN": feature_id}, + } + + +def test_mol_provisioner_uses_official_identity_and_exact_boundary_clipping() -> None: + module = load_provisioner() + boundary = Polygon([(5.0, 51.0), (5.2, 51.0), (5.2, 51.2), (5.0, 51.2), (5.0, 51.0)]) + inside = feature("GBG.inside", [[[5.05, 51.05], [5.1, 51.05], [5.1, 51.1], [5.05, 51.1], [5.05, 51.05]]]) + crossing = feature("GBG.crossing", [[[5.18, 51.08], [5.22, 51.08], [5.22, 51.12], [5.18, 51.12], [5.18, 51.08]]]) + outside = feature("GBG.outside", [[[5.3, 51.3], [5.31, 51.3], [5.31, 51.31], [5.3, 51.31], [5.3, 51.3]]]) + pages = [ + ( + {"type": "FeatureCollection", "features": [inside, crossing, outside, inside]}, + "https://geo.api.vlaanderen.be/GRB/page-1", + ) + ] + + buildings, summary = module.build_municipality_buildings(pages, boundary, max_features=10) + + assert module.MUNICIPALITY_NIS_CODE == "13025" + assert module.PROJECT_NAME == "Mol Municipality Workbench" + assert len(buildings) == 2 + assert summary["bbox_feature_count"] == 3 + assert summary["outside_boundary_count"] == 1 + assert summary["clipped_at_boundary_count"] == 1 + assert summary["reference_truncated"] is False + assert all(shape(item["geometry"]).within(boundary) for item in buildings) + assert buildings[0]["properties"]["coverage_scope"] == "municipality" + assert buildings[0]["properties"]["source_name"] == "grb" + assert buildings[0]["properties"]["reference_layer_name"] == "buildings" + assert buildings[1]["properties"]["clipped_to_municipality"] is True + + +def test_mol_provisioner_refuses_a_truncated_municipality_dataset() -> None: + module = load_provisioner() + boundary = Polygon([(5.0, 51.0), (5.2, 51.0), (5.2, 51.2), (5.0, 51.2), (5.0, 51.0)]) + pages = [ + ( + { + "type": "FeatureCollection", + "features": [ + feature("GBG.1", [[[5.01, 51.01], [5.02, 51.01], [5.02, 51.02], [5.01, 51.02], [5.01, 51.01]]]), + feature("GBG.2", [[[5.03, 51.03], [5.04, 51.03], [5.04, 51.04], [5.03, 51.04], [5.03, 51.03]]]), + ], + }, + "https://geo.api.vlaanderen.be/GRB/page-1", + ) + ] + + with pytest.raises(RuntimeError, match="refusing a truncated municipality dataset"): + module.build_municipality_buildings(pages, boundary, max_features=1) + + +def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> None: + class FakeSession: + def __init__(self) -> None: + self.added = [] + self.flushes = 0 + self.commits = 0 + self.refreshes = 0 + + def add(self, item) -> None: + self.added.append(item) + + def flush(self) -> None: + self.flushes += 1 + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, _item) -> None: + self.refreshes += 1 + + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": f"GBG.{index}", + "properties": {"layer_type": "building"}, + "geometry": { + "type": "Polygon", + "coordinates": [[[5.0, 51.0], [5.001, 51.0], [5.001, 51.001], [5.0, 51.001], [5.0, 51.0]]], + }, + } + for index in range(250) + ], + } + db = FakeSession() + + persisted = VectorFeatureService.persist_geojson_features(db, uuid4(), payload, feature_class="buildings") + + assert len(persisted) == 250 + assert all(isinstance(item, VectorFeature) for item in persisted) + assert db.flushes == 1 + assert db.commits == 1 + assert db.refreshes == 0 + + +def test_municipality_workspace_is_wired_into_runtime_and_frontend_priority() -> None: + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + focus = (ROOT / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8") + project_hook = (ROOT / "frontend" / "src" / "hooks" / "useProjectWorkspace.ts").read_text(encoding="utf-8") + dataset_hook = (ROOT / "frontend" / "src" / "hooks" / "useDatasetWorkflow.ts").read_text(encoding="utf-8") + map_source = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(encoding="utf-8") + map_workspace = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8") + + assert "py_compile scripts/provision_mol_municipality_workspace.py" in readiness + assert "COPY scripts/provision_mol_municipality_workspace.py" in dockerfile + assert "PRIMARY_FOCUS_MUNICIPALITY_PROJECT_NAME = 'Mol Municipality Workbench'" in focus + assert "items.find(isPrimaryFocusMunicipalityProject)" in project_hook + assert "datasets.find(isPrimaryFocusMunicipalityBoundaryDataset)" in dataset_hook + assert "featureCollectionBounds(featureCollection)" in map_source + assert "useMemo(() => getFeatureCollectionBBox(mapFeatureCollection)" in map_workspace + assert "Math.min(...xs)" not in map_source + diff --git a/backend/tests/test_sprint7a_persistence_foundation.py b/backend/tests/test_sprint7a_persistence_foundation.py index 59356eb4..f55dc4dc 100644 --- a/backend/tests/test_sprint7a_persistence_foundation.py +++ b/backend/tests/test_sprint7a_persistence_foundation.py @@ -79,7 +79,9 @@ def test_vector_feature_service_persists_geojson_features_with_properties() -> N assert persisted[0].source_feature_id == "building-1" assert persisted[0].properties_json == {"class": "building", "height": 7} assert db.added == persisted + assert db.flushes == 1 assert db.commits == 1 + assert db.refreshes == [] def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None: diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index dd0411fc..a3ffb582 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -72,6 +72,7 @@ RUN python scripts/gis_import_smoke.py \ && mkdir -p /app/storage /run/nginx /var/log/nginx COPY scripts/prepare_operator_real_data_samples.py /app/scripts/prepare_operator_real_data_samples.py +COPY scripts/provision_mol_municipality_workspace.py /app/scripts/provision_mol_municipality_workspace.py COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 7d2a37e6..d0e138f1 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7395,3 +7395,41 @@ Open: ## Next pass - Harden detection QA coverage and matching diagnostics before making a retraining decision; keep that work separate from this presentation-only sprint. + +# Sprint 181 - Complete Mol municipality workspace + +## Implementation + +- Added an explicit operator provisioner for the official VRBG `Refgem` + municipality geometry for Mol (NIS `13025`) and the complete paged GRB `GBG` + building collection clipped to that exact boundary. +- Added deterministic persistent source artefacts and a manifest containing + source URLs, checksums, page/feature counts, boundary bounds and area, and an + explicit truncation flag. Pagination and identity checks fail closed. +- Kept the provider boundary honest: provisioning is an operator action and + imports through canonical Project, Area and Dataset HTTP routes. It does not + enable the dormant live GRB provider or write directly to `vector_features`. +- Optimized the existing vector persistence path by replacing one ORM refresh + per feature with a single flush and commit. Persistence shape and API + behavior remain unchanged. +- Made `Mol Municipality Workbench` the preferred fresh-session context once + its official boundary is ready. The boundary opens first; the much larger + building layer remains explicitly selectable from the Map database-layer + control. +- Replaced spread-based map extent calculations with a streaming, memoized + GeoJSON bounds helper and added municipality/building layer styling. This + avoids large coordinate arrays while keeping the existing MapLibre path. + +## Initial validation + +- Exact clipping, pagination/truncation, persistence-scaling and frontend + wiring regression coverage passed locally. +- No API route, ORM model, migration, QA metric, detection result or model + configuration changed. + +## Next pass + +- Deploy and execute the explicit provisioner on Tower, verify full PostGIS + feature counts and exact Mol bounds, then audit the complete municipality in + MapLibre before acquiring raster imagery for a deliberately selected Mol + analysis zone. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 5a260207..9e719a82 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -13,6 +13,22 @@ Dit document verzamelt concrete databronnen voor GeoIntel Kempen. - Prioriteit: V1 - Opmerking: belangrijkste officiële referentiebron voor QA/QC. +### Volledige gemeente Mol + +De operationele Mol-workspace gebruikt twee expliciete Digitaal Vlaanderen +OGC API-bronnen: + +- `VRBG/Refgem`, gefilterd op `NAAM='Mol'` en gecontroleerd op NIS-code + `13025`, als officiële gemeentegrens. +- `GRB/GBG`, volledig gepagineerd en exact tegen die grens gesneden, als + gemeentebrede gebouwreferentie. + +Deze bronnen worden uitsluitend via het expliciete operatorcommando +`scripts/provision_mol_municipality_workspace.py` opgehaald. De applicatie +start geen verborgen providerfetch. De resulterende GeoJSON-artefacten, +checksums en bron-URL's worden onder persistent operator storage bewaard en +via de bestaande DatasetService/vectorfeature-flow geïmporteerd. + ## OSM - Naam: OpenStreetMap diff --git a/docs/TODO.md b/docs/TODO.md index 3e69c259..304554d0 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -513,6 +513,18 @@ This file now starts with the current implementation status. Older preparation/b - [x] Train and reject `geointel-building-yolov8s-aoi1024cleanpx12vis035e50-pt` through the positive/background promotion gate. - [x] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation. +# Sprint 181 - Complete Mol municipality workspace + +- [x] Persist the official VRBG municipality boundary for Mol (NIS `13025`). +- [x] Page, deduplicate and boundary-clip the complete GRB GBG building layer. +- [x] Import municipality layers through the existing DatasetService and + VectorFeatureService path, with no direct provider-to-database write. +- [x] Prefer the complete municipality workspace on a fresh session while + keeping the lightweight boundary as the initially rendered layer. +- [x] Make GeoJSON bounds calculation safe for municipality-scale layers. +- [ ] Acquire and tile a georeferenced raster only for an explicitly selected + Mol analysis zone before running the next configured-YOLO validation. + # Sprint 171 - Positive AOI expansion and small-building recovery - [x] Reject cross-model false-negative comparisons when reference populations differ. diff --git a/frontend/README.md b/frontend/README.md index f7e5b3cd..a70bcbec 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -2,7 +2,7 @@ React + TypeScript + MapLibre foundation for project/area/dataset workflow. -Mol is the primary operating context. On a fresh session the workbench prefers a persisted Mol project or dataset, centers an empty MapLibre view on Mol and pre-fills a compact Mol AOI. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available. +Mol is the primary operating context. On a fresh session the workbench first prefers the complete persisted `Mol Municipality Workbench` when its official NIS `13025` boundary is ready. Its lightweight municipality boundary opens before the complete GRB building reference layer, so initial map framing remains responsive while all municipality buildings stay selectable from the database-layer control. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available. The workbench uses a task-based shell instead of a single long panel stack. `App.tsx` still owns shared orchestration state, but the UI is organized into Overview, Data, Map, QA/QC, AI Labs, Exports and System workspaces with a persistent top context bar and an optional selection-detail drawer. @@ -14,6 +14,8 @@ Wide and ultrawide screens keep a readable sidebar and centered work area, expan The Map workspace defaults to an OpenStreetMap road basemap with visible attribution so uploaded vectors, AOIs and QA overlays appear on a real street context. Set `VITE_MAP_STYLE_URL` to a managed MapLibre style URL to override this for production or high-volume deployments. +Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, GRB buildings use cyan/teal and existing detection/change-result colors remain distinct. + When the public OpenStreetMap fallback is active, the Map workspace shows a basemap usage notice. This keeps the local/demo default honest and reminds operators to configure a managed style URL before production or heavier tile traffic. Operational GIS testing is now available directly in the Map workspace. Users can choose a persisted vector database layer, load it on the map, reuse the selected AOI or active layer extent, run the existing persisted `vector_features` bbox query, save the result as a derived dataset, export the selection GeoJSON, choose a reference dataset and launch QA/QC without creating fake data or a parallel backend path. The guided workflow also includes a one-click full run action that executes query, derived dataset save, GeoJSON export and optional QA/QC in sequence with visible status. For repeated review, switch the run mode from `Create new dataset/export` to `Reuse latest saved dataset for QA`; this reruns QA/QC against the latest saved derived dataset without creating another dataset/export pair. diff --git a/frontend/src/components/GeoMap.tsx b/frontend/src/components/GeoMap.tsx index a85e2d80..9f418a57 100644 --- a/frontend/src/components/GeoMap.tsx +++ b/frontend/src/components/GeoMap.tsx @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from 'react' import maplibregl from 'maplibre-gl' import 'maplibre-gl/dist/maplibre-gl.css' import { PRIMARY_FOCUS_CENTER } from '../config/primaryFocus' +import { featureCollectionBounds } from '../lib/geojsonBounds' interface GeoMapProps { data: GeoJSON.FeatureCollection | null @@ -49,36 +50,13 @@ function defaultMapStyle(): string | maplibregl.StyleSpecification { } function collectCoordinates(featureCollection: GeoJSON.FeatureCollection): maplibregl.LngLatBoundsLike | null { - const coordinates: [number, number][] = [] - const walk = (coords: unknown) => { - if (!Array.isArray(coords)) { - return - } - if (coords.length === 2 && typeof coords[0] === 'number' && typeof coords[1] === 'number') { - coordinates.push([coords[0], coords[1]]) - return - } - for (const item of coords) { - walk(item) - } - } - - for (const feature of featureCollection.features) { - const geometry = feature.geometry as any - if (geometry && geometry.coordinates) { - walk(geometry.coordinates) - } - } - - if (coordinates.length === 0) { + const bounds = featureCollectionBounds(featureCollection) + if (!bounds) { return null } - - const xs = coordinates.map((point) => point[0]) - const ys = coordinates.map((point) => point[1]) return [ - [Math.min(...xs), Math.min(...ys)], - [Math.max(...xs), Math.max(...ys)], + [bounds.minX, bounds.minY], + [bounds.maxX, bounds.maxY], ] } @@ -244,15 +222,22 @@ function GeoMap({ source: 'dataset', paint: { 'fill-color': [ - 'match', - ['get', 'change_type'], - 'added', - '#16a34a', - 'removed', - '#dc2626', - 'unchanged', - '#2563eb', - '#f97316', + 'case', + ['==', ['get', 'layer_type'], 'municipality_boundary'], + '#0f766e', + ['==', ['get', 'layer_type'], 'building'], + '#0891b2', + [ + 'match', + ['get', 'change_type'], + 'added', + '#16a34a', + 'removed', + '#dc2626', + 'unchanged', + '#2563eb', + '#f97316', + ], ], 'fill-opacity': 0.4, }, @@ -263,17 +248,24 @@ function GeoMap({ source: 'dataset', paint: { 'line-color': [ - 'match', - ['get', 'change_type'], - 'added', - '#15803d', - 'removed', - '#b91c1c', - 'unchanged', - '#1d4ed8', - '#ea580c', + 'case', + ['==', ['get', 'layer_type'], 'municipality_boundary'], + '#0f5f59', + ['==', ['get', 'layer_type'], 'building'], + '#0e7490', + [ + 'match', + ['get', 'change_type'], + 'added', + '#15803d', + 'removed', + '#b91c1c', + 'unchanged', + '#1d4ed8', + '#ea580c', + ], ], - 'line-width': 2, + 'line-width': ['interpolate', ['linear'], ['zoom'], 8, 0.25, 12, 0.8, 16, 2], }, }) } diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index d197adf8..31679337 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -1,6 +1,7 @@ -import { useEffect, useState } from 'react' +import { useEffect, useMemo, useState } from 'react' import GeoMap from '../GeoMap' import type { AreaRead, DatasetCreateResponse, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types' +import { featureCollectionBounds } from '../../lib/geojsonBounds' const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson' const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson' @@ -55,17 +56,15 @@ function getFeatureGeometrySummary(feature: GeoJSON.Feature | null) { } function getFeatureCollectionBBox(collection: GeoJSON.FeatureCollection | null): VectorSelectionBBox | null { - const points = collection?.features.flatMap((feature) => collectGeometryPoints(feature.geometry)) ?? [] - if (points.length === 0) { + const bounds = featureCollectionBounds(collection) + if (!bounds) { return null } - const xs = points.map((point) => point[0]) - const ys = points.map((point) => point[1]) return { - min_x: Math.min(...xs), - min_y: Math.min(...ys), - max_x: Math.max(...xs), - max_y: Math.max(...ys), + min_x: bounds.minX, + min_y: bounds.minY, + max_x: bounds.maxX, + max_y: bounds.maxY, crs: 'EPSG:4326', } } @@ -303,9 +302,9 @@ export function MapWorkspace({ const featureExtractionEntries = featureProperties ? Object.entries(featureProperties).slice(0, 48) : [] const featureGeometrySummary = getFeatureGeometrySummary(selectedMapFeature) const selectedFeatureGeoJson = selectedMapFeature ? selectedFeatureCollection(selectedMapFeature) : null - const selectedFeatureBbox = getFeatureBBox(selectedMapFeature) - const activeLayerBbox = getFeatureCollectionBBox(mapFeatureCollection) - const selectedAreaBbox = getFeatureCollectionBBox(areaFeatureCollection) + const selectedFeatureBbox = useMemo(() => getFeatureBBox(selectedMapFeature), [selectedMapFeature]) + const activeLayerBbox = useMemo(() => getFeatureCollectionBBox(mapFeatureCollection), [mapFeatureCollection]) + const selectedAreaBbox = useMemo(() => getFeatureCollectionBBox(areaFeatureCollection), [areaFeatureCollection]) const currentSelectionBbox = parseBboxInput(bboxInput) const areaSelectionFeatures = mapSelectionResult?.geojson.features ?? [] const areaSelectionPreviewFeatures = areaSelectionFeatures.slice(0, 12) diff --git a/frontend/src/config/primaryFocus.ts b/frontend/src/config/primaryFocus.ts index 9713c505..16399a31 100644 --- a/frontend/src/config/primaryFocus.ts +++ b/frontend/src/config/primaryFocus.ts @@ -4,6 +4,7 @@ export const PRIMARY_FOCUS_LABEL = 'Mol' export const PRIMARY_FOCUS_REGION = 'Mol, Kempen' export const PRIMARY_FOCUS_CENTER: [number, number] = [5.1167, 51.1919] export const PRIMARY_FOCUS_AREA_NAME = 'Mol primary AOI' +export const PRIMARY_FOCUS_MUNICIPALITY_PROJECT_NAME = 'Mol Municipality Workbench' export const PRIMARY_FOCUS_AREA_GEOJSON = '{"type":"MultiPolygon","coordinates":[[[[5.109476746222376,51.18745107550715],[5.123924622861509,51.18745107550715],[5.123924622861509,51.19634848029239],[5.109476746222376,51.19634848029239],[5.109476746222376,51.18745107550715]]]]}' @@ -17,6 +18,20 @@ export function isPrimaryFocusProject(project: ProjectRead): boolean { return containsPrimaryFocus([project.name, project.description, project.region]) } +export function isPrimaryFocusMunicipalityProject(project: ProjectRead): boolean { + return project.name === PRIMARY_FOCUS_MUNICIPALITY_PROJECT_NAME +} + +export function isPrimaryFocusMunicipalityBoundaryDataset(dataset: DatasetCreateResponse): boolean { + return ( + dataset.status === 'ready' && + (dataset.dataset_type === 'vector' || dataset.dataset_type === 'geojson') && + dataset.source_name === 'vrbg' && + dataset.source_metadata?.coverage_scope === 'municipality' && + dataset.source_metadata?.nis_code === '13025' + ) +} + export function isPrimaryFocusProjectData( project: ProjectRead, datasets: DatasetCreateResponse[], diff --git a/frontend/src/hooks/useDatasetWorkflow.ts b/frontend/src/hooks/useDatasetWorkflow.ts index ca03c12d..ff408606 100644 --- a/frontend/src/hooks/useDatasetWorkflow.ts +++ b/frontend/src/hooks/useDatasetWorkflow.ts @@ -11,6 +11,7 @@ import type { VectorSummary, } from '../types' import { formatError } from '../lib/formatError' +import { isPrimaryFocusMunicipalityBoundaryDataset } from '../config/primaryFocus' interface DatasetWorkflowOptions { selectedProjectId: string | null @@ -128,6 +129,7 @@ export function useDatasetWorkflow({ } // Auto-open the first usable dataset so Data, Map and Exports start with real context. const defaultDataset = + datasets.find(isPrimaryFocusMunicipalityBoundaryDataset) ?? datasets.find((dataset) => isVectorDatasetType(dataset.dataset_type) && dataset.status === 'ready') ?? datasets.find((dataset) => dataset.status === 'ready') ?? datasets[0] diff --git a/frontend/src/hooks/useProjectWorkspace.ts b/frontend/src/hooks/useProjectWorkspace.ts index f1ce8707..28a134a4 100644 --- a/frontend/src/hooks/useProjectWorkspace.ts +++ b/frontend/src/hooks/useProjectWorkspace.ts @@ -3,6 +3,8 @@ import { PRIMARY_FOCUS_AREA_GEOJSON, PRIMARY_FOCUS_AREA_NAME, PRIMARY_FOCUS_REGION, + isPrimaryFocusMunicipalityBoundaryDataset, + isPrimaryFocusMunicipalityProject, isPrimaryFocusProject, isPrimaryFocusProjectData, } from '../config/primaryFocus' @@ -76,6 +78,17 @@ export function useProjectWorkspace() { if (selectedProjectId && items.some((project) => project.id === selectedProjectId)) { return selectedProjectId } + const municipalityProject = items.find(isPrimaryFocusMunicipalityProject) + if (municipalityProject) { + try { + const data = await fetchProjectData(municipalityProject.id) + if (data.areas.length > 0 && data.datasets.some(isPrimaryFocusMunicipalityBoundaryDataset)) { + return municipalityProject.id + } + } catch { + // Continue with the normal completeness ranking if the canonical workspace is temporarily unavailable. + } + } const primaryProjects = items.filter(isPrimaryFocusProject) const demoProjects = items.filter(isDemoProject) const candidates = Array.from( diff --git a/frontend/src/lib/geojsonBounds.ts b/frontend/src/lib/geojsonBounds.ts new file mode 100644 index 00000000..c3a8bbd5 --- /dev/null +++ b/frontend/src/lib/geojsonBounds.ts @@ -0,0 +1,75 @@ +export interface GeoJsonBounds { + minX: number + minY: number + maxX: number + maxY: number +} + +function extendBounds(bounds: GeoJsonBounds | null, x: number, y: number): GeoJsonBounds { + if (!bounds) { + return { minX: x, minY: y, maxX: x, maxY: y } + } + bounds.minX = Math.min(bounds.minX, x) + bounds.minY = Math.min(bounds.minY, y) + bounds.maxX = Math.max(bounds.maxX, x) + bounds.maxY = Math.max(bounds.maxY, y) + return bounds +} + +function scanCoordinates(coordinates: unknown, initial: GeoJsonBounds | null): GeoJsonBounds | null { + let bounds = initial + const stack: unknown[] = [coordinates] + while (stack.length > 0) { + const value = stack.pop() + if (!Array.isArray(value)) { + continue + } + if (value.length >= 2 && typeof value[0] === 'number' && typeof value[1] === 'number') { + if (Number.isFinite(value[0]) && Number.isFinite(value[1])) { + bounds = extendBounds(bounds, value[0], value[1]) + } + continue + } + for (const child of value) { + stack.push(child) + } + } + return bounds +} + +export function geometryBounds(geometry: GeoJSON.Geometry | null | undefined): GeoJsonBounds | null { + if (!geometry) { + return null + } + if (geometry.type === 'GeometryCollection') { + return geometry.geometries.reduce( + (bounds, child) => mergeBounds(bounds, geometryBounds(child)), + null, + ) + } + return scanCoordinates(geometry.coordinates, null) +} + +export function mergeBounds(left: GeoJsonBounds | null, right: GeoJsonBounds | null): GeoJsonBounds | null { + if (!left) { + return right ? { ...right } : null + } + if (!right) { + return left + } + left.minX = Math.min(left.minX, right.minX) + left.minY = Math.min(left.minY, right.minY) + left.maxX = Math.max(left.maxX, right.maxX) + left.maxY = Math.max(left.maxY, right.maxY) + return left +} + +export function featureCollectionBounds(collection: GeoJSON.FeatureCollection | null | undefined): GeoJsonBounds | null { + if (!collection) { + return null + } + return collection.features.reduce( + (bounds, feature) => mergeBounds(bounds, geometryBounds(feature.geometry)), + null, + ) +} diff --git a/scripts/README.md b/scripts/README.md index 8a38a620..908e8e6f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1131,6 +1131,32 @@ thresholds without changing the script. The main readiness gate checks this script's syntax; run it explicitly against Docker/PostGIS when validating a live deployment. +## Complete Mol municipality workspace + +Provision the official municipality boundary and every GRB GBG building that +intersects it through the existing project, area, dataset and vector-feature +persistence paths: + +```bash +docker exec -it geointel python3 \ + /app/scripts/provision_mol_municipality_workspace.py +``` + +The command queries `VRBG/Refgem` for municipality `Mol`, verifies NIS code +`13025`, follows every `GRB/GBG` pagination link and clips the resulting +buildings to the official boundary. It writes source artefacts and a checksum +manifest below `/app/storage/operator-data/mol-municipality`, then creates or +reuses `Mol Municipality Workbench` and imports both layers through the public +API. It never writes directly to PostGIS and never runs implicitly at startup. + +Completed artefacts and ready datasets are reused on a repeat run. Use +`--force` only when an operator deliberately wants to refetch and replace the +local source artefacts. Use `--fetch-only` to prepare and inspect the manifest +without changing application persistence. The internal API default is +`http://127.0.0.1:8000`, avoiding proxy timeouts during the large vector +import; override it with `--base-url` when running outside the all-in-one +container. + ## Tower deployment Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime: diff --git a/scripts/provision_mol_municipality_workspace.py b/scripts/provision_mol_municipality_workspace.py new file mode 100644 index 00000000..ff56b75e --- /dev/null +++ b/scripts/provision_mol_municipality_workspace.py @@ -0,0 +1,604 @@ +"""Provision a complete, authoritative Mol municipality map workspace. + +The script is an explicit operator tool. It fetches the official Mol boundary +and GRB buildings, writes auditable artifacts, and imports them through the +existing GeoIntel API. It is never called automatically by application startup. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +import requests +from pyproj import Transformer +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape +from shapely.ops import transform, unary_union +from shapely.validation import make_valid + + +MUNICIPALITY_NAME = "Mol" +MUNICIPALITY_NIS_CODE = "13025" +PROJECT_NAME = "Mol Municipality Workbench" +PROJECT_REGION = "Mol, Kempen" +AREA_NAME = "Gemeente Mol - officiële grens" +BOUNDARY_FILENAME = "mol_municipality_boundary.geojson" +BUILDINGS_FILENAME = "mol_grb_gbg_buildings.geojson" +MANIFEST_FILENAME = "mol_municipality_manifest.json" +VRBG_ITEMS_URL = "https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items" +GRB_GBG_ITEMS_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" +VRBG_ATTRIBUTION = "Bron: Voorlopig referentiebestand gemeentegrenzen, Digitaal Vlaanderen" +GRB_ATTRIBUTION = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen" +DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/mol-municipality") +DEFAULT_API_URL = "http://127.0.0.1:8000" +DEFAULT_PAGE_LIMIT = 1000 +DEFAULT_MAX_FEATURES = 100000 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Fetch and provision the complete official Mol municipality workspace.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(os.environ.get("MOL_MUNICIPALITY_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)), + help="Persistent directory for the boundary, GRB buildings and provenance manifest.", + ) + parser.add_argument( + "--base-url", + default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL), + help="GeoIntel backend URL. The in-container direct backend avoids reverse-proxy timeouts during large imports.", + ) + parser.add_argument( + "--project-name", + default=PROJECT_NAME, + help="Exact idempotent project name used for the municipality workspace.", + ) + parser.add_argument( + "--page-limit", + type=int, + default=int(os.environ.get("MOL_GRB_PAGE_LIMIT", str(DEFAULT_PAGE_LIMIT))), + help="GRB OGC API page size.", + ) + parser.add_argument( + "--max-features", + type=int, + default=int(os.environ.get("MOL_GRB_MAX_FEATURES", str(DEFAULT_MAX_FEATURES))), + help="Safety cap. The script fails instead of writing a truncated municipality dataset.", + ) + parser.add_argument( + "--request-timeout", + type=int, + default=int(os.environ.get("MOL_MUNICIPALITY_REQUEST_TIMEOUT", "180")), + help="Timeout per official source request in seconds.", + ) + parser.add_argument( + "--import-timeout", + type=int, + default=int(os.environ.get("MOL_MUNICIPALITY_IMPORT_TIMEOUT", "1800")), + help="Timeout for each GeoIntel API import request in seconds.", + ) + parser.add_argument("--force", action="store_true", help="Refetch official source artifacts even when complete local artifacts exist.") + parser.add_argument("--fetch-only", action="store_true", help="Prepare artifacts without creating or updating the GeoIntel workspace.") + return parser.parse_args() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None: + path.write_text( + json.dumps( + payload, + ensure_ascii=False, + indent=2 if pretty else None, + separators=None if pretty else (",", ":"), + sort_keys=pretty, + ), + encoding="utf-8", + ) + + +def next_page_url(payload: dict[str, Any]) -> str | None: + links = payload.get("links") or [] + for link in links: + if link.get("rel") == "next" and "geo+json" in str(link.get("type", "")).lower(): + return str(link["href"]) + for link in links: + if link.get("rel") == "next" and link.get("href"): + return str(link["href"]) + return None + + +def normalize_polygonal(geometry): + if geometry.is_empty: + return None + if not geometry.is_valid: + geometry = make_valid(geometry) + if isinstance(geometry, (Polygon, MultiPolygon)): + return geometry + if isinstance(geometry, GeometryCollection): + polygons = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty] + if polygons: + merged = unary_union(polygons) + return merged if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty else None + return None + + +def fetch_mol_boundary(session: requests.Session, timeout: int) -> tuple[dict[str, Any], Any, str]: + params = { + "f": "application/geo+json", + "limit": "10", + "filter": "NAAM='Mol'", + "filter-lang": "cql2-text", + } + response = session.get(VRBG_ITEMS_URL, params=params, timeout=timeout) + response.raise_for_status() + payload = response.json() + matches = [ + feature + for feature in payload.get("features") or [] + if str((feature.get("properties") or {}).get("NAAM", "")).casefold() == MUNICIPALITY_NAME.casefold() + ] + if len(matches) != 1: + raise RuntimeError(f"Expected exactly one official Mol boundary, received {len(matches)}") + + source_feature = matches[0] + properties = dict(source_feature.get("properties") or {}) + if str(properties.get("NISCODE")) != MUNICIPALITY_NIS_CODE: + raise RuntimeError( + f"Official Mol boundary NIS code drifted: expected {MUNICIPALITY_NIS_CODE}, received {properties.get('NISCODE')}" + ) + boundary = normalize_polygonal(shape(source_feature.get("geometry"))) + if boundary is None or not boundary.is_valid: + raise RuntimeError("Official Mol boundary is empty, non-polygonal or invalid") + + source_url = response.url + properties.update( + { + "source_name": "vrbg", + "source_feature_id": str(source_feature.get("id") or MUNICIPALITY_NIS_CODE), + "layer_type": "municipality_boundary", + "authority_level": "authoritative", + "coverage_scope": "municipality", + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "attribution": VRBG_ATTRIBUTION, + "source_url": source_url, + } + ) + feature = { + "type": "Feature", + "id": str(source_feature.get("id") or f"Refgem.{MUNICIPALITY_NIS_CODE}"), + "geometry": mapping(boundary), + "properties": properties, + } + return feature, boundary, source_url + + +def iter_grb_pages( + session: requests.Session, + boundary_bounds: tuple[float, float, float, float], + *, + page_limit: int, + timeout: int, +) -> Iterable[tuple[dict[str, Any], str]]: + params = { + "f": "application/geo+json", + "limit": str(page_limit), + "bbox": ",".join(f"{value:.8f}" for value in boundary_bounds), + } + url: str | None = GRB_GBG_ITEMS_URL + seen_urls: set[str] = set() + first_request = True + while url: + if url in seen_urls: + raise RuntimeError(f"GRB pagination loop detected: {url}") + seen_urls.add(url) + response = session.get(url, params=params if first_request else None, timeout=timeout) + first_request = False + response.raise_for_status() + payload = response.json() + yield payload, response.url + url = next_page_url(payload) + + +def build_municipality_buildings( + pages: Iterable[tuple[dict[str, Any], str]], + boundary, + *, + max_features: int, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + features: list[dict[str, Any]] = [] + source_urls: list[str] = [] + seen_ids: set[str] = set() + bbox_feature_count = 0 + outside_boundary_count = 0 + clipped_at_boundary_count = 0 + + for page, source_url in pages: + source_urls.append(source_url) + for source_feature in page.get("features") or []: + feature_id = str(source_feature.get("id") or "") + if not feature_id: + feature_id = hashlib.sha256( + json.dumps(source_feature.get("geometry"), sort_keys=True).encode("utf-8") + ).hexdigest() + if feature_id in seen_ids: + continue + seen_ids.add(feature_id) + bbox_feature_count += 1 + + source_geometry = normalize_polygonal(shape(source_feature.get("geometry"))) + if source_geometry is None or not source_geometry.intersects(boundary): + outside_boundary_count += 1 + continue + within_boundary = source_geometry.within(boundary) + clipped_geometry = source_geometry if within_boundary else normalize_polygonal(source_geometry.intersection(boundary)) + if clipped_geometry is None: + outside_boundary_count += 1 + continue + if not within_boundary: + clipped_at_boundary_count += 1 + if len(features) >= max_features: + raise RuntimeError( + f"Mol contains more than the configured {max_features} GRB features; refusing a truncated municipality dataset" + ) + + properties = dict(source_feature.get("properties") or {}) + properties.update( + { + "source_name": "grb", + "source_feature_id": feature_id, + "reference_layer_name": "buildings", + "layer_type": "building", + "authority_level": "authoritative", + "coverage_scope": "municipality", + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "clipped_to_municipality": not within_boundary, + "attribution": GRB_ATTRIBUTION, + } + ) + features.append( + { + "type": "Feature", + "id": feature_id, + "geometry": mapping(clipped_geometry), + "properties": properties, + } + ) + + if not features: + raise RuntimeError("No GRB buildings intersect the official Mol municipality boundary") + return features, { + "pages_fetched": len(source_urls), + "source_urls": source_urls, + "bbox_feature_count": bbox_feature_count, + "municipality_feature_count": len(features), + "outside_boundary_count": outside_boundary_count, + "clipped_at_boundary_count": clipped_at_boundary_count, + "reference_truncated": False, + } + + +def municipality_area_km2(boundary) -> float: + transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + return float(transform(transformer.transform, boundary).area / 1_000_000) + + +def prepare_artifacts(args: argparse.Namespace) -> tuple[Path, Path, dict[str, Any]]: + output_dir: Path = args.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + boundary_path = output_dir / BOUNDARY_FILENAME + buildings_path = output_dir / BUILDINGS_FILENAME + manifest_path = output_dir / MANIFEST_FILENAME + + if not args.force and boundary_path.exists() and buildings_path.exists() and manifest_path.exists(): + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("status") == "complete" and int(manifest.get("municipality_feature_count") or 0) > 0: + return boundary_path, buildings_path, manifest + + if args.page_limit <= 0 or args.max_features <= 0: + raise RuntimeError("--page-limit and --max-features must be positive integers") + + with requests.Session() as session: + session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"}) + boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout) + buildings, building_summary = build_municipality_buildings( + iter_grb_pages( + session, + boundary.bounds, + page_limit=args.page_limit, + timeout=args.request_timeout, + ), + boundary, + max_features=args.max_features, + ) + + generated_at = utc_now() + boundary_payload = { + "type": "FeatureCollection", + "name": "Official municipality boundary - Mol", + "features": [boundary_feature], + "source": "Digitaal Vlaanderen VRBG OGC API Features collection Refgem", + "source_url": boundary_source_url, + "attribution": VRBG_ATTRIBUTION, + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "coverage_scope": "municipality", + "generated_at": generated_at, + } + buildings_payload = { + "type": "FeatureCollection", + "name": "GRB buildings - complete municipality Mol", + "features": buildings, + "source": "Digitaal Vlaanderen GRB OGC API Features collection GBG", + "source_url": building_summary["source_urls"][0], + "attribution": GRB_ATTRIBUTION, + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "coverage_scope": "municipality", + "reference_truncated": False, + "generated_at": generated_at, + } + write_json(boundary_path, boundary_payload) + write_json(buildings_path, buildings_payload) + + manifest = { + "schema_version": 1, + "status": "complete", + "generated_at": generated_at, + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "area_km2": municipality_area_km2(boundary), + "wgs84_bbox": list(boundary.bounds), + "boundary_path": str(boundary_path), + "boundary_sha256": sha256_file(boundary_path), + "buildings_path": str(buildings_path), + "buildings_sha256": sha256_file(buildings_path), + "page_limit": args.page_limit, + "max_features": args.max_features, + "boundary_source_url": boundary_source_url, + "attribution": {"boundary": VRBG_ATTRIBUTION, "buildings": GRB_ATTRIBUTION}, + **building_summary, + } + write_json(manifest_path, manifest, pretty=True) + return boundary_path, buildings_path, manifest + + +def response_data(response: requests.Response) -> Any: + try: + payload = response.json() + except ValueError as exc: + raise RuntimeError(f"GeoIntel API returned non-JSON response ({response.status_code}): {response.text[:500]}") from exc + if not response.ok: + raise RuntimeError(f"GeoIntel API request failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:1000]}") + if not isinstance(payload, dict) or "data" not in payload: + raise RuntimeError("GeoIntel API response does not use the canonical data envelope") + return payload["data"] + + +def find_or_create_project(session: requests.Session, base_url: str, project_name: str, timeout: int) -> dict[str, Any]: + projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout)) + existing = next((item for item in projects.get("items") or [] if item.get("name") == project_name), None) + if existing: + return existing + return response_data( + session.post( + f"{base_url}/api/v1/projects", + json={ + "name": project_name, + "description": ( + "Complete municipality workspace for Mol using the official Digitaal Vlaanderen municipality boundary " + "and the full GRB GBG building reference layer clipped to NIS 13025." + ), + "region": PROJECT_REGION, + }, + timeout=timeout, + ) + ) + + +def find_or_create_area( + session: requests.Session, + base_url: str, + project_id: str, + boundary_path: Path, + timeout: int, +) -> dict[str, Any]: + areas = response_data( + session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout) + ) + existing = next((item for item in areas.get("items") or [] if item.get("name") == AREA_NAME), None) + if existing: + return existing + boundary_payload = json.loads(boundary_path.read_text(encoding="utf-8")) + geometry = boundary_payload["features"][0]["geometry"] + return response_data( + session.post( + f"{base_url}/api/v1/projects/{project_id}/areas", + json={"name": AREA_NAME, "crs": "EPSG:4326", "geometry": geometry}, + timeout=timeout, + ) + ) + + +def upload_dataset( + session: requests.Session, + base_url: str, + project_id: str, + area_id: str, + path: Path, + *, + dataset_role: str, + source_name: str, + reference_layer_name: str | None, + source_metadata: dict[str, Any], + provenance_metadata: dict[str, Any], + timeout: int, +) -> dict[str, Any]: + form = { + "dataset_type": "vector", + "source": "operator_official_import", + "dataset_role": dataset_role, + "source_name": source_name, + "source_metadata_json": json.dumps(source_metadata, ensure_ascii=False), + "provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False), + "area_id": area_id, + } + if reference_layer_name: + form["reference_layer_name"] = reference_layer_name + with path.open("rb") as handle: + response = session.post( + f"{base_url}/api/v1/projects/{project_id}/datasets/upload", + data=form, + files={"file": (path.name, handle, "application/geo+json")}, + timeout=timeout, + ) + return response_data(response) + + +def provision_workspace( + args: argparse.Namespace, + boundary_path: Path, + buildings_path: Path, + manifest: dict[str, Any], +) -> dict[str, Any]: + base_url = args.base_url.rstrip("/") + with requests.Session() as session: + project = find_or_create_project(session, base_url, args.project_name, args.import_timeout) + project_id = str(project["id"]) + area = find_or_create_area(session, base_url, project_id, boundary_path, args.import_timeout) + area_id = str(area["id"]) + dataset_list = response_data( + session.get( + f"{base_url}/api/v1/projects/{project_id}/datasets", + params={"limit": 200}, + timeout=args.import_timeout, + ) + ) + datasets = list(dataset_list.get("items") or []) + + common_provenance = { + "operator_tool": "provision_mol_municipality_workspace.py", + "operator_explicit_fetch": True, + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "coverage_scope": "municipality", + "manifest_path": str(args.output_dir / MANIFEST_FILENAME), + "manifest_generated_at": manifest["generated_at"], + "reference_truncated": False, + } + + building_dataset = next((item for item in datasets if item.get("original_filename") == buildings_path.name), None) + if not building_dataset: + building_dataset = upload_dataset( + session, + base_url, + project_id, + area_id, + buildings_path, + dataset_role="reference", + source_name="grb", + reference_layer_name="buildings", + source_metadata={ + "provider": "Digitaal Vlaanderen", + "collection": "GRB/GBG", + "authority_level": "authoritative", + "coverage_scope": "municipality", + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "feature_count": manifest["municipality_feature_count"], + "pages_fetched": manifest["pages_fetched"], + "attribution": GRB_ATTRIBUTION, + }, + provenance_metadata={ + **common_provenance, + "source_url": GRB_GBG_ITEMS_URL, + "artifact_sha256": manifest["buildings_sha256"], + }, + timeout=args.import_timeout, + ) + + boundary_dataset = next((item for item in datasets if item.get("original_filename") == boundary_path.name), None) + if not boundary_dataset: + boundary_dataset = upload_dataset( + session, + base_url, + project_id, + area_id, + boundary_path, + dataset_role="source", + source_name="vrbg", + reference_layer_name=None, + source_metadata={ + "provider": "Digitaal Vlaanderen", + "collection": "VRBG/Refgem", + "authority_level": "authoritative", + "coverage_scope": "municipality", + "layer_type": "municipality_boundary", + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "attribution": VRBG_ATTRIBUTION, + }, + provenance_metadata={ + **common_provenance, + "source_url": manifest["boundary_source_url"], + "artifact_sha256": manifest["boundary_sha256"], + }, + timeout=args.import_timeout, + ) + + return { + "project": project, + "area": area, + "boundary_dataset": boundary_dataset, + "building_dataset": building_dataset, + } + + +def main() -> int: + args = parse_args() + try: + boundary_path, buildings_path, manifest = prepare_artifacts(args) + workspace = None if args.fetch_only else provision_workspace(args, boundary_path, buildings_path, manifest) + except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError) as exc: + print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 1 + + result = { + "status": "ok", + "mode": "fetch_only" if args.fetch_only else "provisioned", + "manifest_path": str(args.output_dir / MANIFEST_FILENAME), + "municipality": MUNICIPALITY_NAME, + "nis_code": MUNICIPALITY_NIS_CODE, + "area_km2": manifest["area_km2"], + "wgs84_bbox": manifest["wgs84_bbox"], + "municipality_feature_count": manifest["municipality_feature_count"], + "pages_fetched": manifest["pages_fetched"], + "reference_truncated": manifest["reference_truncated"], + "workspace": workspace, + } + print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 15f787c1..34df6b3b 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -42,6 +42,7 @@ ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py +${PYTHON_BIN} -m py_compile scripts/provision_mol_municipality_workspace.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py ${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py