feat: map the complete municipality of Mol
This commit is contained in:
@@ -7,6 +7,16 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 181 Complete Mol municipality workspace (2026-07-14)
|
||||
|
||||
- Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary.
|
||||
- Added auditable source artifacts and a manifest with page count, checksums, exact WGS84 bounds, municipality area, feature totals and truncation state; incomplete pagination now fails closed.
|
||||
- Provisioning remains explicit and imports Project, Area and Dataset records through existing canonical API routes and DatasetService/VectorFeatureService persistence rather than writing directly to PostGIS.
|
||||
- Made the complete `Mol Municipality Workbench` the preferred fresh-session context and the official municipality boundary its lightweight default layer, ahead of historical Postel validation projects.
|
||||
- Replaced large coordinate arrays and spread-based bounds calculations with streaming, memoized GeoJSON bounds so municipality-scale vector layers remain safe in MapLibre.
|
||||
- Removed per-feature ORM refreshes after vector import while retaining one flush and commit, avoiding tens of thousands of redundant queries for full-municipality datasets.
|
||||
- Added focused municipality clipping, truncation, persistence-scaling, runtime wiring and frontend-priority regression coverage. No migration or API contract changed.
|
||||
|
||||
## Sprint 180 Premium workbench UX hardening (2026-07-14)
|
||||
|
||||
- Rebuilt the workbench presentation hierarchy around grouped task navigation, a compact Mol context header and an optional selection-detail drawer instead of a permanent third column.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
GeoIntel Kempen is a GeoAI Workbench for the Belgian Kempen. It is designed as a portfolio-grade project combining GIS, remote sensing, raster/vector processing, computer vision, QA/QC and geospatial exports.
|
||||
|
||||
The primary operating focus is Mol. New workbench contexts, AOIs and operator samples start there, while the broader Kempen remains fully supported for cross-area validation and regional interoperability.
|
||||
The primary operating focus is Mol. New workbench contexts, AOIs and operator samples start there, while the broader Kempen remains fully supported for cross-area validation and regional interoperability. The live operator environment can provision a complete `Mol Municipality Workbench` with the official NIS `13025` municipality boundary and every intersecting GRB GBG building; smaller Mol zones remain analysis and validation contexts instead of the default municipal map.
|
||||
|
||||
GeoIntel is not a generic dashboard or chatbot. The core product is:
|
||||
|
||||
|
||||
@@ -490,6 +490,14 @@ background control. Prepare and execute that pack with the documented
|
||||
`prepare_operator_real_data_samples.py` and
|
||||
`run_mol_operational_validation.sh` commands in `scripts/README.md`.
|
||||
|
||||
For municipality-wide navigation, run
|
||||
`/app/scripts/provision_mol_municipality_workspace.py` inside the all-in-one
|
||||
container. It verifies the official Mol boundary (NIS `13025`), pages and clips
|
||||
all GRB GBG buildings, records checksums/provenance under persistent operator
|
||||
storage and imports both datasets through the existing HTTP service boundary.
|
||||
The command is explicit and idempotent; it is never executed during backend
|
||||
startup. See `scripts/README.md` for exact usage and refresh controls.
|
||||
|
||||
The current recommended local building model is
|
||||
`geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt` with tile size
|
||||
`512`, overlap `64` and confidence threshold `0.15`. Its SHA256 is
|
||||
|
||||
@@ -173,7 +173,6 @@ class VectorFeatureService:
|
||||
persisted.append(row)
|
||||
|
||||
if commit:
|
||||
db.flush()
|
||||
db.commit()
|
||||
for row in persisted:
|
||||
db.refresh(row)
|
||||
return persisted
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import Polygon, shape
|
||||
|
||||
from app.models import VectorFeature
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_provisioner():
|
||||
script_path = ROOT / "scripts" / "provision_mol_municipality_workspace.py"
|
||||
spec = importlib.util.spec_from_file_location("mol_municipality_provisioner", script_path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def feature(feature_id: str, coordinates: list[list[list[float]]]):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": {"type": "Polygon", "coordinates": coordinates},
|
||||
"properties": {"UIDN": feature_id},
|
||||
}
|
||||
|
||||
|
||||
def test_mol_provisioner_uses_official_identity_and_exact_boundary_clipping() -> None:
|
||||
module = load_provisioner()
|
||||
boundary = Polygon([(5.0, 51.0), (5.2, 51.0), (5.2, 51.2), (5.0, 51.2), (5.0, 51.0)])
|
||||
inside = feature("GBG.inside", [[[5.05, 51.05], [5.1, 51.05], [5.1, 51.1], [5.05, 51.1], [5.05, 51.05]]])
|
||||
crossing = feature("GBG.crossing", [[[5.18, 51.08], [5.22, 51.08], [5.22, 51.12], [5.18, 51.12], [5.18, 51.08]]])
|
||||
outside = feature("GBG.outside", [[[5.3, 51.3], [5.31, 51.3], [5.31, 51.31], [5.3, 51.31], [5.3, 51.3]]])
|
||||
pages = [
|
||||
(
|
||||
{"type": "FeatureCollection", "features": [inside, crossing, outside, inside]},
|
||||
"https://geo.api.vlaanderen.be/GRB/page-1",
|
||||
)
|
||||
]
|
||||
|
||||
buildings, summary = module.build_municipality_buildings(pages, boundary, max_features=10)
|
||||
|
||||
assert module.MUNICIPALITY_NIS_CODE == "13025"
|
||||
assert module.PROJECT_NAME == "Mol Municipality Workbench"
|
||||
assert len(buildings) == 2
|
||||
assert summary["bbox_feature_count"] == 3
|
||||
assert summary["outside_boundary_count"] == 1
|
||||
assert summary["clipped_at_boundary_count"] == 1
|
||||
assert summary["reference_truncated"] is False
|
||||
assert all(shape(item["geometry"]).within(boundary) for item in buildings)
|
||||
assert buildings[0]["properties"]["coverage_scope"] == "municipality"
|
||||
assert buildings[0]["properties"]["source_name"] == "grb"
|
||||
assert buildings[0]["properties"]["reference_layer_name"] == "buildings"
|
||||
assert buildings[1]["properties"]["clipped_to_municipality"] is True
|
||||
|
||||
|
||||
def test_mol_provisioner_refuses_a_truncated_municipality_dataset() -> None:
|
||||
module = load_provisioner()
|
||||
boundary = Polygon([(5.0, 51.0), (5.2, 51.0), (5.2, 51.2), (5.0, 51.2), (5.0, 51.0)])
|
||||
pages = [
|
||||
(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
feature("GBG.1", [[[5.01, 51.01], [5.02, 51.01], [5.02, 51.02], [5.01, 51.02], [5.01, 51.01]]]),
|
||||
feature("GBG.2", [[[5.03, 51.03], [5.04, 51.03], [5.04, 51.04], [5.03, 51.04], [5.03, 51.03]]]),
|
||||
],
|
||||
},
|
||||
"https://geo.api.vlaanderen.be/GRB/page-1",
|
||||
)
|
||||
]
|
||||
|
||||
with pytest.raises(RuntimeError, match="refusing a truncated municipality dataset"):
|
||||
module.build_municipality_buildings(pages, boundary, max_features=1)
|
||||
|
||||
|
||||
def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> None:
|
||||
class FakeSession:
|
||||
def __init__(self) -> None:
|
||||
self.added = []
|
||||
self.flushes = 0
|
||||
self.commits = 0
|
||||
self.refreshes = 0
|
||||
|
||||
def add(self, item) -> None:
|
||||
self.added.append(item)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def refresh(self, _item) -> None:
|
||||
self.refreshes += 1
|
||||
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"GBG.{index}",
|
||||
"properties": {"layer_type": "building"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[5.0, 51.0], [5.001, 51.0], [5.001, 51.001], [5.0, 51.001], [5.0, 51.0]]],
|
||||
},
|
||||
}
|
||||
for index in range(250)
|
||||
],
|
||||
}
|
||||
db = FakeSession()
|
||||
|
||||
persisted = VectorFeatureService.persist_geojson_features(db, uuid4(), payload, feature_class="buildings")
|
||||
|
||||
assert len(persisted) == 250
|
||||
assert all(isinstance(item, VectorFeature) for item in persisted)
|
||||
assert db.flushes == 1
|
||||
assert db.commits == 1
|
||||
assert db.refreshes == 0
|
||||
|
||||
|
||||
def test_municipality_workspace_is_wired_into_runtime_and_frontend_priority() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
focus = (ROOT / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8")
|
||||
project_hook = (ROOT / "frontend" / "src" / "hooks" / "useProjectWorkspace.ts").read_text(encoding="utf-8")
|
||||
dataset_hook = (ROOT / "frontend" / "src" / "hooks" / "useDatasetWorkflow.ts").read_text(encoding="utf-8")
|
||||
map_source = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(encoding="utf-8")
|
||||
map_workspace = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "py_compile scripts/provision_mol_municipality_workspace.py" in readiness
|
||||
assert "COPY scripts/provision_mol_municipality_workspace.py" in dockerfile
|
||||
assert "PRIMARY_FOCUS_MUNICIPALITY_PROJECT_NAME = 'Mol Municipality Workbench'" in focus
|
||||
assert "items.find(isPrimaryFocusMunicipalityProject)" in project_hook
|
||||
assert "datasets.find(isPrimaryFocusMunicipalityBoundaryDataset)" in dataset_hook
|
||||
assert "featureCollectionBounds(featureCollection)" in map_source
|
||||
assert "useMemo(() => getFeatureCollectionBBox(mapFeatureCollection)" in map_workspace
|
||||
assert "Math.min(...xs)" not in map_source
|
||||
|
||||
@@ -79,7 +79,9 @@ def test_vector_feature_service_persists_geojson_features_with_properties() -> N
|
||||
assert persisted[0].source_feature_id == "building-1"
|
||||
assert persisted[0].properties_json == {"class": "building", "height": 7}
|
||||
assert db.added == persisted
|
||||
assert db.flushes == 1
|
||||
assert db.commits == 1
|
||||
assert db.refreshes == []
|
||||
|
||||
|
||||
def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
|
||||
|
||||
@@ -72,6 +72,7 @@ RUN python scripts/gis_import_smoke.py \
|
||||
&& mkdir -p /app/storage /run/nginx /var/log/nginx
|
||||
|
||||
COPY scripts/prepare_operator_real_data_samples.py /app/scripts/prepare_operator_real_data_samples.py
|
||||
COPY scripts/provision_mol_municipality_workspace.py /app/scripts/provision_mol_municipality_workspace.py
|
||||
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
|
||||
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
|
||||
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
|
||||
|
||||
@@ -7395,3 +7395,41 @@ Open:
|
||||
## Next pass
|
||||
|
||||
- Harden detection QA coverage and matching diagnostics before making a retraining decision; keep that work separate from this presentation-only sprint.
|
||||
|
||||
# Sprint 181 - Complete Mol municipality workspace
|
||||
|
||||
## Implementation
|
||||
|
||||
- Added an explicit operator provisioner for the official VRBG `Refgem`
|
||||
municipality geometry for Mol (NIS `13025`) and the complete paged GRB `GBG`
|
||||
building collection clipped to that exact boundary.
|
||||
- Added deterministic persistent source artefacts and a manifest containing
|
||||
source URLs, checksums, page/feature counts, boundary bounds and area, and an
|
||||
explicit truncation flag. Pagination and identity checks fail closed.
|
||||
- Kept the provider boundary honest: provisioning is an operator action and
|
||||
imports through canonical Project, Area and Dataset HTTP routes. It does not
|
||||
enable the dormant live GRB provider or write directly to `vector_features`.
|
||||
- Optimized the existing vector persistence path by replacing one ORM refresh
|
||||
per feature with a single flush and commit. Persistence shape and API
|
||||
behavior remain unchanged.
|
||||
- Made `Mol Municipality Workbench` the preferred fresh-session context once
|
||||
its official boundary is ready. The boundary opens first; the much larger
|
||||
building layer remains explicitly selectable from the Map database-layer
|
||||
control.
|
||||
- Replaced spread-based map extent calculations with a streaming, memoized
|
||||
GeoJSON bounds helper and added municipality/building layer styling. This
|
||||
avoids large coordinate arrays while keeping the existing MapLibre path.
|
||||
|
||||
## Initial validation
|
||||
|
||||
- Exact clipping, pagination/truncation, persistence-scaling and frontend
|
||||
wiring regression coverage passed locally.
|
||||
- No API route, ORM model, migration, QA metric, detection result or model
|
||||
configuration changed.
|
||||
|
||||
## Next pass
|
||||
|
||||
- Deploy and execute the explicit provisioner on Tower, verify full PostGIS
|
||||
feature counts and exact Mol bounds, then audit the complete municipality in
|
||||
MapLibre before acquiring raster imagery for a deliberately selected Mol
|
||||
analysis zone.
|
||||
|
||||
@@ -13,6 +13,22 @@ Dit document verzamelt concrete databronnen voor GeoIntel Kempen.
|
||||
- Prioriteit: V1
|
||||
- Opmerking: belangrijkste officiële referentiebron voor QA/QC.
|
||||
|
||||
### Volledige gemeente Mol
|
||||
|
||||
De operationele Mol-workspace gebruikt twee expliciete Digitaal Vlaanderen
|
||||
OGC API-bronnen:
|
||||
|
||||
- `VRBG/Refgem`, gefilterd op `NAAM='Mol'` en gecontroleerd op NIS-code
|
||||
`13025`, als officiële gemeentegrens.
|
||||
- `GRB/GBG`, volledig gepagineerd en exact tegen die grens gesneden, als
|
||||
gemeentebrede gebouwreferentie.
|
||||
|
||||
Deze bronnen worden uitsluitend via het expliciete operatorcommando
|
||||
`scripts/provision_mol_municipality_workspace.py` opgehaald. De applicatie
|
||||
start geen verborgen providerfetch. De resulterende GeoJSON-artefacten,
|
||||
checksums en bron-URL's worden onder persistent operator storage bewaard en
|
||||
via de bestaande DatasetService/vectorfeature-flow geïmporteerd.
|
||||
|
||||
## OSM
|
||||
|
||||
- Naam: OpenStreetMap
|
||||
|
||||
@@ -513,6 +513,18 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Train and reject `geointel-building-yolov8s-aoi1024cleanpx12vis035e50-pt` through the positive/background promotion gate.
|
||||
- [x] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation.
|
||||
|
||||
# Sprint 181 - Complete Mol municipality workspace
|
||||
|
||||
- [x] Persist the official VRBG municipality boundary for Mol (NIS `13025`).
|
||||
- [x] Page, deduplicate and boundary-clip the complete GRB GBG building layer.
|
||||
- [x] Import municipality layers through the existing DatasetService and
|
||||
VectorFeatureService path, with no direct provider-to-database write.
|
||||
- [x] Prefer the complete municipality workspace on a fresh session while
|
||||
keeping the lightweight boundary as the initially rendered layer.
|
||||
- [x] Make GeoJSON bounds calculation safe for municipality-scale layers.
|
||||
- [ ] Acquire and tile a georeferenced raster only for an explicitly selected
|
||||
Mol analysis zone before running the next configured-YOLO validation.
|
||||
|
||||
# Sprint 171 - Positive AOI expansion and small-building recovery
|
||||
|
||||
- [x] Reject cross-model false-negative comparisons when reference populations differ.
|
||||
|
||||
+3
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
React + TypeScript + MapLibre foundation for project/area/dataset workflow.
|
||||
|
||||
Mol is the primary operating context. On a fresh session the workbench prefers a persisted Mol project or dataset, centers an empty MapLibre view on Mol and pre-fills a compact Mol AOI. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available.
|
||||
Mol is the primary operating context. On a fresh session the workbench first prefers the complete persisted `Mol Municipality Workbench` when its official NIS `13025` boundary is ready. Its lightweight municipality boundary opens before the complete GRB building reference layer, so initial map framing remains responsive while all municipality buildings stay selectable from the database-layer control. Explicit project and dataset selections remain authoritative, and all broader Kempen workflows remain available.
|
||||
|
||||
The workbench uses a task-based shell instead of a single long panel stack. `App.tsx` still owns shared orchestration state, but the UI is organized into Overview, Data, Map, QA/QC, AI Labs, Exports and System workspaces with a persistent top context bar and an optional selection-detail drawer.
|
||||
|
||||
@@ -14,6 +14,8 @@ Wide and ultrawide screens keep a readable sidebar and centered work area, expan
|
||||
|
||||
The Map workspace defaults to an OpenStreetMap road basemap with visible attribution so uploaded vectors, AOIs and QA overlays appear on a real street context. Set `VITE_MAP_STYLE_URL` to a managed MapLibre style URL to override this for production or high-volume deployments.
|
||||
|
||||
Municipality-scale GeoJSON bounds are scanned incrementally and memoized instead of materializing coordinate arrays. This supports the complete Mol boundary and roughly 37k persisted GRB buildings without JavaScript argument-spread failures. The official boundary uses a dark teal map treatment, GRB buildings use cyan/teal and existing detection/change-result colors remain distinct.
|
||||
|
||||
When the public OpenStreetMap fallback is active, the Map workspace shows a basemap usage notice. This keeps the local/demo default honest and reminds operators to configure a managed style URL before production or heavier tile traffic.
|
||||
|
||||
Operational GIS testing is now available directly in the Map workspace. Users can choose a persisted vector database layer, load it on the map, reuse the selected AOI or active layer extent, run the existing persisted `vector_features` bbox query, save the result as a derived dataset, export the selection GeoJSON, choose a reference dataset and launch QA/QC without creating fake data or a parallel backend path. The guided workflow also includes a one-click full run action that executes query, derived dataset save, GeoJSON export and optional QA/QC in sequence with visible status. For repeated review, switch the run mode from `Create new dataset/export` to `Reuse latest saved dataset for QA`; this reruns QA/QC against the latest saved derived dataset without creating another dataset/export pair.
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from 'react'
|
||||
import maplibregl from 'maplibre-gl'
|
||||
import 'maplibre-gl/dist/maplibre-gl.css'
|
||||
import { PRIMARY_FOCUS_CENTER } from '../config/primaryFocus'
|
||||
import { featureCollectionBounds } from '../lib/geojsonBounds'
|
||||
|
||||
interface GeoMapProps {
|
||||
data: GeoJSON.FeatureCollection | null
|
||||
@@ -49,36 +50,13 @@ function defaultMapStyle(): string | maplibregl.StyleSpecification {
|
||||
}
|
||||
|
||||
function collectCoordinates(featureCollection: GeoJSON.FeatureCollection): maplibregl.LngLatBoundsLike | null {
|
||||
const coordinates: [number, number][] = []
|
||||
const walk = (coords: unknown) => {
|
||||
if (!Array.isArray(coords)) {
|
||||
return
|
||||
}
|
||||
if (coords.length === 2 && typeof coords[0] === 'number' && typeof coords[1] === 'number') {
|
||||
coordinates.push([coords[0], coords[1]])
|
||||
return
|
||||
}
|
||||
for (const item of coords) {
|
||||
walk(item)
|
||||
}
|
||||
}
|
||||
|
||||
for (const feature of featureCollection.features) {
|
||||
const geometry = feature.geometry as any
|
||||
if (geometry && geometry.coordinates) {
|
||||
walk(geometry.coordinates)
|
||||
}
|
||||
}
|
||||
|
||||
if (coordinates.length === 0) {
|
||||
const bounds = featureCollectionBounds(featureCollection)
|
||||
if (!bounds) {
|
||||
return null
|
||||
}
|
||||
|
||||
const xs = coordinates.map((point) => point[0])
|
||||
const ys = coordinates.map((point) => point[1])
|
||||
return [
|
||||
[Math.min(...xs), Math.min(...ys)],
|
||||
[Math.max(...xs), Math.max(...ys)],
|
||||
[bounds.minX, bounds.minY],
|
||||
[bounds.maxX, bounds.maxY],
|
||||
]
|
||||
}
|
||||
|
||||
@@ -244,15 +222,22 @@ function GeoMap({
|
||||
source: 'dataset',
|
||||
paint: {
|
||||
'fill-color': [
|
||||
'match',
|
||||
['get', 'change_type'],
|
||||
'added',
|
||||
'#16a34a',
|
||||
'removed',
|
||||
'#dc2626',
|
||||
'unchanged',
|
||||
'#2563eb',
|
||||
'#f97316',
|
||||
'case',
|
||||
['==', ['get', 'layer_type'], 'municipality_boundary'],
|
||||
'#0f766e',
|
||||
['==', ['get', 'layer_type'], 'building'],
|
||||
'#0891b2',
|
||||
[
|
||||
'match',
|
||||
['get', 'change_type'],
|
||||
'added',
|
||||
'#16a34a',
|
||||
'removed',
|
||||
'#dc2626',
|
||||
'unchanged',
|
||||
'#2563eb',
|
||||
'#f97316',
|
||||
],
|
||||
],
|
||||
'fill-opacity': 0.4,
|
||||
},
|
||||
@@ -263,17 +248,24 @@ function GeoMap({
|
||||
source: 'dataset',
|
||||
paint: {
|
||||
'line-color': [
|
||||
'match',
|
||||
['get', 'change_type'],
|
||||
'added',
|
||||
'#15803d',
|
||||
'removed',
|
||||
'#b91c1c',
|
||||
'unchanged',
|
||||
'#1d4ed8',
|
||||
'#ea580c',
|
||||
'case',
|
||||
['==', ['get', 'layer_type'], 'municipality_boundary'],
|
||||
'#0f5f59',
|
||||
['==', ['get', 'layer_type'], 'building'],
|
||||
'#0e7490',
|
||||
[
|
||||
'match',
|
||||
['get', 'change_type'],
|
||||
'added',
|
||||
'#15803d',
|
||||
'removed',
|
||||
'#b91c1c',
|
||||
'unchanged',
|
||||
'#1d4ed8',
|
||||
'#ea580c',
|
||||
],
|
||||
],
|
||||
'line-width': 2,
|
||||
'line-width': ['interpolate', ['linear'], ['zoom'], 8, 0.25, 12, 0.8, 16, 2],
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import GeoMap from '../GeoMap'
|
||||
import type { AreaRead, DatasetCreateResponse, QaComparisonResult, VectorSelectionBBox, VectorSelectionResponse } from '../../types'
|
||||
import { featureCollectionBounds } from '../../lib/geojsonBounds'
|
||||
|
||||
const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
|
||||
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
|
||||
@@ -55,17 +56,15 @@ function getFeatureGeometrySummary(feature: GeoJSON.Feature | null) {
|
||||
}
|
||||
|
||||
function getFeatureCollectionBBox(collection: GeoJSON.FeatureCollection | null): VectorSelectionBBox | null {
|
||||
const points = collection?.features.flatMap((feature) => collectGeometryPoints(feature.geometry)) ?? []
|
||||
if (points.length === 0) {
|
||||
const bounds = featureCollectionBounds(collection)
|
||||
if (!bounds) {
|
||||
return null
|
||||
}
|
||||
const xs = points.map((point) => point[0])
|
||||
const ys = points.map((point) => point[1])
|
||||
return {
|
||||
min_x: Math.min(...xs),
|
||||
min_y: Math.min(...ys),
|
||||
max_x: Math.max(...xs),
|
||||
max_y: Math.max(...ys),
|
||||
min_x: bounds.minX,
|
||||
min_y: bounds.minY,
|
||||
max_x: bounds.maxX,
|
||||
max_y: bounds.maxY,
|
||||
crs: 'EPSG:4326',
|
||||
}
|
||||
}
|
||||
@@ -303,9 +302,9 @@ export function MapWorkspace({
|
||||
const featureExtractionEntries = featureProperties ? Object.entries(featureProperties).slice(0, 48) : []
|
||||
const featureGeometrySummary = getFeatureGeometrySummary(selectedMapFeature)
|
||||
const selectedFeatureGeoJson = selectedMapFeature ? selectedFeatureCollection(selectedMapFeature) : null
|
||||
const selectedFeatureBbox = getFeatureBBox(selectedMapFeature)
|
||||
const activeLayerBbox = getFeatureCollectionBBox(mapFeatureCollection)
|
||||
const selectedAreaBbox = getFeatureCollectionBBox(areaFeatureCollection)
|
||||
const selectedFeatureBbox = useMemo(() => getFeatureBBox(selectedMapFeature), [selectedMapFeature])
|
||||
const activeLayerBbox = useMemo(() => getFeatureCollectionBBox(mapFeatureCollection), [mapFeatureCollection])
|
||||
const selectedAreaBbox = useMemo(() => getFeatureCollectionBBox(areaFeatureCollection), [areaFeatureCollection])
|
||||
const currentSelectionBbox = parseBboxInput(bboxInput)
|
||||
const areaSelectionFeatures = mapSelectionResult?.geojson.features ?? []
|
||||
const areaSelectionPreviewFeatures = areaSelectionFeatures.slice(0, 12)
|
||||
|
||||
@@ -4,6 +4,7 @@ export const PRIMARY_FOCUS_LABEL = 'Mol'
|
||||
export const PRIMARY_FOCUS_REGION = 'Mol, Kempen'
|
||||
export const PRIMARY_FOCUS_CENTER: [number, number] = [5.1167, 51.1919]
|
||||
export const PRIMARY_FOCUS_AREA_NAME = 'Mol primary AOI'
|
||||
export const PRIMARY_FOCUS_MUNICIPALITY_PROJECT_NAME = 'Mol Municipality Workbench'
|
||||
export const PRIMARY_FOCUS_AREA_GEOJSON =
|
||||
'{"type":"MultiPolygon","coordinates":[[[[5.109476746222376,51.18745107550715],[5.123924622861509,51.18745107550715],[5.123924622861509,51.19634848029239],[5.109476746222376,51.19634848029239],[5.109476746222376,51.18745107550715]]]]}'
|
||||
|
||||
@@ -17,6 +18,20 @@ export function isPrimaryFocusProject(project: ProjectRead): boolean {
|
||||
return containsPrimaryFocus([project.name, project.description, project.region])
|
||||
}
|
||||
|
||||
export function isPrimaryFocusMunicipalityProject(project: ProjectRead): boolean {
|
||||
return project.name === PRIMARY_FOCUS_MUNICIPALITY_PROJECT_NAME
|
||||
}
|
||||
|
||||
export function isPrimaryFocusMunicipalityBoundaryDataset(dataset: DatasetCreateResponse): boolean {
|
||||
return (
|
||||
dataset.status === 'ready' &&
|
||||
(dataset.dataset_type === 'vector' || dataset.dataset_type === 'geojson') &&
|
||||
dataset.source_name === 'vrbg' &&
|
||||
dataset.source_metadata?.coverage_scope === 'municipality' &&
|
||||
dataset.source_metadata?.nis_code === '13025'
|
||||
)
|
||||
}
|
||||
|
||||
export function isPrimaryFocusProjectData(
|
||||
project: ProjectRead,
|
||||
datasets: DatasetCreateResponse[],
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
VectorSummary,
|
||||
} from '../types'
|
||||
import { formatError } from '../lib/formatError'
|
||||
import { isPrimaryFocusMunicipalityBoundaryDataset } from '../config/primaryFocus'
|
||||
|
||||
interface DatasetWorkflowOptions {
|
||||
selectedProjectId: string | null
|
||||
@@ -128,6 +129,7 @@ export function useDatasetWorkflow({
|
||||
}
|
||||
// Auto-open the first usable dataset so Data, Map and Exports start with real context.
|
||||
const defaultDataset =
|
||||
datasets.find(isPrimaryFocusMunicipalityBoundaryDataset) ??
|
||||
datasets.find((dataset) => isVectorDatasetType(dataset.dataset_type) && dataset.status === 'ready') ??
|
||||
datasets.find((dataset) => dataset.status === 'ready') ??
|
||||
datasets[0]
|
||||
|
||||
@@ -3,6 +3,8 @@ import {
|
||||
PRIMARY_FOCUS_AREA_GEOJSON,
|
||||
PRIMARY_FOCUS_AREA_NAME,
|
||||
PRIMARY_FOCUS_REGION,
|
||||
isPrimaryFocusMunicipalityBoundaryDataset,
|
||||
isPrimaryFocusMunicipalityProject,
|
||||
isPrimaryFocusProject,
|
||||
isPrimaryFocusProjectData,
|
||||
} from '../config/primaryFocus'
|
||||
@@ -76,6 +78,17 @@ export function useProjectWorkspace() {
|
||||
if (selectedProjectId && items.some((project) => project.id === selectedProjectId)) {
|
||||
return selectedProjectId
|
||||
}
|
||||
const municipalityProject = items.find(isPrimaryFocusMunicipalityProject)
|
||||
if (municipalityProject) {
|
||||
try {
|
||||
const data = await fetchProjectData(municipalityProject.id)
|
||||
if (data.areas.length > 0 && data.datasets.some(isPrimaryFocusMunicipalityBoundaryDataset)) {
|
||||
return municipalityProject.id
|
||||
}
|
||||
} catch {
|
||||
// Continue with the normal completeness ranking if the canonical workspace is temporarily unavailable.
|
||||
}
|
||||
}
|
||||
const primaryProjects = items.filter(isPrimaryFocusProject)
|
||||
const demoProjects = items.filter(isDemoProject)
|
||||
const candidates = Array.from(
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
export interface GeoJsonBounds {
|
||||
minX: number
|
||||
minY: number
|
||||
maxX: number
|
||||
maxY: number
|
||||
}
|
||||
|
||||
function extendBounds(bounds: GeoJsonBounds | null, x: number, y: number): GeoJsonBounds {
|
||||
if (!bounds) {
|
||||
return { minX: x, minY: y, maxX: x, maxY: y }
|
||||
}
|
||||
bounds.minX = Math.min(bounds.minX, x)
|
||||
bounds.minY = Math.min(bounds.minY, y)
|
||||
bounds.maxX = Math.max(bounds.maxX, x)
|
||||
bounds.maxY = Math.max(bounds.maxY, y)
|
||||
return bounds
|
||||
}
|
||||
|
||||
function scanCoordinates(coordinates: unknown, initial: GeoJsonBounds | null): GeoJsonBounds | null {
|
||||
let bounds = initial
|
||||
const stack: unknown[] = [coordinates]
|
||||
while (stack.length > 0) {
|
||||
const value = stack.pop()
|
||||
if (!Array.isArray(value)) {
|
||||
continue
|
||||
}
|
||||
if (value.length >= 2 && typeof value[0] === 'number' && typeof value[1] === 'number') {
|
||||
if (Number.isFinite(value[0]) && Number.isFinite(value[1])) {
|
||||
bounds = extendBounds(bounds, value[0], value[1])
|
||||
}
|
||||
continue
|
||||
}
|
||||
for (const child of value) {
|
||||
stack.push(child)
|
||||
}
|
||||
}
|
||||
return bounds
|
||||
}
|
||||
|
||||
export function geometryBounds(geometry: GeoJSON.Geometry | null | undefined): GeoJsonBounds | null {
|
||||
if (!geometry) {
|
||||
return null
|
||||
}
|
||||
if (geometry.type === 'GeometryCollection') {
|
||||
return geometry.geometries.reduce<GeoJsonBounds | null>(
|
||||
(bounds, child) => mergeBounds(bounds, geometryBounds(child)),
|
||||
null,
|
||||
)
|
||||
}
|
||||
return scanCoordinates(geometry.coordinates, null)
|
||||
}
|
||||
|
||||
export function mergeBounds(left: GeoJsonBounds | null, right: GeoJsonBounds | null): GeoJsonBounds | null {
|
||||
if (!left) {
|
||||
return right ? { ...right } : null
|
||||
}
|
||||
if (!right) {
|
||||
return left
|
||||
}
|
||||
left.minX = Math.min(left.minX, right.minX)
|
||||
left.minY = Math.min(left.minY, right.minY)
|
||||
left.maxX = Math.max(left.maxX, right.maxX)
|
||||
left.maxY = Math.max(left.maxY, right.maxY)
|
||||
return left
|
||||
}
|
||||
|
||||
export function featureCollectionBounds(collection: GeoJSON.FeatureCollection | null | undefined): GeoJsonBounds | null {
|
||||
if (!collection) {
|
||||
return null
|
||||
}
|
||||
return collection.features.reduce<GeoJsonBounds | null>(
|
||||
(bounds, feature) => mergeBounds(bounds, geometryBounds(feature.geometry)),
|
||||
null,
|
||||
)
|
||||
}
|
||||
@@ -1131,6 +1131,32 @@ thresholds without changing the script. The main readiness gate checks this
|
||||
script's syntax; run it explicitly against Docker/PostGIS when validating a
|
||||
live deployment.
|
||||
|
||||
## Complete Mol municipality workspace
|
||||
|
||||
Provision the official municipality boundary and every GRB GBG building that
|
||||
intersects it through the existing project, area, dataset and vector-feature
|
||||
persistence paths:
|
||||
|
||||
```bash
|
||||
docker exec -it geointel python3 \
|
||||
/app/scripts/provision_mol_municipality_workspace.py
|
||||
```
|
||||
|
||||
The command queries `VRBG/Refgem` for municipality `Mol`, verifies NIS code
|
||||
`13025`, follows every `GRB/GBG` pagination link and clips the resulting
|
||||
buildings to the official boundary. It writes source artefacts and a checksum
|
||||
manifest below `/app/storage/operator-data/mol-municipality`, then creates or
|
||||
reuses `Mol Municipality Workbench` and imports both layers through the public
|
||||
API. It never writes directly to PostGIS and never runs implicitly at startup.
|
||||
|
||||
Completed artefacts and ready datasets are reused on a repeat run. Use
|
||||
`--force` only when an operator deliberately wants to refetch and replace the
|
||||
local source artefacts. Use `--fetch-only` to prepare and inspect the manifest
|
||||
without changing application persistence. The internal API default is
|
||||
`http://127.0.0.1:8000`, avoiding proxy timeouts during the large vector
|
||||
import; override it with `--base-url` when running outside the all-in-one
|
||||
container.
|
||||
|
||||
## Tower deployment
|
||||
|
||||
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
|
||||
|
||||
@@ -0,0 +1,604 @@
|
||||
"""Provision a complete, authoritative Mol municipality map workspace.
|
||||
|
||||
The script is an explicit operator tool. It fetches the official Mol boundary
|
||||
and GRB buildings, writes auditable artifacts, and imports them through the
|
||||
existing GeoIntel API. It is never called automatically by application startup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
import requests
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
||||
from shapely.ops import transform, unary_union
|
||||
from shapely.validation import make_valid
|
||||
|
||||
|
||||
MUNICIPALITY_NAME = "Mol"
|
||||
MUNICIPALITY_NIS_CODE = "13025"
|
||||
PROJECT_NAME = "Mol Municipality Workbench"
|
||||
PROJECT_REGION = "Mol, Kempen"
|
||||
AREA_NAME = "Gemeente Mol - officiële grens"
|
||||
BOUNDARY_FILENAME = "mol_municipality_boundary.geojson"
|
||||
BUILDINGS_FILENAME = "mol_grb_gbg_buildings.geojson"
|
||||
MANIFEST_FILENAME = "mol_municipality_manifest.json"
|
||||
VRBG_ITEMS_URL = "https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items"
|
||||
GRB_GBG_ITEMS_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
|
||||
VRBG_ATTRIBUTION = "Bron: Voorlopig referentiebestand gemeentegrenzen, Digitaal Vlaanderen"
|
||||
GRB_ATTRIBUTION = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
|
||||
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/mol-municipality")
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_PAGE_LIMIT = 1000
|
||||
DEFAULT_MAX_FEATURES = 100000
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Fetch and provision the complete official Mol municipality workspace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("MOL_MUNICIPALITY_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)),
|
||||
help="Persistent directory for the boundary, GRB buildings and provenance manifest.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL),
|
||||
help="GeoIntel backend URL. The in-container direct backend avoids reverse-proxy timeouts during large imports.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project-name",
|
||||
default=PROJECT_NAME,
|
||||
help="Exact idempotent project name used for the municipality workspace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--page-limit",
|
||||
type=int,
|
||||
default=int(os.environ.get("MOL_GRB_PAGE_LIMIT", str(DEFAULT_PAGE_LIMIT))),
|
||||
help="GRB OGC API page size.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-features",
|
||||
type=int,
|
||||
default=int(os.environ.get("MOL_GRB_MAX_FEATURES", str(DEFAULT_MAX_FEATURES))),
|
||||
help="Safety cap. The script fails instead of writing a truncated municipality dataset.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--request-timeout",
|
||||
type=int,
|
||||
default=int(os.environ.get("MOL_MUNICIPALITY_REQUEST_TIMEOUT", "180")),
|
||||
help="Timeout per official source request in seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--import-timeout",
|
||||
type=int,
|
||||
default=int(os.environ.get("MOL_MUNICIPALITY_IMPORT_TIMEOUT", "1800")),
|
||||
help="Timeout for each GeoIntel API import request in seconds.",
|
||||
)
|
||||
parser.add_argument("--force", action="store_true", help="Refetch official source artifacts even when complete local artifacts exist.")
|
||||
parser.add_argument("--fetch-only", action="store_true", help="Prepare artifacts without creating or updating the GeoIntel workspace.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
indent=2 if pretty else None,
|
||||
separators=None if pretty else (",", ":"),
|
||||
sort_keys=pretty,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def next_page_url(payload: dict[str, Any]) -> str | None:
|
||||
links = payload.get("links") or []
|
||||
for link in links:
|
||||
if link.get("rel") == "next" and "geo+json" in str(link.get("type", "")).lower():
|
||||
return str(link["href"])
|
||||
for link in links:
|
||||
if link.get("rel") == "next" and link.get("href"):
|
||||
return str(link["href"])
|
||||
return None
|
||||
|
||||
|
||||
def normalize_polygonal(geometry):
|
||||
if geometry.is_empty:
|
||||
return None
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if isinstance(geometry, (Polygon, MultiPolygon)):
|
||||
return geometry
|
||||
if isinstance(geometry, GeometryCollection):
|
||||
polygons = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty]
|
||||
if polygons:
|
||||
merged = unary_union(polygons)
|
||||
return merged if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty else None
|
||||
return None
|
||||
|
||||
|
||||
def fetch_mol_boundary(session: requests.Session, timeout: int) -> tuple[dict[str, Any], Any, str]:
|
||||
params = {
|
||||
"f": "application/geo+json",
|
||||
"limit": "10",
|
||||
"filter": "NAAM='Mol'",
|
||||
"filter-lang": "cql2-text",
|
||||
}
|
||||
response = session.get(VRBG_ITEMS_URL, params=params, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
matches = [
|
||||
feature
|
||||
for feature in payload.get("features") or []
|
||||
if str((feature.get("properties") or {}).get("NAAM", "")).casefold() == MUNICIPALITY_NAME.casefold()
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError(f"Expected exactly one official Mol boundary, received {len(matches)}")
|
||||
|
||||
source_feature = matches[0]
|
||||
properties = dict(source_feature.get("properties") or {})
|
||||
if str(properties.get("NISCODE")) != MUNICIPALITY_NIS_CODE:
|
||||
raise RuntimeError(
|
||||
f"Official Mol boundary NIS code drifted: expected {MUNICIPALITY_NIS_CODE}, received {properties.get('NISCODE')}"
|
||||
)
|
||||
boundary = normalize_polygonal(shape(source_feature.get("geometry")))
|
||||
if boundary is None or not boundary.is_valid:
|
||||
raise RuntimeError("Official Mol boundary is empty, non-polygonal or invalid")
|
||||
|
||||
source_url = response.url
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "vrbg",
|
||||
"source_feature_id": str(source_feature.get("id") or MUNICIPALITY_NIS_CODE),
|
||||
"layer_type": "municipality_boundary",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": "municipality",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
"source_url": source_url,
|
||||
}
|
||||
)
|
||||
feature = {
|
||||
"type": "Feature",
|
||||
"id": str(source_feature.get("id") or f"Refgem.{MUNICIPALITY_NIS_CODE}"),
|
||||
"geometry": mapping(boundary),
|
||||
"properties": properties,
|
||||
}
|
||||
return feature, boundary, source_url
|
||||
|
||||
|
||||
def iter_grb_pages(
|
||||
session: requests.Session,
|
||||
boundary_bounds: tuple[float, float, float, float],
|
||||
*,
|
||||
page_limit: int,
|
||||
timeout: int,
|
||||
) -> Iterable[tuple[dict[str, Any], str]]:
|
||||
params = {
|
||||
"f": "application/geo+json",
|
||||
"limit": str(page_limit),
|
||||
"bbox": ",".join(f"{value:.8f}" for value in boundary_bounds),
|
||||
}
|
||||
url: str | None = GRB_GBG_ITEMS_URL
|
||||
seen_urls: set[str] = set()
|
||||
first_request = True
|
||||
while url:
|
||||
if url in seen_urls:
|
||||
raise RuntimeError(f"GRB pagination loop detected: {url}")
|
||||
seen_urls.add(url)
|
||||
response = session.get(url, params=params if first_request else None, timeout=timeout)
|
||||
first_request = False
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
yield payload, response.url
|
||||
url = next_page_url(payload)
|
||||
|
||||
|
||||
def build_municipality_buildings(
|
||||
pages: Iterable[tuple[dict[str, Any], str]],
|
||||
boundary,
|
||||
*,
|
||||
max_features: int,
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
features: list[dict[str, Any]] = []
|
||||
source_urls: list[str] = []
|
||||
seen_ids: set[str] = set()
|
||||
bbox_feature_count = 0
|
||||
outside_boundary_count = 0
|
||||
clipped_at_boundary_count = 0
|
||||
|
||||
for page, source_url in pages:
|
||||
source_urls.append(source_url)
|
||||
for source_feature in page.get("features") or []:
|
||||
feature_id = str(source_feature.get("id") or "")
|
||||
if not feature_id:
|
||||
feature_id = hashlib.sha256(
|
||||
json.dumps(source_feature.get("geometry"), sort_keys=True).encode("utf-8")
|
||||
).hexdigest()
|
||||
if feature_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(feature_id)
|
||||
bbox_feature_count += 1
|
||||
|
||||
source_geometry = normalize_polygonal(shape(source_feature.get("geometry")))
|
||||
if source_geometry is None or not source_geometry.intersects(boundary):
|
||||
outside_boundary_count += 1
|
||||
continue
|
||||
within_boundary = source_geometry.within(boundary)
|
||||
clipped_geometry = source_geometry if within_boundary else normalize_polygonal(source_geometry.intersection(boundary))
|
||||
if clipped_geometry is None:
|
||||
outside_boundary_count += 1
|
||||
continue
|
||||
if not within_boundary:
|
||||
clipped_at_boundary_count += 1
|
||||
if len(features) >= max_features:
|
||||
raise RuntimeError(
|
||||
f"Mol contains more than the configured {max_features} GRB features; refusing a truncated municipality dataset"
|
||||
)
|
||||
|
||||
properties = dict(source_feature.get("properties") or {})
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "grb",
|
||||
"source_feature_id": feature_id,
|
||||
"reference_layer_name": "buildings",
|
||||
"layer_type": "building",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": "municipality",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"clipped_to_municipality": not within_boundary,
|
||||
"attribution": GRB_ATTRIBUTION,
|
||||
}
|
||||
)
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": mapping(clipped_geometry),
|
||||
"properties": properties,
|
||||
}
|
||||
)
|
||||
|
||||
if not features:
|
||||
raise RuntimeError("No GRB buildings intersect the official Mol municipality boundary")
|
||||
return features, {
|
||||
"pages_fetched": len(source_urls),
|
||||
"source_urls": source_urls,
|
||||
"bbox_feature_count": bbox_feature_count,
|
||||
"municipality_feature_count": len(features),
|
||||
"outside_boundary_count": outside_boundary_count,
|
||||
"clipped_at_boundary_count": clipped_at_boundary_count,
|
||||
"reference_truncated": False,
|
||||
}
|
||||
|
||||
|
||||
def municipality_area_km2(boundary) -> float:
|
||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
return float(transform(transformer.transform, boundary).area / 1_000_000)
|
||||
|
||||
|
||||
def prepare_artifacts(args: argparse.Namespace) -> tuple[Path, Path, dict[str, Any]]:
|
||||
output_dir: Path = args.output_dir
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
boundary_path = output_dir / BOUNDARY_FILENAME
|
||||
buildings_path = output_dir / BUILDINGS_FILENAME
|
||||
manifest_path = output_dir / MANIFEST_FILENAME
|
||||
|
||||
if not args.force and boundary_path.exists() and buildings_path.exists() and manifest_path.exists():
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if manifest.get("status") == "complete" and int(manifest.get("municipality_feature_count") or 0) > 0:
|
||||
return boundary_path, buildings_path, manifest
|
||||
|
||||
if args.page_limit <= 0 or args.max_features <= 0:
|
||||
raise RuntimeError("--page-limit and --max-features must be positive integers")
|
||||
|
||||
with requests.Session() as session:
|
||||
session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
|
||||
boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout)
|
||||
buildings, building_summary = build_municipality_buildings(
|
||||
iter_grb_pages(
|
||||
session,
|
||||
boundary.bounds,
|
||||
page_limit=args.page_limit,
|
||||
timeout=args.request_timeout,
|
||||
),
|
||||
boundary,
|
||||
max_features=args.max_features,
|
||||
)
|
||||
|
||||
generated_at = utc_now()
|
||||
boundary_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"name": "Official municipality boundary - Mol",
|
||||
"features": [boundary_feature],
|
||||
"source": "Digitaal Vlaanderen VRBG OGC API Features collection Refgem",
|
||||
"source_url": boundary_source_url,
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"coverage_scope": "municipality",
|
||||
"generated_at": generated_at,
|
||||
}
|
||||
buildings_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"name": "GRB buildings - complete municipality Mol",
|
||||
"features": buildings,
|
||||
"source": "Digitaal Vlaanderen GRB OGC API Features collection GBG",
|
||||
"source_url": building_summary["source_urls"][0],
|
||||
"attribution": GRB_ATTRIBUTION,
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"coverage_scope": "municipality",
|
||||
"reference_truncated": False,
|
||||
"generated_at": generated_at,
|
||||
}
|
||||
write_json(boundary_path, boundary_payload)
|
||||
write_json(buildings_path, buildings_payload)
|
||||
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "complete",
|
||||
"generated_at": generated_at,
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"area_km2": municipality_area_km2(boundary),
|
||||
"wgs84_bbox": list(boundary.bounds),
|
||||
"boundary_path": str(boundary_path),
|
||||
"boundary_sha256": sha256_file(boundary_path),
|
||||
"buildings_path": str(buildings_path),
|
||||
"buildings_sha256": sha256_file(buildings_path),
|
||||
"page_limit": args.page_limit,
|
||||
"max_features": args.max_features,
|
||||
"boundary_source_url": boundary_source_url,
|
||||
"attribution": {"boundary": VRBG_ATTRIBUTION, "buildings": GRB_ATTRIBUTION},
|
||||
**building_summary,
|
||||
}
|
||||
write_json(manifest_path, manifest, pretty=True)
|
||||
return boundary_path, buildings_path, manifest
|
||||
|
||||
|
||||
def response_data(response: requests.Response) -> Any:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"GeoIntel API returned non-JSON response ({response.status_code}): {response.text[:500]}") from exc
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"GeoIntel API request failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:1000]}")
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def find_or_create_project(session: requests.Session, base_url: str, project_name: str, timeout: int) -> dict[str, Any]:
|
||||
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
|
||||
existing = next((item for item in projects.get("items") or [] if item.get("name") == project_name), None)
|
||||
if existing:
|
||||
return existing
|
||||
return response_data(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects",
|
||||
json={
|
||||
"name": project_name,
|
||||
"description": (
|
||||
"Complete municipality workspace for Mol using the official Digitaal Vlaanderen municipality boundary "
|
||||
"and the full GRB GBG building reference layer clipped to NIS 13025."
|
||||
),
|
||||
"region": PROJECT_REGION,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def find_or_create_area(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
boundary_path: Path,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
areas = response_data(
|
||||
session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout)
|
||||
)
|
||||
existing = next((item for item in areas.get("items") or [] if item.get("name") == AREA_NAME), None)
|
||||
if existing:
|
||||
return existing
|
||||
boundary_payload = json.loads(boundary_path.read_text(encoding="utf-8"))
|
||||
geometry = boundary_payload["features"][0]["geometry"]
|
||||
return response_data(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/areas",
|
||||
json={"name": AREA_NAME, "crs": "EPSG:4326", "geometry": geometry},
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upload_dataset(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area_id: str,
|
||||
path: Path,
|
||||
*,
|
||||
dataset_role: str,
|
||||
source_name: str,
|
||||
reference_layer_name: str | None,
|
||||
source_metadata: dict[str, Any],
|
||||
provenance_metadata: dict[str, Any],
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
form = {
|
||||
"dataset_type": "vector",
|
||||
"source": "operator_official_import",
|
||||
"dataset_role": dataset_role,
|
||||
"source_name": source_name,
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
}
|
||||
if reference_layer_name:
|
||||
form["reference_layer_name"] = reference_layer_name
|
||||
with path.open("rb") as handle:
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||
data=form,
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
)
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def provision_workspace(
|
||||
args: argparse.Namespace,
|
||||
boundary_path: Path,
|
||||
buildings_path: Path,
|
||||
manifest: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
base_url = args.base_url.rstrip("/")
|
||||
with requests.Session() as session:
|
||||
project = find_or_create_project(session, base_url, args.project_name, args.import_timeout)
|
||||
project_id = str(project["id"])
|
||||
area = find_or_create_area(session, base_url, project_id, boundary_path, args.import_timeout)
|
||||
area_id = str(area["id"])
|
||||
dataset_list = response_data(
|
||||
session.get(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets",
|
||||
params={"limit": 200},
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
)
|
||||
datasets = list(dataset_list.get("items") or [])
|
||||
|
||||
common_provenance = {
|
||||
"operator_tool": "provision_mol_municipality_workspace.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"coverage_scope": "municipality",
|
||||
"manifest_path": str(args.output_dir / MANIFEST_FILENAME),
|
||||
"manifest_generated_at": manifest["generated_at"],
|
||||
"reference_truncated": False,
|
||||
}
|
||||
|
||||
building_dataset = next((item for item in datasets if item.get("original_filename") == buildings_path.name), None)
|
||||
if not building_dataset:
|
||||
building_dataset = upload_dataset(
|
||||
session,
|
||||
base_url,
|
||||
project_id,
|
||||
area_id,
|
||||
buildings_path,
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"provider": "Digitaal Vlaanderen",
|
||||
"collection": "GRB/GBG",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": "municipality",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"feature_count": manifest["municipality_feature_count"],
|
||||
"pages_fetched": manifest["pages_fetched"],
|
||||
"attribution": GRB_ATTRIBUTION,
|
||||
},
|
||||
provenance_metadata={
|
||||
**common_provenance,
|
||||
"source_url": GRB_GBG_ITEMS_URL,
|
||||
"artifact_sha256": manifest["buildings_sha256"],
|
||||
},
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
|
||||
boundary_dataset = next((item for item in datasets if item.get("original_filename") == boundary_path.name), None)
|
||||
if not boundary_dataset:
|
||||
boundary_dataset = upload_dataset(
|
||||
session,
|
||||
base_url,
|
||||
project_id,
|
||||
area_id,
|
||||
boundary_path,
|
||||
dataset_role="source",
|
||||
source_name="vrbg",
|
||||
reference_layer_name=None,
|
||||
source_metadata={
|
||||
"provider": "Digitaal Vlaanderen",
|
||||
"collection": "VRBG/Refgem",
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": "municipality",
|
||||
"layer_type": "municipality_boundary",
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
},
|
||||
provenance_metadata={
|
||||
**common_provenance,
|
||||
"source_url": manifest["boundary_source_url"],
|
||||
"artifact_sha256": manifest["boundary_sha256"],
|
||||
},
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
|
||||
return {
|
||||
"project": project,
|
||||
"area": area,
|
||||
"boundary_dataset": boundary_dataset,
|
||||
"building_dataset": building_dataset,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
boundary_path, buildings_path, manifest = prepare_artifacts(args)
|
||||
workspace = None if args.fetch_only else provision_workspace(args, boundary_path, buildings_path, manifest)
|
||||
except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
"mode": "fetch_only" if args.fetch_only else "provisioned",
|
||||
"manifest_path": str(args.output_dir / MANIFEST_FILENAME),
|
||||
"municipality": MUNICIPALITY_NAME,
|
||||
"nis_code": MUNICIPALITY_NIS_CODE,
|
||||
"area_km2": manifest["area_km2"],
|
||||
"wgs84_bbox": manifest["wgs84_bbox"],
|
||||
"municipality_feature_count": manifest["municipality_feature_count"],
|
||||
"pages_fetched": manifest["pages_fetched"],
|
||||
"reference_truncated": manifest["reference_truncated"],
|
||||
"workspace": workspace,
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -42,6 +42,7 @@ ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
|
||||
${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
|
||||
${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
|
||||
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_municipality_workspace.py
|
||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||
|
||||
Reference in New Issue
Block a user