feat: add official modern Mol land-use series
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-14 17:51:24 +02:00
parent 8f27c5cc6f
commit b35f135fea
14 changed files with 1199 additions and 20 deletions
+9
View File
@@ -7,6 +7,15 @@
# Changelog
## Sprint 188 Official modern Mol land-use series (2026-07-14)
- Added an explicit, reusable MercatorNet WCS operator for official Departement Omgeving land-use snapshots in 2013, 2016, 2019, 2022 and 2025.
- Validated categorical integer GeoTIFF input at 10 m in EPSG:31370, retained raw rasters/checksums/manifests and polygonized only documented class 12 (`Bos`).
- Imported forest polygons through the existing DatasetService/VectorFeatureService path with canonical EPSG:4326 geometry and hectare intersection metrics; no startup fetch or direct PostGIS write was added.
- Kept the modern 2013-2025 series separate from historical 1778/1873/1969 cartography and added a compact temporal-series selector when both are available.
- Made the map-first forest theme prefer the authoritative modern source and its latest 2025 snapshot while preserving historical access.
- Added focused raster, CRS, class, provenance, packaging and frontend contract tests plus readiness compilation coverage.
## Sprint 187 Temporal Mol explorer (2026-07-14)
- Added first-class temporal dataset metadata and immutable dataset-version provenance for uploaded and derived vector/raster datasets.
+9 -3
View File
@@ -947,12 +947,15 @@ snapshots explicitly:
```bash
docker exec geointel python /app/scripts/provision_mol_population_history.py
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py
docker exec geointel python /app/scripts/provision_official_landuse_timeseries.py
```
The first command imports Statbel sector population for 2021-2025. The second
imports Digitaal Vlaanderen historical land use for 1778, 1873 and 1969. Both
are idempotent, use the normal API/DatasetService flow and retain fetched
artifacts in persistent operator storage. They never run on app startup.
imports Digitaal Vlaanderen historical land use for 1778, 1873 and 1969. The
third imports the Departement Omgeving 10 m forest class for 2013, 2016, 2019,
2022 and 2025. All commands are idempotent, use the normal
API/DatasetService flow and retain fetched artifacts in persistent operator
storage. They never run on app startup.
Historical land-use work can be bounded explicitly:
@@ -964,6 +967,9 @@ docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --y
`POST /api/v1/projects/{project_id}/temporal/compare` compares two snapshots
inside one EPSG:4326 bbox. Partial statistical sectors are estimates; old map
editions without stable identities do not produce invented object changes.
Modern raster-derived forest polygons have the same identity limitation. Their
area is measured in EPSG:31370 and is exact within the 10 m source
representation, not a cadastral forest survey.
## Helpful repository scripts
@@ -0,0 +1,155 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
from types import SimpleNamespace
import sys
import numpy as np
from pyproj import Transformer
import rasterio
from rasterio.transform import from_origin
from shapely.geometry import Polygon, shape
from shapely.ops import transform
ROOT = Path(__file__).resolve().parents[2]
def load_provisioner():
script_path = ROOT / "scripts" / "provision_official_landuse_timeseries.py"
spec = importlib.util.spec_from_file_location("official_landuse_provisioner", script_path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_official_landuse_wcs_contract_is_categorical_and_deterministic() -> None:
module = load_provisioner()
params = module.build_wcs_params(2025, (196594.1, 205064.2, 210910.7, 223974.9))
assert module.SUPPORTED_YEARS == (2013, 2016, 2019, 2022, 2025)
assert module.LAND_USE_CLASSES[12] == "Bos"
assert params == {
"SERVICE": "WCS",
"VERSION": "1.0.0",
"REQUEST": "GetCoverage",
"COVERAGE": "lu:lu_landgebruik_vlaa_2025_v3",
"CRS": "EPSG:31370",
"BBOX": "196590.000,205060.000,210920.000,223980.000",
"RESX": "10",
"RESY": "10",
"FORMAT": "image/tiff",
"RESPONSE_CRS": "EPSG:31370",
}
assert module.series_key(module.THEMES[0], "Mol") == "department-omgeving:land-use:forest:mol"
def test_official_landuse_polygonization_clips_and_preserves_provenance(tmp_path: Path) -> None:
module = load_provisioner()
raster_path = tmp_path / "landuse.tif"
values = np.array(
[
[1, 1, 1, 1, 1, 1],
[1, 12, 12, 1, 1, 1],
[1, 12, 12, 1, 12, 1],
[1, 1, 1, 1, 12, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
],
dtype="int32",
)
with rasterio.open(
raster_path,
"w",
driver="GTiff",
width=6,
height=6,
count=1,
dtype="int32",
crs="EPSG:31370",
transform=from_origin(200000, 210000, 10, 10),
nodata=-9999,
) as destination:
destination.write(values, 1)
to_wgs84 = Transformer.from_crs(31370, 4326, always_xy=True)
boundary_metric = Polygon(
[(200005, 209945), (200055, 209945), (200055, 209995), (200005, 209995), (200005, 209945)]
)
boundary = transform(to_wgs84.transform, boundary_metric)
payload, stats = module.polygonize_snapshot(
raster_path=raster_path,
boundary=boundary,
year=2025,
theme=module.THEMES[0],
municipality_name="Mol",
nis_code="13025",
scope_key="mol",
max_features=100,
)
assert payload["type"] == "FeatureCollection"
assert payload["crs"]["properties"]["name"] == "EPSG:4326"
assert payload["source_coverage_id"] == "lu:lu_landgebruik_vlaa_2025_v3"
assert stats["source_pixel_count"] == 6
assert stats["source_pixel_area_m2"] == 600
assert stats["feature_count"] == 2
assert 0 < stats["polygon_area_m2"] <= 600
assert stats["class_histogram"] == {"1": 30, "12": 6}
for feature in payload["features"]:
geometry = shape(feature["geometry"])
properties = feature["properties"]
assert geometry.is_valid
assert geometry.within(boundary.buffer(1e-9))
assert properties["source_name"] == "department_omgeving_land_use"
assert properties["land_use_class_ids"] == [12]
assert properties["source_resolution_m"] == 10.0
assert properties["source_raster_sha256"] == stats["raster_sha256"]
def test_official_landuse_metadata_keeps_modern_series_separate(tmp_path: Path) -> None:
module = load_provisioner()
theme = module.THEMES[0]
snapshot = module.PreparedSnapshot(
year=2022,
theme=theme,
raster_path=tmp_path / "source.tif",
vector_path=tmp_path / "forest.geojson",
manifest_path=tmp_path / "forest.manifest.json",
feature_count=42,
raster_sha256="a" * 64,
vector_sha256="b" * 64,
)
args = SimpleNamespace(scope_key="mol", municipality_name="Mol", nis_code="13025")
source = module.build_source_metadata(args, snapshot)
provenance = module.build_provenance_metadata(args, snapshot)
assert source["temporal_series_label"] == "Moderne landgebruikskaart (10 m)"
assert source["selection_aggregation"]["method"] == "intersection_area"
assert source["identity_stable"] is False
assert source["land_use_class_names"] == ["Bos"]
assert "10 m" in source["selection_aggregation"]["warning"]
assert provenance["operator_explicit_fetch"] is True
assert provenance["coverage_id"] == "lu:lu_landgebruik_vlaa_2022_v3"
assert "historical-landuse" not in module.series_key(theme, "mol")
def test_official_landuse_operator_is_packaged_and_readiness_checked() -> None:
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
assert "py_compile scripts/provision_official_landuse_timeseries.py" in readiness
assert "COPY scripts/provision_official_landuse_timeseries.py" in dockerfile
assert "department_omgeving_land_use' ? 90_000" in workspace
assert "activeTemporalSeriesGroups.length > 1" in workspace
assert "Moderne landgebruikskaart (10 m)" in (ROOT / "scripts/provision_official_landuse_timeseries.py").read_text(
encoding="utf-8"
)
+1
View File
@@ -76,6 +76,7 @@ COPY scripts/provision_mol_municipality_workspace.py /app/scripts/provision_mol_
COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_layers.py
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
+24
View File
@@ -7819,3 +7819,27 @@ Remaining source limitations:
- The latest official population snapshot in this workspace is 2025.
- The forest/green evolution series currently represents the available historical land-use editions through 1969; it must not be presented as current forest cover.
- Partial-sector population results remain area-weighted estimates because no finer authoritative population surface has been ingested.
## Sprint 188 Official modern Mol land-use series (2026-07-14)
Implemented:
- Added `provision_official_landuse_timeseries.py` for explicit MercatorNet WCS subsets of the official Departement Omgeving version 3 land-use maps for 2013, 2016, 2019, 2022 and 2025.
- Validated one-band integer GeoTIFF input, EPSG:31370, 10 m resolution and the documented 1-19 class domain before processing.
- Preserved every raw raster, exact request URL, catalogue URL, raster/vector checksum, class histogram and processing manifest in operator storage.
- Polygonized documented class 12 (`Bos`) in metric CRS, clipped it to the official Mol boundary, normalized it to EPSG:4326 and uploaded it through the existing API/DatasetService/vector-feature path.
- Added source-governed hectare aggregation metadata, 10 m/non-cadastral limitations and unstable raster-polygon identity declarations.
- Kept `department-omgeving:land-use:forest:mol` separate from the historical land-use series and added a compact frontend series selector.
Source validation:
- Live WCS capabilities exposed all five expected coverages through `lu:lu_landgebruik_vlaa_<year>_v3`.
- A full-Mol fetch-only run produced 3,768 / 3,823 / 3,953 / 3,615 / 3,676 forest polygons for 2013 / 2016 / 2019 / 2022 / 2025 without truncation.
- Polygonized forest area was 3,723.17 / 3,615.39 / 3,592.27 / 3,648.98 / 3,626.56 ha. These are measurements within the official 10 m representation, not cadastral forest areas.
- Every raster cell touching the municipality is considered before exact vector clipping. The 916 NoData edge cells in each WCS subset were explicitly excluded and recorded rather than assigned a class.
Validation:
- New focused backend suite passed 4 tests.
- Frontend TypeScript typecheck passed after temporal-series selection wiring.
- Full readiness and live Tower/PostGIS/browser validation remain the final steps of this pass.
Next:
- Deploy and provision all five modern snapshots on Tower, then verify current forest selection, both temporal series and a drawn rectangle in the internal browser.
+27
View File
@@ -61,6 +61,33 @@ OSM is used for fast, broad, fallback context.
### OSM caveat
OSM is community-maintained and may be incomplete. UI and reports must describe it as contextual/fallback data, not official ground truth.
## Official modern land-use strategy
The Departement Omgeving version 3 land-use maps are the authoritative modern
source series for categorical land use. GeoIntel retrieves only explicit
boundary subsets from MercatorNet WCS; no whole-Flanders raster is fetched at
startup or during an interactive map query.
- Source years: 2013, 2016, 2019, 2022 and 2025.
- Native representation: one integer category per 10 m cell in EPSG:31370.
- Preserved evidence: raw GeoTIFF, WCS request, catalogue URL, checksum, class
histogram and processing manifest.
- Operational representation: source cells polygonized in EPSG:31370, clipped
to the requested boundary and normalized to EPSG:4326 `vector_features`.
- Boundary behavior: retain every source cell touching the boundary, then
perform the exact metric geometry intersection; exclude and report NoData.
- Selection metric: exact polygon intersection area within the source's 10 m
representation, reported in hectares with a non-cadastral warning.
- Identity: raster polygons are not stable source objects; temporal object
lineage is unavailable.
Modern land-use series must remain separate from the 1778, 1873 and 1969
historical cartographic series. Comparisons are valid within each persisted
series and must retain the source methodology warning. Extension from Mol to
Kempen reuses the same operator with an explicit approved boundary, project,
area and scope key; an ambiguous regional label is never converted into an
invented boundary.
## User-uploaded raster strategy
V1 must support controlled local datasets because public raster access and model compatibility can be difficult.
+35
View File
@@ -63,6 +63,41 @@ The operator uses standards-compliant WFS 2.0 XML POST requests. This keeps
the spatial/class filters server-side without exposing a long XML filter in a
GET query, which the public gateway rejects.
### Mol modern land use
`scripts/provision_official_landuse_timeseries.py` uses the public Departement
Omgeving/MercatorNet WCS to retrieve the harmonized version 3 land-use maps for
2013, 2016, 2019, 2022 and 2025. The source is a categorical 10 m GeoTIFF in
Belgian Lambert 72 (`EPSG:31370`) with 19 documented classes. GeoIntel currently
derives only class `12` (`Bos`) as the operational forest theme.
Every source raster is clipped against the explicit official boundary,
validated for integer classes, CRS and resolution, checksummed and retained in
persistent operator storage. Forest cells are polygonized and clipped in
`EPSG:31370`, then transformed to canonical `EPSG:4326` and uploaded through
DatasetService/VectorFeatureService. The raw raster remains the provenance
artifact; the persisted polygons provide rectangle selection, PostGIS area
aggregation and temporal comparison without a parallel query path.
All cells touching the requested boundary are considered before the exact
metric geometry clip; NoData cells are excluded and counted in the manifest.
The modern series key is
`department-omgeving:land-use:forest:mol`. It remains separate from
`digitaal-vlaanderen:historical-landuse:forest:mol`: the 1778-1969
cartographic editions and the harmonized 2013-2025 10 m land-use maps are not
presented as one continuous measurement method. Raster-derived polygon
identities are unstable, so GeoIntel compares hectares and never invents
added/removed forest objects.
Official catalogues:
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2013
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2016
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2019
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2022
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2025
- https://www.vlaanderen.be/statistiek-vlaanderen/ruimtegebruik/landgebruik/metadata-landgebruik
## OSM
- Naam: OpenStreetMap
+3 -2
View File
@@ -6,8 +6,9 @@
- [x] Add drag rectangle selection with automatic persisted theme queries.
- [x] Separate exact bbox intersection totals from the bounded map preview.
- [x] Add official Mol GRB roads, water and parcels provisioning support.
- [ ] Select and validate an official Mol population/statistical-sector source before enabling the population theme.
- [ ] Select and validate an authoritative Flemish land-cover source before enabling the forest/green theme.
- [x] Select, validate and provision official Statbel 2021-2025 population/statistical-sector snapshots for Mol.
- [x] Select, validate and provision official Departement Omgeving 2013-2025 forest snapshots from the 10 m land-use map.
- [ ] Define the exact administrative Kempen scope before provisioning municipality or regional copies of the proven source series.
This file now starts with the current implementation status. Older preparation/backlog sections are preserved below as historical planning context and should not be treated as the live sprint board without checking `docs/CODEX_EXECUTION_LOG.md`.
+4
View File
@@ -10,6 +10,10 @@ current, while `Evolution` lets the operator compare an earlier and later
snapshot from the same series over a drawn rectangle. Results show units,
absolute/percentage change, estimate status and source limitations.
Added/removed/modified overlays only appear for stable source identities.
When a theme has multiple valid methodologies, a compact series selector keeps
them explicit. Forest therefore defaults to the official modern 2013-2025
10 m series, while the separate 1778-1969 historical map series remains
selectable and is never merged into the same trend.
Selection results use dataset-specific PostGIS summaries. Object layers show
intersecting counts, population shows inhabitants with partial-sector
+67 -12
View File
@@ -7,6 +7,7 @@ import { useTemporalComparison } from '../../hooks/useTemporalComparison'
const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = []
type DataThemeId = 'buildings' | 'population' | 'forest' | 'water' | 'roads' | 'parcels'
@@ -18,6 +19,12 @@ interface DataTheme {
tokens: string[]
}
interface TemporalSeriesGroup {
key: string
label: string
items: DatasetCreateResponse[]
}
const DATA_THEMES: DataTheme[] = [
{
id: 'buildings',
@@ -90,6 +97,7 @@ function pickThemeDataset(datasets: DatasetCreateResponse[], theme: DataTheme):
const score = (dataset: DatasetCreateResponse) =>
(dataset.reference_layer_name && theme.tokens.includes(dataset.reference_layer_name.toLowerCase()) ? 1_000_000 : 0) +
(dataset.source_name === 'grb' ? 100_000 : 0) +
(dataset.source_name === 'department_omgeving_land_use' ? 90_000 : 0) +
(dataset.dataset_role === 'reference' ? 10_000 : 0) +
(dataset.observed_at ? new Date(dataset.observed_at).getTime() / 100_000_000 : 0) +
(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0)
@@ -98,7 +106,21 @@ function pickThemeDataset(datasets: DatasetCreateResponse[], theme: DataTheme):
return candidates[0] ?? null
}
function pickThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataTheme): DatasetCreateResponse[] {
function temporalSeriesLabel(items: DatasetCreateResponse[]): string {
const configuredLabel = items.find((item) => typeof item.source_metadata?.['temporal_series_label'] === 'string')
?.source_metadata?.['temporal_series_label']
if (typeof configuredLabel === 'string' && configuredLabel.trim()) {
return configuredLabel
}
const first = items[0]
const source = first?.source_name ?? first?.source ?? 'Tijdreeks'
const firstYear = first?.observed_at ? new Date(first.observed_at).getUTCFullYear() : null
const last = items[items.length - 1]
const lastYear = last?.observed_at ? new Date(last.observed_at).getUTCFullYear() : null
return firstYear && lastYear ? `${source} (${firstYear}-${lastYear})` : source
}
function listThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataTheme): TemporalSeriesGroup[] {
const groups = new Map<string, DatasetCreateResponse[]>()
for (const dataset of datasets) {
if (!datasetMatchesTheme(dataset, theme) || !dataset.temporal_series_key || !dataset.observed_at) {
@@ -108,16 +130,21 @@ function pickThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataT
items.push(dataset)
groups.set(dataset.temporal_series_key, items)
}
return Array.from(groups.values())
.filter((items) => items.length >= 2)
return Array.from(groups.entries())
.filter(([, items]) => items.length >= 2)
.map(([key, items]) => {
const ordered = [...items].sort(
(left, right) => new Date(left.observed_at ?? 0).getTime() - new Date(right.observed_at ?? 0).getTime(),
)
return { key, label: temporalSeriesLabel(ordered), items: ordered }
})
.sort((left, right) => {
if (right.length !== left.length) {
return right.length - left.length
if (right.items.length !== left.items.length) {
return right.items.length - left.items.length
}
const latest = (items: DatasetCreateResponse[]) => Math.max(...items.map((item) => new Date(item.observed_at ?? 0).getTime()))
const latest = (group: TemporalSeriesGroup) => Math.max(...group.items.map((item) => new Date(item.observed_at ?? 0).getTime()))
return latest(right) - latest(left)
})[0]
?.sort((left, right) => new Date(left.observed_at ?? 0).getTime() - new Date(right.observed_at ?? 0).getTime()) ?? []
})
}
function formatObservationDate(value: string | null | undefined): string {
@@ -496,6 +523,7 @@ export function MapWorkspace({
clearTemporalComparison,
} = useTemporalComparison(selectedProjectId)
const [analysisMode, setAnalysisMode] = useState<'current' | 'evolution'>('current')
const [selectedTemporalSeriesKey, setSelectedTemporalSeriesKey] = useState('')
const [earlierDatasetId, setEarlierDatasetId] = useState('')
const [laterDatasetId, setLaterDatasetId] = useState('')
const [bboxSelectionMode, setBboxSelectionMode] = useState(false)
@@ -536,10 +564,13 @@ export function MapWorkspace({
)
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
const activeThemeDataset = themeDatasetMap[activeTheme.id]
const activeTemporalSeries = useMemo(
() => pickThemeTemporalSeries(availableMapDatasets, activeTheme),
const activeTemporalSeriesGroups = useMemo(
() => listThemeTemporalSeries(availableMapDatasets, activeTheme),
[activeTheme, availableMapDatasets],
)
const activeTemporalSeriesGroup = activeTemporalSeriesGroups.find((group) => group.key === selectedTemporalSeriesKey)
?? activeTemporalSeriesGroups[0]
const activeTemporalSeries = activeTemporalSeriesGroup?.items ?? EMPTY_TEMPORAL_SERIES
const themeResults = useMemo(
() =>
themeInsights.flatMap((insight) => {
@@ -592,6 +623,14 @@ export function MapWorkspace({
setBboxInput(bboxToInputState(mapSelectionBbox))
}, [mapSelectionBbox])
useEffect(() => {
setSelectedTemporalSeriesKey((current) =>
activeTemporalSeriesGroups.some((group) => group.key === current)
? current
: activeTemporalSeriesGroups[0]?.key ?? '',
)
}, [activeTemporalSeriesGroups])
useEffect(() => {
const first = activeTemporalSeries[0]
const last = activeTemporalSeries[activeTemporalSeries.length - 1]
@@ -912,7 +951,7 @@ export function MapWorkspace({
<span>{analysisMode === 'evolution' ? 'Tijdreeks' : 'Actieve bron'}</span>
<strong>
{analysisMode === 'evolution'
? activeTemporalSeries[0]?.temporal_series_key ?? 'Geen tijdreeks beschikbaar'
? activeTemporalSeriesGroup?.label ?? 'Geen tijdreeks beschikbaar'
: activeThemeDataset?.name ?? 'Geen databron beschikbaar'}
</strong>
<small>
@@ -928,6 +967,22 @@ export function MapWorkspace({
{analysisMode === 'evolution' ? (
<div className="geo-time-controls" aria-label="Meetmomenten vergelijken">
{activeTemporalSeriesGroups.length > 1 ? (
<label className="geo-series-control">
Reeks
<select
value={activeTemporalSeriesGroup?.key ?? ''}
onChange={(event) => {
setSelectedTemporalSeriesKey(event.target.value)
clearTemporalComparison()
}}
>
{activeTemporalSeriesGroups.map((group) => (
<option key={group.key} value={group.key}>{group.label}</option>
))}
</select>
</label>
) : null}
<label>
Van
<select value={earlierDatasetId} onChange={(event) => { setEarlierDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
@@ -1208,7 +1263,7 @@ export function MapWorkspace({
<span>
<strong>Bron:</strong>{' '}
{analysisMode === 'evolution'
? activeTemporalSeries[0]?.temporal_series_key ?? 'geen vergelijkbare tijdreeks'
? activeTemporalSeriesGroup?.label ?? 'geen vergelijkbare tijdreeks'
: activeThemeDataset
? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.name}`
: 'niet beschikbaar'}
+4
View File
@@ -5675,6 +5675,10 @@ section {
font-size: 0.68rem;
}
.geo-time-controls .geo-series-control {
grid-column: 1 / -1;
}
.geo-time-controls button {
grid-column: 1 / -1;
min-height: 2.25rem;
+41 -3
View File
@@ -1212,9 +1212,47 @@ normal dataset API. Artifacts and manifests are retained below
datasets; use `--force` only for an explicit source refresh. Use
`--layers roads,water` or `--fetch-only` for a bounded operator run.
The context provisioner does not add population or forest values. Those themes
remain visibly unavailable until an authoritative/statistically appropriate
source is imported; zero is never substituted for missing source data.
The context provisioner itself does not add population or forest values. Use
the dedicated official-source operators below; zero is never substituted for
missing source data.
## Official Mol temporal sources
After the municipality workspace exists, provision official Statbel population
and both independently modelled land-use series:
```bash
docker exec -it geointel python3 /app/scripts/provision_mol_population_history.py
docker exec -it geointel python3 /app/scripts/provision_mol_historical_landuse.py
docker exec -it geointel python3 /app/scripts/provision_official_landuse_timeseries.py
```
The modern land-use command checks the MercatorNet WCS capabilities, downloads
only the Mol bounding subset of each 10 m `EPSG:31370` raster, validates the
categorical integer grid, clips against the official boundary and polygonizes
class `12` (`Bos`). Raw rasters, vector artifacts and checksum manifests are
stored under `/app/storage/operator-data/official-landuse/mol`. The resulting
2013, 2016, 2019, 2022 and 2025 vectors are uploaded through the canonical API
as `department-omgeving:land-use:forest:mol`.
Prepare and inspect artifacts without changing the database:
```bash
docker exec -it geointel python3 \
/app/scripts/provision_official_landuse_timeseries.py --fetch-only
```
Use `--force` only to refetch and rebuild local artifacts. Existing persisted
snapshots remain immutable and are reused by year/series. To use the operator
for another approved region, pass all scope inputs explicitly, for example
`--boundary-path`, `--project-name`, `--area-name`, `--municipality-name`,
`--nis-code`, `--scope-key` and `--output-dir`. GeoIntel does not infer what
"Kempen" means administratively.
The 2013-2025 series is methodologically separate from the historical
1778/1873/1969 series. The map-first Evolution view exposes a series selector
when both exist; it never calculates one continuous trend across those source
families.
## Tower deployment
@@ -0,0 +1,817 @@
"""Provision official modern Flemish land-use snapshots for an explicit area.
The operator downloads categorical 10 metre GeoTIFF subsets from the public
Departement Omgeving MercatorNet WCS, clips them to a supplied boundary and
polygonizes only explicitly supported classes. Raw rasters and checksum
manifests remain provenance artifacts. Vector output is imported through the
normal GeoIntel dataset API and is never written directly to PostGIS.
Mol is the safe default. Other scopes must provide their own boundary, project,
area name and identity explicitly.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import sys
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import numpy as np
import rasterio
import requests
from pyproj import Transformer
from rasterio.features import geometry_mask, shapes
from requests.adapters import HTTPAdapter
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
from shapely.ops import transform, unary_union
from shapely.validation import make_valid
from urllib3.util.retry import Retry
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_PROJECT_NAME = "Mol Municipality Workbench"
DEFAULT_AREA_NAME = "Gemeente Mol"
DEFAULT_MUNICIPALITY_NAME = "Mol"
DEFAULT_NIS_CODE = "13025"
DEFAULT_SCOPE_KEY = "mol"
DEFAULT_BOUNDARY_PATH = Path("/app/storage/operator-data/mol-municipality/mol_municipality_boundary.geojson")
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/official-landuse/mol")
WCS_URL = "https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs"
WCS_VERSION = "1.0.0"
SOURCE_CRS = "EPSG:31370"
OUTPUT_CRS = "EPSG:4326"
SOURCE_RESOLUTION_METRES = 10.0
SUPPORTED_YEARS = (2013, 2016, 2019, 2022, 2025)
ATTRIBUTION = "Bron: Landgebruik Vlaanderen, Departement Omgeving"
SERIES_LABEL = "Moderne landgebruikskaart (10 m)"
CATALOGUE_URLS = {
year: f"https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-{year}"
for year in SUPPORTED_YEARS
}
LAND_USE_CLASSES = {
1: "Huizen en tuinen",
2: "Industrie",
3: "Commerciele doeleinden",
4: "Diensten",
5: "Transportinfrastructuur",
6: "Recreatie",
7: "Landbouwgebouwen en -infrastructuur",
8: "Overige bebouwde terreinen",
9: "Overige onbebouwde terreinen",
10: "Actieve groeves",
11: "Luchthavens",
12: "Bos",
13: "Akker",
14: "Grasland in landbouwgebruik",
15: "Struikgewas",
16: "Braakliggend en duinen",
17: "Water",
18: "Moeras",
19: "Overige graslanden",
}
@dataclass(frozen=True)
class ThemeDefinition:
key: str
label: str
class_ids: tuple[int, ...]
@dataclass(frozen=True)
class PreparedSnapshot:
year: int
theme: ThemeDefinition
raster_path: Path
vector_path: Path
manifest_path: Path
feature_count: int
raster_sha256: str
vector_sha256: str
THEMES = (ThemeDefinition("forest", "Bos", (12,)),)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision official 2013-2025 Flemish land-use snapshots.")
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
parser.add_argument("--area-name", default=DEFAULT_AREA_NAME, help="Case-insensitive fragment identifying the persisted Area.")
parser.add_argument("--municipality-name", default=DEFAULT_MUNICIPALITY_NAME)
parser.add_argument("--nis-code", default=DEFAULT_NIS_CODE)
parser.add_argument("--scope-key", default=DEFAULT_SCOPE_KEY)
parser.add_argument("--years", default=",".join(str(year) for year in SUPPORTED_YEARS))
parser.add_argument("--themes", default="forest")
parser.add_argument(
"--boundary-path",
type=Path,
default=Path(os.environ.get("OFFICIAL_LANDUSE_BOUNDARY_PATH", DEFAULT_BOUNDARY_PATH)),
)
parser.add_argument(
"--output-dir",
type=Path,
default=Path(os.environ.get("OFFICIAL_LANDUSE_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)),
)
parser.add_argument("--request-timeout", type=int, default=240)
parser.add_argument("--import-timeout", type=int, default=1800)
parser.add_argument("--max-features", type=int, default=100000)
parser.add_argument("--force", action="store_true", help="Refetch and rebuild local source artifacts; persisted datasets stay immutable.")
parser.add_argument("--fetch-only", action="store_true", help="Prepare and verify artifacts without changing GeoIntel persistence.")
return parser.parse_args()
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def write_json_atomic(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
temporary = path.with_suffix(f"{path.suffix}.partial")
temporary.write_text(
json.dumps(
payload,
ensure_ascii=False,
indent=2 if pretty else None,
separators=None if pretty else (",", ":"),
sort_keys=pretty,
),
encoding="utf-8",
)
temporary.replace(path)
def build_session() -> requests.Session:
retry = Retry(
total=5,
connect=5,
read=5,
status=5,
backoff_factor=1.0,
status_forcelist=(429, 500, 502, 503, 504),
allowed_methods=frozenset({"GET"}),
raise_on_status=True,
)
session = requests.Session()
session.headers.update({"User-Agent": "GeoIntel-Official-Landuse-Operator/1.0"})
adapter = HTTPAdapter(max_retries=retry)
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
def coverage_id(year: int) -> str:
return f"lu:lu_landgebruik_vlaa_{year}_v3"
def series_key(theme: ThemeDefinition, scope_key: str) -> str:
return f"department-omgeving:land-use:{theme.key}:{scope_key.strip().lower()}"
def load_boundary(path: Path):
if not path.exists():
raise RuntimeError(f"Boundary artifact is missing at {path}")
payload = json.loads(path.read_text(encoding="utf-8"))
features = payload.get("features") or []
if len(features) != 1:
raise RuntimeError("Boundary artifact must contain exactly one feature")
boundary = normalize_polygonal(shape(features[0].get("geometry")))
if boundary is None:
raise RuntimeError("Boundary artifact is empty, invalid or non-polygonal")
return boundary
def normalize_polygonal(geometry):
if geometry is None or geometry.is_empty:
return None
if not geometry.is_valid:
geometry = make_valid(geometry)
if isinstance(geometry, (Polygon, MultiPolygon)):
return geometry
if isinstance(geometry, GeometryCollection):
polygons = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty]
if not polygons:
return None
merged = unary_union(polygons)
return merged if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty else None
return None
def metric_boundary(boundary):
transformer = Transformer.from_crs(OUTPUT_CRS, SOURCE_CRS, always_xy=True)
projected = normalize_polygonal(transform(transformer.transform, boundary))
if projected is None:
raise RuntimeError("Boundary could not be projected to EPSG:31370")
return projected
def snapped_bounds(bounds: tuple[float, float, float, float]) -> tuple[float, float, float, float]:
min_x, min_y, max_x, max_y = bounds
resolution = SOURCE_RESOLUTION_METRES
return (
math.floor(min_x / resolution) * resolution,
math.floor(min_y / resolution) * resolution,
math.ceil(max_x / resolution) * resolution,
math.ceil(max_y / resolution) * resolution,
)
def build_wcs_params(year: int, bounds: tuple[float, float, float, float]) -> dict[str, str]:
min_x, min_y, max_x, max_y = snapped_bounds(bounds)
return {
"SERVICE": "WCS",
"VERSION": WCS_VERSION,
"REQUEST": "GetCoverage",
"COVERAGE": coverage_id(year),
"CRS": SOURCE_CRS,
"BBOX": f"{min_x:.3f},{min_y:.3f},{max_x:.3f},{max_y:.3f}",
"RESX": str(int(SOURCE_RESOLUTION_METRES)),
"RESY": str(int(SOURCE_RESOLUTION_METRES)),
"FORMAT": "image/tiff",
"RESPONSE_CRS": SOURCE_CRS,
}
def verify_coverages(session: requests.Session, years: list[int], timeout: int) -> None:
response = session.get(
WCS_URL,
params={"SERVICE": "WCS", "VERSION": WCS_VERSION, "REQUEST": "GetCapabilities"},
timeout=timeout,
)
response.raise_for_status()
missing = [coverage_id(year) for year in years if coverage_id(year) not in response.text]
if missing:
raise RuntimeError(f"Official WCS is missing expected coverages: {', '.join(missing)}")
def validate_raster(path: Path) -> dict[str, Any]:
try:
with rasterio.open(path) as dataset:
epsg = dataset.crs.to_epsg() if dataset.crs else None
resolution = (abs(float(dataset.res[0])), abs(float(dataset.res[1])))
if epsg != 31370:
raise RuntimeError(f"Expected EPSG:31370 source raster, received {dataset.crs}")
if dataset.count != 1:
raise RuntimeError(f"Expected one categorical raster band, received {dataset.count}")
if any(abs(value - SOURCE_RESOLUTION_METRES) > 0.01 for value in resolution):
raise RuntimeError(f"Expected 10 metre source resolution, received {resolution}")
if not np.issubdtype(np.dtype(dataset.dtypes[0]), np.integer):
raise RuntimeError(f"Expected integer land-use classes, received {dataset.dtypes[0]}")
return {
"width": dataset.width,
"height": dataset.height,
"dtype": dataset.dtypes[0],
"nodata": dataset.nodata,
"crs": SOURCE_CRS,
"resolution_metres": SOURCE_RESOLUTION_METRES,
"bounds": list(dataset.bounds),
}
except rasterio.errors.RasterioError as exc:
raise RuntimeError(f"Official WCS response is not a readable GeoTIFF: {exc}") from exc
def download_raster(
session: requests.Session,
*,
year: int,
bounds: tuple[float, float, float, float],
path: Path,
timeout: int,
) -> dict[str, Any]:
response = session.get(WCS_URL, params=build_wcs_params(year, bounds), timeout=timeout, stream=True)
response.raise_for_status()
content_type = str(response.headers.get("content-type") or "").lower()
if "tiff" not in content_type:
preview = response.content[:500].decode("utf-8", errors="replace")
raise RuntimeError(f"Official WCS returned {content_type or 'unknown content'} instead of GeoTIFF: {preview}")
temporary = path.with_suffix(f"{path.suffix}.partial")
try:
with temporary.open("wb") as handle:
for chunk in response.iter_content(chunk_size=1024 * 1024):
if chunk:
handle.write(chunk)
profile = validate_raster(temporary)
temporary.replace(path)
except Exception:
temporary.unlink(missing_ok=True)
raise
return {**profile, "request_url": response.url, "retrieved_at": utc_now()}
def polygonize_snapshot(
*,
raster_path: Path,
boundary,
year: int,
theme: ThemeDefinition,
municipality_name: str,
nis_code: str,
scope_key: str,
max_features: int,
) -> tuple[dict[str, Any], dict[str, Any]]:
boundary_metric = metric_boundary(boundary)
to_output = Transformer.from_crs(SOURCE_CRS, OUTPUT_CRS, always_xy=True)
raster_sha256 = sha256_file(raster_path)
with rasterio.open(raster_path) as dataset:
validate_raster(raster_path)
values = dataset.read(1)
inside = geometry_mask(
[mapping(boundary_metric)],
out_shape=values.shape,
transform=dataset.transform,
invert=True,
# Exact clipping happens after polygonization, so retain every
# classified source cell that overlaps the requested boundary.
all_touched=True,
)
valid_inside = inside.copy()
nodata_count = 0
if dataset.nodata is not None:
nodata = np.isclose(values, dataset.nodata)
nodata_count = int(np.count_nonzero(inside & nodata))
valid_inside &= ~nodata
available_values, available_counts = np.unique(values[valid_inside], return_counts=True)
class_histogram = {str(int(value)): int(count) for value, count in zip(available_values, available_counts)}
unknown_classes = sorted(int(value) for value in available_values if int(value) not in LAND_USE_CLASSES)
if unknown_classes:
raise RuntimeError(f"Land-use raster {year} contains undocumented classes: {unknown_classes}")
class_mask = np.isin(values, np.asarray(theme.class_ids)) & valid_inside
source_pixel_count = int(np.count_nonzero(class_mask))
if source_pixel_count == 0:
raise RuntimeError(f"Land-use raster {year} contains no {theme.label} cells inside the boundary")
features: list[dict[str, Any]] = []
polygon_area_m2 = 0.0
for raw_geometry, raw_value in shapes(
class_mask.astype("uint8"),
mask=class_mask,
transform=dataset.transform,
connectivity=4,
):
if int(raw_value) != 1:
continue
geometry_metric = normalize_polygonal(shape(raw_geometry).intersection(boundary_metric))
if geometry_metric is None or geometry_metric.area <= 0:
continue
geometry_output = normalize_polygonal(transform(to_output.transform, geometry_metric))
if geometry_output is None:
continue
if len(features) >= max_features:
raise RuntimeError(
f"Land-use {theme.key} {year} exceeds the {max_features} feature safety limit; refusing truncation"
)
feature_hash = hashlib.sha256(
f"{year}:{theme.key}:".encode("utf-8") + geometry_metric.wkb
).hexdigest()[:24]
feature_id = f"land-use-{year}-{theme.key}-{feature_hash}"
area_m2 = float(geometry_metric.area)
polygon_area_m2 += area_m2
features.append(
{
"type": "Feature",
"id": feature_id,
"geometry": mapping(geometry_output),
"properties": {
"source_name": "department_omgeving_land_use",
"source_feature_id": feature_id,
"reference_layer_name": theme.key,
"layer_type": theme.key,
"authority_level": "authoritative",
"coverage_scope": scope_key,
"municipality": municipality_name,
"nis_code": nis_code,
"observation_year": year,
"land_use_class_ids": list(theme.class_ids),
"land_use_class_names": [LAND_USE_CLASSES[class_id] for class_id in theme.class_ids],
"source_resolution_m": SOURCE_RESOLUTION_METRES,
"polygon_area_m2": round(area_m2, 3),
"source_coverage_id": coverage_id(year),
"source_raster_sha256": raster_sha256,
"attribution": ATTRIBUTION,
},
}
)
payload = {
"type": "FeatureCollection",
"name": f"{theme.label} - {municipality_name} {year}",
"crs": {"type": "name", "properties": {"name": OUTPUT_CRS}},
"municipality": municipality_name,
"nis_code": nis_code,
"scope_key": scope_key,
"observation_year": year,
"source_coverage_id": coverage_id(year),
"source_resolution_m": SOURCE_RESOLUTION_METRES,
"attribution": ATTRIBUTION,
"features": features,
}
stats = {
"feature_count": len(features),
"source_pixel_count": source_pixel_count,
"source_pixel_area_m2": source_pixel_count * SOURCE_RESOLUTION_METRES**2,
"polygon_area_m2": polygon_area_m2,
"nodata_pixels_inside_boundary": nodata_count,
"class_histogram": class_histogram,
"raster_sha256": raster_sha256,
}
return payload, stats
def existing_snapshot(
*,
year: int,
theme: ThemeDefinition,
raster_path: Path,
vector_path: Path,
manifest_path: Path,
) -> PreparedSnapshot | None:
if not raster_path.exists() or not vector_path.exists() or not manifest_path.exists():
return None
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
raster_sha256 = sha256_file(raster_path)
vector_sha256 = sha256_file(vector_path)
if (
manifest.get("year") != year
or manifest.get("theme") != theme.key
or manifest.get("coverage_id") != coverage_id(year)
or manifest.get("class_ids") != list(theme.class_ids)
or manifest.get("raster_sha256") != raster_sha256
or manifest.get("vector_sha256") != vector_sha256
):
return None
validate_raster(raster_path)
feature_count = int(manifest["feature_count"])
if feature_count <= 0:
return None
return PreparedSnapshot(
year=year,
theme=theme,
raster_path=raster_path,
vector_path=vector_path,
manifest_path=manifest_path,
feature_count=feature_count,
raster_sha256=raster_sha256,
vector_sha256=vector_sha256,
)
except (OSError, ValueError, KeyError, RuntimeError, rasterio.errors.RasterioError):
return None
def prepare_snapshot(
session: requests.Session,
*,
args: argparse.Namespace,
boundary,
boundary_metric,
year: int,
theme: ThemeDefinition,
) -> PreparedSnapshot:
stem = f"{args.scope_key}_land_use_{theme.key}_{year}"
raster_path = args.output_dir / f"{stem}.tif"
vector_path = args.output_dir / f"{stem}.geojson"
manifest_path = args.output_dir / f"{stem}.manifest.json"
if not args.force:
prepared = existing_snapshot(
year=year,
theme=theme,
raster_path=raster_path,
vector_path=vector_path,
manifest_path=manifest_path,
)
if prepared:
return prepared
raster_profile: dict[str, Any]
if args.force or not raster_path.exists():
raster_profile = download_raster(
session,
year=year,
bounds=boundary_metric.bounds,
path=raster_path,
timeout=args.request_timeout,
)
else:
raster_profile = validate_raster(raster_path)
raster_profile["request_url"] = requests.Request(
"GET", WCS_URL, params=build_wcs_params(year, boundary_metric.bounds)
).prepare().url
raster_profile["retrieved_at"] = None
payload, stats = polygonize_snapshot(
raster_path=raster_path,
boundary=boundary,
year=year,
theme=theme,
municipality_name=args.municipality_name,
nis_code=args.nis_code,
scope_key=args.scope_key,
max_features=args.max_features,
)
write_json_atomic(vector_path, payload)
vector_sha256 = sha256_file(vector_path)
manifest = {
"schema_version": 1,
"year": year,
"theme": theme.key,
"class_ids": list(theme.class_ids),
"class_names": [LAND_USE_CLASSES[class_id] for class_id in theme.class_ids],
"coverage_id": coverage_id(year),
"catalogue_url": CATALOGUE_URLS[year],
"wcs_url": WCS_URL,
"wcs_version": WCS_VERSION,
"wcs_request_url": raster_profile.get("request_url"),
"source_crs": SOURCE_CRS,
"output_crs": OUTPUT_CRS,
"source_resolution_metres": SOURCE_RESOLUTION_METRES,
"boundary_path": str(args.boundary_path),
"municipality": args.municipality_name,
"nis_code": args.nis_code,
"scope_key": args.scope_key,
"raster_path": str(raster_path),
"vector_path": str(vector_path),
"raster_profile": raster_profile,
"raster_sha256": stats["raster_sha256"],
"vector_sha256": vector_sha256,
"feature_count": stats["feature_count"],
"source_pixel_count": stats["source_pixel_count"],
"source_pixel_area_m2": stats["source_pixel_area_m2"],
"polygon_area_m2": stats["polygon_area_m2"],
"nodata_pixels_inside_boundary": stats["nodata_pixels_inside_boundary"],
"class_histogram": stats["class_histogram"],
"generated_at": utc_now(),
}
write_json_atomic(manifest_path, manifest, pretty=True)
return PreparedSnapshot(
year=year,
theme=theme,
raster_path=raster_path,
vector_path=vector_path,
manifest_path=manifest_path,
feature_count=int(stats["feature_count"]),
raster_sha256=str(stats["raster_sha256"]),
vector_sha256=vector_sha256,
)
def response_data(response: requests.Response) -> Any:
try:
payload = response.json()
except ValueError as exc:
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
if not response.ok:
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:800]}")
if not isinstance(payload, dict) or "data" not in payload:
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
return payload["data"]
def locate_workspace(session: requests.Session, base_url: str, args: argparse.Namespace):
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=args.import_timeout))
project = next((item for item in projects.get("items") or [] if item.get("name") == args.project_name), None)
if not project:
raise RuntimeError(f"Project {args.project_name!r} is missing")
project_id = str(project["id"])
areas = response_data(
session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=args.import_timeout)
)
area_fragment = args.area_name.strip().casefold()
matches = [item for item in areas.get("items") or [] if area_fragment in str(item.get("name") or "").casefold()]
if len(matches) != 1:
raise RuntimeError(f"Expected one Area matching {args.area_name!r}, received {len(matches)}")
datasets = response_data(
session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 500}, timeout=args.import_timeout)
)
return project_id, str(matches[0]["id"]), list(datasets.get("items") or [])
def build_source_metadata(args: argparse.Namespace, snapshot: PreparedSnapshot) -> dict[str, Any]:
return {
"provider": "Departement Omgeving",
"source_title": f"Landgebruik - Vlaanderen - toestand {snapshot.year}",
"catalogue_url": CATALOGUE_URLS[snapshot.year],
"coverage_id": coverage_id(snapshot.year),
"authority_level": "authoritative",
"coverage_scope": args.scope_key,
"municipality": args.municipality_name,
"nis_code": args.nis_code,
"attribution": ATTRIBUTION,
"license_note": "Publieke Vlaamse overheidsdata; raadpleeg de toegangs- en gebruiksvoorwaarden in de bronmetadata.",
"methodology_version": "3",
"source_resolution_metres": SOURCE_RESOLUTION_METRES,
"source_crs": SOURCE_CRS,
"polygon_crs": OUTPUT_CRS,
"land_use_class_ids": list(snapshot.theme.class_ids),
"land_use_class_names": [LAND_USE_CLASSES[class_id] for class_id in snapshot.theme.class_ids],
"temporal_series_label": SERIES_LABEL,
"observation_date_precision": "year",
"identity_stable": False,
"identity_limitation": "Raster-derived polygons can split or merge between source editions; object lineage is not inferred.",
"selection_aggregation": {
"method": "intersection_area",
"label": "Oppervlakte",
"unit": "ha",
"is_estimate": False,
"warning": "Oppervlakte is exact binnen de officiele 10 m rasterrepresentatie en is niet perceelsnauwkeurig.",
},
"comparison_limitation": "Compare editions as 10 m land-use states; source inputs and methodology can evolve between publication years.",
}
def build_provenance_metadata(args: argparse.Namespace, snapshot: PreparedSnapshot) -> dict[str, Any]:
return {
"operator_tool": "provision_official_landuse_timeseries.py",
"operator_explicit_fetch": True,
"wcs_url": WCS_URL,
"wcs_version": WCS_VERSION,
"coverage_id": coverage_id(snapshot.year),
"catalogue_url": CATALOGUE_URLS[snapshot.year],
"raw_raster_path": str(snapshot.raster_path),
"polygon_artifact_path": str(snapshot.vector_path),
"manifest_path": str(snapshot.manifest_path),
"raster_sha256": snapshot.raster_sha256,
"vector_sha256": snapshot.vector_sha256,
"source_crs": SOURCE_CRS,
"output_crs": OUTPUT_CRS,
"source_resolution_metres": SOURCE_RESOLUTION_METRES,
"generated_at": utc_now(),
}
def upload_snapshot(
session: requests.Session,
*,
base_url: str,
project_id: str,
area_id: str,
args: argparse.Namespace,
snapshot: PreparedSnapshot,
) -> dict[str, Any]:
observed_at = f"{snapshot.year}-01-01T00:00:00Z"
with snapshot.vector_path.open("rb") as handle:
response = session.post(
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
data={
"dataset_type": "vector",
"source": "operator_official_import",
"dataset_role": "reference",
"source_name": "department_omgeving_land_use",
"reference_layer_name": snapshot.theme.key,
"source_metadata_json": json.dumps(build_source_metadata(args, snapshot), ensure_ascii=False),
"provenance_metadata_json": json.dumps(build_provenance_metadata(args, snapshot), ensure_ascii=False),
"area_id": area_id,
"temporal_series_key": series_key(snapshot.theme, args.scope_key),
"observed_at": observed_at,
"valid_from": observed_at,
"temporal_granularity": "year",
"source_version": f"{snapshot.year}-v3",
},
files={"file": (snapshot.vector_path.name, handle, "application/geo+json")},
timeout=args.import_timeout,
)
return response_data(response)
def main() -> int:
args = parse_args()
try:
years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()})
except ValueError:
print(json.dumps({"status": "error", "message": "Years must be comma-separated integers"}), file=sys.stderr)
return 2
requested_themes = {value.strip().lower() for value in args.themes.split(",") if value.strip()}
definitions = [definition for definition in THEMES if definition.key in requested_themes]
unsupported_years = [year for year in years if year not in SUPPORTED_YEARS]
unsupported_themes = requested_themes - {definition.key for definition in THEMES}
if unsupported_years or unsupported_themes or not years or not definitions:
print(
json.dumps(
{"status": "error", "message": f"Unsupported years={unsupported_years}, themes={sorted(unsupported_themes)}"}
),
file=sys.stderr,
)
return 2
if not args.scope_key.strip() or not args.project_name.strip() or not args.area_name.strip():
print(json.dumps({"status": "error", "message": "scope-key, project-name and area-name are required"}), file=sys.stderr)
return 2
args.output_dir.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
try:
boundary = load_boundary(args.boundary_path)
boundary_metric = metric_boundary(boundary)
prepared: list[PreparedSnapshot] = []
with build_session() as source_session:
verify_coverages(source_session, years, args.request_timeout)
for year in years:
for theme in definitions:
prepared.append(
prepare_snapshot(
source_session,
args=args,
boundary=boundary,
boundary_metric=boundary_metric,
year=year,
theme=theme,
)
)
if args.fetch_only:
results = [
{
"year": item.year,
"theme": item.theme.key,
"status": "prepared",
"feature_count": item.feature_count,
"raster_path": str(item.raster_path),
"vector_path": str(item.vector_path),
"manifest_path": str(item.manifest_path),
}
for item in prepared
]
else:
base_url = args.base_url.rstrip("/")
with requests.Session() as api_session:
project_id, area_id, existing = locate_workspace(api_session, base_url, args)
for item in prepared:
key = series_key(item.theme, args.scope_key)
observed_date = f"{item.year}-01-01"
dataset = next(
(
candidate
for candidate in existing
if candidate.get("temporal_series_key") == key
and str(candidate.get("observed_at") or "").startswith(observed_date)
),
None,
)
if dataset:
results.append(
{
"year": item.year,
"theme": item.theme.key,
"status": "existing",
"dataset_id": dataset["id"],
"feature_count": dataset.get("feature_count"),
}
)
continue
dataset = upload_snapshot(
api_session,
base_url=base_url,
project_id=project_id,
area_id=area_id,
args=args,
snapshot=item,
)
existing.append(dataset)
results.append(
{
"year": item.year,
"theme": item.theme.key,
"status": "imported",
"dataset_id": dataset["id"],
"feature_count": dataset.get("feature_count"),
}
)
except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError, json.JSONDecodeError) as exc:
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 1
print(
json.dumps(
{
"status": "ok",
"scope": args.scope_key,
"municipality": args.municipality_name,
"series": [series_key(theme, args.scope_key) for theme in definitions],
"snapshots": results,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
sys.exit(main())
+3
View File
@@ -44,6 +44,9 @@ ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_municipality_workspace.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_context_layers.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py