Add regional historical land-use operator
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-15 23:25:15 +02:00
parent acac63a3dd
commit 3a8f6e6cef
13 changed files with 1165 additions and 11 deletions
+13
View File
@@ -7,6 +7,19 @@
# Changelog
## Sprint 209 Regional historical land-use expansion (2026-07-15)
- Added an explicit operator for the official 1778, 1873 and 1969 historical
building, water and road land-use classes across all 28 approved Kempen
transport-region municipalities.
- Avoided the official WFS 10,000-result regional cap through resumable VRBG
municipality partitions, exact raw-response gzip retention and SHA256
source/output manifests.
- Added exact municipality clipping, partition-unique evidence ids, bounded
feature limits and one canonical regional Dataset upload per theme/year.
- Extended the regional time-series coordinator and release packaging without
changing API contracts, database migrations or frontend architecture.
## Sprint 208 Governed VMM flood-hazard scenarios (2026-07-15)
- Audited official water-depth and bathymetry sources and found no public,
+17 -2
View File
@@ -1046,7 +1046,7 @@ storage. They never run on app startup.
Historical land-use work can be bounded explicitly:
```bash
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778 1969 --themes forest water
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778,1969 --themes forest,water
```
`GET /api/v1/projects/{project_id}/temporal/series` discovers the series and
@@ -1066,7 +1066,8 @@ docker exec geointel python /app/scripts/provision_regional_timeseries.py
This resolves the retained official boundary and imports five Statbel
population snapshots plus five modern forest, water, built-function and
transport-infrastructure snapshots into
transport-infrastructure snapshots, followed by the 1778/1873/1969 historical
building, water and road snapshots, into
`Kempen Regional Workbench`. Mol and regional series keys remain separate and
existing immutable datasets are reused. Complete statistical sectors use exact
published totals; a rectangle cutting a sector remains an area-weighted
@@ -1076,6 +1077,20 @@ The regional forest path partitions WCS requests by official municipality to
stay within upstream response limits, then builds one retained 10 m mosaic and
one normal regional vector Dataset. A failed source request leaves completed
partition artifacts reusable and never lowers source resolution silently.
Historical WFS retrieval is likewise partitioned by all 28 municipality
boundaries because broad WFS counts stop at 10,000. Exact source responses are
retained as checksummed gzip artifacts before clipping and regional assembly.
Run that stage independently when needed:
```bash
docker exec geointel python /app/scripts/provision_regional_historical_landuse.py
```
Use `--fetch-only` for source/artifact validation without persistence. The
historical building class represents mapped built land-use surfaces, not
individual building footprints; water remains surface area, not depth or
volume; historical roads are mapped road surfaces, not present-day centerline
length.
Official operator datasets record that their geometries were clipped to the
persisted Area. When that exact Area is selected, vector totals and aggregate
@@ -90,11 +90,12 @@ def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Pat
assert module.resolve_boundary_path(args, scope) == boundary
def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp_path: Path) -> None:
def test_regional_coordinator_builds_explicit_population_and_landuse_commands(tmp_path: Path) -> None:
module = load_script("provision_regional_timeseries.py")
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
args = argparse.Namespace(
output_root=tmp_path / "time-series",
scope_output_root=tmp_path / "scopes",
fetch_only=False,
force=False,
skip_population=False,
@@ -102,15 +103,19 @@ def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp
base_url="http://backend:8000",
population_years="2021,2025",
landuse_years="2013,2025",
historical_years="1778,1873,1969",
historical_themes="buildings,water,roads",
request_timeout=300,
import_timeout=3600,
max_landuse_features=500000,
max_historical_features=500000,
skip_historical=False,
)
members_path = tmp_path / "municipalities.geojson"
commands = dict(module.build_operator_commands(args, scope, tmp_path / "boundary.geojson", members_path))
assert set(commands) == {"population", "forest"}
assert set(commands) == {"population", "forest", "historical_landuse"}
assert commands["population"][0] == sys.executable
assert "--scope" in commands["population"]
assert "kempen-transport-region" in commands["population"]
@@ -122,6 +127,10 @@ def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp
assert ",".join(scope.nis_codes) in commands["forest"]
assert "--force" not in commands["population"]
assert "--fetch-only" not in commands["forest"]
assert commands["historical_landuse"][0] == sys.executable
assert "provision_regional_historical_landuse.py" in commands["historical_landuse"][1]
assert "buildings,water,roads" in commands["historical_landuse"]
assert "--scope-output-root" in commands["historical_landuse"]
def test_regional_forest_provenance_does_not_claim_one_municipality() -> None:
@@ -0,0 +1,239 @@
from __future__ import annotations
import gzip
import importlib.util
import json
from pathlib import Path
import sys
from shapely.geometry import box, mapping, shape
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
def load_script():
path = SCRIPTS / "provision_regional_historical_landuse.py"
spec = importlib.util.spec_from_file_location("test_provision_regional_historical_landuse", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
class FakeResponse:
status_code = 200
ok = True
text = ""
def __init__(self, payload):
self.payload = payload
self.content = json.dumps(payload, separators=(",", ":")).encode("utf-8")
def raise_for_status(self):
return None
def json(self):
return self.payload
class FakeSession:
def __init__(self, responses):
self.responses = list(responses)
self.calls = []
def post(self, url, **kwargs):
self.calls.append((url, kwargs))
if not self.responses:
raise AssertionError("Unexpected source/API request")
return self.responses.pop(0)
def source_feature(feature_id: str, geometry, landuse_class: str = "bebouwing"):
return {
"type": "Feature",
"id": feature_id,
"geometry": mapping(geometry),
"properties": {"KLASSE": landuse_class},
}
def test_member_boundaries_require_every_approved_municipality(tmp_path: Path) -> None:
module = load_script()
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
features = []
for index, member in enumerate(scope.members):
features.append(
{
"type": "Feature",
"geometry": mapping(box(index, 0, index + 0.9, 0.9)),
"properties": {"nis_code": member.nis_code, "municipality": member.name},
}
)
path = tmp_path / "members.geojson"
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
boundaries = module.load_member_boundaries(path, scope)
assert list(boundaries) == list(scope.nis_codes)
assert len(boundaries) == 28
assert boundaries["13025"][0].name == "Mol"
def test_partition_retains_exact_source_response_and_clips_to_municipality(tmp_path: Path) -> None:
module = load_script()
definition = next(item for item in module.THEMES if item.key == "buildings")
member = module.ScopeMember("Mol", "13025")
boundary = box(5.0, 51.0, 5.1, 51.1)
payload = {
"type": "FeatureCollection",
"features": [source_feature("Lgbrk1778.1", box(4.95, 51.02, 5.05, 51.08))],
}
session = FakeSession([FakeResponse(payload)])
manifest = module.prepare_partition(
session,
output_root=tmp_path,
year=1778,
definition=definition,
scope_key="kempen-transport-region",
member=member,
boundary=boundary,
page_size=50,
max_features=100,
simplify_tolerance_degrees=0.0,
timeout=30,
force=False,
)
output = json.loads(Path(manifest["output_path"]).read_text(encoding="utf-8"))
feature = output["features"][0]
raw_path = Path(manifest["output_path"]).parent / manifest["raw_pages"][0]["artifact_path"]
assert manifest["feature_count"] == 1
assert manifest["source_feature_count"] == 1
assert feature["id"] == "Lgbrk1778.1:13025"
assert feature["properties"]["original_source_feature_id"] == "Lgbrk1778.1"
assert feature["properties"]["coverage_scope"] == "kempen-transport-region"
assert shape(feature["geometry"]).bounds == (5.0, 51.02, 5.05, 51.08)
assert gzip.decompress(raw_path.read_bytes()) == FakeResponse(payload).content
cached = module.prepare_partition(
FakeSession([]),
output_root=tmp_path,
year=1778,
definition=definition,
scope_key="kempen-transport-region",
member=member,
boundary=boundary,
page_size=50,
max_features=100,
simplify_tolerance_degrees=0.0,
timeout=30,
force=False,
)
assert cached["output_sha256"] == manifest["output_sha256"]
def test_regional_snapshot_assembles_unique_partition_features(tmp_path: Path) -> None:
module = load_script()
scope = module.GeographicScope(
key="test-region",
display_name="Test region",
project_name="Test",
project_region="Test",
area_name="Test area",
authority_name="Authority",
authority_url="https://example.test",
scope_type="test",
limitation_message="Test only",
members=(module.ScopeMember("Left", "10001"), module.ScopeMember("Right", "10002")),
)
definition = next(item for item in module.THEMES if item.key == "water")
partitions = []
for index, member in enumerate(scope.members):
path, _manifest_path, _raw_dir = module.partition_paths(tmp_path, 1873, definition.key, member.nis_code)
feature = source_feature(f"water.{index}:{member.nis_code}", box(index, 0, index + 0.5, 0.5), "water")
module.atomic_write_json(path, {"type": "FeatureCollection", "features": [feature]})
partitions.append(
{
"municipality": member.name,
"nis_code": member.nis_code,
"source_feature_count": 1,
"feature_count": 1,
"raw_pages": [{"artifact_path": "unused"}],
"output_path": str(path),
"output_sha256": module.sha256_file(path),
}
)
output_path, manifest = module.assemble_snapshot(
output_root=tmp_path,
scope=scope,
year=1873,
definition=definition,
partitions=partitions,
max_total_features=10,
)
payload = json.loads(output_path.read_text(encoding="utf-8"))
assert manifest["coverage_complete"] is True
assert manifest["feature_count"] == 2
assert manifest["empty_partitions"] == []
assert len({feature["id"] for feature in payload["features"]}) == 2
def test_upload_contract_is_regional_temporal_and_partition_audited(tmp_path: Path) -> None:
module = load_script()
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
definition = next(item for item in module.THEMES if item.key == "roads")
path = tmp_path / "roads.geojson"
path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
response_payload = {"data": {"id": "dataset-id", "feature_count": 42}}
session = FakeSession([FakeResponse(response_payload)])
manifest = {
"coverage_complete": True,
"empty_partitions": [],
"partitions": [{} for _ in scope.members],
"partition_identity_sha256": "partition-hash",
"output_sha256": "output-hash",
"generated_at": "2026-07-15T00:00:00+00:00",
}
result = module.upload_snapshot(
session,
base_url="http://backend:8000",
project_id="project-id",
area_id="area-id",
scope=scope,
year=1969,
definition=definition,
path=path,
manifest=manifest,
simplify_tolerance_degrees=0.00001,
timeout=30,
)
data = session.calls[0][1]["data"]
source_metadata = json.loads(data["source_metadata_json"])
provenance = json.loads(data["provenance_metadata_json"])
assert result["id"] == "dataset-id"
assert data["area_id"] == "area-id"
assert data["temporal_series_key"].endswith(":roads:kempen-transport-region")
assert data["observed_at"] == "1969-01-01T00:00:00Z"
assert source_metadata["member_count"] == 28
assert source_metadata["partitioned_source_audit"] is True
assert source_metadata["identity_stable"] is False
assert provenance["partition_count"] == 28
assert provenance["raw_source_responses_retained"] is True
def test_regional_historical_operator_is_packaged_and_release_checked() -> None:
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
assert "COPY scripts/provision_regional_historical_landuse.py" in dockerfile
assert "py_compile scripts/provision_regional_historical_landuse.py" in readiness
+1
View File
@@ -78,6 +78,7 @@ COPY scripts/provision_mol_dhmv.py /app/scripts/provision_mol_dhmv.py
COPY scripts/provision_mol_flood_hazards.py /app/scripts/provision_mol_flood_hazards.py
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_regional_historical_landuse.py
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
+44
View File
@@ -8808,3 +8808,47 @@ Next:
value derivable from DHMV or flood-hazard maps. Continue the independent P5
waterbody linkage/uncertainty design only when governed depth profiles exist;
otherwise proceed with the regional historical buildings/water/roads backlog.
## Sprint 209 - Regional historical land-use expansion (2026-07-15)
Implemented:
- Audited the production Digitaal Vlaanderen Historical Land Use WFS and its
1778, 1873 and 1969 collections. Broad regional hit counts stop at 10,000,
confirming that one transport-region bounding request is not a defensible
completeness strategy.
- Added `scripts/provision_regional_historical_landuse.py` for historical
building, water and road land-use classes over the 28 registered transport-
region municipalities. It requires the checksummed VRBG scope artifact and
never infers or broadens the regional definition.
- Added resumable municipality partitions with exact WFS JSON response bytes
retained as deterministic gzip artifacts, source/artifact SHA256, request
pagination evidence, boundary hashes, clipping diagnostics and combined
regional output manifests.
- Clipped and validated polygonal geometry per municipality, suffixing source
ids with NIS codes to prevent cross-boundary collisions. Source identities
remain explicitly unstable and do not enable object lineage.
- Added one canonical Dataset upload per theme/year with regional Area scope,
immutable temporal identity, hectare aggregation and methodological warning.
No API route, database migration or direct PostGIS write was added.
- Extended `provision_regional_timeseries.py`, all-in-one packaging, readiness
compilation and operator/source documentation.
Validation evidence before live provisioning:
- Fourteen focused partition, raw-evidence, clipping, assembly, upload,
coordinator and packaging tests pass.
- The official source catalog still reports the production WFS and the three
governed historical collections. Live partition provisioning and browser
Evolution validation follow in this pass.
Known limitations:
- Historical `buildings` are mapped built land-use surfaces, not individual
building footprints. Historical `roads` and `water` are mapped surfaces,
not current centerlines, bathymetry or volume.
- The three editions use different source maps and cartographic methods.
Comparisons are exploratory hectare changes within one series, never a
continuous equivalent to modern GRB or 10 m land-use products.
Next:
- Run the complete nine-snapshot operator on live Tower/PostGIS, verify all 28
partition manifests and then validate regional/municipality Evolution in the
browser before marking the backlog item complete.
+6
View File
@@ -300,3 +300,9 @@ observation/validity dates, checksum and source/provenance JSON. The
Time-series comparison is read-only and aggregates persisted geometry inside a
requested bbox. Object-level added/removed/modified evidence is only valid for
sources that explicitly declare stable source feature identifiers.
Regional 1778/1873/1969 historical land-use snapshots use the same tables.
Their `provenance_metadata` records 28-partition identity, combined output
checksum and retained raw-response status. Municipality-clipped source ids are
partition-suffixed for uniqueness, while `identity_stable=false` prohibits
object-lineage interpretation. No partition or temporal shadow table is added.
+20 -3
View File
@@ -129,7 +129,7 @@ Statistical-sector codes and boundaries can change between editions. GeoIntel
therefore compares population metrics but does not present sector additions,
removals or boundary changes as population-object lineage.
### Mol historical land use
### Historical land use for Mol and the approved region
`scripts/provision_mol_historical_landuse.py` uses the official Digitaal
Vlaanderen Historical Land Use WFS for the 1778, 1873 and 1969 collections.
@@ -146,6 +146,17 @@ The operator uses standards-compliant WFS 2.0 XML POST requests. This keeps
the spatial/class filters server-side without exposing a long XML filter in a
GET query, which the public gateway rejects.
`scripts/provision_regional_historical_landuse.py` extends the building,
water and road series to the approved 28-municipality transport-region scope.
Broad regional WFS counts are capped at 10,000 by the public service, so the
operator uses the retained official VRBG municipality boundaries as 28
explicit source partitions. Each exact WFS JSON response is retained as a
deterministically compressed artifact with response and artifact SHA256,
then clipped to its municipality before one regional snapshot is assembled.
Partition-suffixed feature ids prevent cross-boundary source polygons from
colliding. Those ids remain processing evidence and are not historical object
lineage.
### Mol modern land use
`scripts/provision_official_landuse_timeseries.py` uses the public Departement
@@ -181,14 +192,18 @@ added/removed forest objects.
### Regional official time series
`scripts/provision_regional_timeseries.py` applies the same population and
modern forest operators to the approved 28-municipality
`scripts/provision_regional_timeseries.py` applies the population, modern
land-use and partitioned historical land-use operators to the approved 28-municipality
`kempen-transport-region` scope. It resolves the checksummed VRBG union
boundary and writes to `Kempen Regional Workbench` through the normal dataset
API. Regional keys are
`statbel:population-statistical-sector:kempen-transport-region` and
`department-omgeving:land-use:{theme}:kempen-transport-region`, so Mol datasets
remain independent observations rather than aliases.
Historical buildings, water and roads use
`digitaal-vlaanderen:historical-landuse:{theme}:kempen-transport-region` and
remain separate from both current GRB geometry and the harmonized 2013-2025
land-use rasters.
The command is explicit and operator-triggered. No source fetch happens during
startup or map interaction. Partial statistical sectors remain area-weighted
@@ -201,6 +216,8 @@ exact regional union clip before polygon persistence.
Official catalogues:
- https://www.vlaanderen.be/datavindplaats/catalogus/wfs-historisch-landgebruik
- https://www.vlaanderen.be/datavindplaats/catalogus/digitalisatie-historisch-landgebruik-en-landgebruiksveranderingen-in-vlaanderen-1778-2022
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2013
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2016
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2019
+3
View File
@@ -28,6 +28,9 @@ When a theme has multiple valid methodologies, a compact series selector keeps
them explicit. Forest therefore defaults to the official modern 2013-2025
10 m series, while the separate 1778-1969 historical map series remains
selectable and is never merged into the same trend.
Regional building, water and road evolution uses the same selector after the
partition-audited 1778/1873/1969 snapshots are provisioned. Historical built
surfaces never replace current GRB building footprints in Latest state.
Selection results use dataset-specific PostGIS summaries with end-user units.
Building footprints, forest, water surfaces and parcels show intersected
+17
View File
@@ -1283,6 +1283,23 @@ The 2013-2025 series is methodologically separate from the historical
when both exist; it never calculates one continuous trend across those source
families.
### Regional historical buildings, water and roads
Provision the three historical editions for the complete approved region:
```bash
docker exec geointel python /app/scripts/provision_regional_historical_landuse.py
```
The public WFS caps broad regional result counts at 10,000. The operator uses
the 28 retained official municipality boundaries instead, stores every exact
WFS response as a checksummed gzip artifact, clips polygonal geometry per
municipality and assembles nine regional GeoJSON snapshots. Completed
partitions are reused after checksum validation. `--fetch-only` prepares and
audits artifacts without uploading; `--force` explicitly refreshes the source
partitions. Historical identities are declared unstable and support hectare
comparison only, not object lineage.
## Official Kempen operational scope
GeoIntel defines its regional `Kempen` workspace as the official Vlaamse
@@ -0,0 +1,756 @@
"""Provision partition-audited historical land use for an approved region.
The official historical land-use WFS caps broad regional result counts. This
operator therefore fetches and clips each approved municipality separately,
retains the exact source responses as checksummed gzip artifacts, assembles one
regional snapshot per theme/year and persists only through the dataset API.
"""
from __future__ import annotations
import argparse
import gzip
import hashlib
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import requests
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
from shapely.ops import unary_union
from shapely.validation import make_valid
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope, ScopeMember
from provision_mol_historical_landuse import (
ATTRIBUTION,
COLLECTIONS,
THEMES,
WFS_URL,
ThemeDefinition,
build_session,
response_data,
wfs_filter_xml,
wfs_request_xml,
)
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_SCOPE_KEY = "kempen-transport-region"
DEFAULT_SCOPE_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/regional-historical-landuse")
SOURCE_CATALOG_URL = (
"https://www.vlaanderen.be/datavindplaats/catalogus/"
"digitalisatie-historisch-landgebruik-en-landgebruiksveranderingen-in-vlaanderen-1778-2022"
)
SUPPORTED_THEME_KEYS = frozenset({"buildings", "water", "roads"})
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision regional historical buildings, water and roads.")
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--years", default="1778,1873,1969")
parser.add_argument("--themes", default="buildings,water,roads")
parser.add_argument(
"--scope-output-root",
type=Path,
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_SCOPE_OUTPUT_ROOT)),
)
parser.add_argument(
"--output-root",
type=Path,
default=Path(os.environ.get("GEOINTEL_REGIONAL_HISTORICAL_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
)
parser.add_argument("--page-size", type=int, default=500)
parser.add_argument("--max-features-per-partition", type=int, default=100_000)
parser.add_argument("--max-total-features", type=int, default=500_000)
parser.add_argument("--simplify-tolerance-degrees", type=float, default=0.00001)
parser.add_argument("--request-timeout", type=int, default=300)
parser.add_argument("--import-timeout", type=int, default=3600)
parser.add_argument("--force", action="store_true")
parser.add_argument("--fetch-only", action="store_true")
return parser.parse_args()
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def sha256_bytes(content: bytes) -> str:
return hashlib.sha256(content).hexdigest()
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def atomic_write_bytes(path: Path, content: bytes) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(f"{path.suffix}.partial")
temporary.write_bytes(content)
temporary.replace(path)
def atomic_write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
content = json.dumps(
payload,
ensure_ascii=False,
indent=2 if pretty else None,
separators=None if pretty else (",", ":"),
sort_keys=pretty,
).encode("utf-8")
atomic_write_bytes(path, content)
def normalize_polygonal(geometry):
if geometry is None or geometry.is_empty:
return None
if not geometry.is_valid:
geometry = make_valid(geometry)
if isinstance(geometry, (Polygon, MultiPolygon)):
return geometry
if isinstance(geometry, GeometryCollection):
polygons = [
part
for part in geometry.geoms
if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty
]
if polygons:
merged = unary_union(polygons)
if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty:
return merged
return None
def resolve_member_boundaries(scope: GeographicScope, scope_output_root: Path) -> Path:
scope_dir = scope_output_root / scope.key
manifest_path = scope_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json"
if not manifest_path.is_file():
raise RuntimeError(
f"Official scope manifest is missing at {manifest_path}; run provision_geographic_scope.py first"
)
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if (
manifest.get("status") != "complete"
or manifest.get("scope_key") != scope.key
or int(manifest.get("member_count") or 0) != len(scope.members)
):
raise RuntimeError(f"Official scope manifest at {manifest_path} is incomplete or inconsistent")
members_path = scope_dir / str(manifest.get("municipalities_filename") or "")
if not members_path.is_file() or sha256_file(members_path) != manifest.get("municipalities_sha256"):
raise RuntimeError("Official municipality-boundary artifact is missing or fails its scope checksum")
return members_path
def load_member_boundaries(path: Path, scope: GeographicScope) -> dict[str, tuple[ScopeMember, Any]]:
payload = json.loads(path.read_text(encoding="utf-8"))
features = payload.get("features") if isinstance(payload, dict) else None
if not isinstance(features, list):
raise RuntimeError("Municipality-boundary artifact is not a GeoJSON FeatureCollection")
expected = {member.nis_code: member for member in scope.members}
selected: dict[str, tuple[ScopeMember, Any]] = {}
for feature in features:
properties = feature.get("properties") or {}
nis_code = str(properties.get("nis_code") or properties.get("NISCODE") or "")
if nis_code not in expected:
continue
if nis_code in selected:
raise RuntimeError(f"Municipality-boundary artifact contains duplicate NIS code {nis_code}")
geometry = normalize_polygonal(shape(feature.get("geometry")))
if geometry is None:
raise RuntimeError(f"Municipality boundary for {expected[nis_code].name} is invalid")
selected[nis_code] = (expected[nis_code], geometry)
missing = [member.name for member in scope.members if member.nis_code not in selected]
if missing:
raise RuntimeError(f"Municipality-boundary artifact is missing: {', '.join(missing)}")
return {member.nis_code: selected[member.nis_code] for member in scope.members}
def fetch_source_page(
session: requests.Session,
*,
collection: str,
filter_xml: str,
page_size: int,
start_index: int,
timeout: int,
) -> tuple[bytes, list[dict[str, Any]]]:
response = session.post(
WFS_URL,
data=wfs_request_xml(
collection=collection,
filter_xml=filter_xml,
page_size=page_size,
start_index=start_index,
).encode("utf-8"),
headers={"Content-Type": "application/xml; charset=UTF-8", "Accept": "application/json"},
timeout=timeout,
)
response.raise_for_status()
raw_content = response.content
payload = response.json()
features = payload.get("features") if isinstance(payload, dict) else None
if not isinstance(features, list):
raise RuntimeError("Historical land-use WFS returned an invalid FeatureCollection")
return raw_content, features
def partition_paths(output_root: Path, year: int, theme: str, nis_code: str) -> tuple[Path, Path, Path]:
directory = output_root / "partitions" / str(year) / theme
output_path = directory / f"{nis_code}.geojson"
return output_path, output_path.with_suffix(".manifest.json"), directory / f"{nis_code}.raw"
def cached_partition(
output_path: Path,
manifest_path: Path,
*,
year: int,
theme: str,
nis_code: str,
page_size: int,
simplify_tolerance_degrees: float,
boundary_geometry_sha256: str,
):
if not output_path.is_file() or not manifest_path.is_file():
return None
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
if (
manifest.get("status") != "complete"
or manifest.get("year") != year
or manifest.get("theme") != theme
or manifest.get("nis_code") != nis_code
or int(manifest.get("page_size") or 0) != page_size
or float(manifest.get("geometry_simplification_tolerance_degrees") or 0.0)
!= simplify_tolerance_degrees
or manifest.get("boundary_geometry_sha256") != boundary_geometry_sha256
or sha256_file(output_path) != manifest.get("output_sha256")
):
return None
for page in manifest.get("raw_pages") or []:
raw_path = manifest_path.parent / str(page.get("artifact_path") or "")
if not raw_path.is_file() or sha256_file(raw_path) != page.get("artifact_sha256"):
return None
return manifest
def prepare_partition(
session: requests.Session,
*,
output_root: Path,
year: int,
definition: ThemeDefinition,
scope_key: str,
member: ScopeMember,
boundary,
page_size: int,
max_features: int,
simplify_tolerance_degrees: float,
timeout: int,
force: bool,
) -> dict[str, Any]:
output_path, manifest_path, raw_dir = partition_paths(output_root, year, definition.key, member.nis_code)
boundary_geometry_sha256 = sha256_bytes(boundary.wkb)
if not force:
cached = cached_partition(
output_path,
manifest_path,
year=year,
theme=definition.key,
nis_code=member.nis_code,
page_size=page_size,
simplify_tolerance_degrees=simplify_tolerance_degrees,
boundary_geometry_sha256=boundary_geometry_sha256,
)
if cached:
return cached
collection = COLLECTIONS[year]
filter_xml = wfs_filter_xml(definition, boundary.bounds)
source_seen: set[str] = set()
output_features: list[dict[str, Any]] = []
raw_pages: list[dict[str, Any]] = []
invalid_geometry_count = 0
outside_boundary_count = 0
start_index = 0
while True:
raw_content, page = fetch_source_page(
session,
collection=collection,
filter_xml=filter_xml,
page_size=page_size,
start_index=start_index,
timeout=timeout,
)
compressed = gzip.compress(raw_content, compresslevel=6, mtime=0)
raw_path = raw_dir / f"page_{start_index:09d}.geojson.gz"
atomic_write_bytes(raw_path, compressed)
raw_pages.append(
{
"start_index": start_index,
"feature_count": len(page),
"response_sha256": sha256_bytes(raw_content),
"response_size_bytes": len(raw_content),
"artifact_path": str(raw_path.relative_to(manifest_path.parent)),
"artifact_sha256": sha256_bytes(compressed),
"artifact_size_bytes": len(compressed),
}
)
new_source_ids = 0
for raw_feature in page:
source_feature_id = str(raw_feature.get("id") or "")
if not source_feature_id or source_feature_id in source_seen:
continue
source_seen.add(source_feature_id)
new_source_ids += 1
properties = dict(raw_feature.get("properties") or {})
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
if not definition.matches(landuse_class):
continue
try:
source_geometry = normalize_polygonal(shape(raw_feature.get("geometry")))
except (AttributeError, TypeError, ValueError):
source_geometry = None
if source_geometry is None:
invalid_geometry_count += 1
continue
clipped = normalize_polygonal(source_geometry.intersection(boundary))
if clipped is None:
outside_boundary_count += 1
continue
if simplify_tolerance_degrees > 0:
clipped = normalize_polygonal(
clipped.simplify(simplify_tolerance_degrees, preserve_topology=True)
)
if clipped is None:
invalid_geometry_count += 1
continue
partition_feature_id = f"{source_feature_id}:{member.nis_code}"
properties.update(
{
"source_name": "historical_landuse",
"source_feature_id": partition_feature_id,
"original_source_feature_id": source_feature_id,
"reference_layer_name": definition.key,
"authority_level": "authoritative",
"coverage_scope": scope_key,
"municipality": member.name,
"nis_code": member.nis_code,
"observation_year": year,
"historical_landuse_class": landuse_class,
"attribution": ATTRIBUTION,
}
)
output_features.append(
{
"type": "Feature",
"id": partition_feature_id,
"geometry": mapping(clipped),
"properties": properties,
}
)
if len(output_features) > max_features:
raise RuntimeError(
f"Historical {definition.key} {year} exceeds {max_features} clipped features in {member.name}"
)
if len(source_seen) > max_features:
raise RuntimeError(
f"Historical {definition.key} {year} exceeds {max_features} source features in {member.name}"
)
if len(page) < page_size:
break
if new_source_ids == 0:
raise RuntimeError(
f"Historical WFS pagination stalled for {definition.key} {year} in {member.name}"
)
start_index += len(page)
payload = {
"type": "FeatureCollection",
"name": f"{definition.label} - {member.name} {year}",
"crs": GEOJSON_CRS,
"features": output_features,
"observation_year": year,
"municipality": member.name,
"nis_code": member.nis_code,
"attribution": ATTRIBUTION,
}
atomic_write_json(output_path, payload)
manifest = {
"status": "complete",
"year": year,
"theme": definition.key,
"collection": collection,
"municipality": member.name,
"nis_code": member.nis_code,
"boundary_bbox": [float(value) for value in boundary.bounds],
"boundary_geometry_sha256": boundary_geometry_sha256,
"page_size": page_size,
"source_feature_count": len(source_seen),
"feature_count": len(output_features),
"invalid_geometry_count": invalid_geometry_count,
"outside_boundary_count": outside_boundary_count,
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
"raw_pages": raw_pages,
"output_path": str(output_path),
"output_sha256": sha256_file(output_path),
"output_size_bytes": output_path.stat().st_size,
"generated_at": utc_now(),
}
atomic_write_json(manifest_path, manifest, pretty=True)
return manifest
def snapshot_paths(output_root: Path, scope: GeographicScope, year: int, theme: str) -> tuple[Path, Path]:
directory = output_root / "snapshots"
stem = f"{scope.key.replace('-', '_')}_historical_{theme}_{year}"
output_path = directory / f"{stem}.geojson"
return output_path, directory / f"{stem}.manifest.json"
def assemble_snapshot(
*,
output_root: Path,
scope: GeographicScope,
year: int,
definition: ThemeDefinition,
partitions: list[dict[str, Any]],
max_total_features: int,
) -> tuple[Path, dict[str, Any]]:
if len(partitions) != len(scope.members):
raise RuntimeError(f"Expected {len(scope.members)} partition manifests, received {len(partitions)}")
output_path, manifest_path = snapshot_paths(output_root, scope, year, definition.key)
partition_identity = sha256_bytes(
json.dumps(
[(item["nis_code"], item["output_sha256"]) for item in partitions],
separators=(",", ":"),
).encode("utf-8")
)
if output_path.is_file() and manifest_path.is_file():
existing = json.loads(manifest_path.read_text(encoding="utf-8"))
if (
existing.get("partition_identity_sha256") == partition_identity
and sha256_file(output_path) == existing.get("output_sha256")
):
return output_path, existing
output_path.parent.mkdir(parents=True, exist_ok=True)
temporary = output_path.with_suffix(f"{output_path.suffix}.partial")
feature_count = 0
seen_ids: set[str] = set()
with temporary.open("w", encoding="utf-8") as output:
header = {
"type": "FeatureCollection",
"name": f"{definition.label} - {scope.display_name} {year}",
"crs": GEOJSON_CRS,
"scope_key": scope.key,
"member_count": len(scope.members),
"observation_year": year,
"attribution": ATTRIBUTION,
}
output.write(json.dumps(header, ensure_ascii=False, separators=(",", ":"))[:-1])
output.write(',"features":[')
first = True
for partition in partitions:
partition_path = Path(str(partition["output_path"]))
payload = json.loads(partition_path.read_text(encoding="utf-8"))
features = payload.get("features") if isinstance(payload, dict) else None
if not isinstance(features, list) or len(features) != int(partition["feature_count"]):
raise RuntimeError(f"Partition feature count drift for NIS {partition['nis_code']}")
for feature in features:
feature_id = str(feature.get("id") or "")
if not feature_id or feature_id in seen_ids:
raise RuntimeError(f"Duplicate or missing partition feature id in {definition.key} {year}")
seen_ids.add(feature_id)
if not first:
output.write(",")
output.write(json.dumps(feature, ensure_ascii=False, separators=(",", ":")))
first = False
feature_count += 1
if feature_count > max_total_features:
raise RuntimeError(
f"Regional historical {definition.key} {year} exceeds {max_total_features} features"
)
output.write("]}")
if feature_count == 0:
temporary.unlink(missing_ok=True)
raise RuntimeError(f"Regional historical {definition.key} {year} contains no clipped features")
temporary.replace(output_path)
empty_partitions = [item["nis_code"] for item in partitions if int(item["feature_count"]) == 0]
manifest = {
"status": "complete",
"scope_key": scope.key,
"scope_type": scope.scope_type,
"member_count": len(scope.members),
"year": year,
"theme": definition.key,
"collection": COLLECTIONS[year],
"feature_count": feature_count,
"coverage_complete": True,
"empty_partitions": empty_partitions,
"partition_identity_sha256": partition_identity,
"partitions": [
{
"municipality": item["municipality"],
"nis_code": item["nis_code"],
"source_feature_count": item["source_feature_count"],
"feature_count": item["feature_count"],
"raw_page_count": len(item.get("raw_pages") or []),
"output_sha256": item["output_sha256"],
}
for item in partitions
],
"output_path": str(output_path),
"output_sha256": sha256_file(output_path),
"output_size_bytes": output_path.stat().st_size,
"generated_at": utc_now(),
}
atomic_write_json(manifest_path, manifest, pretty=True)
return output_path, manifest
def locate_workspace(session: requests.Session, base_url: str, scope: GeographicScope, timeout: int):
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
project = next((item for item in projects.get("items") or [] if item.get("name") == scope.project_name), None)
if not project:
raise RuntimeError(f"Project {scope.project_name!r} is missing")
project_id = str(project["id"])
areas = response_data(
session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout)
)
area = next((item for item in areas.get("items") or [] if item.get("name") == scope.area_name), None)
if not area:
raise RuntimeError(f"Official scope Area {scope.area_name!r} is missing")
datasets = response_data(
session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 200}, timeout=timeout)
)
return project_id, str(area["id"]), list(datasets.get("items") or [])
def upload_snapshot(
session: requests.Session,
*,
base_url: str,
project_id: str,
area_id: str,
scope: GeographicScope,
year: int,
definition: ThemeDefinition,
path: Path,
manifest: dict[str, Any],
simplify_tolerance_degrees: float,
timeout: int,
) -> dict[str, Any]:
observed_at = f"{year}-01-01T00:00:00Z"
source_metadata = {
"provider": "Digitaal Vlaanderen",
"collection": COLLECTIONS[year],
"authority_level": "authoritative",
"coverage_scope": scope.key,
"scope_type": scope.scope_type,
"scope_display_name": scope.display_name,
"member_count": len(scope.members),
"member_nis_codes": list(scope.nis_codes),
"partitioned_source_audit": True,
"coverage_complete": bool(manifest["coverage_complete"]),
"empty_partition_nis_codes": manifest["empty_partitions"],
"attribution": ATTRIBUTION,
"source_catalog_url": SOURCE_CATALOG_URL,
"identity_stable": False,
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
"selection_aggregation": {
"method": "intersection_area",
"label": "Oppervlakte",
"unit": "ha",
"is_estimate": False,
"warning": (
"Historische kaartklassen, bronkaarten en karteermethodes verschillen per bronjaar; "
"interpreteer evoluties binnen die methodologische context."
),
},
}
provenance_metadata = {
"operator_tool": "provision_regional_historical_landuse.py",
"operator_explicit_fetch": True,
"wfs_url": WFS_URL,
"collection": COLLECTIONS[year],
"partition_count": len(manifest["partitions"]),
"partition_identity_sha256": manifest["partition_identity_sha256"],
"combined_output_sha256": manifest["output_sha256"],
"raw_source_responses_retained": True,
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
"generated_at": manifest["generated_at"],
}
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:{scope.key}"
with path.open("rb") as handle:
response = session.post(
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
data={
"dataset_type": "vector",
"source": "operator_official_import",
"dataset_role": "reference",
"source_name": "historical_landuse",
"reference_layer_name": definition.key,
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
"area_id": area_id,
"temporal_series_key": series_key,
"observed_at": observed_at,
"valid_from": observed_at,
"temporal_granularity": "year",
"source_version": str(year),
},
files={"file": (path.name, handle, "application/geo+json")},
timeout=timeout,
)
return response_data(response)
def requested_configuration(args: argparse.Namespace) -> tuple[list[int], list[ThemeDefinition]]:
try:
years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()})
except ValueError as exc:
raise ValueError("Years must be comma-separated integers") from exc
theme_keys = {value.strip().lower() for value in args.themes.split(",") if value.strip()}
definitions = [item for item in THEMES if item.key in theme_keys and item.key in SUPPORTED_THEME_KEYS]
unsupported_years = sorted(set(years) - set(COLLECTIONS))
unsupported_themes = sorted(theme_keys - SUPPORTED_THEME_KEYS)
if not years or not definitions or unsupported_years or unsupported_themes:
raise ValueError(f"Unsupported years={unsupported_years}, themes={unsupported_themes}")
return years, definitions
def main() -> int:
args = parse_args()
scope = GEOGRAPHIC_SCOPES[args.scope]
try:
if args.page_size <= 0 or args.max_features_per_partition <= 0 or args.max_total_features <= 0:
raise ValueError("Page and feature safety limits must be greater than zero")
years, definitions = requested_configuration(args)
members_path = resolve_member_boundaries(scope, args.scope_output_root)
boundaries = load_member_boundaries(members_path, scope)
output_root = args.output_root / scope.key
prepared: list[tuple[int, ThemeDefinition, Path, dict[str, Any]]] = []
with build_session() as source_session:
for year in years:
for definition in definitions:
partitions = [
prepare_partition(
source_session,
output_root=output_root,
year=year,
definition=definition,
scope_key=scope.key,
member=member,
boundary=boundaries[member.nis_code][1],
page_size=args.page_size,
max_features=args.max_features_per_partition,
simplify_tolerance_degrees=args.simplify_tolerance_degrees,
timeout=args.request_timeout,
force=args.force,
)
for member in scope.members
]
path, manifest = assemble_snapshot(
output_root=output_root,
scope=scope,
year=year,
definition=definition,
partitions=partitions,
max_total_features=args.max_total_features,
)
prepared.append((year, definition, path, manifest))
results: list[dict[str, Any]] = []
if args.fetch_only:
results = [
{
"year": year,
"theme": definition.key,
"feature_count": manifest["feature_count"],
"partition_count": len(manifest["partitions"]),
"empty_partitions": manifest["empty_partitions"],
"path": str(path),
"status": "prepared",
}
for year, definition, path, manifest in prepared
]
else:
base_url = args.base_url.rstrip("/")
with requests.Session() as api_session:
project_id, area_id, existing = locate_workspace(api_session, base_url, scope, args.import_timeout)
for year, definition, path, manifest in prepared:
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:{scope.key}"
dataset = next(
(
item
for item in existing
if item.get("temporal_series_key") == series_key
and str(item.get("observed_at") or "").startswith(str(year))
),
None,
)
if dataset:
if int(dataset.get("feature_count") or 0) != int(manifest["feature_count"]):
raise RuntimeError(
f"Persisted {definition.key} {year} count differs from the audited source artifact"
)
status = "existing"
else:
dataset = upload_snapshot(
api_session,
base_url=base_url,
project_id=project_id,
area_id=area_id,
scope=scope,
year=year,
definition=definition,
path=path,
manifest=manifest,
simplify_tolerance_degrees=args.simplify_tolerance_degrees,
timeout=args.import_timeout,
)
existing.append(dataset)
status = "imported"
results.append(
{
"year": year,
"theme": definition.key,
"dataset_id": dataset["id"],
"feature_count": dataset.get("feature_count"),
"partition_count": len(manifest["partitions"]),
"empty_partitions": manifest["empty_partitions"],
"status": status,
}
)
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException, json.JSONDecodeError) as exc:
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 1
print(
json.dumps(
{
"status": "ok",
"mode": "fetch_only" if args.fetch_only else "synchronized",
"scope": scope.key,
"display_name": scope.display_name,
"member_count": len(scope.members),
"snapshots": results,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
sys.exit(main())
+37 -4
View File
@@ -1,4 +1,4 @@
"""Synchronize official population and forest time series for one approved scope.
"""Synchronize official population and land-use time series for one approved scope.
This is an explicit operator command, never an application-startup task. It
coordinates the existing Statbel and Departement Omgeving import paths and
@@ -30,6 +30,8 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--population-years", default="2021,2022,2023,2024,2025")
parser.add_argument("--landuse-years", default="2013,2016,2019,2022,2025")
parser.add_argument("--historical-years", default="1778,1873,1969")
parser.add_argument("--historical-themes", default="buildings,water,roads")
parser.add_argument(
"--scope-output-root",
type=Path,
@@ -41,10 +43,12 @@ def parse_args() -> argparse.Namespace:
default=Path(os.environ.get("GEOINTEL_REGIONAL_TIMESERIES_OUTPUT_ROOT", DEFAULT_TIMESERIES_OUTPUT_ROOT)),
)
parser.add_argument("--max-landuse-features", type=int, default=500000)
parser.add_argument("--max-historical-features", type=int, default=500000)
parser.add_argument("--request-timeout", type=int, default=300)
parser.add_argument("--import-timeout", type=int, default=3600)
parser.add_argument("--skip-population", action="store_true")
parser.add_argument("--skip-landuse", action="store_true")
parser.add_argument("--skip-historical", action="store_true")
parser.add_argument("--fetch-only", action="store_true")
parser.add_argument("--force", action="store_true")
return parser.parse_args()
@@ -154,8 +158,37 @@ def build_operator_commands(
],
)
)
if not args.skip_historical:
commands.append(
(
"historical_landuse",
[
sys.executable,
str(scripts_dir / "provision_regional_historical_landuse.py"),
"--scope",
scope.key,
"--base-url",
args.base_url,
"--years",
args.historical_years,
"--themes",
args.historical_themes,
"--scope-output-root",
str(args.scope_output_root),
"--output-root",
str(args.output_root / "historical"),
"--request-timeout",
str(args.request_timeout),
"--import-timeout",
str(args.import_timeout),
"--max-total-features",
str(args.max_historical_features),
*common_flags,
],
)
)
if not commands:
raise ValueError("At least one of population or land-use synchronization must remain enabled")
raise ValueError("At least one regional time-series synchronization must remain enabled")
return commands
@@ -177,8 +210,8 @@ def main() -> int:
args = parse_args()
scope = GEOGRAPHIC_SCOPES[args.scope]
try:
if args.max_landuse_features <= 0:
raise ValueError("max-landuse-features must be greater than zero")
if args.max_landuse_features <= 0 or args.max_historical_features <= 0:
raise ValueError("land-use feature safety limits must be greater than zero")
boundary_path, members_path, manifest_path = resolve_boundary(scope, args.scope_output_root)
commands = build_operator_commands(args, scope, boundary_path, members_path)
results = {label: run_operator(label, command) for label, command in commands}
+1
View File
@@ -46,6 +46,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_municipality_workspace.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_context_layers.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py