Add regional historical land-use operator
This commit is contained in:
@@ -7,6 +7,19 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 209 Regional historical land-use expansion (2026-07-15)
|
||||
|
||||
- Added an explicit operator for the official 1778, 1873 and 1969 historical
|
||||
building, water and road land-use classes across all 28 approved Kempen
|
||||
transport-region municipalities.
|
||||
- Avoided the official WFS 10,000-result regional cap through resumable VRBG
|
||||
municipality partitions, exact raw-response gzip retention and SHA256
|
||||
source/output manifests.
|
||||
- Added exact municipality clipping, partition-unique evidence ids, bounded
|
||||
feature limits and one canonical regional Dataset upload per theme/year.
|
||||
- Extended the regional time-series coordinator and release packaging without
|
||||
changing API contracts, database migrations or frontend architecture.
|
||||
|
||||
## Sprint 208 Governed VMM flood-hazard scenarios (2026-07-15)
|
||||
|
||||
- Audited official water-depth and bathymetry sources and found no public,
|
||||
|
||||
+17
-2
@@ -1046,7 +1046,7 @@ storage. They never run on app startup.
|
||||
Historical land-use work can be bounded explicitly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778 1969 --themes forest water
|
||||
docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778,1969 --themes forest,water
|
||||
```
|
||||
|
||||
`GET /api/v1/projects/{project_id}/temporal/series` discovers the series and
|
||||
@@ -1066,7 +1066,8 @@ docker exec geointel python /app/scripts/provision_regional_timeseries.py
|
||||
|
||||
This resolves the retained official boundary and imports five Statbel
|
||||
population snapshots plus five modern forest, water, built-function and
|
||||
transport-infrastructure snapshots into
|
||||
transport-infrastructure snapshots, followed by the 1778/1873/1969 historical
|
||||
building, water and road snapshots, into
|
||||
`Kempen Regional Workbench`. Mol and regional series keys remain separate and
|
||||
existing immutable datasets are reused. Complete statistical sectors use exact
|
||||
published totals; a rectangle cutting a sector remains an area-weighted
|
||||
@@ -1076,6 +1077,20 @@ The regional forest path partitions WCS requests by official municipality to
|
||||
stay within upstream response limits, then builds one retained 10 m mosaic and
|
||||
one normal regional vector Dataset. A failed source request leaves completed
|
||||
partition artifacts reusable and never lowers source resolution silently.
|
||||
Historical WFS retrieval is likewise partitioned by all 28 municipality
|
||||
boundaries because broad WFS counts stop at 10,000. Exact source responses are
|
||||
retained as checksummed gzip artifacts before clipping and regional assembly.
|
||||
Run that stage independently when needed:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_historical_landuse.py
|
||||
```
|
||||
|
||||
Use `--fetch-only` for source/artifact validation without persistence. The
|
||||
historical building class represents mapped built land-use surfaces, not
|
||||
individual building footprints; water remains surface area, not depth or
|
||||
volume; historical roads are mapped road surfaces, not present-day centerline
|
||||
length.
|
||||
|
||||
Official operator datasets record that their geometries were clipped to the
|
||||
persisted Area. When that exact Area is selected, vector totals and aggregate
|
||||
|
||||
@@ -90,11 +90,12 @@ def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Pat
|
||||
assert module.resolve_boundary_path(args, scope) == boundary
|
||||
|
||||
|
||||
def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp_path: Path) -> None:
|
||||
def test_regional_coordinator_builds_explicit_population_and_landuse_commands(tmp_path: Path) -> None:
|
||||
module = load_script("provision_regional_timeseries.py")
|
||||
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
|
||||
args = argparse.Namespace(
|
||||
output_root=tmp_path / "time-series",
|
||||
scope_output_root=tmp_path / "scopes",
|
||||
fetch_only=False,
|
||||
force=False,
|
||||
skip_population=False,
|
||||
@@ -102,15 +103,19 @@ def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp
|
||||
base_url="http://backend:8000",
|
||||
population_years="2021,2025",
|
||||
landuse_years="2013,2025",
|
||||
historical_years="1778,1873,1969",
|
||||
historical_themes="buildings,water,roads",
|
||||
request_timeout=300,
|
||||
import_timeout=3600,
|
||||
max_landuse_features=500000,
|
||||
max_historical_features=500000,
|
||||
skip_historical=False,
|
||||
)
|
||||
|
||||
members_path = tmp_path / "municipalities.geojson"
|
||||
commands = dict(module.build_operator_commands(args, scope, tmp_path / "boundary.geojson", members_path))
|
||||
|
||||
assert set(commands) == {"population", "forest"}
|
||||
assert set(commands) == {"population", "forest", "historical_landuse"}
|
||||
assert commands["population"][0] == sys.executable
|
||||
assert "--scope" in commands["population"]
|
||||
assert "kempen-transport-region" in commands["population"]
|
||||
@@ -122,6 +127,10 @@ def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp
|
||||
assert ",".join(scope.nis_codes) in commands["forest"]
|
||||
assert "--force" not in commands["population"]
|
||||
assert "--fetch-only" not in commands["forest"]
|
||||
assert commands["historical_landuse"][0] == sys.executable
|
||||
assert "provision_regional_historical_landuse.py" in commands["historical_landuse"][1]
|
||||
assert "buildings,water,roads" in commands["historical_landuse"]
|
||||
assert "--scope-output-root" in commands["historical_landuse"]
|
||||
|
||||
|
||||
def test_regional_forest_provenance_does_not_claim_one_municipality() -> None:
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from shapely.geometry import box, mapping, shape
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
|
||||
def load_script():
|
||||
path = SCRIPTS / "provision_regional_historical_landuse.py"
|
||||
spec = importlib.util.spec_from_file_location("test_provision_regional_historical_landuse", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 200
|
||||
ok = True
|
||||
text = ""
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.content = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, responses):
|
||||
self.responses = list(responses)
|
||||
self.calls = []
|
||||
|
||||
def post(self, url, **kwargs):
|
||||
self.calls.append((url, kwargs))
|
||||
if not self.responses:
|
||||
raise AssertionError("Unexpected source/API request")
|
||||
return self.responses.pop(0)
|
||||
|
||||
|
||||
def source_feature(feature_id: str, geometry, landuse_class: str = "bebouwing"):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": mapping(geometry),
|
||||
"properties": {"KLASSE": landuse_class},
|
||||
}
|
||||
|
||||
|
||||
def test_member_boundaries_require_every_approved_municipality(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
|
||||
features = []
|
||||
for index, member in enumerate(scope.members):
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": mapping(box(index, 0, index + 0.9, 0.9)),
|
||||
"properties": {"nis_code": member.nis_code, "municipality": member.name},
|
||||
}
|
||||
)
|
||||
path = tmp_path / "members.geojson"
|
||||
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
|
||||
|
||||
boundaries = module.load_member_boundaries(path, scope)
|
||||
|
||||
assert list(boundaries) == list(scope.nis_codes)
|
||||
assert len(boundaries) == 28
|
||||
assert boundaries["13025"][0].name == "Mol"
|
||||
|
||||
|
||||
def test_partition_retains_exact_source_response_and_clips_to_municipality(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
definition = next(item for item in module.THEMES if item.key == "buildings")
|
||||
member = module.ScopeMember("Mol", "13025")
|
||||
boundary = box(5.0, 51.0, 5.1, 51.1)
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [source_feature("Lgbrk1778.1", box(4.95, 51.02, 5.05, 51.08))],
|
||||
}
|
||||
session = FakeSession([FakeResponse(payload)])
|
||||
|
||||
manifest = module.prepare_partition(
|
||||
session,
|
||||
output_root=tmp_path,
|
||||
year=1778,
|
||||
definition=definition,
|
||||
scope_key="kempen-transport-region",
|
||||
member=member,
|
||||
boundary=boundary,
|
||||
page_size=50,
|
||||
max_features=100,
|
||||
simplify_tolerance_degrees=0.0,
|
||||
timeout=30,
|
||||
force=False,
|
||||
)
|
||||
|
||||
output = json.loads(Path(manifest["output_path"]).read_text(encoding="utf-8"))
|
||||
feature = output["features"][0]
|
||||
raw_path = Path(manifest["output_path"]).parent / manifest["raw_pages"][0]["artifact_path"]
|
||||
assert manifest["feature_count"] == 1
|
||||
assert manifest["source_feature_count"] == 1
|
||||
assert feature["id"] == "Lgbrk1778.1:13025"
|
||||
assert feature["properties"]["original_source_feature_id"] == "Lgbrk1778.1"
|
||||
assert feature["properties"]["coverage_scope"] == "kempen-transport-region"
|
||||
assert shape(feature["geometry"]).bounds == (5.0, 51.02, 5.05, 51.08)
|
||||
assert gzip.decompress(raw_path.read_bytes()) == FakeResponse(payload).content
|
||||
|
||||
cached = module.prepare_partition(
|
||||
FakeSession([]),
|
||||
output_root=tmp_path,
|
||||
year=1778,
|
||||
definition=definition,
|
||||
scope_key="kempen-transport-region",
|
||||
member=member,
|
||||
boundary=boundary,
|
||||
page_size=50,
|
||||
max_features=100,
|
||||
simplify_tolerance_degrees=0.0,
|
||||
timeout=30,
|
||||
force=False,
|
||||
)
|
||||
assert cached["output_sha256"] == manifest["output_sha256"]
|
||||
|
||||
|
||||
def test_regional_snapshot_assembles_unique_partition_features(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
scope = module.GeographicScope(
|
||||
key="test-region",
|
||||
display_name="Test region",
|
||||
project_name="Test",
|
||||
project_region="Test",
|
||||
area_name="Test area",
|
||||
authority_name="Authority",
|
||||
authority_url="https://example.test",
|
||||
scope_type="test",
|
||||
limitation_message="Test only",
|
||||
members=(module.ScopeMember("Left", "10001"), module.ScopeMember("Right", "10002")),
|
||||
)
|
||||
definition = next(item for item in module.THEMES if item.key == "water")
|
||||
partitions = []
|
||||
for index, member in enumerate(scope.members):
|
||||
path, _manifest_path, _raw_dir = module.partition_paths(tmp_path, 1873, definition.key, member.nis_code)
|
||||
feature = source_feature(f"water.{index}:{member.nis_code}", box(index, 0, index + 0.5, 0.5), "water")
|
||||
module.atomic_write_json(path, {"type": "FeatureCollection", "features": [feature]})
|
||||
partitions.append(
|
||||
{
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"source_feature_count": 1,
|
||||
"feature_count": 1,
|
||||
"raw_pages": [{"artifact_path": "unused"}],
|
||||
"output_path": str(path),
|
||||
"output_sha256": module.sha256_file(path),
|
||||
}
|
||||
)
|
||||
|
||||
output_path, manifest = module.assemble_snapshot(
|
||||
output_root=tmp_path,
|
||||
scope=scope,
|
||||
year=1873,
|
||||
definition=definition,
|
||||
partitions=partitions,
|
||||
max_total_features=10,
|
||||
)
|
||||
payload = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
|
||||
assert manifest["coverage_complete"] is True
|
||||
assert manifest["feature_count"] == 2
|
||||
assert manifest["empty_partitions"] == []
|
||||
assert len({feature["id"] for feature in payload["features"]}) == 2
|
||||
|
||||
|
||||
def test_upload_contract_is_regional_temporal_and_partition_audited(tmp_path: Path) -> None:
|
||||
module = load_script()
|
||||
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
|
||||
definition = next(item for item in module.THEMES if item.key == "roads")
|
||||
path = tmp_path / "roads.geojson"
|
||||
path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
||||
response_payload = {"data": {"id": "dataset-id", "feature_count": 42}}
|
||||
session = FakeSession([FakeResponse(response_payload)])
|
||||
manifest = {
|
||||
"coverage_complete": True,
|
||||
"empty_partitions": [],
|
||||
"partitions": [{} for _ in scope.members],
|
||||
"partition_identity_sha256": "partition-hash",
|
||||
"output_sha256": "output-hash",
|
||||
"generated_at": "2026-07-15T00:00:00+00:00",
|
||||
}
|
||||
|
||||
result = module.upload_snapshot(
|
||||
session,
|
||||
base_url="http://backend:8000",
|
||||
project_id="project-id",
|
||||
area_id="area-id",
|
||||
scope=scope,
|
||||
year=1969,
|
||||
definition=definition,
|
||||
path=path,
|
||||
manifest=manifest,
|
||||
simplify_tolerance_degrees=0.00001,
|
||||
timeout=30,
|
||||
)
|
||||
data = session.calls[0][1]["data"]
|
||||
source_metadata = json.loads(data["source_metadata_json"])
|
||||
provenance = json.loads(data["provenance_metadata_json"])
|
||||
|
||||
assert result["id"] == "dataset-id"
|
||||
assert data["area_id"] == "area-id"
|
||||
assert data["temporal_series_key"].endswith(":roads:kempen-transport-region")
|
||||
assert data["observed_at"] == "1969-01-01T00:00:00Z"
|
||||
assert source_metadata["member_count"] == 28
|
||||
assert source_metadata["partitioned_source_audit"] is True
|
||||
assert source_metadata["identity_stable"] is False
|
||||
assert provenance["partition_count"] == 28
|
||||
assert provenance["raw_source_responses_retained"] is True
|
||||
|
||||
|
||||
def test_regional_historical_operator_is_packaged_and_release_checked() -> None:
|
||||
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "COPY scripts/provision_regional_historical_landuse.py" in dockerfile
|
||||
assert "py_compile scripts/provision_regional_historical_landuse.py" in readiness
|
||||
@@ -78,6 +78,7 @@ COPY scripts/provision_mol_dhmv.py /app/scripts/provision_mol_dhmv.py
|
||||
COPY scripts/provision_mol_flood_hazards.py /app/scripts/provision_mol_flood_hazards.py
|
||||
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
|
||||
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
|
||||
COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_regional_historical_landuse.py
|
||||
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
||||
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
|
||||
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
||||
|
||||
@@ -8808,3 +8808,47 @@ Next:
|
||||
value derivable from DHMV or flood-hazard maps. Continue the independent P5
|
||||
waterbody linkage/uncertainty design only when governed depth profiles exist;
|
||||
otherwise proceed with the regional historical buildings/water/roads backlog.
|
||||
|
||||
## Sprint 209 - Regional historical land-use expansion (2026-07-15)
|
||||
|
||||
Implemented:
|
||||
- Audited the production Digitaal Vlaanderen Historical Land Use WFS and its
|
||||
1778, 1873 and 1969 collections. Broad regional hit counts stop at 10,000,
|
||||
confirming that one transport-region bounding request is not a defensible
|
||||
completeness strategy.
|
||||
- Added `scripts/provision_regional_historical_landuse.py` for historical
|
||||
building, water and road land-use classes over the 28 registered transport-
|
||||
region municipalities. It requires the checksummed VRBG scope artifact and
|
||||
never infers or broadens the regional definition.
|
||||
- Added resumable municipality partitions with exact WFS JSON response bytes
|
||||
retained as deterministic gzip artifacts, source/artifact SHA256, request
|
||||
pagination evidence, boundary hashes, clipping diagnostics and combined
|
||||
regional output manifests.
|
||||
- Clipped and validated polygonal geometry per municipality, suffixing source
|
||||
ids with NIS codes to prevent cross-boundary collisions. Source identities
|
||||
remain explicitly unstable and do not enable object lineage.
|
||||
- Added one canonical Dataset upload per theme/year with regional Area scope,
|
||||
immutable temporal identity, hectare aggregation and methodological warning.
|
||||
No API route, database migration or direct PostGIS write was added.
|
||||
- Extended `provision_regional_timeseries.py`, all-in-one packaging, readiness
|
||||
compilation and operator/source documentation.
|
||||
|
||||
Validation evidence before live provisioning:
|
||||
- Fourteen focused partition, raw-evidence, clipping, assembly, upload,
|
||||
coordinator and packaging tests pass.
|
||||
- The official source catalog still reports the production WFS and the three
|
||||
governed historical collections. Live partition provisioning and browser
|
||||
Evolution validation follow in this pass.
|
||||
|
||||
Known limitations:
|
||||
- Historical `buildings` are mapped built land-use surfaces, not individual
|
||||
building footprints. Historical `roads` and `water` are mapped surfaces,
|
||||
not current centerlines, bathymetry or volume.
|
||||
- The three editions use different source maps and cartographic methods.
|
||||
Comparisons are exploratory hectare changes within one series, never a
|
||||
continuous equivalent to modern GRB or 10 m land-use products.
|
||||
|
||||
Next:
|
||||
- Run the complete nine-snapshot operator on live Tower/PostGIS, verify all 28
|
||||
partition manifests and then validate regional/municipality Evolution in the
|
||||
browser before marking the backlog item complete.
|
||||
|
||||
@@ -300,3 +300,9 @@ observation/validity dates, checksum and source/provenance JSON. The
|
||||
Time-series comparison is read-only and aggregates persisted geometry inside a
|
||||
requested bbox. Object-level added/removed/modified evidence is only valid for
|
||||
sources that explicitly declare stable source feature identifiers.
|
||||
|
||||
Regional 1778/1873/1969 historical land-use snapshots use the same tables.
|
||||
Their `provenance_metadata` records 28-partition identity, combined output
|
||||
checksum and retained raw-response status. Municipality-clipped source ids are
|
||||
partition-suffixed for uniqueness, while `identity_stable=false` prohibits
|
||||
object-lineage interpretation. No partition or temporal shadow table is added.
|
||||
|
||||
+20
-3
@@ -129,7 +129,7 @@ Statistical-sector codes and boundaries can change between editions. GeoIntel
|
||||
therefore compares population metrics but does not present sector additions,
|
||||
removals or boundary changes as population-object lineage.
|
||||
|
||||
### Mol historical land use
|
||||
### Historical land use for Mol and the approved region
|
||||
|
||||
`scripts/provision_mol_historical_landuse.py` uses the official Digitaal
|
||||
Vlaanderen Historical Land Use WFS for the 1778, 1873 and 1969 collections.
|
||||
@@ -146,6 +146,17 @@ The operator uses standards-compliant WFS 2.0 XML POST requests. This keeps
|
||||
the spatial/class filters server-side without exposing a long XML filter in a
|
||||
GET query, which the public gateway rejects.
|
||||
|
||||
`scripts/provision_regional_historical_landuse.py` extends the building,
|
||||
water and road series to the approved 28-municipality transport-region scope.
|
||||
Broad regional WFS counts are capped at 10,000 by the public service, so the
|
||||
operator uses the retained official VRBG municipality boundaries as 28
|
||||
explicit source partitions. Each exact WFS JSON response is retained as a
|
||||
deterministically compressed artifact with response and artifact SHA256,
|
||||
then clipped to its municipality before one regional snapshot is assembled.
|
||||
Partition-suffixed feature ids prevent cross-boundary source polygons from
|
||||
colliding. Those ids remain processing evidence and are not historical object
|
||||
lineage.
|
||||
|
||||
### Mol modern land use
|
||||
|
||||
`scripts/provision_official_landuse_timeseries.py` uses the public Departement
|
||||
@@ -181,14 +192,18 @@ added/removed forest objects.
|
||||
|
||||
### Regional official time series
|
||||
|
||||
`scripts/provision_regional_timeseries.py` applies the same population and
|
||||
modern forest operators to the approved 28-municipality
|
||||
`scripts/provision_regional_timeseries.py` applies the population, modern
|
||||
land-use and partitioned historical land-use operators to the approved 28-municipality
|
||||
`kempen-transport-region` scope. It resolves the checksummed VRBG union
|
||||
boundary and writes to `Kempen Regional Workbench` through the normal dataset
|
||||
API. Regional keys are
|
||||
`statbel:population-statistical-sector:kempen-transport-region` and
|
||||
`department-omgeving:land-use:{theme}:kempen-transport-region`, so Mol datasets
|
||||
remain independent observations rather than aliases.
|
||||
Historical buildings, water and roads use
|
||||
`digitaal-vlaanderen:historical-landuse:{theme}:kempen-transport-region` and
|
||||
remain separate from both current GRB geometry and the harmonized 2013-2025
|
||||
land-use rasters.
|
||||
|
||||
The command is explicit and operator-triggered. No source fetch happens during
|
||||
startup or map interaction. Partial statistical sectors remain area-weighted
|
||||
@@ -201,6 +216,8 @@ exact regional union clip before polygon persistence.
|
||||
|
||||
Official catalogues:
|
||||
|
||||
- https://www.vlaanderen.be/datavindplaats/catalogus/wfs-historisch-landgebruik
|
||||
- https://www.vlaanderen.be/datavindplaats/catalogus/digitalisatie-historisch-landgebruik-en-landgebruiksveranderingen-in-vlaanderen-1778-2022
|
||||
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2013
|
||||
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2016
|
||||
- https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2019
|
||||
|
||||
@@ -28,6 +28,9 @@ When a theme has multiple valid methodologies, a compact series selector keeps
|
||||
them explicit. Forest therefore defaults to the official modern 2013-2025
|
||||
10 m series, while the separate 1778-1969 historical map series remains
|
||||
selectable and is never merged into the same trend.
|
||||
Regional building, water and road evolution uses the same selector after the
|
||||
partition-audited 1778/1873/1969 snapshots are provisioned. Historical built
|
||||
surfaces never replace current GRB building footprints in Latest state.
|
||||
|
||||
Selection results use dataset-specific PostGIS summaries with end-user units.
|
||||
Building footprints, forest, water surfaces and parcels show intersected
|
||||
|
||||
@@ -1283,6 +1283,23 @@ The 2013-2025 series is methodologically separate from the historical
|
||||
when both exist; it never calculates one continuous trend across those source
|
||||
families.
|
||||
|
||||
### Regional historical buildings, water and roads
|
||||
|
||||
Provision the three historical editions for the complete approved region:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_historical_landuse.py
|
||||
```
|
||||
|
||||
The public WFS caps broad regional result counts at 10,000. The operator uses
|
||||
the 28 retained official municipality boundaries instead, stores every exact
|
||||
WFS response as a checksummed gzip artifact, clips polygonal geometry per
|
||||
municipality and assembles nine regional GeoJSON snapshots. Completed
|
||||
partitions are reused after checksum validation. `--fetch-only` prepares and
|
||||
audits artifacts without uploading; `--force` explicitly refreshes the source
|
||||
partitions. Historical identities are declared unstable and support hectare
|
||||
comparison only, not object lineage.
|
||||
|
||||
## Official Kempen operational scope
|
||||
|
||||
GeoIntel defines its regional `Kempen` workspace as the official Vlaamse
|
||||
|
||||
@@ -0,0 +1,756 @@
|
||||
"""Provision partition-audited historical land use for an approved region.
|
||||
|
||||
The official historical land-use WFS caps broad regional result counts. This
|
||||
operator therefore fetches and clips each approved municipality separately,
|
||||
retains the exact source responses as checksummed gzip artifacts, assembles one
|
||||
regional snapshot per theme/year and persists only through the dataset API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
||||
from shapely.ops import unary_union
|
||||
from shapely.validation import make_valid
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope, ScopeMember
|
||||
from provision_mol_historical_landuse import (
|
||||
ATTRIBUTION,
|
||||
COLLECTIONS,
|
||||
THEMES,
|
||||
WFS_URL,
|
||||
ThemeDefinition,
|
||||
build_session,
|
||||
response_data,
|
||||
wfs_filter_xml,
|
||||
wfs_request_xml,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_SCOPE_KEY = "kempen-transport-region"
|
||||
DEFAULT_SCOPE_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/regional-historical-landuse")
|
||||
SOURCE_CATALOG_URL = (
|
||||
"https://www.vlaanderen.be/datavindplaats/catalogus/"
|
||||
"digitalisatie-historisch-landgebruik-en-landgebruiksveranderingen-in-vlaanderen-1778-2022"
|
||||
)
|
||||
SUPPORTED_THEME_KEYS = frozenset({"buildings", "water", "roads"})
|
||||
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision regional historical buildings, water and roads.")
|
||||
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--years", default="1778,1873,1969")
|
||||
parser.add_argument("--themes", default="buildings,water,roads")
|
||||
parser.add_argument(
|
||||
"--scope-output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_SCOPE_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_REGIONAL_HISTORICAL_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument("--page-size", type=int, default=500)
|
||||
parser.add_argument("--max-features-per-partition", type=int, default=100_000)
|
||||
parser.add_argument("--max-total-features", type=int, default=500_000)
|
||||
parser.add_argument("--simplify-tolerance-degrees", type=float, default=0.00001)
|
||||
parser.add_argument("--request-timeout", type=int, default=300)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def sha256_bytes(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def atomic_write_bytes(path: Path, content: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(f"{path.suffix}.partial")
|
||||
temporary.write_bytes(content)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
|
||||
content = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
indent=2 if pretty else None,
|
||||
separators=None if pretty else (",", ":"),
|
||||
sort_keys=pretty,
|
||||
).encode("utf-8")
|
||||
atomic_write_bytes(path, content)
|
||||
|
||||
|
||||
def normalize_polygonal(geometry):
|
||||
if geometry is None or geometry.is_empty:
|
||||
return None
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if isinstance(geometry, (Polygon, MultiPolygon)):
|
||||
return geometry
|
||||
if isinstance(geometry, GeometryCollection):
|
||||
polygons = [
|
||||
part
|
||||
for part in geometry.geoms
|
||||
if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty
|
||||
]
|
||||
if polygons:
|
||||
merged = unary_union(polygons)
|
||||
if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty:
|
||||
return merged
|
||||
return None
|
||||
|
||||
|
||||
def resolve_member_boundaries(scope: GeographicScope, scope_output_root: Path) -> Path:
|
||||
scope_dir = scope_output_root / scope.key
|
||||
manifest_path = scope_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
raise RuntimeError(
|
||||
f"Official scope manifest is missing at {manifest_path}; run provision_geographic_scope.py first"
|
||||
)
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if (
|
||||
manifest.get("status") != "complete"
|
||||
or manifest.get("scope_key") != scope.key
|
||||
or int(manifest.get("member_count") or 0) != len(scope.members)
|
||||
):
|
||||
raise RuntimeError(f"Official scope manifest at {manifest_path} is incomplete or inconsistent")
|
||||
members_path = scope_dir / str(manifest.get("municipalities_filename") or "")
|
||||
if not members_path.is_file() or sha256_file(members_path) != manifest.get("municipalities_sha256"):
|
||||
raise RuntimeError("Official municipality-boundary artifact is missing or fails its scope checksum")
|
||||
return members_path
|
||||
|
||||
|
||||
def load_member_boundaries(path: Path, scope: GeographicScope) -> dict[str, tuple[ScopeMember, Any]]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
if not isinstance(features, list):
|
||||
raise RuntimeError("Municipality-boundary artifact is not a GeoJSON FeatureCollection")
|
||||
expected = {member.nis_code: member for member in scope.members}
|
||||
selected: dict[str, tuple[ScopeMember, Any]] = {}
|
||||
for feature in features:
|
||||
properties = feature.get("properties") or {}
|
||||
nis_code = str(properties.get("nis_code") or properties.get("NISCODE") or "")
|
||||
if nis_code not in expected:
|
||||
continue
|
||||
if nis_code in selected:
|
||||
raise RuntimeError(f"Municipality-boundary artifact contains duplicate NIS code {nis_code}")
|
||||
geometry = normalize_polygonal(shape(feature.get("geometry")))
|
||||
if geometry is None:
|
||||
raise RuntimeError(f"Municipality boundary for {expected[nis_code].name} is invalid")
|
||||
selected[nis_code] = (expected[nis_code], geometry)
|
||||
missing = [member.name for member in scope.members if member.nis_code not in selected]
|
||||
if missing:
|
||||
raise RuntimeError(f"Municipality-boundary artifact is missing: {', '.join(missing)}")
|
||||
return {member.nis_code: selected[member.nis_code] for member in scope.members}
|
||||
|
||||
|
||||
def fetch_source_page(
|
||||
session: requests.Session,
|
||||
*,
|
||||
collection: str,
|
||||
filter_xml: str,
|
||||
page_size: int,
|
||||
start_index: int,
|
||||
timeout: int,
|
||||
) -> tuple[bytes, list[dict[str, Any]]]:
|
||||
response = session.post(
|
||||
WFS_URL,
|
||||
data=wfs_request_xml(
|
||||
collection=collection,
|
||||
filter_xml=filter_xml,
|
||||
page_size=page_size,
|
||||
start_index=start_index,
|
||||
).encode("utf-8"),
|
||||
headers={"Content-Type": "application/xml; charset=UTF-8", "Accept": "application/json"},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
raw_content = response.content
|
||||
payload = response.json()
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
if not isinstance(features, list):
|
||||
raise RuntimeError("Historical land-use WFS returned an invalid FeatureCollection")
|
||||
return raw_content, features
|
||||
|
||||
|
||||
def partition_paths(output_root: Path, year: int, theme: str, nis_code: str) -> tuple[Path, Path, Path]:
|
||||
directory = output_root / "partitions" / str(year) / theme
|
||||
output_path = directory / f"{nis_code}.geojson"
|
||||
return output_path, output_path.with_suffix(".manifest.json"), directory / f"{nis_code}.raw"
|
||||
|
||||
|
||||
def cached_partition(
|
||||
output_path: Path,
|
||||
manifest_path: Path,
|
||||
*,
|
||||
year: int,
|
||||
theme: str,
|
||||
nis_code: str,
|
||||
page_size: int,
|
||||
simplify_tolerance_degrees: float,
|
||||
boundary_geometry_sha256: str,
|
||||
):
|
||||
if not output_path.is_file() or not manifest_path.is_file():
|
||||
return None
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if (
|
||||
manifest.get("status") != "complete"
|
||||
or manifest.get("year") != year
|
||||
or manifest.get("theme") != theme
|
||||
or manifest.get("nis_code") != nis_code
|
||||
or int(manifest.get("page_size") or 0) != page_size
|
||||
or float(manifest.get("geometry_simplification_tolerance_degrees") or 0.0)
|
||||
!= simplify_tolerance_degrees
|
||||
or manifest.get("boundary_geometry_sha256") != boundary_geometry_sha256
|
||||
or sha256_file(output_path) != manifest.get("output_sha256")
|
||||
):
|
||||
return None
|
||||
for page in manifest.get("raw_pages") or []:
|
||||
raw_path = manifest_path.parent / str(page.get("artifact_path") or "")
|
||||
if not raw_path.is_file() or sha256_file(raw_path) != page.get("artifact_sha256"):
|
||||
return None
|
||||
return manifest
|
||||
|
||||
|
||||
def prepare_partition(
|
||||
session: requests.Session,
|
||||
*,
|
||||
output_root: Path,
|
||||
year: int,
|
||||
definition: ThemeDefinition,
|
||||
scope_key: str,
|
||||
member: ScopeMember,
|
||||
boundary,
|
||||
page_size: int,
|
||||
max_features: int,
|
||||
simplify_tolerance_degrees: float,
|
||||
timeout: int,
|
||||
force: bool,
|
||||
) -> dict[str, Any]:
|
||||
output_path, manifest_path, raw_dir = partition_paths(output_root, year, definition.key, member.nis_code)
|
||||
boundary_geometry_sha256 = sha256_bytes(boundary.wkb)
|
||||
if not force:
|
||||
cached = cached_partition(
|
||||
output_path,
|
||||
manifest_path,
|
||||
year=year,
|
||||
theme=definition.key,
|
||||
nis_code=member.nis_code,
|
||||
page_size=page_size,
|
||||
simplify_tolerance_degrees=simplify_tolerance_degrees,
|
||||
boundary_geometry_sha256=boundary_geometry_sha256,
|
||||
)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
collection = COLLECTIONS[year]
|
||||
filter_xml = wfs_filter_xml(definition, boundary.bounds)
|
||||
source_seen: set[str] = set()
|
||||
output_features: list[dict[str, Any]] = []
|
||||
raw_pages: list[dict[str, Any]] = []
|
||||
invalid_geometry_count = 0
|
||||
outside_boundary_count = 0
|
||||
start_index = 0
|
||||
while True:
|
||||
raw_content, page = fetch_source_page(
|
||||
session,
|
||||
collection=collection,
|
||||
filter_xml=filter_xml,
|
||||
page_size=page_size,
|
||||
start_index=start_index,
|
||||
timeout=timeout,
|
||||
)
|
||||
compressed = gzip.compress(raw_content, compresslevel=6, mtime=0)
|
||||
raw_path = raw_dir / f"page_{start_index:09d}.geojson.gz"
|
||||
atomic_write_bytes(raw_path, compressed)
|
||||
raw_pages.append(
|
||||
{
|
||||
"start_index": start_index,
|
||||
"feature_count": len(page),
|
||||
"response_sha256": sha256_bytes(raw_content),
|
||||
"response_size_bytes": len(raw_content),
|
||||
"artifact_path": str(raw_path.relative_to(manifest_path.parent)),
|
||||
"artifact_sha256": sha256_bytes(compressed),
|
||||
"artifact_size_bytes": len(compressed),
|
||||
}
|
||||
)
|
||||
new_source_ids = 0
|
||||
for raw_feature in page:
|
||||
source_feature_id = str(raw_feature.get("id") or "")
|
||||
if not source_feature_id or source_feature_id in source_seen:
|
||||
continue
|
||||
source_seen.add(source_feature_id)
|
||||
new_source_ids += 1
|
||||
properties = dict(raw_feature.get("properties") or {})
|
||||
landuse_class = str(properties.get("KLASSE") or "").strip().lower()
|
||||
if not definition.matches(landuse_class):
|
||||
continue
|
||||
try:
|
||||
source_geometry = normalize_polygonal(shape(raw_feature.get("geometry")))
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
source_geometry = None
|
||||
if source_geometry is None:
|
||||
invalid_geometry_count += 1
|
||||
continue
|
||||
clipped = normalize_polygonal(source_geometry.intersection(boundary))
|
||||
if clipped is None:
|
||||
outside_boundary_count += 1
|
||||
continue
|
||||
if simplify_tolerance_degrees > 0:
|
||||
clipped = normalize_polygonal(
|
||||
clipped.simplify(simplify_tolerance_degrees, preserve_topology=True)
|
||||
)
|
||||
if clipped is None:
|
||||
invalid_geometry_count += 1
|
||||
continue
|
||||
partition_feature_id = f"{source_feature_id}:{member.nis_code}"
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "historical_landuse",
|
||||
"source_feature_id": partition_feature_id,
|
||||
"original_source_feature_id": source_feature_id,
|
||||
"reference_layer_name": definition.key,
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": scope_key,
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"observation_year": year,
|
||||
"historical_landuse_class": landuse_class,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
)
|
||||
output_features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": partition_feature_id,
|
||||
"geometry": mapping(clipped),
|
||||
"properties": properties,
|
||||
}
|
||||
)
|
||||
if len(output_features) > max_features:
|
||||
raise RuntimeError(
|
||||
f"Historical {definition.key} {year} exceeds {max_features} clipped features in {member.name}"
|
||||
)
|
||||
if len(source_seen) > max_features:
|
||||
raise RuntimeError(
|
||||
f"Historical {definition.key} {year} exceeds {max_features} source features in {member.name}"
|
||||
)
|
||||
if len(page) < page_size:
|
||||
break
|
||||
if new_source_ids == 0:
|
||||
raise RuntimeError(
|
||||
f"Historical WFS pagination stalled for {definition.key} {year} in {member.name}"
|
||||
)
|
||||
start_index += len(page)
|
||||
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"{definition.label} - {member.name} {year}",
|
||||
"crs": GEOJSON_CRS,
|
||||
"features": output_features,
|
||||
"observation_year": year,
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
atomic_write_json(output_path, payload)
|
||||
manifest = {
|
||||
"status": "complete",
|
||||
"year": year,
|
||||
"theme": definition.key,
|
||||
"collection": collection,
|
||||
"municipality": member.name,
|
||||
"nis_code": member.nis_code,
|
||||
"boundary_bbox": [float(value) for value in boundary.bounds],
|
||||
"boundary_geometry_sha256": boundary_geometry_sha256,
|
||||
"page_size": page_size,
|
||||
"source_feature_count": len(source_seen),
|
||||
"feature_count": len(output_features),
|
||||
"invalid_geometry_count": invalid_geometry_count,
|
||||
"outside_boundary_count": outside_boundary_count,
|
||||
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
|
||||
"raw_pages": raw_pages,
|
||||
"output_path": str(output_path),
|
||||
"output_sha256": sha256_file(output_path),
|
||||
"output_size_bytes": output_path.stat().st_size,
|
||||
"generated_at": utc_now(),
|
||||
}
|
||||
atomic_write_json(manifest_path, manifest, pretty=True)
|
||||
return manifest
|
||||
|
||||
|
||||
def snapshot_paths(output_root: Path, scope: GeographicScope, year: int, theme: str) -> tuple[Path, Path]:
|
||||
directory = output_root / "snapshots"
|
||||
stem = f"{scope.key.replace('-', '_')}_historical_{theme}_{year}"
|
||||
output_path = directory / f"{stem}.geojson"
|
||||
return output_path, directory / f"{stem}.manifest.json"
|
||||
|
||||
|
||||
def assemble_snapshot(
|
||||
*,
|
||||
output_root: Path,
|
||||
scope: GeographicScope,
|
||||
year: int,
|
||||
definition: ThemeDefinition,
|
||||
partitions: list[dict[str, Any]],
|
||||
max_total_features: int,
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
if len(partitions) != len(scope.members):
|
||||
raise RuntimeError(f"Expected {len(scope.members)} partition manifests, received {len(partitions)}")
|
||||
output_path, manifest_path = snapshot_paths(output_root, scope, year, definition.key)
|
||||
partition_identity = sha256_bytes(
|
||||
json.dumps(
|
||||
[(item["nis_code"], item["output_sha256"]) for item in partitions],
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
)
|
||||
if output_path.is_file() and manifest_path.is_file():
|
||||
existing = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if (
|
||||
existing.get("partition_identity_sha256") == partition_identity
|
||||
and sha256_file(output_path) == existing.get("output_sha256")
|
||||
):
|
||||
return output_path, existing
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = output_path.with_suffix(f"{output_path.suffix}.partial")
|
||||
feature_count = 0
|
||||
seen_ids: set[str] = set()
|
||||
with temporary.open("w", encoding="utf-8") as output:
|
||||
header = {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"{definition.label} - {scope.display_name} {year}",
|
||||
"crs": GEOJSON_CRS,
|
||||
"scope_key": scope.key,
|
||||
"member_count": len(scope.members),
|
||||
"observation_year": year,
|
||||
"attribution": ATTRIBUTION,
|
||||
}
|
||||
output.write(json.dumps(header, ensure_ascii=False, separators=(",", ":"))[:-1])
|
||||
output.write(',"features":[')
|
||||
first = True
|
||||
for partition in partitions:
|
||||
partition_path = Path(str(partition["output_path"]))
|
||||
payload = json.loads(partition_path.read_text(encoding="utf-8"))
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
if not isinstance(features, list) or len(features) != int(partition["feature_count"]):
|
||||
raise RuntimeError(f"Partition feature count drift for NIS {partition['nis_code']}")
|
||||
for feature in features:
|
||||
feature_id = str(feature.get("id") or "")
|
||||
if not feature_id or feature_id in seen_ids:
|
||||
raise RuntimeError(f"Duplicate or missing partition feature id in {definition.key} {year}")
|
||||
seen_ids.add(feature_id)
|
||||
if not first:
|
||||
output.write(",")
|
||||
output.write(json.dumps(feature, ensure_ascii=False, separators=(",", ":")))
|
||||
first = False
|
||||
feature_count += 1
|
||||
if feature_count > max_total_features:
|
||||
raise RuntimeError(
|
||||
f"Regional historical {definition.key} {year} exceeds {max_total_features} features"
|
||||
)
|
||||
output.write("]}")
|
||||
if feature_count == 0:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise RuntimeError(f"Regional historical {definition.key} {year} contains no clipped features")
|
||||
temporary.replace(output_path)
|
||||
empty_partitions = [item["nis_code"] for item in partitions if int(item["feature_count"]) == 0]
|
||||
manifest = {
|
||||
"status": "complete",
|
||||
"scope_key": scope.key,
|
||||
"scope_type": scope.scope_type,
|
||||
"member_count": len(scope.members),
|
||||
"year": year,
|
||||
"theme": definition.key,
|
||||
"collection": COLLECTIONS[year],
|
||||
"feature_count": feature_count,
|
||||
"coverage_complete": True,
|
||||
"empty_partitions": empty_partitions,
|
||||
"partition_identity_sha256": partition_identity,
|
||||
"partitions": [
|
||||
{
|
||||
"municipality": item["municipality"],
|
||||
"nis_code": item["nis_code"],
|
||||
"source_feature_count": item["source_feature_count"],
|
||||
"feature_count": item["feature_count"],
|
||||
"raw_page_count": len(item.get("raw_pages") or []),
|
||||
"output_sha256": item["output_sha256"],
|
||||
}
|
||||
for item in partitions
|
||||
],
|
||||
"output_path": str(output_path),
|
||||
"output_sha256": sha256_file(output_path),
|
||||
"output_size_bytes": output_path.stat().st_size,
|
||||
"generated_at": utc_now(),
|
||||
}
|
||||
atomic_write_json(manifest_path, manifest, pretty=True)
|
||||
return output_path, manifest
|
||||
|
||||
|
||||
def locate_workspace(session: requests.Session, base_url: str, scope: GeographicScope, timeout: int):
|
||||
projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
|
||||
project = next((item for item in projects.get("items") or [] if item.get("name") == scope.project_name), None)
|
||||
if not project:
|
||||
raise RuntimeError(f"Project {scope.project_name!r} is missing")
|
||||
project_id = str(project["id"])
|
||||
areas = response_data(
|
||||
session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout)
|
||||
)
|
||||
area = next((item for item in areas.get("items") or [] if item.get("name") == scope.area_name), None)
|
||||
if not area:
|
||||
raise RuntimeError(f"Official scope Area {scope.area_name!r} is missing")
|
||||
datasets = response_data(
|
||||
session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 200}, timeout=timeout)
|
||||
)
|
||||
return project_id, str(area["id"]), list(datasets.get("items") or [])
|
||||
|
||||
|
||||
def upload_snapshot(
|
||||
session: requests.Session,
|
||||
*,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area_id: str,
|
||||
scope: GeographicScope,
|
||||
year: int,
|
||||
definition: ThemeDefinition,
|
||||
path: Path,
|
||||
manifest: dict[str, Any],
|
||||
simplify_tolerance_degrees: float,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
observed_at = f"{year}-01-01T00:00:00Z"
|
||||
source_metadata = {
|
||||
"provider": "Digitaal Vlaanderen",
|
||||
"collection": COLLECTIONS[year],
|
||||
"authority_level": "authoritative",
|
||||
"coverage_scope": scope.key,
|
||||
"scope_type": scope.scope_type,
|
||||
"scope_display_name": scope.display_name,
|
||||
"member_count": len(scope.members),
|
||||
"member_nis_codes": list(scope.nis_codes),
|
||||
"partitioned_source_audit": True,
|
||||
"coverage_complete": bool(manifest["coverage_complete"]),
|
||||
"empty_partition_nis_codes": manifest["empty_partitions"],
|
||||
"attribution": ATTRIBUTION,
|
||||
"source_catalog_url": SOURCE_CATALOG_URL,
|
||||
"identity_stable": False,
|
||||
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
|
||||
"selection_aggregation": {
|
||||
"method": "intersection_area",
|
||||
"label": "Oppervlakte",
|
||||
"unit": "ha",
|
||||
"is_estimate": False,
|
||||
"warning": (
|
||||
"Historische kaartklassen, bronkaarten en karteermethodes verschillen per bronjaar; "
|
||||
"interpreteer evoluties binnen die methodologische context."
|
||||
),
|
||||
},
|
||||
}
|
||||
provenance_metadata = {
|
||||
"operator_tool": "provision_regional_historical_landuse.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"wfs_url": WFS_URL,
|
||||
"collection": COLLECTIONS[year],
|
||||
"partition_count": len(manifest["partitions"]),
|
||||
"partition_identity_sha256": manifest["partition_identity_sha256"],
|
||||
"combined_output_sha256": manifest["output_sha256"],
|
||||
"raw_source_responses_retained": True,
|
||||
"geometry_simplification_tolerance_degrees": simplify_tolerance_degrees,
|
||||
"generated_at": manifest["generated_at"],
|
||||
}
|
||||
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:{scope.key}"
|
||||
with path.open("rb") as handle:
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||
data={
|
||||
"dataset_type": "vector",
|
||||
"source": "operator_official_import",
|
||||
"dataset_role": "reference",
|
||||
"source_name": "historical_landuse",
|
||||
"reference_layer_name": definition.key,
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": series_key,
|
||||
"observed_at": observed_at,
|
||||
"valid_from": observed_at,
|
||||
"temporal_granularity": "year",
|
||||
"source_version": str(year),
|
||||
},
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
)
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def requested_configuration(args: argparse.Namespace) -> tuple[list[int], list[ThemeDefinition]]:
|
||||
try:
|
||||
years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()})
|
||||
except ValueError as exc:
|
||||
raise ValueError("Years must be comma-separated integers") from exc
|
||||
theme_keys = {value.strip().lower() for value in args.themes.split(",") if value.strip()}
|
||||
definitions = [item for item in THEMES if item.key in theme_keys and item.key in SUPPORTED_THEME_KEYS]
|
||||
unsupported_years = sorted(set(years) - set(COLLECTIONS))
|
||||
unsupported_themes = sorted(theme_keys - SUPPORTED_THEME_KEYS)
|
||||
if not years or not definitions or unsupported_years or unsupported_themes:
|
||||
raise ValueError(f"Unsupported years={unsupported_years}, themes={unsupported_themes}")
|
||||
return years, definitions
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||
try:
|
||||
if args.page_size <= 0 or args.max_features_per_partition <= 0 or args.max_total_features <= 0:
|
||||
raise ValueError("Page and feature safety limits must be greater than zero")
|
||||
years, definitions = requested_configuration(args)
|
||||
members_path = resolve_member_boundaries(scope, args.scope_output_root)
|
||||
boundaries = load_member_boundaries(members_path, scope)
|
||||
output_root = args.output_root / scope.key
|
||||
prepared: list[tuple[int, ThemeDefinition, Path, dict[str, Any]]] = []
|
||||
with build_session() as source_session:
|
||||
for year in years:
|
||||
for definition in definitions:
|
||||
partitions = [
|
||||
prepare_partition(
|
||||
source_session,
|
||||
output_root=output_root,
|
||||
year=year,
|
||||
definition=definition,
|
||||
scope_key=scope.key,
|
||||
member=member,
|
||||
boundary=boundaries[member.nis_code][1],
|
||||
page_size=args.page_size,
|
||||
max_features=args.max_features_per_partition,
|
||||
simplify_tolerance_degrees=args.simplify_tolerance_degrees,
|
||||
timeout=args.request_timeout,
|
||||
force=args.force,
|
||||
)
|
||||
for member in scope.members
|
||||
]
|
||||
path, manifest = assemble_snapshot(
|
||||
output_root=output_root,
|
||||
scope=scope,
|
||||
year=year,
|
||||
definition=definition,
|
||||
partitions=partitions,
|
||||
max_total_features=args.max_total_features,
|
||||
)
|
||||
prepared.append((year, definition, path, manifest))
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
if args.fetch_only:
|
||||
results = [
|
||||
{
|
||||
"year": year,
|
||||
"theme": definition.key,
|
||||
"feature_count": manifest["feature_count"],
|
||||
"partition_count": len(manifest["partitions"]),
|
||||
"empty_partitions": manifest["empty_partitions"],
|
||||
"path": str(path),
|
||||
"status": "prepared",
|
||||
}
|
||||
for year, definition, path, manifest in prepared
|
||||
]
|
||||
else:
|
||||
base_url = args.base_url.rstrip("/")
|
||||
with requests.Session() as api_session:
|
||||
project_id, area_id, existing = locate_workspace(api_session, base_url, scope, args.import_timeout)
|
||||
for year, definition, path, manifest in prepared:
|
||||
series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:{scope.key}"
|
||||
dataset = next(
|
||||
(
|
||||
item
|
||||
for item in existing
|
||||
if item.get("temporal_series_key") == series_key
|
||||
and str(item.get("observed_at") or "").startswith(str(year))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if dataset:
|
||||
if int(dataset.get("feature_count") or 0) != int(manifest["feature_count"]):
|
||||
raise RuntimeError(
|
||||
f"Persisted {definition.key} {year} count differs from the audited source artifact"
|
||||
)
|
||||
status = "existing"
|
||||
else:
|
||||
dataset = upload_snapshot(
|
||||
api_session,
|
||||
base_url=base_url,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
scope=scope,
|
||||
year=year,
|
||||
definition=definition,
|
||||
path=path,
|
||||
manifest=manifest,
|
||||
simplify_tolerance_degrees=args.simplify_tolerance_degrees,
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
existing.append(dataset)
|
||||
status = "imported"
|
||||
results.append(
|
||||
{
|
||||
"year": year,
|
||||
"theme": definition.key,
|
||||
"dataset_id": dataset["id"],
|
||||
"feature_count": dataset.get("feature_count"),
|
||||
"partition_count": len(manifest["partitions"]),
|
||||
"empty_partitions": manifest["empty_partitions"],
|
||||
"status": status,
|
||||
}
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException, json.JSONDecodeError) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"mode": "fetch_only" if args.fetch_only else "synchronized",
|
||||
"scope": scope.key,
|
||||
"display_name": scope.display_name,
|
||||
"member_count": len(scope.members),
|
||||
"snapshots": results,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Synchronize official population and forest time series for one approved scope.
|
||||
"""Synchronize official population and land-use time series for one approved scope.
|
||||
|
||||
This is an explicit operator command, never an application-startup task. It
|
||||
coordinates the existing Statbel and Departement Omgeving import paths and
|
||||
@@ -30,6 +30,8 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--population-years", default="2021,2022,2023,2024,2025")
|
||||
parser.add_argument("--landuse-years", default="2013,2016,2019,2022,2025")
|
||||
parser.add_argument("--historical-years", default="1778,1873,1969")
|
||||
parser.add_argument("--historical-themes", default="buildings,water,roads")
|
||||
parser.add_argument(
|
||||
"--scope-output-root",
|
||||
type=Path,
|
||||
@@ -41,10 +43,12 @@ def parse_args() -> argparse.Namespace:
|
||||
default=Path(os.environ.get("GEOINTEL_REGIONAL_TIMESERIES_OUTPUT_ROOT", DEFAULT_TIMESERIES_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument("--max-landuse-features", type=int, default=500000)
|
||||
parser.add_argument("--max-historical-features", type=int, default=500000)
|
||||
parser.add_argument("--request-timeout", type=int, default=300)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
parser.add_argument("--skip-population", action="store_true")
|
||||
parser.add_argument("--skip-landuse", action="store_true")
|
||||
parser.add_argument("--skip-historical", action="store_true")
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
parser.add_argument("--force", action="store_true")
|
||||
return parser.parse_args()
|
||||
@@ -154,8 +158,37 @@ def build_operator_commands(
|
||||
],
|
||||
)
|
||||
)
|
||||
if not args.skip_historical:
|
||||
commands.append(
|
||||
(
|
||||
"historical_landuse",
|
||||
[
|
||||
sys.executable,
|
||||
str(scripts_dir / "provision_regional_historical_landuse.py"),
|
||||
"--scope",
|
||||
scope.key,
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
"--years",
|
||||
args.historical_years,
|
||||
"--themes",
|
||||
args.historical_themes,
|
||||
"--scope-output-root",
|
||||
str(args.scope_output_root),
|
||||
"--output-root",
|
||||
str(args.output_root / "historical"),
|
||||
"--request-timeout",
|
||||
str(args.request_timeout),
|
||||
"--import-timeout",
|
||||
str(args.import_timeout),
|
||||
"--max-total-features",
|
||||
str(args.max_historical_features),
|
||||
*common_flags,
|
||||
],
|
||||
)
|
||||
)
|
||||
if not commands:
|
||||
raise ValueError("At least one of population or land-use synchronization must remain enabled")
|
||||
raise ValueError("At least one regional time-series synchronization must remain enabled")
|
||||
return commands
|
||||
|
||||
|
||||
@@ -177,8 +210,8 @@ def main() -> int:
|
||||
args = parse_args()
|
||||
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||
try:
|
||||
if args.max_landuse_features <= 0:
|
||||
raise ValueError("max-landuse-features must be greater than zero")
|
||||
if args.max_landuse_features <= 0 or args.max_historical_features <= 0:
|
||||
raise ValueError("land-use feature safety limits must be greater than zero")
|
||||
boundary_path, members_path, manifest_path = resolve_boundary(scope, args.scope_output_root)
|
||||
commands = build_operator_commands(args, scope, boundary_path, members_path)
|
||||
results = {label: run_operator(label, command) for label, command in commands}
|
||||
|
||||
@@ -46,6 +46,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_municipality_workspace.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_context_layers.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||
|
||||
Reference in New Issue
Block a user