diff --git a/CHANGELOG.md b/CHANGELOG.md index 35ff244f..705c5e3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ # Changelog +## Sprint 209 Regional historical land-use expansion (2026-07-15) + +- Added an explicit operator for the official 1778, 1873 and 1969 historical + building, water and road land-use classes across all 28 approved Kempen + transport-region municipalities. +- Avoided the official WFS 10,000-result regional cap through resumable VRBG + municipality partitions, exact raw-response gzip retention and SHA256 + source/output manifests. +- Added exact municipality clipping, partition-unique evidence ids, bounded + feature limits and one canonical regional Dataset upload per theme/year. +- Extended the regional time-series coordinator and release packaging without + changing API contracts, database migrations or frontend architecture. + ## Sprint 208 Governed VMM flood-hazard scenarios (2026-07-15) - Audited official water-depth and bathymetry sources and found no public, diff --git a/backend/README.md b/backend/README.md index a8062dda..e61e1925 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1046,7 +1046,7 @@ storage. They never run on app startup. Historical land-use work can be bounded explicitly: ```bash -docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778 1969 --themes forest water +docker exec geointel python /app/scripts/provision_mol_historical_landuse.py --years 1778,1969 --themes forest,water ``` `GET /api/v1/projects/{project_id}/temporal/series` discovers the series and @@ -1066,7 +1066,8 @@ docker exec geointel python /app/scripts/provision_regional_timeseries.py This resolves the retained official boundary and imports five Statbel population snapshots plus five modern forest, water, built-function and -transport-infrastructure snapshots into +transport-infrastructure snapshots, followed by the 1778/1873/1969 historical +building, water and road snapshots, into `Kempen Regional Workbench`. Mol and regional series keys remain separate and existing immutable datasets are reused. Complete statistical sectors use exact published totals; a rectangle cutting a sector remains an area-weighted @@ -1076,6 +1077,20 @@ The regional forest path partitions WCS requests by official municipality to stay within upstream response limits, then builds one retained 10 m mosaic and one normal regional vector Dataset. A failed source request leaves completed partition artifacts reusable and never lowers source resolution silently. +Historical WFS retrieval is likewise partitioned by all 28 municipality +boundaries because broad WFS counts stop at 10,000. Exact source responses are +retained as checksummed gzip artifacts before clipping and regional assembly. +Run that stage independently when needed: + +```bash +docker exec geointel python /app/scripts/provision_regional_historical_landuse.py +``` + +Use `--fetch-only` for source/artifact validation without persistence. The +historical building class represents mapped built land-use surfaces, not +individual building footprints; water remains surface area, not depth or +volume; historical roads are mapped road surfaces, not present-day centerline +length. Official operator datasets record that their geometries were clipped to the persisted Area. When that exact Area is selected, vector totals and aggregate diff --git a/backend/tests/test_sprint194_regional_timeseries.py b/backend/tests/test_sprint194_regional_timeseries.py index e97fab02..a2b3d133 100644 --- a/backend/tests/test_sprint194_regional_timeseries.py +++ b/backend/tests/test_sprint194_regional_timeseries.py @@ -90,11 +90,12 @@ def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Pat assert module.resolve_boundary_path(args, scope) == boundary -def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp_path: Path) -> None: +def test_regional_coordinator_builds_explicit_population_and_landuse_commands(tmp_path: Path) -> None: module = load_script("provision_regional_timeseries.py") scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] args = argparse.Namespace( output_root=tmp_path / "time-series", + scope_output_root=tmp_path / "scopes", fetch_only=False, force=False, skip_population=False, @@ -102,15 +103,19 @@ def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp base_url="http://backend:8000", population_years="2021,2025", landuse_years="2013,2025", + historical_years="1778,1873,1969", + historical_themes="buildings,water,roads", request_timeout=300, import_timeout=3600, max_landuse_features=500000, + max_historical_features=500000, + skip_historical=False, ) members_path = tmp_path / "municipalities.geojson" commands = dict(module.build_operator_commands(args, scope, tmp_path / "boundary.geojson", members_path)) - assert set(commands) == {"population", "forest"} + assert set(commands) == {"population", "forest", "historical_landuse"} assert commands["population"][0] == sys.executable assert "--scope" in commands["population"] assert "kempen-transport-region" in commands["population"] @@ -122,6 +127,10 @@ def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp assert ",".join(scope.nis_codes) in commands["forest"] assert "--force" not in commands["population"] assert "--fetch-only" not in commands["forest"] + assert commands["historical_landuse"][0] == sys.executable + assert "provision_regional_historical_landuse.py" in commands["historical_landuse"][1] + assert "buildings,water,roads" in commands["historical_landuse"] + assert "--scope-output-root" in commands["historical_landuse"] def test_regional_forest_provenance_does_not_claim_one_municipality() -> None: diff --git a/backend/tests/test_sprint209_regional_historical_landuse.py b/backend/tests/test_sprint209_regional_historical_landuse.py new file mode 100644 index 00000000..2bf8f1a9 --- /dev/null +++ b/backend/tests/test_sprint209_regional_historical_landuse.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import gzip +import importlib.util +import json +from pathlib import Path +import sys + +from shapely.geometry import box, mapping, shape + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + + +def load_script(): + path = SCRIPTS / "provision_regional_historical_landuse.py" + spec = importlib.util.spec_from_file_location("test_provision_regional_historical_landuse", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class FakeResponse: + status_code = 200 + ok = True + text = "" + + def __init__(self, payload): + self.payload = payload + self.content = json.dumps(payload, separators=(",", ":")).encode("utf-8") + + def raise_for_status(self): + return None + + def json(self): + return self.payload + + +class FakeSession: + def __init__(self, responses): + self.responses = list(responses) + self.calls = [] + + def post(self, url, **kwargs): + self.calls.append((url, kwargs)) + if not self.responses: + raise AssertionError("Unexpected source/API request") + return self.responses.pop(0) + + +def source_feature(feature_id: str, geometry, landuse_class: str = "bebouwing"): + return { + "type": "Feature", + "id": feature_id, + "geometry": mapping(geometry), + "properties": {"KLASSE": landuse_class}, + } + + +def test_member_boundaries_require_every_approved_municipality(tmp_path: Path) -> None: + module = load_script() + scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] + features = [] + for index, member in enumerate(scope.members): + features.append( + { + "type": "Feature", + "geometry": mapping(box(index, 0, index + 0.9, 0.9)), + "properties": {"nis_code": member.nis_code, "municipality": member.name}, + } + ) + path = tmp_path / "members.geojson" + path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8") + + boundaries = module.load_member_boundaries(path, scope) + + assert list(boundaries) == list(scope.nis_codes) + assert len(boundaries) == 28 + assert boundaries["13025"][0].name == "Mol" + + +def test_partition_retains_exact_source_response_and_clips_to_municipality(tmp_path: Path) -> None: + module = load_script() + definition = next(item for item in module.THEMES if item.key == "buildings") + member = module.ScopeMember("Mol", "13025") + boundary = box(5.0, 51.0, 5.1, 51.1) + payload = { + "type": "FeatureCollection", + "features": [source_feature("Lgbrk1778.1", box(4.95, 51.02, 5.05, 51.08))], + } + session = FakeSession([FakeResponse(payload)]) + + manifest = module.prepare_partition( + session, + output_root=tmp_path, + year=1778, + definition=definition, + scope_key="kempen-transport-region", + member=member, + boundary=boundary, + page_size=50, + max_features=100, + simplify_tolerance_degrees=0.0, + timeout=30, + force=False, + ) + + output = json.loads(Path(manifest["output_path"]).read_text(encoding="utf-8")) + feature = output["features"][0] + raw_path = Path(manifest["output_path"]).parent / manifest["raw_pages"][0]["artifact_path"] + assert manifest["feature_count"] == 1 + assert manifest["source_feature_count"] == 1 + assert feature["id"] == "Lgbrk1778.1:13025" + assert feature["properties"]["original_source_feature_id"] == "Lgbrk1778.1" + assert feature["properties"]["coverage_scope"] == "kempen-transport-region" + assert shape(feature["geometry"]).bounds == (5.0, 51.02, 5.05, 51.08) + assert gzip.decompress(raw_path.read_bytes()) == FakeResponse(payload).content + + cached = module.prepare_partition( + FakeSession([]), + output_root=tmp_path, + year=1778, + definition=definition, + scope_key="kempen-transport-region", + member=member, + boundary=boundary, + page_size=50, + max_features=100, + simplify_tolerance_degrees=0.0, + timeout=30, + force=False, + ) + assert cached["output_sha256"] == manifest["output_sha256"] + + +def test_regional_snapshot_assembles_unique_partition_features(tmp_path: Path) -> None: + module = load_script() + scope = module.GeographicScope( + key="test-region", + display_name="Test region", + project_name="Test", + project_region="Test", + area_name="Test area", + authority_name="Authority", + authority_url="https://example.test", + scope_type="test", + limitation_message="Test only", + members=(module.ScopeMember("Left", "10001"), module.ScopeMember("Right", "10002")), + ) + definition = next(item for item in module.THEMES if item.key == "water") + partitions = [] + for index, member in enumerate(scope.members): + path, _manifest_path, _raw_dir = module.partition_paths(tmp_path, 1873, definition.key, member.nis_code) + feature = source_feature(f"water.{index}:{member.nis_code}", box(index, 0, index + 0.5, 0.5), "water") + module.atomic_write_json(path, {"type": "FeatureCollection", "features": [feature]}) + partitions.append( + { + "municipality": member.name, + "nis_code": member.nis_code, + "source_feature_count": 1, + "feature_count": 1, + "raw_pages": [{"artifact_path": "unused"}], + "output_path": str(path), + "output_sha256": module.sha256_file(path), + } + ) + + output_path, manifest = module.assemble_snapshot( + output_root=tmp_path, + scope=scope, + year=1873, + definition=definition, + partitions=partitions, + max_total_features=10, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + + assert manifest["coverage_complete"] is True + assert manifest["feature_count"] == 2 + assert manifest["empty_partitions"] == [] + assert len({feature["id"] for feature in payload["features"]}) == 2 + + +def test_upload_contract_is_regional_temporal_and_partition_audited(tmp_path: Path) -> None: + module = load_script() + scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] + definition = next(item for item in module.THEMES if item.key == "roads") + path = tmp_path / "roads.geojson" + path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8") + response_payload = {"data": {"id": "dataset-id", "feature_count": 42}} + session = FakeSession([FakeResponse(response_payload)]) + manifest = { + "coverage_complete": True, + "empty_partitions": [], + "partitions": [{} for _ in scope.members], + "partition_identity_sha256": "partition-hash", + "output_sha256": "output-hash", + "generated_at": "2026-07-15T00:00:00+00:00", + } + + result = module.upload_snapshot( + session, + base_url="http://backend:8000", + project_id="project-id", + area_id="area-id", + scope=scope, + year=1969, + definition=definition, + path=path, + manifest=manifest, + simplify_tolerance_degrees=0.00001, + timeout=30, + ) + data = session.calls[0][1]["data"] + source_metadata = json.loads(data["source_metadata_json"]) + provenance = json.loads(data["provenance_metadata_json"]) + + assert result["id"] == "dataset-id" + assert data["area_id"] == "area-id" + assert data["temporal_series_key"].endswith(":roads:kempen-transport-region") + assert data["observed_at"] == "1969-01-01T00:00:00Z" + assert source_metadata["member_count"] == 28 + assert source_metadata["partitioned_source_audit"] is True + assert source_metadata["identity_stable"] is False + assert provenance["partition_count"] == 28 + assert provenance["raw_source_responses_retained"] is True + + +def test_regional_historical_operator_is_packaged_and_release_checked() -> None: + dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8") + + assert "COPY scripts/provision_regional_historical_landuse.py" in dockerfile + assert "py_compile scripts/provision_regional_historical_landuse.py" in readiness diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index d5d91dab..55a064b5 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -78,6 +78,7 @@ COPY scripts/provision_mol_dhmv.py /app/scripts/provision_mol_dhmv.py COPY scripts/provision_mol_flood_hazards.py /app/scripts/provision_mol_flood_hazards.py COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py +COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_regional_historical_landuse.py COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 04a1d6b3..d1b3eee0 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8808,3 +8808,47 @@ Next: value derivable from DHMV or flood-hazard maps. Continue the independent P5 waterbody linkage/uncertainty design only when governed depth profiles exist; otherwise proceed with the regional historical buildings/water/roads backlog. + +## Sprint 209 - Regional historical land-use expansion (2026-07-15) + +Implemented: +- Audited the production Digitaal Vlaanderen Historical Land Use WFS and its + 1778, 1873 and 1969 collections. Broad regional hit counts stop at 10,000, + confirming that one transport-region bounding request is not a defensible + completeness strategy. +- Added `scripts/provision_regional_historical_landuse.py` for historical + building, water and road land-use classes over the 28 registered transport- + region municipalities. It requires the checksummed VRBG scope artifact and + never infers or broadens the regional definition. +- Added resumable municipality partitions with exact WFS JSON response bytes + retained as deterministic gzip artifacts, source/artifact SHA256, request + pagination evidence, boundary hashes, clipping diagnostics and combined + regional output manifests. +- Clipped and validated polygonal geometry per municipality, suffixing source + ids with NIS codes to prevent cross-boundary collisions. Source identities + remain explicitly unstable and do not enable object lineage. +- Added one canonical Dataset upload per theme/year with regional Area scope, + immutable temporal identity, hectare aggregation and methodological warning. + No API route, database migration or direct PostGIS write was added. +- Extended `provision_regional_timeseries.py`, all-in-one packaging, readiness + compilation and operator/source documentation. + +Validation evidence before live provisioning: +- Fourteen focused partition, raw-evidence, clipping, assembly, upload, + coordinator and packaging tests pass. +- The official source catalog still reports the production WFS and the three + governed historical collections. Live partition provisioning and browser + Evolution validation follow in this pass. + +Known limitations: +- Historical `buildings` are mapped built land-use surfaces, not individual + building footprints. Historical `roads` and `water` are mapped surfaces, + not current centerlines, bathymetry or volume. +- The three editions use different source maps and cartographic methods. + Comparisons are exploratory hectare changes within one series, never a + continuous equivalent to modern GRB or 10 m land-use products. + +Next: +- Run the complete nine-snapshot operator on live Tower/PostGIS, verify all 28 + partition manifests and then validate regional/municipality Evolution in the + browser before marking the backlog item complete. diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index 38e81fbd..2d585a5a 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -300,3 +300,9 @@ observation/validity dates, checksum and source/provenance JSON. The Time-series comparison is read-only and aggregates persisted geometry inside a requested bbox. Object-level added/removed/modified evidence is only valid for sources that explicitly declare stable source feature identifiers. + +Regional 1778/1873/1969 historical land-use snapshots use the same tables. +Their `provenance_metadata` records 28-partition identity, combined output +checksum and retained raw-response status. Municipality-clipped source ids are +partition-suffixed for uniqueness, while `identity_stable=false` prohibits +object-lineage interpretation. No partition or temporal shadow table is added. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 7e0411d1..e4fbab62 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -129,7 +129,7 @@ Statistical-sector codes and boundaries can change between editions. GeoIntel therefore compares population metrics but does not present sector additions, removals or boundary changes as population-object lineage. -### Mol historical land use +### Historical land use for Mol and the approved region `scripts/provision_mol_historical_landuse.py` uses the official Digitaal Vlaanderen Historical Land Use WFS for the 1778, 1873 and 1969 collections. @@ -146,6 +146,17 @@ The operator uses standards-compliant WFS 2.0 XML POST requests. This keeps the spatial/class filters server-side without exposing a long XML filter in a GET query, which the public gateway rejects. +`scripts/provision_regional_historical_landuse.py` extends the building, +water and road series to the approved 28-municipality transport-region scope. +Broad regional WFS counts are capped at 10,000 by the public service, so the +operator uses the retained official VRBG municipality boundaries as 28 +explicit source partitions. Each exact WFS JSON response is retained as a +deterministically compressed artifact with response and artifact SHA256, +then clipped to its municipality before one regional snapshot is assembled. +Partition-suffixed feature ids prevent cross-boundary source polygons from +colliding. Those ids remain processing evidence and are not historical object +lineage. + ### Mol modern land use `scripts/provision_official_landuse_timeseries.py` uses the public Departement @@ -181,14 +192,18 @@ added/removed forest objects. ### Regional official time series -`scripts/provision_regional_timeseries.py` applies the same population and -modern forest operators to the approved 28-municipality +`scripts/provision_regional_timeseries.py` applies the population, modern +land-use and partitioned historical land-use operators to the approved 28-municipality `kempen-transport-region` scope. It resolves the checksummed VRBG union boundary and writes to `Kempen Regional Workbench` through the normal dataset API. Regional keys are `statbel:population-statistical-sector:kempen-transport-region` and `department-omgeving:land-use:{theme}:kempen-transport-region`, so Mol datasets remain independent observations rather than aliases. +Historical buildings, water and roads use +`digitaal-vlaanderen:historical-landuse:{theme}:kempen-transport-region` and +remain separate from both current GRB geometry and the harmonized 2013-2025 +land-use rasters. The command is explicit and operator-triggered. No source fetch happens during startup or map interaction. Partial statistical sectors remain area-weighted @@ -201,6 +216,8 @@ exact regional union clip before polygon persistence. Official catalogues: +- https://www.vlaanderen.be/datavindplaats/catalogus/wfs-historisch-landgebruik +- https://www.vlaanderen.be/datavindplaats/catalogus/digitalisatie-historisch-landgebruik-en-landgebruiksveranderingen-in-vlaanderen-1778-2022 - https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2013 - https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2016 - https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2019 diff --git a/frontend/README.md b/frontend/README.md index 7f10654e..c44bdbdb 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -28,6 +28,9 @@ When a theme has multiple valid methodologies, a compact series selector keeps them explicit. Forest therefore defaults to the official modern 2013-2025 10 m series, while the separate 1778-1969 historical map series remains selectable and is never merged into the same trend. +Regional building, water and road evolution uses the same selector after the +partition-audited 1778/1873/1969 snapshots are provisioned. Historical built +surfaces never replace current GRB building footprints in Latest state. Selection results use dataset-specific PostGIS summaries with end-user units. Building footprints, forest, water surfaces and parcels show intersected diff --git a/scripts/README.md b/scripts/README.md index 3a6c0288..c368fd75 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1283,6 +1283,23 @@ The 2013-2025 series is methodologically separate from the historical when both exist; it never calculates one continuous trend across those source families. +### Regional historical buildings, water and roads + +Provision the three historical editions for the complete approved region: + +```bash +docker exec geointel python /app/scripts/provision_regional_historical_landuse.py +``` + +The public WFS caps broad regional result counts at 10,000. The operator uses +the 28 retained official municipality boundaries instead, stores every exact +WFS response as a checksummed gzip artifact, clips polygonal geometry per +municipality and assembles nine regional GeoJSON snapshots. Completed +partitions are reused after checksum validation. `--fetch-only` prepares and +audits artifacts without uploading; `--force` explicitly refreshes the source +partitions. Historical identities are declared unstable and support hectare +comparison only, not object lineage. + ## Official Kempen operational scope GeoIntel defines its regional `Kempen` workspace as the official Vlaamse diff --git a/scripts/provision_regional_historical_landuse.py b/scripts/provision_regional_historical_landuse.py new file mode 100644 index 00000000..25d0eb0b --- /dev/null +++ b/scripts/provision_regional_historical_landuse.py @@ -0,0 +1,756 @@ +"""Provision partition-audited historical land use for an approved region. + +The official historical land-use WFS caps broad regional result counts. This +operator therefore fetches and clips each approved municipality separately, +retains the exact source responses as checksummed gzip artifacts, assembles one +regional snapshot per theme/year and persists only through the dataset API. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import requests +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape +from shapely.ops import unary_union +from shapely.validation import make_valid + +from geographic_scopes import GEOGRAPHIC_SCOPES, GeographicScope, ScopeMember +from provision_mol_historical_landuse import ( + ATTRIBUTION, + COLLECTIONS, + THEMES, + WFS_URL, + ThemeDefinition, + build_session, + response_data, + wfs_filter_xml, + wfs_request_xml, +) + + +DEFAULT_API_URL = "http://127.0.0.1:8000" +DEFAULT_SCOPE_KEY = "kempen-transport-region" +DEFAULT_SCOPE_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes") +DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/regional-historical-landuse") +SOURCE_CATALOG_URL = ( + "https://www.vlaanderen.be/datavindplaats/catalogus/" + "digitalisatie-historisch-landgebruik-en-landgebruiksveranderingen-in-vlaanderen-1778-2022" +) +SUPPORTED_THEME_KEYS = frozenset({"buildings", "water", "roads"}) +GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Provision regional historical buildings, water and roads.") + parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY) + parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL)) + parser.add_argument("--years", default="1778,1873,1969") + parser.add_argument("--themes", default="buildings,water,roads") + parser.add_argument( + "--scope-output-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_SCOPE_OUTPUT_ROOT)), + ) + parser.add_argument( + "--output-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_REGIONAL_HISTORICAL_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)), + ) + parser.add_argument("--page-size", type=int, default=500) + parser.add_argument("--max-features-per-partition", type=int, default=100_000) + parser.add_argument("--max-total-features", type=int, default=500_000) + parser.add_argument("--simplify-tolerance-degrees", type=float, default=0.00001) + parser.add_argument("--request-timeout", type=int, default=300) + parser.add_argument("--import-timeout", type=int, default=3600) + parser.add_argument("--force", action="store_true") + parser.add_argument("--fetch-only", action="store_true") + return parser.parse_args() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def sha256_bytes(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_write_bytes(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(f"{path.suffix}.partial") + temporary.write_bytes(content) + temporary.replace(path) + + +def atomic_write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None: + content = json.dumps( + payload, + ensure_ascii=False, + indent=2 if pretty else None, + separators=None if pretty else (",", ":"), + sort_keys=pretty, + ).encode("utf-8") + atomic_write_bytes(path, content) + + +def normalize_polygonal(geometry): + if geometry is None or geometry.is_empty: + return None + if not geometry.is_valid: + geometry = make_valid(geometry) + if isinstance(geometry, (Polygon, MultiPolygon)): + return geometry + if isinstance(geometry, GeometryCollection): + polygons = [ + part + for part in geometry.geoms + if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty + ] + if polygons: + merged = unary_union(polygons) + if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty: + return merged + return None + + +def resolve_member_boundaries(scope: GeographicScope, scope_output_root: Path) -> Path: + scope_dir = scope_output_root / scope.key + manifest_path = scope_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json" + if not manifest_path.is_file(): + raise RuntimeError( + f"Official scope manifest is missing at {manifest_path}; run provision_geographic_scope.py first" + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if ( + manifest.get("status") != "complete" + or manifest.get("scope_key") != scope.key + or int(manifest.get("member_count") or 0) != len(scope.members) + ): + raise RuntimeError(f"Official scope manifest at {manifest_path} is incomplete or inconsistent") + members_path = scope_dir / str(manifest.get("municipalities_filename") or "") + if not members_path.is_file() or sha256_file(members_path) != manifest.get("municipalities_sha256"): + raise RuntimeError("Official municipality-boundary artifact is missing or fails its scope checksum") + return members_path + + +def load_member_boundaries(path: Path, scope: GeographicScope) -> dict[str, tuple[ScopeMember, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(features, list): + raise RuntimeError("Municipality-boundary artifact is not a GeoJSON FeatureCollection") + expected = {member.nis_code: member for member in scope.members} + selected: dict[str, tuple[ScopeMember, Any]] = {} + for feature in features: + properties = feature.get("properties") or {} + nis_code = str(properties.get("nis_code") or properties.get("NISCODE") or "") + if nis_code not in expected: + continue + if nis_code in selected: + raise RuntimeError(f"Municipality-boundary artifact contains duplicate NIS code {nis_code}") + geometry = normalize_polygonal(shape(feature.get("geometry"))) + if geometry is None: + raise RuntimeError(f"Municipality boundary for {expected[nis_code].name} is invalid") + selected[nis_code] = (expected[nis_code], geometry) + missing = [member.name for member in scope.members if member.nis_code not in selected] + if missing: + raise RuntimeError(f"Municipality-boundary artifact is missing: {', '.join(missing)}") + return {member.nis_code: selected[member.nis_code] for member in scope.members} + + +def fetch_source_page( + session: requests.Session, + *, + collection: str, + filter_xml: str, + page_size: int, + start_index: int, + timeout: int, +) -> tuple[bytes, list[dict[str, Any]]]: + response = session.post( + WFS_URL, + data=wfs_request_xml( + collection=collection, + filter_xml=filter_xml, + page_size=page_size, + start_index=start_index, + ).encode("utf-8"), + headers={"Content-Type": "application/xml; charset=UTF-8", "Accept": "application/json"}, + timeout=timeout, + ) + response.raise_for_status() + raw_content = response.content + payload = response.json() + features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(features, list): + raise RuntimeError("Historical land-use WFS returned an invalid FeatureCollection") + return raw_content, features + + +def partition_paths(output_root: Path, year: int, theme: str, nis_code: str) -> tuple[Path, Path, Path]: + directory = output_root / "partitions" / str(year) / theme + output_path = directory / f"{nis_code}.geojson" + return output_path, output_path.with_suffix(".manifest.json"), directory / f"{nis_code}.raw" + + +def cached_partition( + output_path: Path, + manifest_path: Path, + *, + year: int, + theme: str, + nis_code: str, + page_size: int, + simplify_tolerance_degrees: float, + boundary_geometry_sha256: str, +): + if not output_path.is_file() or not manifest_path.is_file(): + return None + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if ( + manifest.get("status") != "complete" + or manifest.get("year") != year + or manifest.get("theme") != theme + or manifest.get("nis_code") != nis_code + or int(manifest.get("page_size") or 0) != page_size + or float(manifest.get("geometry_simplification_tolerance_degrees") or 0.0) + != simplify_tolerance_degrees + or manifest.get("boundary_geometry_sha256") != boundary_geometry_sha256 + or sha256_file(output_path) != manifest.get("output_sha256") + ): + return None + for page in manifest.get("raw_pages") or []: + raw_path = manifest_path.parent / str(page.get("artifact_path") or "") + if not raw_path.is_file() or sha256_file(raw_path) != page.get("artifact_sha256"): + return None + return manifest + + +def prepare_partition( + session: requests.Session, + *, + output_root: Path, + year: int, + definition: ThemeDefinition, + scope_key: str, + member: ScopeMember, + boundary, + page_size: int, + max_features: int, + simplify_tolerance_degrees: float, + timeout: int, + force: bool, +) -> dict[str, Any]: + output_path, manifest_path, raw_dir = partition_paths(output_root, year, definition.key, member.nis_code) + boundary_geometry_sha256 = sha256_bytes(boundary.wkb) + if not force: + cached = cached_partition( + output_path, + manifest_path, + year=year, + theme=definition.key, + nis_code=member.nis_code, + page_size=page_size, + simplify_tolerance_degrees=simplify_tolerance_degrees, + boundary_geometry_sha256=boundary_geometry_sha256, + ) + if cached: + return cached + + collection = COLLECTIONS[year] + filter_xml = wfs_filter_xml(definition, boundary.bounds) + source_seen: set[str] = set() + output_features: list[dict[str, Any]] = [] + raw_pages: list[dict[str, Any]] = [] + invalid_geometry_count = 0 + outside_boundary_count = 0 + start_index = 0 + while True: + raw_content, page = fetch_source_page( + session, + collection=collection, + filter_xml=filter_xml, + page_size=page_size, + start_index=start_index, + timeout=timeout, + ) + compressed = gzip.compress(raw_content, compresslevel=6, mtime=0) + raw_path = raw_dir / f"page_{start_index:09d}.geojson.gz" + atomic_write_bytes(raw_path, compressed) + raw_pages.append( + { + "start_index": start_index, + "feature_count": len(page), + "response_sha256": sha256_bytes(raw_content), + "response_size_bytes": len(raw_content), + "artifact_path": str(raw_path.relative_to(manifest_path.parent)), + "artifact_sha256": sha256_bytes(compressed), + "artifact_size_bytes": len(compressed), + } + ) + new_source_ids = 0 + for raw_feature in page: + source_feature_id = str(raw_feature.get("id") or "") + if not source_feature_id or source_feature_id in source_seen: + continue + source_seen.add(source_feature_id) + new_source_ids += 1 + properties = dict(raw_feature.get("properties") or {}) + landuse_class = str(properties.get("KLASSE") or "").strip().lower() + if not definition.matches(landuse_class): + continue + try: + source_geometry = normalize_polygonal(shape(raw_feature.get("geometry"))) + except (AttributeError, TypeError, ValueError): + source_geometry = None + if source_geometry is None: + invalid_geometry_count += 1 + continue + clipped = normalize_polygonal(source_geometry.intersection(boundary)) + if clipped is None: + outside_boundary_count += 1 + continue + if simplify_tolerance_degrees > 0: + clipped = normalize_polygonal( + clipped.simplify(simplify_tolerance_degrees, preserve_topology=True) + ) + if clipped is None: + invalid_geometry_count += 1 + continue + partition_feature_id = f"{source_feature_id}:{member.nis_code}" + properties.update( + { + "source_name": "historical_landuse", + "source_feature_id": partition_feature_id, + "original_source_feature_id": source_feature_id, + "reference_layer_name": definition.key, + "authority_level": "authoritative", + "coverage_scope": scope_key, + "municipality": member.name, + "nis_code": member.nis_code, + "observation_year": year, + "historical_landuse_class": landuse_class, + "attribution": ATTRIBUTION, + } + ) + output_features.append( + { + "type": "Feature", + "id": partition_feature_id, + "geometry": mapping(clipped), + "properties": properties, + } + ) + if len(output_features) > max_features: + raise RuntimeError( + f"Historical {definition.key} {year} exceeds {max_features} clipped features in {member.name}" + ) + if len(source_seen) > max_features: + raise RuntimeError( + f"Historical {definition.key} {year} exceeds {max_features} source features in {member.name}" + ) + if len(page) < page_size: + break + if new_source_ids == 0: + raise RuntimeError( + f"Historical WFS pagination stalled for {definition.key} {year} in {member.name}" + ) + start_index += len(page) + + payload = { + "type": "FeatureCollection", + "name": f"{definition.label} - {member.name} {year}", + "crs": GEOJSON_CRS, + "features": output_features, + "observation_year": year, + "municipality": member.name, + "nis_code": member.nis_code, + "attribution": ATTRIBUTION, + } + atomic_write_json(output_path, payload) + manifest = { + "status": "complete", + "year": year, + "theme": definition.key, + "collection": collection, + "municipality": member.name, + "nis_code": member.nis_code, + "boundary_bbox": [float(value) for value in boundary.bounds], + "boundary_geometry_sha256": boundary_geometry_sha256, + "page_size": page_size, + "source_feature_count": len(source_seen), + "feature_count": len(output_features), + "invalid_geometry_count": invalid_geometry_count, + "outside_boundary_count": outside_boundary_count, + "geometry_simplification_tolerance_degrees": simplify_tolerance_degrees, + "raw_pages": raw_pages, + "output_path": str(output_path), + "output_sha256": sha256_file(output_path), + "output_size_bytes": output_path.stat().st_size, + "generated_at": utc_now(), + } + atomic_write_json(manifest_path, manifest, pretty=True) + return manifest + + +def snapshot_paths(output_root: Path, scope: GeographicScope, year: int, theme: str) -> tuple[Path, Path]: + directory = output_root / "snapshots" + stem = f"{scope.key.replace('-', '_')}_historical_{theme}_{year}" + output_path = directory / f"{stem}.geojson" + return output_path, directory / f"{stem}.manifest.json" + + +def assemble_snapshot( + *, + output_root: Path, + scope: GeographicScope, + year: int, + definition: ThemeDefinition, + partitions: list[dict[str, Any]], + max_total_features: int, +) -> tuple[Path, dict[str, Any]]: + if len(partitions) != len(scope.members): + raise RuntimeError(f"Expected {len(scope.members)} partition manifests, received {len(partitions)}") + output_path, manifest_path = snapshot_paths(output_root, scope, year, definition.key) + partition_identity = sha256_bytes( + json.dumps( + [(item["nis_code"], item["output_sha256"]) for item in partitions], + separators=(",", ":"), + ).encode("utf-8") + ) + if output_path.is_file() and manifest_path.is_file(): + existing = json.loads(manifest_path.read_text(encoding="utf-8")) + if ( + existing.get("partition_identity_sha256") == partition_identity + and sha256_file(output_path) == existing.get("output_sha256") + ): + return output_path, existing + + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary = output_path.with_suffix(f"{output_path.suffix}.partial") + feature_count = 0 + seen_ids: set[str] = set() + with temporary.open("w", encoding="utf-8") as output: + header = { + "type": "FeatureCollection", + "name": f"{definition.label} - {scope.display_name} {year}", + "crs": GEOJSON_CRS, + "scope_key": scope.key, + "member_count": len(scope.members), + "observation_year": year, + "attribution": ATTRIBUTION, + } + output.write(json.dumps(header, ensure_ascii=False, separators=(",", ":"))[:-1]) + output.write(',"features":[') + first = True + for partition in partitions: + partition_path = Path(str(partition["output_path"])) + payload = json.loads(partition_path.read_text(encoding="utf-8")) + features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(features, list) or len(features) != int(partition["feature_count"]): + raise RuntimeError(f"Partition feature count drift for NIS {partition['nis_code']}") + for feature in features: + feature_id = str(feature.get("id") or "") + if not feature_id or feature_id in seen_ids: + raise RuntimeError(f"Duplicate or missing partition feature id in {definition.key} {year}") + seen_ids.add(feature_id) + if not first: + output.write(",") + output.write(json.dumps(feature, ensure_ascii=False, separators=(",", ":"))) + first = False + feature_count += 1 + if feature_count > max_total_features: + raise RuntimeError( + f"Regional historical {definition.key} {year} exceeds {max_total_features} features" + ) + output.write("]}") + if feature_count == 0: + temporary.unlink(missing_ok=True) + raise RuntimeError(f"Regional historical {definition.key} {year} contains no clipped features") + temporary.replace(output_path) + empty_partitions = [item["nis_code"] for item in partitions if int(item["feature_count"]) == 0] + manifest = { + "status": "complete", + "scope_key": scope.key, + "scope_type": scope.scope_type, + "member_count": len(scope.members), + "year": year, + "theme": definition.key, + "collection": COLLECTIONS[year], + "feature_count": feature_count, + "coverage_complete": True, + "empty_partitions": empty_partitions, + "partition_identity_sha256": partition_identity, + "partitions": [ + { + "municipality": item["municipality"], + "nis_code": item["nis_code"], + "source_feature_count": item["source_feature_count"], + "feature_count": item["feature_count"], + "raw_page_count": len(item.get("raw_pages") or []), + "output_sha256": item["output_sha256"], + } + for item in partitions + ], + "output_path": str(output_path), + "output_sha256": sha256_file(output_path), + "output_size_bytes": output_path.stat().st_size, + "generated_at": utc_now(), + } + atomic_write_json(manifest_path, manifest, pretty=True) + return output_path, manifest + + +def locate_workspace(session: requests.Session, base_url: str, scope: GeographicScope, timeout: int): + projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout)) + project = next((item for item in projects.get("items") or [] if item.get("name") == scope.project_name), None) + if not project: + raise RuntimeError(f"Project {scope.project_name!r} is missing") + project_id = str(project["id"]) + areas = response_data( + session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout) + ) + area = next((item for item in areas.get("items") or [] if item.get("name") == scope.area_name), None) + if not area: + raise RuntimeError(f"Official scope Area {scope.area_name!r} is missing") + datasets = response_data( + session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 200}, timeout=timeout) + ) + return project_id, str(area["id"]), list(datasets.get("items") or []) + + +def upload_snapshot( + session: requests.Session, + *, + base_url: str, + project_id: str, + area_id: str, + scope: GeographicScope, + year: int, + definition: ThemeDefinition, + path: Path, + manifest: dict[str, Any], + simplify_tolerance_degrees: float, + timeout: int, +) -> dict[str, Any]: + observed_at = f"{year}-01-01T00:00:00Z" + source_metadata = { + "provider": "Digitaal Vlaanderen", + "collection": COLLECTIONS[year], + "authority_level": "authoritative", + "coverage_scope": scope.key, + "scope_type": scope.scope_type, + "scope_display_name": scope.display_name, + "member_count": len(scope.members), + "member_nis_codes": list(scope.nis_codes), + "partitioned_source_audit": True, + "coverage_complete": bool(manifest["coverage_complete"]), + "empty_partition_nis_codes": manifest["empty_partitions"], + "attribution": ATTRIBUTION, + "source_catalog_url": SOURCE_CATALOG_URL, + "identity_stable": False, + "geometry_simplification_tolerance_degrees": simplify_tolerance_degrees, + "selection_aggregation": { + "method": "intersection_area", + "label": "Oppervlakte", + "unit": "ha", + "is_estimate": False, + "warning": ( + "Historische kaartklassen, bronkaarten en karteermethodes verschillen per bronjaar; " + "interpreteer evoluties binnen die methodologische context." + ), + }, + } + provenance_metadata = { + "operator_tool": "provision_regional_historical_landuse.py", + "operator_explicit_fetch": True, + "wfs_url": WFS_URL, + "collection": COLLECTIONS[year], + "partition_count": len(manifest["partitions"]), + "partition_identity_sha256": manifest["partition_identity_sha256"], + "combined_output_sha256": manifest["output_sha256"], + "raw_source_responses_retained": True, + "geometry_simplification_tolerance_degrees": simplify_tolerance_degrees, + "generated_at": manifest["generated_at"], + } + series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:{scope.key}" + with path.open("rb") as handle: + response = session.post( + f"{base_url}/api/v1/projects/{project_id}/datasets/upload", + data={ + "dataset_type": "vector", + "source": "operator_official_import", + "dataset_role": "reference", + "source_name": "historical_landuse", + "reference_layer_name": definition.key, + "source_metadata_json": json.dumps(source_metadata, ensure_ascii=False), + "provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False), + "area_id": area_id, + "temporal_series_key": series_key, + "observed_at": observed_at, + "valid_from": observed_at, + "temporal_granularity": "year", + "source_version": str(year), + }, + files={"file": (path.name, handle, "application/geo+json")}, + timeout=timeout, + ) + return response_data(response) + + +def requested_configuration(args: argparse.Namespace) -> tuple[list[int], list[ThemeDefinition]]: + try: + years = sorted({int(value.strip()) for value in args.years.split(",") if value.strip()}) + except ValueError as exc: + raise ValueError("Years must be comma-separated integers") from exc + theme_keys = {value.strip().lower() for value in args.themes.split(",") if value.strip()} + definitions = [item for item in THEMES if item.key in theme_keys and item.key in SUPPORTED_THEME_KEYS] + unsupported_years = sorted(set(years) - set(COLLECTIONS)) + unsupported_themes = sorted(theme_keys - SUPPORTED_THEME_KEYS) + if not years or not definitions or unsupported_years or unsupported_themes: + raise ValueError(f"Unsupported years={unsupported_years}, themes={unsupported_themes}") + return years, definitions + + +def main() -> int: + args = parse_args() + scope = GEOGRAPHIC_SCOPES[args.scope] + try: + if args.page_size <= 0 or args.max_features_per_partition <= 0 or args.max_total_features <= 0: + raise ValueError("Page and feature safety limits must be greater than zero") + years, definitions = requested_configuration(args) + members_path = resolve_member_boundaries(scope, args.scope_output_root) + boundaries = load_member_boundaries(members_path, scope) + output_root = args.output_root / scope.key + prepared: list[tuple[int, ThemeDefinition, Path, dict[str, Any]]] = [] + with build_session() as source_session: + for year in years: + for definition in definitions: + partitions = [ + prepare_partition( + source_session, + output_root=output_root, + year=year, + definition=definition, + scope_key=scope.key, + member=member, + boundary=boundaries[member.nis_code][1], + page_size=args.page_size, + max_features=args.max_features_per_partition, + simplify_tolerance_degrees=args.simplify_tolerance_degrees, + timeout=args.request_timeout, + force=args.force, + ) + for member in scope.members + ] + path, manifest = assemble_snapshot( + output_root=output_root, + scope=scope, + year=year, + definition=definition, + partitions=partitions, + max_total_features=args.max_total_features, + ) + prepared.append((year, definition, path, manifest)) + + results: list[dict[str, Any]] = [] + if args.fetch_only: + results = [ + { + "year": year, + "theme": definition.key, + "feature_count": manifest["feature_count"], + "partition_count": len(manifest["partitions"]), + "empty_partitions": manifest["empty_partitions"], + "path": str(path), + "status": "prepared", + } + for year, definition, path, manifest in prepared + ] + else: + base_url = args.base_url.rstrip("/") + with requests.Session() as api_session: + project_id, area_id, existing = locate_workspace(api_session, base_url, scope, args.import_timeout) + for year, definition, path, manifest in prepared: + series_key = f"digitaal-vlaanderen:historical-landuse:{definition.key}:{scope.key}" + dataset = next( + ( + item + for item in existing + if item.get("temporal_series_key") == series_key + and str(item.get("observed_at") or "").startswith(str(year)) + ), + None, + ) + if dataset: + if int(dataset.get("feature_count") or 0) != int(manifest["feature_count"]): + raise RuntimeError( + f"Persisted {definition.key} {year} count differs from the audited source artifact" + ) + status = "existing" + else: + dataset = upload_snapshot( + api_session, + base_url=base_url, + project_id=project_id, + area_id=area_id, + scope=scope, + year=year, + definition=definition, + path=path, + manifest=manifest, + simplify_tolerance_degrees=args.simplify_tolerance_degrees, + timeout=args.import_timeout, + ) + existing.append(dataset) + status = "imported" + results.append( + { + "year": year, + "theme": definition.key, + "dataset_id": dataset["id"], + "feature_count": dataset.get("feature_count"), + "partition_count": len(manifest["partitions"]), + "empty_partitions": manifest["empty_partitions"], + "status": status, + } + ) + except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException, json.JSONDecodeError) as exc: + print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 1 + + print( + json.dumps( + { + "status": "ok", + "mode": "fetch_only" if args.fetch_only else "synchronized", + "scope": scope.key, + "display_name": scope.display_name, + "member_count": len(scope.members), + "snapshots": results, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/provision_regional_timeseries.py b/scripts/provision_regional_timeseries.py index d66a7da0..90203950 100644 --- a/scripts/provision_regional_timeseries.py +++ b/scripts/provision_regional_timeseries.py @@ -1,4 +1,4 @@ -"""Synchronize official population and forest time series for one approved scope. +"""Synchronize official population and land-use time series for one approved scope. This is an explicit operator command, never an application-startup task. It coordinates the existing Statbel and Departement Omgeving import paths and @@ -30,6 +30,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL)) parser.add_argument("--population-years", default="2021,2022,2023,2024,2025") parser.add_argument("--landuse-years", default="2013,2016,2019,2022,2025") + parser.add_argument("--historical-years", default="1778,1873,1969") + parser.add_argument("--historical-themes", default="buildings,water,roads") parser.add_argument( "--scope-output-root", type=Path, @@ -41,10 +43,12 @@ def parse_args() -> argparse.Namespace: default=Path(os.environ.get("GEOINTEL_REGIONAL_TIMESERIES_OUTPUT_ROOT", DEFAULT_TIMESERIES_OUTPUT_ROOT)), ) parser.add_argument("--max-landuse-features", type=int, default=500000) + parser.add_argument("--max-historical-features", type=int, default=500000) parser.add_argument("--request-timeout", type=int, default=300) parser.add_argument("--import-timeout", type=int, default=3600) parser.add_argument("--skip-population", action="store_true") parser.add_argument("--skip-landuse", action="store_true") + parser.add_argument("--skip-historical", action="store_true") parser.add_argument("--fetch-only", action="store_true") parser.add_argument("--force", action="store_true") return parser.parse_args() @@ -154,8 +158,37 @@ def build_operator_commands( ], ) ) + if not args.skip_historical: + commands.append( + ( + "historical_landuse", + [ + sys.executable, + str(scripts_dir / "provision_regional_historical_landuse.py"), + "--scope", + scope.key, + "--base-url", + args.base_url, + "--years", + args.historical_years, + "--themes", + args.historical_themes, + "--scope-output-root", + str(args.scope_output_root), + "--output-root", + str(args.output_root / "historical"), + "--request-timeout", + str(args.request_timeout), + "--import-timeout", + str(args.import_timeout), + "--max-total-features", + str(args.max_historical_features), + *common_flags, + ], + ) + ) if not commands: - raise ValueError("At least one of population or land-use synchronization must remain enabled") + raise ValueError("At least one regional time-series synchronization must remain enabled") return commands @@ -177,8 +210,8 @@ def main() -> int: args = parse_args() scope = GEOGRAPHIC_SCOPES[args.scope] try: - if args.max_landuse_features <= 0: - raise ValueError("max-landuse-features must be greater than zero") + if args.max_landuse_features <= 0 or args.max_historical_features <= 0: + raise ValueError("land-use feature safety limits must be greater than zero") boundary_path, members_path, manifest_path = resolve_boundary(scope, args.scope_output_root) commands = build_operator_commands(args, scope, boundary_path, members_path) results = {label: run_operator(label, command) for label, command in commands} diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 596c3c4a..b32fdc82 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -46,6 +46,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_municipality_workspace.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_context_layers.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_population_history.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py +${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py ${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py