From a5b4bbdf8f76b52006b427fd3ccf7b23601b8b01 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 22:19:30 +0200 Subject: [PATCH] feat: synchronize regional official time series --- CHANGELOG.md | 9 + backend/README.md | 15 ++ .../test_sprint194_regional_timeseries.py | 146 +++++++++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docs/CODEX_EXECUTION_LOG.md | 18 ++ docs/DATA_SOURCES.md | 16 ++ frontend/README.md | 2 +- frontend/src/components/map/MapWorkspace.tsx | 6 +- frontend/src/lib/datasetDisplay.ts | 14 +- scripts/README.md | 17 ++ scripts/provision_mol_population_history.py | 191 +++++++++++++---- .../provision_official_landuse_timeseries.py | 25 ++- scripts/provision_regional_timeseries.py | 199 ++++++++++++++++++ scripts/run_readiness_check.sh | 1 + 14 files changed, 609 insertions(+), 51 deletions(-) create mode 100644 backend/tests/test_sprint194_regional_timeseries.py create mode 100644 scripts/provision_regional_timeseries.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bf296b96..690f8127 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 194 Regional official time-series synchronization (2026-07-14) + +- Generalized the proven Statbel population operator from a hardcoded Mol import to an approved geographic scope while keeping Mol as the backwards-compatible default. +- Added one explicit regional synchronization command for official 2021-2025 population and 2013-2025 modern forest snapshots. +- Kept every fetch operator-triggered, idempotent and behind the canonical DatasetService upload path; no startup fetch, migration or API contract change was introduced. +- Preserved separate Mol/regional series keys and honest partial-sector population and 10 m forest-area limitations. +- Replaced internal provider identifiers with readable source labels in the primary map. +- Added focused scope filtering, command construction, boundary resolution, multi-NIS provenance, packaging and frontend-label tests. + ## Sprint 193 End-user regional workbench simplification (2026-07-14) - Made the complete `Kempen Regional Workbench` the automatic fresh-session data context and removed the redundant region selector from the primary map. diff --git a/backend/README.md b/backend/README.md index 7315c1b0..6518cc6c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1024,6 +1024,21 @@ Modern raster-derived forest polygons have the same identity limitation. Their area is measured in EPSG:31370 and is exact within the 10 m source representation, not a cadastral forest survey. +The same source-governed operators can synchronize the approved regional +scope in one explicit pass: + +```bash +docker exec geointel python /app/scripts/provision_regional_timeseries.py +``` + +This resolves the retained official boundary and imports five Statbel +population snapshots plus five modern forest snapshots into +`Kempen Regional Workbench`. Mol and regional series keys remain separate and +existing immutable datasets are reused. Complete statistical sectors use exact +published totals; a rectangle cutting a sector remains an area-weighted +estimate. Forest area is measured within the official 10 m representation. +Use `--fetch-only` to validate source artifacts without database mutation. + ## Helpful repository scripts - `bash scripts/backend_install.sh` diff --git a/backend/tests/test_sprint194_regional_timeseries.py b/backend/tests/test_sprint194_regional_timeseries.py new file mode 100644 index 00000000..d89cffbd --- /dev/null +++ b/backend/tests/test_sprint194_regional_timeseries.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import argparse +import importlib.util +import io +import json +from pathlib import Path +import sys +import zipfile + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + + +def load_script(name: str): + path = SCRIPTS / name + module_name = f"test_{path.stem}" + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def population_archive() -> bytes: + text = "\n".join( + ( + "CD_REFNIS|CD_SECTOR|TOTAL|TX_DESCR_SECTOR_NL|TX_DESCR_NL", + "13025|13025A00-|120|Mol centrum|Mol", + "13008|13008A00-|240|Geel centrum|Geel", + "11002|11002A00-|360|Antwerpen centrum|Antwerpen", + ) + ) + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("population.csv", text) + return buffer.getvalue() + + +def test_population_operator_filters_to_the_approved_scope() -> None: + module = load_script("provision_mol_population_history.py") + regional = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] + mol = module.GEOGRAPHIC_SCOPES["mol"] + + regional_rows = module.population_rows(population_archive(), regional) + mol_rows = module.population_rows(population_archive(), mol) + + assert set(regional_rows) == {"13025A00-", "13008A00-"} + assert regional_rows["13008A00-"]["municipality"] == "Geel" + assert regional_rows["13008A00-"]["nis_code"] == "13008" + assert set(mol_rows) == {"13025A00-"} + assert module.series_key(regional) == "statbel:population-statistical-sector:kempen-transport-region" + assert module.series_key(mol) == "statbel:population-statistical-sector:mol" + + +def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Path) -> None: + module = load_script("provision_mol_population_history.py") + scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] + scope_dir = tmp_path / scope.key + scope_dir.mkdir(parents=True) + boundary = scope_dir / "boundary.geojson" + boundary.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8") + manifest = scope_dir / "kempen_transport_region_scope_manifest.json" + manifest.write_text( + json.dumps( + { + "status": "complete", + "scope_key": scope.key, + "boundary_filename": boundary.name, + } + ), + encoding="utf-8", + ) + args = argparse.Namespace(boundary_path=None, scope_output_root=tmp_path) + + assert module.resolve_boundary_path(args, scope) == boundary + + +def test_regional_coordinator_builds_explicit_population_and_forest_commands(tmp_path: Path) -> None: + module = load_script("provision_regional_timeseries.py") + scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] + args = argparse.Namespace( + output_root=tmp_path / "time-series", + fetch_only=False, + force=False, + skip_population=False, + skip_landuse=False, + base_url="http://backend:8000", + population_years="2021,2025", + landuse_years="2013,2025", + request_timeout=300, + import_timeout=3600, + max_landuse_features=500000, + ) + + commands = dict(module.build_operator_commands(args, scope, tmp_path / "boundary.geojson")) + + assert set(commands) == {"population", "forest"} + assert commands["population"][0] == sys.executable + assert "--scope" in commands["population"] + assert "kempen-transport-region" in commands["population"] + assert scope.project_name in commands["population"] + assert commands["forest"][0] == sys.executable + assert "--max-features" in commands["forest"] + assert ",".join(scope.nis_codes) in commands["forest"] + assert "--force" not in commands["population"] + assert "--fetch-only" not in commands["forest"] + + +def test_regional_forest_provenance_does_not_claim_one_municipality() -> None: + module = load_script("provision_official_landuse_timeseries.py") + + regional = module.scope_identity("Kempen (28 gemeenten)", "13001,13008,13025") + municipal = module.scope_identity("Mol", "13025") + + assert regional == { + "scope_display_name": "Kempen (28 gemeenten)", + "member_nis_codes": ["13001", "13008", "13025"], + "municipality": None, + "nis_code": None, + } + assert municipal["municipality"] == "Mol" + assert municipal["nis_code"] == "13025" + + +def test_regional_timeseries_operator_is_packaged_and_release_checked() -> None: + dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8") + + assert "COPY scripts/provision_regional_timeseries.py" in dockerfile + assert "py_compile scripts/provision_regional_timeseries.py" in readiness + + +def test_end_user_dataset_sources_are_human_readable() -> None: + display = (ROOT / "frontend/src/lib/datasetDisplay.ts").read_text(encoding="utf-8") + workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8") + + assert "department_omgeving_land_use: 'Departement Omgeving'" in display + assert "statbel: 'Statbel'" in display + assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace + assert "dataset ? getDatasetSourceDisplayName(dataset)" in workspace diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 55dca368..98280089 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -77,6 +77,7 @@ COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_ COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py +COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py COPY scripts/provision_regional_grb_buildings.py /app/scripts/provision_regional_grb_buildings.py diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index c5816905..8e8324c5 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7960,6 +7960,24 @@ Live operational proof: Next: - Build bounded, idempotent regional theme ingestion in municipality-sized partitions, starting with current buildings and retaining per-member provenance before exposing any Kempen-wide metric in the explorer. +## Sprint 194 - Regional official time-series operator foundation (2026-07-14) + +Implemented: +- Made the existing Statbel population operator scope-aware while retaining Mol defaults and artifact compatibility. +- Added approved-scope NIS filtering, per-feature municipality provenance, checksummed scope-boundary resolution and distinct regional temporal-series keys. +- Added `provision_regional_timeseries.py` as one explicit coordinator for population and modern forest imports into `Kempen Regional Workbench`. +- Reused the current Dataset upload API and existing forest polygonization path; no migration, public endpoint, direct PostGIS write, startup fetch or fabricated value was added. +- Corrected regional forest provenance so a 28-member coverage does not claim to be one municipality. +- Added friendly Statbel/GRB/VRBG/Departement Omgeving labels to the primary map. + +Validation before deployment: +- Source contracts were rechecked against the official Statbel sector-population page and the official 2025 Landgebruik Vlaanderen catalogue. +- The focused regional/temporal suite passed 19 tests. +- Frontend TypeScript typecheck and production build passed. + +Next: +- Deploy the packaged operator, execute the live regional synchronization, verify persisted counts and temporal comparisons, then mark the regional time-series TODO complete only if PostGIS and browser evidence agree. + ## Sprint 190 Regional Kempen GRB buildings (2026-07-14) Implemented: diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 1639b808..457cc02b 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -148,6 +148,22 @@ presented as one continuous measurement method. Raster-derived polygon identities are unstable, so GeoIntel compares hectares and never invents added/removed forest objects. +### Regional official time series + +`scripts/provision_regional_timeseries.py` applies the same population and +modern forest operators to the approved 28-municipality +`kempen-transport-region` scope. It resolves the checksummed VRBG union +boundary and writes to `Kempen Regional Workbench` through the normal dataset +API. Regional keys are +`statbel:population-statistical-sector:kempen-transport-region` and +`department-omgeving:land-use:forest:kempen-transport-region`, so Mol datasets +remain independent observations rather than aliases. + +The command is explicit and operator-triggered. No source fetch happens during +startup or map interaction. Partial statistical sectors remain area-weighted +population estimates; forest hectares remain measurements in the harmonized +10 m source representation and are not cadastral forest boundaries. + Official catalogues: - https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2013 diff --git a/frontend/README.md b/frontend/README.md index 1ce42cb9..ee3fce74 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -2,7 +2,7 @@ React + TypeScript + MapLibre workbench for regional geographic analysis. -The persisted `Kempen Regional Workbench` is the automatic operational data context. Its current GRB layers are loaded once for the complete official 28-municipality Vlaamse vervoerregio; the operator chooses Mol, another municipality or the complete region as a spatial work-area filter. The primary map no longer asks the user to choose a technical project or region before data becomes usable. Mol remains the primary validation and historical-data focus, while explicit project selection stays available under advanced management. +The persisted `Kempen Regional Workbench` is the automatic operational data context. Its datasets are loaded once for the complete official 28-municipality Vlaamse vervoerregio; the operator chooses Mol, another municipality or the complete region as a spatial work-area filter. The primary map no longer asks the user to choose a technical project or region before data becomes usable. Regional population and modern forest snapshots use the same current/evolution flow as Mol, while explicit project selection stays available under advanced management. The user-facing shell is task based: `Kaart`, `Bronnen`, `Kwaliteit`, `Beeldanalyse`, `Downloads`, `Status` and `Beheer`. Internal benchmark projects, raw dataset metadata, provider capabilities, model registry details and QA evidence remain accessible through labelled advanced disclosures instead of competing with the normal workflow. diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index d8ca5465..f81903e1 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -4,7 +4,7 @@ import type { AreaRead, DatasetCreateResponse, MapViewportState, ProjectRead, Qa import { featureCollectionBounds } from '../../lib/geojsonBounds' import { useMapThemeSelectionInsights } from '../../hooks/useMapThemeSelectionInsights' import { useTemporalComparison } from '../../hooks/useTemporalComparison' -import { getDatasetDisplayName } from '../../lib/datasetDisplay' +import { getDatasetDisplayName, getDatasetSourceDisplayName } from '../../lib/datasetDisplay' const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson' const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson' @@ -998,7 +998,7 @@ export function MapWorkspace({ ? `${activeTemporalSeries.length} officiële meetmomenten · ${formatObservationDate(activeTemporalSeries[0].observed_at)} tot ${formatObservationDate(activeTemporalSeries[activeTemporalSeries.length - 1].observed_at)}` : 'Minstens twee expliciet gedateerde snapshots zijn vereist.' : activeThemeDataset - ? `${activeThemeDataset.source_name ?? activeThemeDataset.source} · ${activeThemeDataset.dataset_role ?? 'source'} · ${formatObservationDate(activeThemeDataset.observed_at)}` + ? `${getDatasetSourceDisplayName(activeThemeDataset)} · ${formatObservationDate(activeThemeDataset.observed_at)}` : activeTheme.description} @@ -1247,7 +1247,7 @@ export function MapWorkspace({