from __future__ import annotations import argparse import importlib.util import io import json from pathlib import Path import sys import zipfile from types import SimpleNamespace from uuid import uuid4 import numpy as np import rasterio from rasterio.transform import from_origin from app.models import Dataset from app.services.vector_feature_service import VectorFeatureService from tests.frontend_contract import read_feature ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" if str(SCRIPTS) not in sys.path: sys.path.insert(0, str(SCRIPTS)) def load_script(name: str): path = SCRIPTS / name module_name = f"test_{path.stem}" spec = importlib.util.spec_from_file_location(module_name, path) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules[module_name] = module spec.loader.exec_module(module) return module def population_archive() -> bytes: text = "\n".join( ( "CD_REFNIS|CD_SECTOR|TOTAL|TX_DESCR_SECTOR_NL|TX_DESCR_NL", "13025|13025A00-|120|Mol centrum|Mol", "13008|13008A00-|240|Geel centrum|Geel", "11002|11002A00-|360|Antwerpen centrum|Antwerpen", ) ) buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w") as archive: archive.writestr("population.csv", text) return buffer.getvalue() def test_population_operator_filters_to_the_approved_scope() -> None: module = load_script("provision_mol_population_history.py") regional = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] mol = module.GEOGRAPHIC_SCOPES["mol"] belgium = module.GEOGRAPHIC_SCOPES["belgium"] regional_rows = module.population_rows(population_archive(), regional) mol_rows = module.population_rows(population_archive(), mol) national_rows = module.population_rows(population_archive(), belgium) assert set(regional_rows) == {"13025A00-", "13008A00-"} assert regional_rows["13008A00-"]["municipality"] == "Geel" assert regional_rows["13008A00-"]["nis_code"] == "13008" assert set(mol_rows) == {"13025A00-"} assert set(national_rows) == {"13025A00-", "13008A00-", "11002A00-"} assert national_rows["11002A00-"]["municipality"] == "Antwerpen" assert belgium.all_municipalities is True assert module.series_key(regional) == "statbel:population-statistical-sector:kempen-transport-region" assert module.series_key(mol) == "statbel:population-statistical-sector:mol" assert module.series_key(belgium) == "statbel:population-statistical-sector:belgium" def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Path) -> None: module = load_script("provision_mol_population_history.py") scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] scope_dir = tmp_path / scope.key scope_dir.mkdir(parents=True) boundary = scope_dir / "boundary.geojson" boundary.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8") manifest = scope_dir / "kempen_transport_region_scope_manifest.json" manifest.write_text( json.dumps( { "status": "complete", "scope_key": scope.key, "boundary_filename": boundary.name, } ), encoding="utf-8", ) args = argparse.Namespace(boundary_path=None, scope_output_root=tmp_path) assert module.resolve_boundary_path(args, scope) == boundary def test_population_operator_resolves_checksum_verified_belgium_boundary(tmp_path: Path) -> None: module = load_script("provision_mol_population_history.py") scope = module.GEOGRAPHIC_SCOPES["belgium"] scope_dir = tmp_path / "belgium-north-sea" scope_dir.mkdir(parents=True) boundary = scope_dir / "belgium_land_boundary.geojson" boundary.write_text( json.dumps( { "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": { "type": "Polygon", "coordinates": [[[2.5, 49.5], [6.4, 49.5], [6.4, 51.5], [2.5, 51.5], [2.5, 49.5]]], }, "properties": {}, } ], } ), encoding="utf-8", ) (scope_dir / "manifest.json").write_text( json.dumps( { "scope": "belgium-and-belgian-north-sea", "artifacts": { "belgium_land_boundary": { "sha256": module.sha256_path(boundary), } }, } ), encoding="utf-8", ) args = argparse.Namespace(boundary_path=None, scope_output_root=tmp_path) assert module.resolve_boundary_path(args, scope) == boundary def test_regional_coordinator_builds_explicit_population_and_landuse_commands(tmp_path: Path) -> None: module = load_script("provision_regional_timeseries.py") scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] args = argparse.Namespace( output_root=tmp_path / "time-series", scope_output_root=tmp_path / "scopes", fetch_only=False, force=False, skip_population=False, skip_landuse=False, base_url="http://backend:8000", population_years="2021,2025", landuse_years="2013,2025", historical_years="1778,1873,1969", historical_themes="buildings,water,roads", request_timeout=300, import_timeout=3600, max_landuse_features=500000, max_historical_features=500000, skip_historical=False, ) members_path = tmp_path / "municipalities.geojson" commands = dict(module.build_operator_commands(args, scope, tmp_path / "boundary.geojson", members_path)) assert set(commands) == {"population", "forest", "historical_landuse"} assert commands["population"][0] == sys.executable assert "--scope" in commands["population"] assert "kempen-transport-region" in commands["population"] assert scope.project_name in commands["population"] assert commands["forest"][0] == sys.executable assert "--max-features" in commands["forest"] assert "--partition-boundaries-path" in commands["forest"] assert str(members_path) in commands["forest"] assert ",".join(scope.nis_codes) in commands["forest"] assert "--force" not in commands["population"] assert "--fetch-only" not in commands["forest"] assert commands["historical_landuse"][0] == sys.executable assert "provision_regional_historical_landuse.py" in commands["historical_landuse"][1] assert "buildings,water,roads" in commands["historical_landuse"] assert "--scope-output-root" in commands["historical_landuse"] def test_regional_forest_provenance_does_not_claim_one_municipality() -> None: module = load_script("provision_official_landuse_timeseries.py") regional = module.scope_identity("Kempen (28 gemeenten)", "13001,13008,13025") municipal = module.scope_identity("Mol", "13025") assert regional == { "scope_display_name": "Kempen (28 gemeenten)", "member_nis_codes": ["13001", "13008", "13025"], "municipality": None, "nis_code": None, } assert municipal["municipality"] == "Mol" assert municipal["nis_code"] == "13025" def test_regional_forest_partition_rasters_merge_without_resolution_loss(tmp_path: Path) -> None: module = load_script("provision_official_landuse_timeseries.py") left = tmp_path / "left.tif" right = tmp_path / "right.tif" profile = { "driver": "GTiff", "height": 2, "width": 2, "count": 1, "dtype": "uint8", "crs": "EPSG:31370", "transform": from_origin(100000, 200000, 10, 10), "nodata": 0, } with rasterio.open(left, "w", **profile) as target: target.write(np.full((1, 2, 2), 12, dtype="uint8")) with rasterio.open( right, "w", **{**profile, "transform": from_origin(100020, 200000, 10, 10)}, ) as target: target.write(np.full((1, 2, 2), 17, dtype="uint8")) destination = tmp_path / "regional.tif" result = module.merge_partition_rasters([left, right], destination) assert result["width"] == 4 assert result["height"] == 2 assert result["resolution_metres"] == 10.0 with rasterio.open(destination) as merged: assert merged.read(1).tolist() == [[12, 12, 17, 17], [12, 12, 17, 17]] def test_regional_timeseries_operator_is_packaged_and_release_checked() -> None: dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8") assert "COPY scripts/provision_regional_timeseries.py" in dockerfile assert "py_compile scripts/provision_regional_timeseries.py" in readiness def test_end_user_dataset_sources_are_human_readable() -> None: display = (ROOT / "frontend/src/lib/datasetDisplay.ts").read_text(encoding="utf-8") workspace = read_feature("map_workspace") catalog = read_feature("datasets") status = (ROOT / "frontend/src/components/WorkbenchStatusStrip.tsx").read_text(encoding="utf-8") detection = read_feature("detection") exports = read_feature("exports") assert "department_omgeving_land_use: 'Departement Omgeving'" in display assert "statbel: 'Statbel'" in display assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace assert "getDatasetSourceDisplayName(resultDataset)" in workspace assert "Zoek optioneel een gemeente" in workspace assert "latestDatasetBySeries" in catalog assert "Historische meetmomenten" in catalog assert "getDatasetSourceDisplayName(dataset)" in catalog assert "statusLabel(item.state)" in status assert "Technische modelevaluatie" in detection assert "Nog geen downloads gemaakt." in exports def test_full_area_fast_path_requires_matching_area_and_clipped_operator_provenance() -> None: project_id = uuid4() area_id = uuid4() trusted = Dataset( id=uuid4(), project_id=project_id, area_id=area_id, name="Regional forest", dataset_type="vector", source="operator_official_import", provenance_metadata={"operator_tool": "provision_official_landuse_timeseries.py"}, ) explicit = Dataset( id=uuid4(), project_id=project_id, area_id=area_id, name="Clipped vector", dataset_type="vector", source="manual", source_metadata={"geometry_clipped_to_area": True}, ) regional_historical = Dataset( id=uuid4(), project_id=project_id, area_id=area_id, name="Regional historical buildings", dataset_type="vector", source="operator_official_import", provenance_metadata={"operator_tool": "provision_regional_historical_landuse.py"}, ) untrusted = Dataset( id=uuid4(), project_id=project_id, area_id=area_id, name="Assigned only", dataset_type="vector", source="manual", ) assert VectorFeatureService.can_use_full_area_fast_path(trusted, area_id) is True assert VectorFeatureService.can_use_full_area_fast_path(explicit, area_id) is True assert VectorFeatureService.can_use_full_area_fast_path(regional_historical, area_id) is True assert VectorFeatureService.can_use_full_area_fast_path(untrusted, area_id) is False assert VectorFeatureService.can_use_full_area_fast_path(trusted, uuid4()) is False assert VectorFeatureService.can_use_full_area_fast_path(trusted, None) is False def test_full_area_summary_uses_exact_stored_values_without_partial_intersection() -> None: class ScalarQuery: def __init__(self, value: float) -> None: self.value = value def filter(self, *_args): return self def scalar(self): return self.value class ScalarSession: def __init__(self, value: float) -> None: self.value = value self.query_count = 0 def query(self, *_args): self.query_count += 1 return ScalarQuery(self.value) dataset = Dataset( id=uuid4(), project_id=uuid4(), name="Population", dataset_type="vector", source="operator_official_import", source_metadata={ "selection_aggregation": { "method": "area_weighted_sum", "property": "population_total", "label": "Inwoners", "unit": "inwoners", "warning_only_when_estimate": True, "warning": "Partial-sector estimate", } }, ) session = ScalarSession(506_473.0) result = VectorFeatureService.summarize_features_by_bbox( session, dataset=dataset, bbox={"min_x": 4.5, "min_y": 51.0, "max_x": 5.3, "max_y": 51.6, "crs": "EPSG:4326"}, total_feature_count=733, selection_geometry=SimpleNamespace(), full_dataset_area=True, ) assert result["metric_value"] == 506_473.0 assert result["is_estimate"] is False assert result["warning"] is None assert session.query_count == 1