Files
geointel/backend/tests/test_sprint187_temporal_map_foundation.py
T
JensandClaude Opus 5 39c12822bc extract the map workspace's domain layer out of the component
MapWorkspace.tsx opened with ~590 lines of theme catalogue, dataset matching
and label formatting above a 3.200-line component. None of it is React, all of
it is independently testable, and both render paths read from it, so it belongs
beside the pure helpers that already live in mapWorkspaceUtils.

The contract tests that read MapWorkspace.tsx would have gone red for a move
that changes no behaviour at all — 24 of them. That is the brittleness the
frontend_contract helper exists to remove, so it gains read_map_workspace():
the workspace is one feature spread over several modules, and a contract
belongs to the feature rather than to whichever file currently holds it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-22 18:49:06 +02:00

550 lines
21 KiB
Python

from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import pytest
from geoalchemy2.shape import from_shape, to_shape
from shapely.geometry import Polygon, box
from app.core.errors import AppError
from app.models import Dataset, DatasetVersion
from app.schemas.dataset import DatasetTemporalUpdate
from app.schemas.temporal import TemporalComparisonRequest, TemporalObjectChanges
from app.services.dataset_service import DatasetService
from app.services.temporal_analysis_service import TemporalAnalysisService
from app.services.vector_feature_service import VectorFeatureService
from tests.frontend_contract import read_map_workspace
ROOT = Path(__file__).parents[2]
class ScalarQuery:
def __init__(self, value: float):
self.value = value
def filter(self, *args): # noqa: ANN002, ARG002
return self
def scalar(self):
return self.value
class ScalarSession:
def __init__(self, value: float):
self.value = value
def query(self, *args): # noqa: ANN002, ARG002
return ScalarQuery(self.value)
class SequenceScalarSession:
def __init__(self, values: list[float]):
self.values = iter(values)
def query(self, *args): # noqa: ANN002, ARG002
return ScalarQuery(next(self.values))
class FeatureRowsQuery:
def __init__(self, rows: list[object]):
self.rows = rows
self.row_limit: int | None = None
def filter(self, *args): # noqa: ANN002, ARG002
return self
def order_by(self, *args): # noqa: ANN002, ARG002
return self
def limit(self, value: int):
self.row_limit = value
return self
def all(self):
return self.rows[: self.row_limit]
class SequentialFeatureSession:
def __init__(self, row_sets: list[list[object]]):
self.row_sets = iter(row_sets)
def query(self, _model):
return FeatureRowsQuery(next(self.row_sets))
class VersionQuery:
def __init__(self, latest: DatasetVersion | None):
self.latest = latest
def filter(self, *args): # noqa: ANN002, ARG002
return self
def order_by(self, *args): # noqa: ANN002, ARG002
return self
def first(self):
return self.latest
class TemporalUpdateSession:
def __init__(self, dataset: Dataset, latest: DatasetVersion | None):
self.dataset = dataset
self.latest = latest
self.added: list[object] = []
def get(self, model, item_id): # noqa: ANN001
return self.dataset if model is Dataset and item_id == self.dataset.id else None
def query(self, model): # noqa: ANN001
assert model is DatasetVersion
return VersionQuery(self.latest)
def add(self, item): # noqa: ANN001
self.added.append(item)
def commit(self):
return None
def refresh(self, _item):
return None
def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "feature_count") -> Dataset:
return Dataset(
id=uuid4(),
project_id=project_id,
name=f"snapshot-{observed_year}.geojson",
dataset_type="vector",
source="official",
dataset_role="reference",
temporal_series_key="official:test:mol",
observed_at=datetime(observed_year, 1, 1, tzinfo=timezone.utc),
source_version=str(observed_year),
source_metadata={
"selection_aggregation": {
"method": metric_method,
"label": "Objecten",
"unit": "objecten",
}
},
)
def test_temporal_series_keeps_only_latest_snapshot_per_observation_date() -> None:
project_id = uuid4()
old = temporal_dataset(project_id=project_id, observed_year=2025)
old.imported_at = datetime(2026, 7, 19, tzinfo=timezone.utc)
latest = temporal_dataset(project_id=project_id, observed_year=2025)
latest.imported_at = datetime(2026, 7, 21, tzinfo=timezone.utc)
earlier = temporal_dataset(project_id=project_id, observed_year=2022)
earlier.imported_at = datetime(2026, 7, 21, tzinfo=timezone.utc)
canonical = TemporalAnalysisService._canonical_observation_snapshots([old, latest, earlier])
assert [dataset.id for dataset in canonical] == [earlier.id, latest.id]
def governed_grb_dataset(*, project_id, observed_day: int) -> Dataset:
dataset = temporal_dataset(project_id=project_id, observed_year=2026, metric_method="intersection_area")
dataset.observed_at = datetime(2026, 7, observed_day, tzinfo=timezone.utc)
dataset.source_version = f"2026-07-{observed_day:02d}"
dataset.source_name = "grb"
dataset.reference_layer_name = "buildings"
dataset.temporal_series_key = "grb:buildings:kempen-transport-region"
dataset.source_metadata = {
"authority_level": "authoritative",
"collection": "GRB/GBG",
"coverage_scope": "kempen-transport-region",
"scope_type": "transport_region",
"member_count": 28,
"partition_count": 28,
"partition_strategy": "municipality_bbox_maximum_boundary_intersection",
"selection_aggregation": {
"method": "intersection_area",
"label": "Bebouwde grondoppervlakte",
"unit": "ha",
},
}
dataset.provenance_metadata = {
"operator_tool": "provision_regional_grb_buildings.py",
"reference_truncated": False,
"manifest_path": "/storage/operator/grb/manifest.json",
"source_url": "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items",
"artifact_sha256": "a" * 64,
"partition_checksums": {f"{index:05d}": "b" * 64 for index in range(28)},
}
if observed_day > 14:
dataset.source_metadata["geometry_clipped_to_area"] = True
return dataset
def persisted_feature(dataset_id, source_feature_id: str | None, polygon: Polygon):
return SimpleNamespace(
id=uuid4(),
dataset_id=dataset_id,
source_feature_id=source_feature_id,
properties_json={},
geometry=from_shape(polygon, srid=4326),
)
def test_temporal_migration_and_models_align() -> None:
migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8")
for field in (
"temporal_series_key",
"observed_at",
"valid_from",
"valid_to",
"temporal_granularity",
"source_version",
):
assert field in migration
assert hasattr(Dataset, field)
assert "ix_vector_features_dataset_source_feature" in migration
assert 'down_revision = "202606120900"' in migration
def test_temporal_metadata_requires_an_explicit_series_and_observation_date() -> None:
with pytest.raises(AppError, match="observed_at is required"):
DatasetService._validate_temporal_metadata(
temporal_series_key="official:test:mol",
observed_at=None,
valid_from=None,
valid_to=None,
temporal_granularity="year",
source_version="2024",
)
with pytest.raises(AppError, match="valid_to must be"):
DatasetService._validate_temporal_metadata(
temporal_series_key="official:test:mol",
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
valid_from=datetime(2024, 12, 31, tzinfo=timezone.utc),
valid_to=datetime(2024, 1, 1, tzinfo=timezone.utc),
temporal_granularity="year",
source_version="2024",
)
def test_temporal_metadata_update_appends_provenance_version_and_is_idempotent() -> None:
project_id = uuid4()
dataset = temporal_dataset(project_id=project_id, observed_year=2024)
dataset.status = "ready"
dataset.metadata_json = {}
latest = DatasetVersion(
dataset_id=dataset.id,
version=3,
observed_at=dataset.observed_at,
source_version="2024",
)
session = TemporalUpdateSession(dataset, latest)
payload = DatasetTemporalUpdate(
temporal_series_key="official:test:mol",
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
temporal_granularity="year",
source_version="2025",
)
updated = DatasetService.update_temporal_metadata(session, dataset.id, payload)
assert updated.observed_at == payload.observed_at
assert latest.version == 3
assert latest.observed_at == datetime(2024, 1, 1, tzinfo=timezone.utc)
assert len(session.added) == 2
appended = session.added[1]
assert isinstance(appended, DatasetVersion)
assert appended.version == 4
assert appended.observed_at == payload.observed_at
session.added.clear()
DatasetService.update_temporal_metadata(session, dataset.id, payload)
assert session.added == []
def test_selection_area_aggregation_returns_hectares_without_loading_all_features() -> None:
project_id = uuid4()
dataset = temporal_dataset(project_id=project_id, observed_year=1969, metric_method="intersection_area")
dataset.source_metadata["selection_aggregation"].update({"label": "Oppervlakte", "unit": "ha"})
result = VectorFeatureService.summarize_features_by_bbox(
ScalarSession(125_000.0),
dataset=dataset,
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"},
total_feature_count=40,
)
assert result["metric_value"] == 12.5
assert result["metric_unit"] == "ha"
assert result["feature_count"] == 40
def test_population_area_weighting_is_exact_for_full_features_and_estimated_for_partial_features() -> None:
dataset = temporal_dataset(project_id=uuid4(), observed_year=2025, metric_method="area_weighted_sum")
dataset.source_metadata["selection_aggregation"].update(
{
"property": "population_total",
"label": "Inwoners",
"unit": "inwoners",
"warning": "Partial-sector estimate",
"warning_only_when_estimate": True,
}
)
bbox = {"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"}
full = VectorFeatureService.summarize_features_by_bbox(
SequenceScalarSession([49, 38_675.0]),
dataset=dataset,
bbox=bbox,
total_feature_count=49,
)
partial = VectorFeatureService.summarize_features_by_bbox(
SequenceScalarSession([1, 1_250.5]),
dataset=dataset,
bbox=bbox,
total_feature_count=3,
)
assert full["metric_value"] == 38_675.0
assert full["is_estimate"] is False
assert full["warning"] is None
assert partial["metric_value"] == 1_250.5
assert partial["is_estimate"] is True
assert partial["warning"] == "Partial-sector estimate"
def test_temporal_compare_returns_delta_and_canonical_change_payload(monkeypatch) -> None:
project_id = uuid4()
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
later = temporal_dataset(project_id=project_id, observed_year=2024)
def get_dataset(_db, _project_id, dataset_id, _label):
return earlier if dataset_id == earlier.id else later
def summarize(_db, *, dataset, bbox, disclose_selection_edge=True): # noqa: ARG001
value = 100.0 if dataset.id == earlier.id else 115.0
return {
"metric_label": "Inwoners",
"metric_value": value,
"metric_unit": "inwoners",
"aggregation_method": "area_weighted_sum",
"feature_count": 10,
"is_estimate": True,
"warning": "Areal weighting",
}
monkeypatch.setattr(TemporalAnalysisService, "_get_temporal_dataset", staticmethod(get_dataset))
monkeypatch.setattr(VectorFeatureService, "summarize_features_by_bbox", staticmethod(summarize))
monkeypatch.setattr(
TemporalAnalysisService,
"_compare_identity_features",
staticmethod(
lambda *args, **kwargs: (
TemporalObjectChanges(available=True, added_count=1, removed_count=0, modified_count=2, unchanged_count=7),
{"type": "FeatureCollection", "features": []},
[],
)
),
)
result = TemporalAnalysisService.compare(
SimpleNamespace(),
project_id=project_id,
payload=TemporalComparisonRequest(
earlier_dataset_id=earlier.id,
later_dataset_id=later.id,
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3},
),
)
assert result.metric.absolute_change == 15.0
assert result.metric.percent_change == 15.0
assert result.metric.is_estimate is True
assert result.object_changes.modified_count == 2
assert result.geojson["type"] == "FeatureCollection"
def test_temporal_comparison_clips_cross_boundary_bbox_to_selected_area(monkeypatch) -> None:
project_id = uuid4()
area_id = uuid4()
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
later = temporal_dataset(project_id=project_id, observed_year=2024)
area_shape = box(5.0, 51.0, 5.2, 51.2)
area = SimpleNamespace(
id=area_id,
project_id=project_id,
geometry=from_shape(area_shape, srid=4326),
)
captured_geometries = []
identity_capture = {}
monkeypatch.setattr(
TemporalAnalysisService,
"_get_temporal_dataset",
staticmethod(lambda _db, _project_id, dataset_id, _label: earlier if dataset_id == earlier.id else later),
)
monkeypatch.setattr(
TemporalAnalysisService,
"_get_selection_area",
staticmethod(lambda _db, _project_id, requested_area_id: area if requested_area_id == area_id else None),
)
def summarize(_db, *, dataset, bbox, selection_geometry, full_dataset_area, disclose_selection_edge=True): # noqa: ARG001
captured_geometries.append(selection_geometry)
return {
"metric_label": "Oppervlakte",
"metric_value": 10.0 if dataset.id == earlier.id else 12.0,
"metric_unit": "ha",
"aggregation_method": "intersection_area",
"feature_count": 1,
"is_estimate": False,
"warning": None,
}
def compare_identity(*_args, **kwargs):
identity_capture.update(kwargs)
return (
TemporalObjectChanges(available=False),
{"type": "FeatureCollection", "features": []},
[],
)
monkeypatch.setattr(VectorFeatureService, "summarize_features_by_bbox", staticmethod(summarize))
monkeypatch.setattr(TemporalAnalysisService, "_compare_identity_features", staticmethod(compare_identity))
result = TemporalAnalysisService.compare(
SimpleNamespace(),
project_id=project_id,
payload=TemporalComparisonRequest(
earlier_dataset_id=earlier.id,
later_dataset_id=later.id,
area_id=area_id,
bbox={"min_x": 4.9, "min_y": 51.1, "max_x": 5.1, "max_y": 51.3},
),
)
expected = box(5.0, 51.1, 5.1, 51.2)
assert result.metric.absolute_change == 2.0
assert all(to_shape(geometry).equals(expected) for geometry in captured_geometries)
assert to_shape(identity_capture["selection_geometry"]).equals(expected)
assert identity_capture["earlier_full_dataset_area"] is False
assert identity_capture["later_full_dataset_area"] is False
def test_unstable_temporal_identity_returns_clear_end_user_warning() -> None:
project_id = uuid4()
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
later = temporal_dataset(project_id=project_id, observed_year=2025)
earlier.source_metadata["identity_stable"] = False
later.source_metadata["identity_stable"] = False
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
SimpleNamespace(),
earlier=earlier,
later=later,
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3},
preview_limit=100,
)
assert changes.available is False
assert geojson == {"type": "FeatureCollection", "features": []}
assert warnings == ["Wijzigingen van individuele objecten kunnen voor deze bron niet betrouwbaar worden gevolgd."]
def test_governed_legacy_grb_snapshots_use_verified_official_feature_identity() -> None:
project_id = uuid4()
earlier = governed_grb_dataset(project_id=project_id, observed_day=14)
later = governed_grb_dataset(project_id=project_id, observed_day=15)
original = Polygon([(5.1, 51.1), (5.101, 51.1), (5.101, 51.101), (5.1, 51.101)])
changed = Polygon([(5.1, 51.1), (5.102, 51.1), (5.102, 51.101), (5.1, 51.101)])
added = Polygon([(5.11, 51.11), (5.111, 51.11), (5.111, 51.111), (5.11, 51.111)])
session = SequentialFeatureSession(
[
[persisted_feature(earlier.id, "GBG.1", original)],
[persisted_feature(later.id, "GBG.1", changed), persisted_feature(later.id, "GBG.2", added)],
]
)
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
session,
earlier=earlier,
later=later,
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.2, "max_y": 51.2},
preview_limit=100,
)
assert changes.available is True
assert changes.added_count == 1
assert changes.removed_count == 0
assert changes.modified_count == 1
assert changes.unchanged_count == 0
assert {feature["properties"]["change_type"] for feature in geojson["features"]} == {"added", "modified"}
assert warnings == []
def test_governed_grb_object_history_fails_closed_for_unverified_identity() -> None:
project_id = uuid4()
earlier = governed_grb_dataset(project_id=project_id, observed_day=14)
later = governed_grb_dataset(project_id=project_id, observed_day=15)
polygon = Polygon([(5.1, 51.1), (5.101, 51.1), (5.101, 51.101), (5.1, 51.101)])
fallback_hash = "a" * 64
session = SequentialFeatureSession(
[
[persisted_feature(earlier.id, fallback_hash, polygon)],
[persisted_feature(later.id, fallback_hash, polygon)],
]
)
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
session,
earlier=earlier,
later=later,
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.2, "max_y": 51.2},
preview_limit=100,
)
assert changes.available is False
assert geojson["features"] == []
assert warnings == ["De geselecteerde objecten bevatten geen volledig verifieerbare stabiele bronidentiteit."]
def test_legacy_grb_identity_requires_complete_partition_evidence() -> None:
dataset = governed_grb_dataset(project_id=uuid4(), observed_day=14)
dataset.provenance_metadata["partition_checksums"] = {"13025": "b" * 64}
assert TemporalAnalysisService._identity_contract(dataset) is None
def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
workspace = read_map_workspace()
temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8")
population = (ROOT / "scripts/provision_mol_population_history.py").read_text(encoding="utf-8")
landuse = (ROOT / "scripts/provision_mol_historical_landuse.py").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
assert "Laatste toestand" in workspace
assert "Evolutie" in workspace
assert "Vergelijk periode" in workspace
assert "temporalRangeLabel" in workspace
assert "Dagelijkse GRB-edities tonen wijzigingen in de officiële registratie" in workspace
assert "/temporal/compare" in temporal_api
assert "Statbel" in population and "area_weighted_sum" in population
assert '"identity_stable": False' in population
assert "HistLandgebruik" in landuse and "intersection_area" in landuse
assert "<wfs:GetFeature" in landuse and "session.post(" in landuse
assert 'lambda value: value.startswith("weg"), "weg*"' in landuse
assert "provision_mol_population_history.py" in dockerfile
assert "provision_mol_historical_landuse.py" in dockerfile
assert "fake" not in population.lower()
def test_tower_deploy_waits_for_startup_migration_before_live_smoke() -> None:
for relative_path in ("scripts/deploy_tower.ps1", "scripts/deploy_tower.sh"):
script = (ROOT / relative_path).read_text(encoding="utf-8")
assert "bash deploy/unraid/deploy-release.sh" in script
release_script = (ROOT / "deploy/unraid/deploy-release.sh").read_text(encoding="utf-8")
wait_position = release_script.index("wait_for_geointel_health")
invocation_position = release_script.index("\n wait_for_geointel_health", wait_position)
smoke_position = release_script.index("LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh")
assert "docker inspect --format" in release_script
assert invocation_position < smoke_position