326 lines
12 KiB
Python
326 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
|
|
from app.core.errors import AppError
|
|
from app.models import Dataset, DatasetVersion
|
|
from app.schemas.dataset import DatasetTemporalUpdate
|
|
from app.schemas.temporal import TemporalComparisonRequest, TemporalObjectChanges
|
|
from app.services.dataset_service import DatasetService
|
|
from app.services.temporal_analysis_service import TemporalAnalysisService
|
|
from app.services.vector_feature_service import VectorFeatureService
|
|
|
|
|
|
ROOT = Path(__file__).parents[2]
|
|
|
|
|
|
class ScalarQuery:
|
|
def __init__(self, value: float):
|
|
self.value = value
|
|
|
|
def filter(self, *args): # noqa: ANN002, ARG002
|
|
return self
|
|
|
|
def scalar(self):
|
|
return self.value
|
|
|
|
|
|
class ScalarSession:
|
|
def __init__(self, value: float):
|
|
self.value = value
|
|
|
|
def query(self, *args): # noqa: ANN002, ARG002
|
|
return ScalarQuery(self.value)
|
|
|
|
|
|
class SequenceScalarSession:
|
|
def __init__(self, values: list[float]):
|
|
self.values = iter(values)
|
|
|
|
def query(self, *args): # noqa: ANN002, ARG002
|
|
return ScalarQuery(next(self.values))
|
|
|
|
|
|
class VersionQuery:
|
|
def __init__(self, latest: DatasetVersion | None):
|
|
self.latest = latest
|
|
|
|
def filter(self, *args): # noqa: ANN002, ARG002
|
|
return self
|
|
|
|
def order_by(self, *args): # noqa: ANN002, ARG002
|
|
return self
|
|
|
|
def first(self):
|
|
return self.latest
|
|
|
|
|
|
class TemporalUpdateSession:
|
|
def __init__(self, dataset: Dataset, latest: DatasetVersion | None):
|
|
self.dataset = dataset
|
|
self.latest = latest
|
|
self.added: list[object] = []
|
|
|
|
def get(self, model, item_id): # noqa: ANN001
|
|
return self.dataset if model is Dataset and item_id == self.dataset.id else None
|
|
|
|
def query(self, model): # noqa: ANN001
|
|
assert model is DatasetVersion
|
|
return VersionQuery(self.latest)
|
|
|
|
def add(self, item): # noqa: ANN001
|
|
self.added.append(item)
|
|
|
|
def commit(self):
|
|
return None
|
|
|
|
def refresh(self, _item):
|
|
return None
|
|
|
|
|
|
def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "feature_count") -> Dataset:
|
|
return Dataset(
|
|
id=uuid4(),
|
|
project_id=project_id,
|
|
name=f"snapshot-{observed_year}.geojson",
|
|
dataset_type="vector",
|
|
source="official",
|
|
dataset_role="reference",
|
|
temporal_series_key="official:test:mol",
|
|
observed_at=datetime(observed_year, 1, 1, tzinfo=timezone.utc),
|
|
source_version=str(observed_year),
|
|
source_metadata={
|
|
"selection_aggregation": {
|
|
"method": metric_method,
|
|
"label": "Objecten",
|
|
"unit": "objecten",
|
|
}
|
|
},
|
|
)
|
|
|
|
|
|
def test_temporal_migration_and_models_align() -> None:
|
|
migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8")
|
|
for field in (
|
|
"temporal_series_key",
|
|
"observed_at",
|
|
"valid_from",
|
|
"valid_to",
|
|
"temporal_granularity",
|
|
"source_version",
|
|
):
|
|
assert field in migration
|
|
assert hasattr(Dataset, field)
|
|
assert "ix_vector_features_dataset_source_feature" in migration
|
|
assert 'down_revision = "202606120900"' in migration
|
|
|
|
|
|
def test_temporal_metadata_requires_an_explicit_series_and_observation_date() -> None:
|
|
with pytest.raises(AppError, match="observed_at is required"):
|
|
DatasetService._validate_temporal_metadata(
|
|
temporal_series_key="official:test:mol",
|
|
observed_at=None,
|
|
valid_from=None,
|
|
valid_to=None,
|
|
temporal_granularity="year",
|
|
source_version="2024",
|
|
)
|
|
with pytest.raises(AppError, match="valid_to must be"):
|
|
DatasetService._validate_temporal_metadata(
|
|
temporal_series_key="official:test:mol",
|
|
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
|
valid_from=datetime(2024, 12, 31, tzinfo=timezone.utc),
|
|
valid_to=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
|
temporal_granularity="year",
|
|
source_version="2024",
|
|
)
|
|
|
|
|
|
def test_temporal_metadata_update_appends_provenance_version_and_is_idempotent() -> None:
|
|
project_id = uuid4()
|
|
dataset = temporal_dataset(project_id=project_id, observed_year=2024)
|
|
dataset.status = "ready"
|
|
dataset.metadata_json = {}
|
|
latest = DatasetVersion(
|
|
dataset_id=dataset.id,
|
|
version=3,
|
|
observed_at=dataset.observed_at,
|
|
source_version="2024",
|
|
)
|
|
session = TemporalUpdateSession(dataset, latest)
|
|
payload = DatasetTemporalUpdate(
|
|
temporal_series_key="official:test:mol",
|
|
observed_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
|
|
temporal_granularity="year",
|
|
source_version="2025",
|
|
)
|
|
|
|
updated = DatasetService.update_temporal_metadata(session, dataset.id, payload)
|
|
|
|
assert updated.observed_at == payload.observed_at
|
|
assert latest.version == 3
|
|
assert latest.observed_at == datetime(2024, 1, 1, tzinfo=timezone.utc)
|
|
assert len(session.added) == 2
|
|
appended = session.added[1]
|
|
assert isinstance(appended, DatasetVersion)
|
|
assert appended.version == 4
|
|
assert appended.observed_at == payload.observed_at
|
|
|
|
session.added.clear()
|
|
DatasetService.update_temporal_metadata(session, dataset.id, payload)
|
|
assert session.added == []
|
|
|
|
|
|
def test_selection_area_aggregation_returns_hectares_without_loading_all_features() -> None:
|
|
project_id = uuid4()
|
|
dataset = temporal_dataset(project_id=project_id, observed_year=1969, metric_method="intersection_area")
|
|
dataset.source_metadata["selection_aggregation"].update({"label": "Oppervlakte", "unit": "ha"})
|
|
result = VectorFeatureService.summarize_features_by_bbox(
|
|
ScalarSession(125_000.0),
|
|
dataset=dataset,
|
|
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"},
|
|
total_feature_count=40,
|
|
)
|
|
assert result["metric_value"] == 12.5
|
|
assert result["metric_unit"] == "ha"
|
|
assert result["feature_count"] == 40
|
|
|
|
|
|
def test_population_area_weighting_is_exact_for_full_features_and_estimated_for_partial_features() -> None:
|
|
dataset = temporal_dataset(project_id=uuid4(), observed_year=2025, metric_method="area_weighted_sum")
|
|
dataset.source_metadata["selection_aggregation"].update(
|
|
{
|
|
"property": "population_total",
|
|
"label": "Inwoners",
|
|
"unit": "inwoners",
|
|
"warning": "Partial-sector estimate",
|
|
"warning_only_when_estimate": True,
|
|
}
|
|
)
|
|
bbox = {"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"}
|
|
|
|
full = VectorFeatureService.summarize_features_by_bbox(
|
|
SequenceScalarSession([38_675.0, 0]),
|
|
dataset=dataset,
|
|
bbox=bbox,
|
|
total_feature_count=49,
|
|
)
|
|
partial = VectorFeatureService.summarize_features_by_bbox(
|
|
SequenceScalarSession([1_250.5, 2]),
|
|
dataset=dataset,
|
|
bbox=bbox,
|
|
total_feature_count=3,
|
|
)
|
|
|
|
assert full["metric_value"] == 38_675.0
|
|
assert full["is_estimate"] is False
|
|
assert full["warning"] is None
|
|
assert partial["metric_value"] == 1_250.5
|
|
assert partial["is_estimate"] is True
|
|
assert partial["warning"] == "Partial-sector estimate"
|
|
|
|
|
|
def test_temporal_compare_returns_delta_and_canonical_change_payload(monkeypatch) -> None:
|
|
project_id = uuid4()
|
|
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
|
|
later = temporal_dataset(project_id=project_id, observed_year=2024)
|
|
|
|
def get_dataset(_db, _project_id, dataset_id, _label):
|
|
return earlier if dataset_id == earlier.id else later
|
|
|
|
def summarize(_db, *, dataset, bbox): # noqa: ARG001
|
|
value = 100.0 if dataset.id == earlier.id else 115.0
|
|
return {
|
|
"metric_label": "Inwoners",
|
|
"metric_value": value,
|
|
"metric_unit": "inwoners",
|
|
"aggregation_method": "area_weighted_sum",
|
|
"feature_count": 10,
|
|
"is_estimate": True,
|
|
"warning": "Areal weighting",
|
|
}
|
|
|
|
monkeypatch.setattr(TemporalAnalysisService, "_get_temporal_dataset", staticmethod(get_dataset))
|
|
monkeypatch.setattr(VectorFeatureService, "summarize_features_by_bbox", staticmethod(summarize))
|
|
monkeypatch.setattr(
|
|
TemporalAnalysisService,
|
|
"_compare_identity_features",
|
|
staticmethod(
|
|
lambda *args, **kwargs: (
|
|
TemporalObjectChanges(available=True, added_count=1, removed_count=0, modified_count=2, unchanged_count=7),
|
|
{"type": "FeatureCollection", "features": []},
|
|
[],
|
|
)
|
|
),
|
|
)
|
|
result = TemporalAnalysisService.compare(
|
|
SimpleNamespace(),
|
|
project_id=project_id,
|
|
payload=TemporalComparisonRequest(
|
|
earlier_dataset_id=earlier.id,
|
|
later_dataset_id=later.id,
|
|
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3},
|
|
),
|
|
)
|
|
assert result.metric.absolute_change == 15.0
|
|
assert result.metric.percent_change == 15.0
|
|
assert result.metric.is_estimate is True
|
|
assert result.object_changes.modified_count == 2
|
|
assert result.geojson["type"] == "FeatureCollection"
|
|
|
|
|
|
def test_unstable_temporal_identity_returns_clear_end_user_warning() -> None:
|
|
project_id = uuid4()
|
|
earlier = temporal_dataset(project_id=project_id, observed_year=2021)
|
|
later = temporal_dataset(project_id=project_id, observed_year=2025)
|
|
earlier.source_metadata["identity_stable"] = False
|
|
later.source_metadata["identity_stable"] = False
|
|
|
|
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
|
|
SimpleNamespace(),
|
|
earlier=earlier,
|
|
later=later,
|
|
bbox={"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3},
|
|
preview_limit=100,
|
|
)
|
|
|
|
assert changes.available is False
|
|
assert geojson == {"type": "FeatureCollection", "features": []}
|
|
assert warnings == ["Wijzigingen van individuele objecten kunnen voor deze bron niet betrouwbaar worden gevolgd."]
|
|
|
|
|
|
def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
|
|
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
|
temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8")
|
|
population = (ROOT / "scripts/provision_mol_population_history.py").read_text(encoding="utf-8")
|
|
landuse = (ROOT / "scripts/provision_mol_historical_landuse.py").read_text(encoding="utf-8")
|
|
dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8")
|
|
|
|
assert "Laatste toestand" in workspace
|
|
assert "Evolutie" in workspace
|
|
assert "Vergelijk periode" in workspace
|
|
assert "/temporal/compare" in temporal_api
|
|
assert "Statbel" in population and "area_weighted_sum" in population
|
|
assert '"identity_stable": False' in population
|
|
assert "HistLandgebruik" in landuse and "intersection_area" in landuse
|
|
assert "<wfs:GetFeature" in landuse and "session.post(" in landuse
|
|
assert 'lambda value: value.startswith("weg"), "weg*"' in landuse
|
|
assert "provision_mol_population_history.py" in dockerfile
|
|
assert "provision_mol_historical_landuse.py" in dockerfile
|
|
assert "fake" not in population.lower()
|
|
|
|
|
|
def test_tower_deploy_waits_for_startup_migration_before_live_smoke() -> None:
|
|
for relative_path in ("scripts/deploy_tower.ps1", "scripts/deploy_tower.sh"):
|
|
script = (ROOT / relative_path).read_text(encoding="utf-8")
|
|
wait_position = script.index("wait_for_geointel_health")
|
|
invocation_position = script.index("\nwait_for_geointel_health", wait_position)
|
|
smoke_position = script.index("LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh")
|
|
assert "docker inspect --format" in script
|
|
assert invocation_position < smoke_position
|