diff --git a/backend/app/schemas/analysis.py b/backend/app/schemas/analysis.py index 02a054f4..4c73c831 100644 --- a/backend/app/schemas/analysis.py +++ b/backend/app/schemas/analysis.py @@ -20,8 +20,12 @@ class ChangeDetectionSummary(BaseModel): target_feature_count: int added_count: int removed_count: int + # A footprint that was redrawn rather than demolished and rebuilt. Without + # this class it appeared as one removal plus one addition. + modified_count: int = 0 unchanged_count: int iou_threshold: float + modified_iou_threshold: float | None = None warnings: list[str] = Field(default_factory=list) generated_at: datetime geojson: dict diff --git a/backend/app/schemas/assistant.py b/backend/app/schemas/assistant.py index cffb78ae..51784902 100644 --- a/backend/app/schemas/assistant.py +++ b/backend/app/schemas/assistant.py @@ -66,12 +66,28 @@ class AssistantTemporalSeries(BaseModel): observation_count: int +class AssistantEstimateDisclosure(BaseModel): + """A value in the answer that the source itself calls an estimate. + + Derived from metric metadata rather than from the generated sentences, so + the disclosure is present whatever wording the model chose. + """ + + theme: str + label: str + unit: str + source: str + dataset_id: UUID + reason: str + + class AssistantQueryResponse(BaseModel): answer: str model: str scope_label: str context_metrics: list[AssistantContextMetric] temporal_series: list[AssistantTemporalSeries] + estimate_disclosures: list[AssistantEstimateDisclosure] = Field(default_factory=list) source_dataset_ids: list[UUID] warnings: list[str] generated_at: datetime diff --git a/backend/app/services/change_detection_service.py b/backend/app/services/change_detection_service.py index 5bc8aac3..b1844fdb 100644 --- a/backend/app/services/change_detection_service.py +++ b/backend/app/services/change_detection_service.py @@ -7,6 +7,7 @@ from uuid import UUID from geoalchemy2.shape import to_shape from shapely.geometry import mapping from shapely.geometry.base import BaseGeometry +from shapely.strtree import STRtree from shapely.validation import make_valid from sqlalchemy.orm import Session @@ -28,11 +29,18 @@ class ChangeDetectionService: target_dataset_id: UUID, iou_threshold: float = 0.8, include_unchanged: bool = True, + modified_threshold: float = 0.3, ) -> ChangeDetectionSummary: if source_dataset_id == target_dataset_id: raise AppError(code="INVALID_PARAMETERS", message="Source and target datasets must differ", status_code=400) if iou_threshold < 0 or iou_threshold > 1: raise AppError(code="INVALID_PARAMETERS", message="iou_threshold must be between 0 and 1", status_code=400) + if modified_threshold < 0 or modified_threshold > iou_threshold: + raise AppError( + code="INVALID_PARAMETERS", + message="modified_threshold must be between 0 and iou_threshold", + status_code=400, + ) source_dataset = ChangeDetectionService._get_project_vector_dataset(db, source_dataset_id, project_id, "Source") target_dataset = ChangeDetectionService._get_project_vector_dataset(db, target_dataset_id, project_id, "Target") @@ -45,80 +53,124 @@ class ChangeDetectionService: if not target_features: raise AppError(code="EMPTY_VECTOR_DATASET", message="Target dataset has no comparable vector features", status_code=422) - matched_target_indices: set[int] = set() - unchanged: list[dict[str, Any]] = [] - removed: list[dict[str, Any]] = [] + classified = ChangeDetectionService._classify_features( + source_features, + target_features, + iou_threshold=iou_threshold, + modified_threshold=modified_threshold, + ) - for source_feature in source_features: - best_iou = 0.0 - best_index: int | None = None - for target_index, target_feature in enumerate(target_features): - if target_index in matched_target_indices: - continue - candidate_iou = ChangeDetectionService._iou(source_feature["geometry"], target_feature["geometry"]) - if candidate_iou > best_iou: - best_iou = candidate_iou - best_index = target_index - - if best_index is not None and best_iou >= iou_threshold: - matched_target_indices.add(best_index) - if include_unchanged: - unchanged.append( - ChangeDetectionService._feature( - geometry=source_feature["geometry"], - change_type="unchanged", - source_dataset_id=source_dataset_id, - target_dataset_id=target_dataset_id, - source_feature_id=source_feature["feature_id"], - target_feature_id=target_features[best_index]["feature_id"], - iou=best_iou, - properties=source_feature["properties"], - ) - ) - else: - removed.append( - ChangeDetectionService._feature( - geometry=source_feature["geometry"], - change_type="removed", - source_dataset_id=source_dataset_id, - target_dataset_id=target_dataset_id, - source_feature_id=source_feature["feature_id"], - target_feature_id=None, - iou=best_iou if best_iou > 0 else None, - properties=source_feature["properties"], - ) + buckets: dict[str, list[dict[str, Any]]] = {"added": [], "removed": [], "modified": [], "unchanged": []} + for item in classified: + buckets[item["change_type"]].append( + ChangeDetectionService._feature( + geometry=item["geometry"], + change_type=item["change_type"], + source_dataset_id=source_dataset_id, + target_dataset_id=target_dataset_id, + source_feature_id=item["source_feature_id"], + target_feature_id=item["target_feature_id"], + iou=item["iou"], + properties=item["properties"], ) - - added = [ - ChangeDetectionService._feature( - geometry=target_feature["geometry"], - change_type="added", - source_dataset_id=source_dataset_id, - target_dataset_id=target_dataset_id, - source_feature_id=None, - target_feature_id=target_feature["feature_id"], - iou=None, - properties=target_feature["properties"], ) - for target_index, target_feature in enumerate(target_features) - if target_index not in matched_target_indices - ] - geojson_features = added + removed + unchanged + unchanged_count = len(buckets["unchanged"]) + if not include_unchanged: + buckets["unchanged"] = [] + + geojson_features = buckets["added"] + buckets["removed"] + buckets["modified"] + buckets["unchanged"] return ChangeDetectionSummary( source_dataset_id=source_dataset_id, target_dataset_id=target_dataset_id, source_feature_count=len(source_features), target_feature_count=len(target_features), - added_count=len(added), - removed_count=len(removed), - unchanged_count=len(unchanged) if include_unchanged else len(matched_target_indices), + added_count=len(buckets["added"]), + removed_count=len(buckets["removed"]), + modified_count=len(buckets["modified"]), + unchanged_count=unchanged_count, iou_threshold=iou_threshold, + modified_iou_threshold=modified_threshold, warnings=source_warnings + target_warnings, generated_at=datetime.now(timezone.utc), geojson={"type": "FeatureCollection", "features": geojson_features}, ) + @staticmethod + def _classify_features( + source_features: list[dict[str, Any]], + target_features: list[dict[str, Any]], + *, + iou_threshold: float, + modified_threshold: float, + ) -> list[dict[str, Any]]: + """Pair source with target footprints and label how each one changed. + + Matching is indexed rather than a full cross product: comparing two + municipal building layers is otherwise hundreds of millions of geometry + intersections. Sources are considered largest first so a big footprint + is not left over after a small neighbour claimed its counterpart. + """ + + target_geometries = [feature["geometry"] for feature in target_features] + tree = STRtree(target_geometries) if target_geometries else None + claimed: set[int] = set() + classified: list[dict[str, Any]] = [] + + order = sorted( + range(len(source_features)), + key=lambda index: (-source_features[index]["geometry"].area, str(source_features[index]["feature_id"])), + ) + for source_index in order: + source_feature = source_features[source_index] + geometry = source_feature["geometry"] + best_iou = 0.0 + best_index: int | None = None + candidates = [] if tree is None else sorted(int(value) for value in tree.query(geometry)) + for target_index in candidates: + if target_index in claimed: + continue + candidate_iou = ChangeDetectionService._iou(geometry, target_geometries[target_index]) + if candidate_iou > best_iou: + best_iou = candidate_iou + best_index = target_index + + if best_index is not None and best_iou >= iou_threshold: + claimed.add(best_index) + change_type = "unchanged" + elif best_index is not None and best_iou >= modified_threshold: + # The same object, redrawn: an annexe, a demolition of one wing, + # or a resurvey. Reporting it as removed + added would hide it. + claimed.add(best_index) + change_type = "modified" + else: + change_type = "removed" + + classified.append( + { + "change_type": change_type, + "geometry": geometry if change_type != "modified" else target_geometries[best_index], + "source_feature_id": source_feature["feature_id"], + "target_feature_id": target_features[best_index]["feature_id"] if change_type != "removed" else None, + "iou": best_iou if best_iou > 0 else None, + "properties": source_feature["properties"], + } + ) + + classified.extend( + { + "change_type": "added", + "geometry": target_feature["geometry"], + "source_feature_id": None, + "target_feature_id": target_feature["feature_id"], + "iou": None, + "properties": target_feature["properties"], + } + for target_index, target_feature in enumerate(target_features) + if target_index not in claimed + ) + return classified + @staticmethod def _get_project_vector_dataset(db: Session, dataset_id: UUID, project_id: UUID, label: str) -> Dataset: dataset = db.get(Dataset, dataset_id) diff --git a/backend/app/services/geo_assistant_service.py b/backend/app/services/geo_assistant_service.py index 57cc8526..7a7ac678 100644 --- a/backend/app/services/geo_assistant_service.py +++ b/backend/app/services/geo_assistant_service.py @@ -16,6 +16,7 @@ from app.core.errors import AppError from app.models import Area, Dataset, Project from app.schemas.assistant import ( AssistantContextMetric, + AssistantEstimateDisclosure, AssistantModelRead, AssistantQueryRequest, AssistantQueryResponse, @@ -112,6 +113,44 @@ class GeoAssistantService: } return themes or None + @classmethod + def estimate_disclosures( + cls, + metrics: list[AssistantContextMetric], + ) -> list[AssistantEstimateDisclosure]: + """List every estimated value behind the answer, straight from metadata. + + ``ensure_estimate_disclosure`` can only add a caveat when it recognises + the phrasing the model produced, which makes the guarantee dependent on + generated text. This derives the same statement from the source + metadata, so it holds regardless of how the answer was written. + """ + + seen: set[tuple[str, UUID]] = set() + disclosures: list[AssistantEstimateDisclosure] = [] + for metric in sorted(metrics, key=lambda item: (item.theme, item.label)): + if not metric.is_estimate: + continue + key = (metric.theme, metric.dataset_id) + if key in seen: + continue + seen.add(key) + topic = cls.ESTIMATE_TOPIC_LABELS.get(metric.theme, metric.label) + disclosures.append( + AssistantEstimateDisclosure( + theme=metric.theme, + label=metric.label, + unit=metric.unit, + source=metric.source, + dataset_id=metric.dataset_id, + reason=( + f"De bronmetadata van {metric.source} markeert {topic} als schatting, " + "geen exacte telling." + ), + ) + ) + return disclosures + @classmethod def ensure_estimate_disclosure( cls, @@ -702,6 +741,7 @@ class GeoAssistantService: scope_label=scope_label, context_metrics=metrics, temporal_series=series, + estimate_disclosures=self.estimate_disclosures(metrics), source_dataset_ids=dataset_ids, warnings=warnings, generated_at=datetime.now(timezone.utc), diff --git a/backend/tests/test_assistant_estimate_disclosure.py b/backend/tests/test_assistant_estimate_disclosure.py new file mode 100644 index 00000000..bd41e2be --- /dev/null +++ b/backend/tests/test_assistant_estimate_disclosure.py @@ -0,0 +1,88 @@ +"""Estimate disclosure must come from the data, not from patching prose. + +``ensure_estimate_disclosure`` rewrites the model's sentences with regular +expressions to insert the word "schatting". That only fires when the generated +text happens to contain one of the phrasings it knows, so whether a number is +labelled an estimate depends on how the language model worded it. The +disclosure is derived from the metric metadata instead, so the honesty of the +answer no longer depends on string matching. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from app.schemas.assistant import AssistantContextMetric +from app.services.geo_assistant_service import GeoAssistantService + + +def _metric(theme: str, label: str, *, is_estimate: bool) -> AssistantContextMetric: + return AssistantContextMetric( + theme=theme, + label=label, + value=36783.0, + unit="inwoners", + source="Statbel", + dataset_id=uuid4(), + observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc), + is_estimate=is_estimate, + ) + + +def test_every_estimated_metric_produces_a_disclosure() -> None: + metrics = [ + _metric("population", "Inwoners", is_estimate=True), + _metric("buildings", "Gebouwen", is_estimate=False), + ] + + disclosures = GeoAssistantService.estimate_disclosures(metrics) + + assert len(disclosures) == 1 + assert disclosures[0].theme == "population" + assert disclosures[0].label == "Inwoners" + assert disclosures[0].source == "Statbel" + assert disclosures[0].dataset_id == metrics[0].dataset_id + assert "schatting" in disclosures[0].reason.casefold() + + +def test_disclosure_does_not_depend_on_the_generated_wording() -> None: + """The regex path only fires on phrasings it recognises; this does not.""" + + metrics = [_metric("population", "Inwoners", is_estimate=True)] + + patched = GeoAssistantService.ensure_estimate_disclosure( + "Er wonen daar 36.783 mensen.", metrics + ) + disclosures = GeoAssistantService.estimate_disclosures(metrics) + + # The prose was left untouched because no known phrase matched... + assert "Datakwaliteit" not in patched + # ...but the structured disclosure is present regardless. + assert len(disclosures) == 1 + + +def test_no_estimates_means_no_disclosures() -> None: + metrics = [_metric("buildings", "Gebouwen", is_estimate=False)] + + assert GeoAssistantService.estimate_disclosures(metrics) == [] + + +def test_disclosures_are_deduplicated_per_theme_and_dataset() -> None: + shared = _metric("population", "Inwoners", is_estimate=True) + duplicate = AssistantContextMetric(**{**shared.model_dump(), "label": "Inwoners (2024)"}) + + disclosures = GeoAssistantService.estimate_disclosures([shared, duplicate]) + + assert len(disclosures) == 1 + + +def test_disclosures_are_ordered_deterministically() -> None: + metrics = [ + _metric("space_occupation", "Ruimtebeslag", is_estimate=True), + _metric("population", "Inwoners", is_estimate=True), + ] + + themes = [item.theme for item in GeoAssistantService.estimate_disclosures(metrics)] + + assert themes == ["population", "space_occupation"] diff --git a/backend/tests/test_change_detection_modified_features.py b/backend/tests/test_change_detection_modified_features.py new file mode 100644 index 00000000..964f707b --- /dev/null +++ b/backend/tests/test_change_detection_modified_features.py @@ -0,0 +1,114 @@ +"""An extended building is a change, not a deletion plus a new building. + +With only added/removed/unchanged, a footprint that grew by an annexe drops +below the IoU threshold and is reported twice: once as removed and once as +added. That hides exactly the category a change-detection product exists to +show, and inflates both counts. +""" + +from __future__ import annotations + +import pytest +from shapely.geometry import box + +from app.services.change_detection_service import ChangeDetectionService + + +def _feature(feature_id: str, geometry): + return {"feature_id": feature_id, "properties": {}, "geometry": geometry} + + +def _classify(source, target, *, iou_threshold=0.8, modified_threshold=0.3): + return ChangeDetectionService._classify_features( + source, + target, + iou_threshold=iou_threshold, + modified_threshold=modified_threshold, + ) + + +def test_an_extended_footprint_is_reported_as_modified() -> None: + source = [_feature("b1", box(0, 0, 10, 10))] + target = [_feature("b1-new", box(0, 0, 10, 14))] # IoU 100/140 = 0.71 + + result = _classify(source, target) + + assert [item["change_type"] for item in result] == ["modified"] + assert result[0]["source_feature_id"] == "b1" + assert result[0]["target_feature_id"] == "b1-new" + assert result[0]["iou"] == pytest.approx(100 / 140) + + +def test_a_nearly_identical_footprint_is_unchanged() -> None: + source = [_feature("b1", box(0, 0, 10, 10))] + target = [_feature("b1", box(0, 0, 10, 10.2))] + + result = _classify(source, target) + + assert [item["change_type"] for item in result] == ["unchanged"] + + +def test_a_genuinely_new_building_stays_added() -> None: + source = [_feature("b1", box(0, 0, 10, 10))] + target = [_feature("b1", box(0, 0, 10, 10)), _feature("b2", box(50, 50, 60, 60))] + + result = _classify(source, target) + + assert sorted(item["change_type"] for item in result) == ["added", "unchanged"] + + +def test_a_demolished_building_stays_removed() -> None: + source = [_feature("b1", box(0, 0, 10, 10)), _feature("b2", box(50, 50, 60, 60))] + target = [_feature("b1", box(0, 0, 10, 10))] + + result = _classify(source, target) + + assert sorted(item["change_type"] for item in result) == ["removed", "unchanged"] + + +def test_barely_overlapping_footprints_are_not_called_modified() -> None: + """Below the modified floor the two are separate objects, not one changed.""" + + source = [_feature("b1", box(0, 0, 10, 10))] + target = [_feature("b2", box(9, 9, 19, 19))] # IoU ~0.005 + + result = _classify(source, target) + + assert sorted(item["change_type"] for item in result) == ["added", "removed"] + + +def test_each_target_is_claimed_at_most_once() -> None: + source = [_feature("a", box(0, 0, 10, 10)), _feature("b", box(0, 0, 10, 12))] + target = [_feature("t", box(0, 0, 10, 10))] + + result = _classify(source, target) + + claimed = [item for item in result if item["target_feature_id"] == "t"] + assert len(claimed) == 1 + + +def test_classification_is_independent_of_input_order() -> None: + source = [_feature("a", box(0, 0, 10, 10)), _feature("b", box(30, 30, 40, 40))] + target = [_feature("a2", box(0, 0, 10, 14)), _feature("c", box(70, 70, 80, 80))] + + forward = _classify(source, target) + reverse = _classify(list(reversed(source)), list(reversed(target))) + + def signature(items): + return sorted( + (item["change_type"], item["source_feature_id"], item["target_feature_id"]) for item in items + ) + + assert signature(forward) == signature(reverse) + + +def test_large_populations_do_not_use_a_full_cross_product() -> None: + """A spatial index keeps a city-sized comparison tractable.""" + + source = [_feature(f"s{i}", box(i * 10, 0, i * 10 + 8, 8)) for i in range(400)] + target = [_feature(f"t{i}", box(i * 10, 0, i * 10 + 8, 8)) for i in range(400)] + + result = _classify(source, target) + + assert all(item["change_type"] == "unchanged" for item in result) + assert len(result) == 400