JensandClaude Opus 5
23d6e0372b
disclosure from data Change detection had only added/removed/unchanged, so a building extended by an annexe dropped below the IoU threshold and was reported twice: once as removed and once as added. That hides exactly the category a change-detection product exists to show and inflates both counts. A "modified" class now covers the band between the modified floor and the unchanged threshold. Matching also ran as a full cross product with no spatial index, unlike the QA matcher beside it: two municipal building layers meant hundreds of millions of geometry intersections. It uses an STRtree and considers larger footprints first, so a big footprint is not left over after a small neighbour claimed its counterpart. The assistant guaranteed honesty about estimated values by rewriting the model's sentences with regular expressions, which only fires when it recognises the phrasing the model happened to produce. estimate_disclosures derives the same statement from the metric metadata, so it holds regardless of how the answer was worded. The prose substitution stays as a second layer. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
89 lines
3.1 KiB
Python
89 lines
3.1 KiB
Python
"""Estimate disclosure must come from the data, not from patching prose.
|
|
|
|
``ensure_estimate_disclosure`` rewrites the model's sentences with regular
|
|
expressions to insert the word "schatting". That only fires when the generated
|
|
text happens to contain one of the phrasings it knows, so whether a number is
|
|
labelled an estimate depends on how the language model worded it. The
|
|
disclosure is derived from the metric metadata instead, so the honesty of the
|
|
answer no longer depends on string matching.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from uuid import uuid4
|
|
|
|
from app.schemas.assistant import AssistantContextMetric
|
|
from app.services.geo_assistant_service import GeoAssistantService
|
|
|
|
|
|
def _metric(theme: str, label: str, *, is_estimate: bool) -> AssistantContextMetric:
|
|
return AssistantContextMetric(
|
|
theme=theme,
|
|
label=label,
|
|
value=36783.0,
|
|
unit="inwoners",
|
|
source="Statbel",
|
|
dataset_id=uuid4(),
|
|
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
|
is_estimate=is_estimate,
|
|
)
|
|
|
|
|
|
def test_every_estimated_metric_produces_a_disclosure() -> None:
|
|
metrics = [
|
|
_metric("population", "Inwoners", is_estimate=True),
|
|
_metric("buildings", "Gebouwen", is_estimate=False),
|
|
]
|
|
|
|
disclosures = GeoAssistantService.estimate_disclosures(metrics)
|
|
|
|
assert len(disclosures) == 1
|
|
assert disclosures[0].theme == "population"
|
|
assert disclosures[0].label == "Inwoners"
|
|
assert disclosures[0].source == "Statbel"
|
|
assert disclosures[0].dataset_id == metrics[0].dataset_id
|
|
assert "schatting" in disclosures[0].reason.casefold()
|
|
|
|
|
|
def test_disclosure_does_not_depend_on_the_generated_wording() -> None:
|
|
"""The regex path only fires on phrasings it recognises; this does not."""
|
|
|
|
metrics = [_metric("population", "Inwoners", is_estimate=True)]
|
|
|
|
patched = GeoAssistantService.ensure_estimate_disclosure(
|
|
"Er wonen daar 36.783 mensen.", metrics
|
|
)
|
|
disclosures = GeoAssistantService.estimate_disclosures(metrics)
|
|
|
|
# The prose was left untouched because no known phrase matched...
|
|
assert "Datakwaliteit" not in patched
|
|
# ...but the structured disclosure is present regardless.
|
|
assert len(disclosures) == 1
|
|
|
|
|
|
def test_no_estimates_means_no_disclosures() -> None:
|
|
metrics = [_metric("buildings", "Gebouwen", is_estimate=False)]
|
|
|
|
assert GeoAssistantService.estimate_disclosures(metrics) == []
|
|
|
|
|
|
def test_disclosures_are_deduplicated_per_theme_and_dataset() -> None:
|
|
shared = _metric("population", "Inwoners", is_estimate=True)
|
|
duplicate = AssistantContextMetric(**{**shared.model_dump(), "label": "Inwoners (2024)"})
|
|
|
|
disclosures = GeoAssistantService.estimate_disclosures([shared, duplicate])
|
|
|
|
assert len(disclosures) == 1
|
|
|
|
|
|
def test_disclosures_are_ordered_deterministically() -> None:
|
|
metrics = [
|
|
_metric("space_occupation", "Ruimtebeslag", is_estimate=True),
|
|
_metric("population", "Inwoners", is_estimate=True),
|
|
]
|
|
|
|
themes = [item.theme for item in GeoAssistantService.estimate_disclosures(metrics)]
|
|
|
|
assert themes == ["population", "space_occupation"]
|