Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
89 lines
3.1 KiB
Python
89 lines
3.1 KiB
Python
"""Estimate disclosure must come from the data, not from patching prose.
|
|
|
|
``ensure_estimate_disclosure`` rewrites the model's sentences with regular
|
|
expressions to insert the word "schatting". That only fires when the generated
|
|
text happens to contain one of the phrasings it knows, so whether a number is
|
|
labelled an estimate depends on how the language model worded it. The
|
|
disclosure is derived from the metric metadata instead, so the honesty of the
|
|
answer no longer depends on string matching.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from uuid import uuid4
|
|
|
|
from app.schemas.assistant import AssistantContextMetric
|
|
from app.services.geo_assistant_service import GeoAssistantService
|
|
|
|
|
|
def _metric(theme: str, label: str, *, is_estimate: bool) -> AssistantContextMetric:
|
|
return AssistantContextMetric(
|
|
theme=theme,
|
|
label=label,
|
|
value=36783.0,
|
|
unit="inwoners",
|
|
source="Statbel",
|
|
dataset_id=uuid4(),
|
|
observed_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
|
is_estimate=is_estimate,
|
|
)
|
|
|
|
|
|
def test_every_estimated_metric_produces_a_disclosure() -> None:
|
|
metrics = [
|
|
_metric("population", "Inwoners", is_estimate=True),
|
|
_metric("buildings", "Gebouwen", is_estimate=False),
|
|
]
|
|
|
|
disclosures = GeoAssistantService.estimate_disclosures(metrics)
|
|
|
|
assert len(disclosures) == 1
|
|
assert disclosures[0].theme == "population"
|
|
assert disclosures[0].label == "Inwoners"
|
|
assert disclosures[0].source == "Statbel"
|
|
assert disclosures[0].dataset_id == metrics[0].dataset_id
|
|
assert "schatting" in disclosures[0].reason.casefold()
|
|
|
|
|
|
def test_disclosure_does_not_depend_on_the_generated_wording() -> None:
|
|
"""The regex path only fires on phrasings it recognises; this does not."""
|
|
|
|
metrics = [_metric("population", "Inwoners", is_estimate=True)]
|
|
|
|
patched = GeoAssistantService.ensure_estimate_disclosure(
|
|
"Er wonen daar 36.783 mensen.", metrics
|
|
)
|
|
disclosures = GeoAssistantService.estimate_disclosures(metrics)
|
|
|
|
# The prose was left untouched because no known phrase matched...
|
|
assert "Datakwaliteit" not in patched
|
|
# ...but the structured disclosure is present regardless.
|
|
assert len(disclosures) == 1
|
|
|
|
|
|
def test_no_estimates_means_no_disclosures() -> None:
|
|
metrics = [_metric("buildings", "Gebouwen", is_estimate=False)]
|
|
|
|
assert GeoAssistantService.estimate_disclosures(metrics) == []
|
|
|
|
|
|
def test_disclosures_are_deduplicated_per_theme_and_dataset() -> None:
|
|
shared = _metric("population", "Inwoners", is_estimate=True)
|
|
duplicate = AssistantContextMetric(**{**shared.model_dump(), "label": "Inwoners (2024)"})
|
|
|
|
disclosures = GeoAssistantService.estimate_disclosures([shared, duplicate])
|
|
|
|
assert len(disclosures) == 1
|
|
|
|
|
|
def test_disclosures_are_ordered_deterministically() -> None:
|
|
metrics = [
|
|
_metric("space_occupation", "Ruimtebeslag", is_estimate=True),
|
|
_metric("population", "Inwoners", is_estimate=True),
|
|
]
|
|
|
|
themes = [item.theme for item in GeoAssistantService.estimate_disclosures(metrics)]
|
|
|
|
assert themes == ["population", "space_occupation"]
|