fix: enforce assistant data quality disclosures
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-15 07:51:23 +02:00
parent 01785a0f95
commit c7faea430f
4 changed files with 98 additions and 0 deletions
+3
View File
@@ -27,6 +27,9 @@
- Tightened generated answers to descriptive source evidence: estimates must
remain labelled, unsupported causal/forecast claims are forbidden and plain
text is requested for the existing chat renderer.
- Added deterministic estimate disclosure for population answers and instructed
the local model to copy governed values literally instead of inventing
averages, rates or derived trends.
## Sprint 201 Semantic area-selection metrics (2026-07-15)
@@ -37,12 +37,48 @@ class GeoAssistantService:
"gedaald",
"gestegen",
)
ESTIMATE_TOPIC_TERMS = {
"population": ("bevolk", "inwoner"),
}
ESTIMATE_TOPIC_LABELS = {
"population": "bevolkingswaarden",
}
@classmethod
def history_requested(cls, question: str) -> bool:
normalized = question.casefold()
return any(keyword in normalized for keyword in cls.HISTORY_KEYWORDS)
@classmethod
def ensure_estimate_disclosure(
cls,
answer: str,
metrics: list[AssistantContextMetric],
) -> str:
normalized = answer.casefold()
if "schat" in normalized:
return answer
disclosed_themes = {
metric.theme
for metric in metrics
if metric.is_estimate
and any(
term in normalized
for term in cls.ESTIMATE_TOPIC_TERMS.get(metric.theme, (metric.label.casefold(),))
)
}
if not disclosed_themes:
return answer
labels = ", ".join(
cls.ESTIMATE_TOPIC_LABELS.get(theme, theme)
for theme in sorted(disclosed_themes)
)
return (
f"Datakwaliteit: {labels} in dit antwoord zijn schattingen volgens de bronmetadata, "
"geen exacte tellingen.\n\n"
f"{answer}"
)
def __init__(self, settings: Settings | None = None):
self.settings = settings or get_settings()
@@ -262,6 +298,9 @@ class GeoAssistantService:
)
context_metrics.append(item)
serialized_metrics.append(item.model_dump(mode="json"))
serialized_metrics[-1]["measurement_quality"] = (
"schatting" if item.is_estimate else "exact_binnen_bronrepresentatie"
)
source_dataset_ids.append(dataset.id)
current_context.append(
{
@@ -307,6 +346,9 @@ class GeoAssistantService:
"value": summary["metric_value"],
"unit": summary["metric_unit"],
"is_estimate": summary["is_estimate"],
"measurement_quality": (
"schatting" if summary["is_estimate"] else "exact_binnen_bronrepresentatie"
),
}
)
if dataset.id not in source_dataset_ids:
@@ -356,6 +398,7 @@ class GeoAssistantService:
"Als is_estimate true is, noem de waarde verplicht een schatting en nooit exact. "
"Objectaantallen zijn ondersteunend; geef betekenisvolle oppervlakte-, lengte- of bevolkingsmetriek voorrang. "
"Beschrijf alleen waargenomen verschillen; verzin geen oorzaak, voorspelling, verzadiging of andere verklaring. "
"Neem waarden en jaren letterlijk over en bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend. "
"Gebruik platte tekst met korte alinea's en opsommingen, zonder Markdown-symbolen. "
"Bereken of suggereer nooit watervolume zonder gekoppelde diepte of bathymetrie. "
"Als de gevraagde informatie niet in de context staat, zeg precies welke bron of meting ontbreekt. "
@@ -393,6 +436,7 @@ class GeoAssistantService:
answer = str(message.get("content") or "").strip()
if not answer:
raise AppError(code="OLLAMA_EMPTY_RESPONSE", message="Ollama gaf geen antwoord terug.", status_code=502)
answer = self.ensure_estimate_disclosure(answer, metrics)
return AssistantQueryResponse(
answer=answer,
model=model,
@@ -98,6 +98,51 @@ def test_geo_assistant_recognizes_dutch_historical_questions(question: str) -> N
assert GeoAssistantService.history_requested(question) is True
def test_geo_assistant_discloses_estimated_population_values() -> None:
metrics = [
AssistantContextMetric(
theme="population",
label="Geschatte bevolking",
value=74_254,
unit="personen",
source="Statbel",
dataset_id=uuid4(),
is_estimate=True,
)
]
answer = GeoAssistantService.ensure_estimate_disclosure(
"De bevolking bedraagt 74.254 personen.",
metrics,
)
assert answer.startswith(
"Datakwaliteit: bevolkingswaarden in dit antwoord zijn schattingen volgens de bronmetadata, "
"geen exacte tellingen."
)
def test_geo_assistant_does_not_add_irrelevant_estimate_disclosure() -> None:
metrics = [
AssistantContextMetric(
theme="population",
label="Geschatte bevolking",
value=74_254,
unit="personen",
source="Statbel",
dataset_id=uuid4(),
is_estimate=True,
)
]
answer = GeoAssistantService.ensure_estimate_disclosure(
"De bosoppervlakte bedraagt 3.626,56 hectare.",
metrics,
)
assert answer == "De bosoppervlakte bedraagt 3.626,56 hectare."
def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch) -> None:
service = GeoAssistantService(ollama_settings())
project_id = uuid4()
@@ -161,6 +206,7 @@ def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch
assert "scope.label is het exact geanalyseerde gebied" in captured["payload"]["messages"][0]["content"]
assert "noem de waarde verplicht een schatting" in captured["payload"]["messages"][0]["content"]
assert "verzin geen oorzaak, voorspelling, verzadiging" in captured["payload"]["messages"][0]["content"]
assert "bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend" in captured["payload"]["messages"][0]["content"]
assert "zonder Markdown-symbolen" in captured["payload"]["messages"][0]["content"]
assert "water_volume_available" in captured["payload"]["messages"][0]["content"]
+5
View File
@@ -8417,6 +8417,11 @@ Validation evidence:
was suggested. The system contract now mandates estimate wording, forbids
causal/forecast claims not present in context, uses deterministic temperature
zero and requests plain text. Full readiness remained green at 617 tests.
- Assistant context now carries explicit measurement quality for current and
historical values. A deterministic response guard prepends the estimate
limitation whenever an answer discusses estimated population data, while the
model prompt forbids independent averages, rates and derived trends. Focused
tests and full readiness passed with 619 backend tests.
Known limitations:
- Water volume remains unavailable until a governed depth/bathymetry source is