fix: enforce assistant data quality disclosures
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-15 07:51:23 +02:00
parent 01785a0f95
commit c7faea430f
4 changed files with 98 additions and 0 deletions
+3
View File
@@ -27,6 +27,9 @@
- Tightened generated answers to descriptive source evidence: estimates must - Tightened generated answers to descriptive source evidence: estimates must
remain labelled, unsupported causal/forecast claims are forbidden and plain remain labelled, unsupported causal/forecast claims are forbidden and plain
text is requested for the existing chat renderer. text is requested for the existing chat renderer.
- Added deterministic estimate disclosure for population answers and instructed
the local model to copy governed values literally instead of inventing
averages, rates or derived trends.
## Sprint 201 Semantic area-selection metrics (2026-07-15) ## Sprint 201 Semantic area-selection metrics (2026-07-15)
@@ -37,12 +37,48 @@ class GeoAssistantService:
"gedaald", "gedaald",
"gestegen", "gestegen",
) )
ESTIMATE_TOPIC_TERMS = {
"population": ("bevolk", "inwoner"),
}
ESTIMATE_TOPIC_LABELS = {
"population": "bevolkingswaarden",
}
@classmethod @classmethod
def history_requested(cls, question: str) -> bool: def history_requested(cls, question: str) -> bool:
normalized = question.casefold() normalized = question.casefold()
return any(keyword in normalized for keyword in cls.HISTORY_KEYWORDS) return any(keyword in normalized for keyword in cls.HISTORY_KEYWORDS)
@classmethod
def ensure_estimate_disclosure(
cls,
answer: str,
metrics: list[AssistantContextMetric],
) -> str:
normalized = answer.casefold()
if "schat" in normalized:
return answer
disclosed_themes = {
metric.theme
for metric in metrics
if metric.is_estimate
and any(
term in normalized
for term in cls.ESTIMATE_TOPIC_TERMS.get(metric.theme, (metric.label.casefold(),))
)
}
if not disclosed_themes:
return answer
labels = ", ".join(
cls.ESTIMATE_TOPIC_LABELS.get(theme, theme)
for theme in sorted(disclosed_themes)
)
return (
f"Datakwaliteit: {labels} in dit antwoord zijn schattingen volgens de bronmetadata, "
"geen exacte tellingen.\n\n"
f"{answer}"
)
def __init__(self, settings: Settings | None = None): def __init__(self, settings: Settings | None = None):
self.settings = settings or get_settings() self.settings = settings or get_settings()
@@ -262,6 +298,9 @@ class GeoAssistantService:
) )
context_metrics.append(item) context_metrics.append(item)
serialized_metrics.append(item.model_dump(mode="json")) serialized_metrics.append(item.model_dump(mode="json"))
serialized_metrics[-1]["measurement_quality"] = (
"schatting" if item.is_estimate else "exact_binnen_bronrepresentatie"
)
source_dataset_ids.append(dataset.id) source_dataset_ids.append(dataset.id)
current_context.append( current_context.append(
{ {
@@ -307,6 +346,9 @@ class GeoAssistantService:
"value": summary["metric_value"], "value": summary["metric_value"],
"unit": summary["metric_unit"], "unit": summary["metric_unit"],
"is_estimate": summary["is_estimate"], "is_estimate": summary["is_estimate"],
"measurement_quality": (
"schatting" if summary["is_estimate"] else "exact_binnen_bronrepresentatie"
),
} }
) )
if dataset.id not in source_dataset_ids: if dataset.id not in source_dataset_ids:
@@ -356,6 +398,7 @@ class GeoAssistantService:
"Als is_estimate true is, noem de waarde verplicht een schatting en nooit exact. " "Als is_estimate true is, noem de waarde verplicht een schatting en nooit exact. "
"Objectaantallen zijn ondersteunend; geef betekenisvolle oppervlakte-, lengte- of bevolkingsmetriek voorrang. " "Objectaantallen zijn ondersteunend; geef betekenisvolle oppervlakte-, lengte- of bevolkingsmetriek voorrang. "
"Beschrijf alleen waargenomen verschillen; verzin geen oorzaak, voorspelling, verzadiging of andere verklaring. " "Beschrijf alleen waargenomen verschillen; verzin geen oorzaak, voorspelling, verzadiging of andere verklaring. "
"Neem waarden en jaren letterlijk over en bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend. "
"Gebruik platte tekst met korte alinea's en opsommingen, zonder Markdown-symbolen. " "Gebruik platte tekst met korte alinea's en opsommingen, zonder Markdown-symbolen. "
"Bereken of suggereer nooit watervolume zonder gekoppelde diepte of bathymetrie. " "Bereken of suggereer nooit watervolume zonder gekoppelde diepte of bathymetrie. "
"Als de gevraagde informatie niet in de context staat, zeg precies welke bron of meting ontbreekt. " "Als de gevraagde informatie niet in de context staat, zeg precies welke bron of meting ontbreekt. "
@@ -393,6 +436,7 @@ class GeoAssistantService:
answer = str(message.get("content") or "").strip() answer = str(message.get("content") or "").strip()
if not answer: if not answer:
raise AppError(code="OLLAMA_EMPTY_RESPONSE", message="Ollama gaf geen antwoord terug.", status_code=502) raise AppError(code="OLLAMA_EMPTY_RESPONSE", message="Ollama gaf geen antwoord terug.", status_code=502)
answer = self.ensure_estimate_disclosure(answer, metrics)
return AssistantQueryResponse( return AssistantQueryResponse(
answer=answer, answer=answer,
model=model, model=model,
@@ -98,6 +98,51 @@ def test_geo_assistant_recognizes_dutch_historical_questions(question: str) -> N
assert GeoAssistantService.history_requested(question) is True assert GeoAssistantService.history_requested(question) is True
def test_geo_assistant_discloses_estimated_population_values() -> None:
metrics = [
AssistantContextMetric(
theme="population",
label="Geschatte bevolking",
value=74_254,
unit="personen",
source="Statbel",
dataset_id=uuid4(),
is_estimate=True,
)
]
answer = GeoAssistantService.ensure_estimate_disclosure(
"De bevolking bedraagt 74.254 personen.",
metrics,
)
assert answer.startswith(
"Datakwaliteit: bevolkingswaarden in dit antwoord zijn schattingen volgens de bronmetadata, "
"geen exacte tellingen."
)
def test_geo_assistant_does_not_add_irrelevant_estimate_disclosure() -> None:
metrics = [
AssistantContextMetric(
theme="population",
label="Geschatte bevolking",
value=74_254,
unit="personen",
source="Statbel",
dataset_id=uuid4(),
is_estimate=True,
)
]
answer = GeoAssistantService.ensure_estimate_disclosure(
"De bosoppervlakte bedraagt 3.626,56 hectare.",
metrics,
)
assert answer == "De bosoppervlakte bedraagt 3.626,56 hectare."
def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch) -> None: def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch) -> None:
service = GeoAssistantService(ollama_settings()) service = GeoAssistantService(ollama_settings())
project_id = uuid4() project_id = uuid4()
@@ -161,6 +206,7 @@ def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch
assert "scope.label is het exact geanalyseerde gebied" in captured["payload"]["messages"][0]["content"] assert "scope.label is het exact geanalyseerde gebied" in captured["payload"]["messages"][0]["content"]
assert "noem de waarde verplicht een schatting" in captured["payload"]["messages"][0]["content"] assert "noem de waarde verplicht een schatting" in captured["payload"]["messages"][0]["content"]
assert "verzin geen oorzaak, voorspelling, verzadiging" in captured["payload"]["messages"][0]["content"] assert "verzin geen oorzaak, voorspelling, verzadiging" in captured["payload"]["messages"][0]["content"]
assert "bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend" in captured["payload"]["messages"][0]["content"]
assert "zonder Markdown-symbolen" in captured["payload"]["messages"][0]["content"] assert "zonder Markdown-symbolen" in captured["payload"]["messages"][0]["content"]
assert "water_volume_available" in captured["payload"]["messages"][0]["content"] assert "water_volume_available" in captured["payload"]["messages"][0]["content"]
+5
View File
@@ -8417,6 +8417,11 @@ Validation evidence:
was suggested. The system contract now mandates estimate wording, forbids was suggested. The system contract now mandates estimate wording, forbids
causal/forecast claims not present in context, uses deterministic temperature causal/forecast claims not present in context, uses deterministic temperature
zero and requests plain text. Full readiness remained green at 617 tests. zero and requests plain text. Full readiness remained green at 617 tests.
- Assistant context now carries explicit measurement quality for current and
historical values. A deterministic response guard prepends the estimate
limitation whenever an answer discusses estimated population data, while the
model prompt forbids independent averages, rates and derived trends. Focused
tests and full readiness passed with 619 backend tests.
Known limitations: Known limitations:
- Water volume remains unavailable until a governed depth/bathymetry source is - Water volume remains unavailable until a governed depth/bathymetry source is