fix: enforce assistant data quality disclosures
This commit is contained in:
@@ -27,6 +27,9 @@
|
||||
- Tightened generated answers to descriptive source evidence: estimates must
|
||||
remain labelled, unsupported causal/forecast claims are forbidden and plain
|
||||
text is requested for the existing chat renderer.
|
||||
- Added deterministic estimate disclosure for population answers and instructed
|
||||
the local model to copy governed values literally instead of inventing
|
||||
averages, rates or derived trends.
|
||||
|
||||
## Sprint 201 Semantic area-selection metrics (2026-07-15)
|
||||
|
||||
|
||||
@@ -37,12 +37,48 @@ class GeoAssistantService:
|
||||
"gedaald",
|
||||
"gestegen",
|
||||
)
|
||||
ESTIMATE_TOPIC_TERMS = {
|
||||
"population": ("bevolk", "inwoner"),
|
||||
}
|
||||
ESTIMATE_TOPIC_LABELS = {
|
||||
"population": "bevolkingswaarden",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def history_requested(cls, question: str) -> bool:
|
||||
normalized = question.casefold()
|
||||
return any(keyword in normalized for keyword in cls.HISTORY_KEYWORDS)
|
||||
|
||||
@classmethod
|
||||
def ensure_estimate_disclosure(
|
||||
cls,
|
||||
answer: str,
|
||||
metrics: list[AssistantContextMetric],
|
||||
) -> str:
|
||||
normalized = answer.casefold()
|
||||
if "schat" in normalized:
|
||||
return answer
|
||||
disclosed_themes = {
|
||||
metric.theme
|
||||
for metric in metrics
|
||||
if metric.is_estimate
|
||||
and any(
|
||||
term in normalized
|
||||
for term in cls.ESTIMATE_TOPIC_TERMS.get(metric.theme, (metric.label.casefold(),))
|
||||
)
|
||||
}
|
||||
if not disclosed_themes:
|
||||
return answer
|
||||
labels = ", ".join(
|
||||
cls.ESTIMATE_TOPIC_LABELS.get(theme, theme)
|
||||
for theme in sorted(disclosed_themes)
|
||||
)
|
||||
return (
|
||||
f"Datakwaliteit: {labels} in dit antwoord zijn schattingen volgens de bronmetadata, "
|
||||
"geen exacte tellingen.\n\n"
|
||||
f"{answer}"
|
||||
)
|
||||
|
||||
def __init__(self, settings: Settings | None = None):
|
||||
self.settings = settings or get_settings()
|
||||
|
||||
@@ -262,6 +298,9 @@ class GeoAssistantService:
|
||||
)
|
||||
context_metrics.append(item)
|
||||
serialized_metrics.append(item.model_dump(mode="json"))
|
||||
serialized_metrics[-1]["measurement_quality"] = (
|
||||
"schatting" if item.is_estimate else "exact_binnen_bronrepresentatie"
|
||||
)
|
||||
source_dataset_ids.append(dataset.id)
|
||||
current_context.append(
|
||||
{
|
||||
@@ -307,6 +346,9 @@ class GeoAssistantService:
|
||||
"value": summary["metric_value"],
|
||||
"unit": summary["metric_unit"],
|
||||
"is_estimate": summary["is_estimate"],
|
||||
"measurement_quality": (
|
||||
"schatting" if summary["is_estimate"] else "exact_binnen_bronrepresentatie"
|
||||
),
|
||||
}
|
||||
)
|
||||
if dataset.id not in source_dataset_ids:
|
||||
@@ -356,6 +398,7 @@ class GeoAssistantService:
|
||||
"Als is_estimate true is, noem de waarde verplicht een schatting en nooit exact. "
|
||||
"Objectaantallen zijn ondersteunend; geef betekenisvolle oppervlakte-, lengte- of bevolkingsmetriek voorrang. "
|
||||
"Beschrijf alleen waargenomen verschillen; verzin geen oorzaak, voorspelling, verzadiging of andere verklaring. "
|
||||
"Neem waarden en jaren letterlijk over en bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend. "
|
||||
"Gebruik platte tekst met korte alinea's en opsommingen, zonder Markdown-symbolen. "
|
||||
"Bereken of suggereer nooit watervolume zonder gekoppelde diepte of bathymetrie. "
|
||||
"Als de gevraagde informatie niet in de context staat, zeg precies welke bron of meting ontbreekt. "
|
||||
@@ -393,6 +436,7 @@ class GeoAssistantService:
|
||||
answer = str(message.get("content") or "").strip()
|
||||
if not answer:
|
||||
raise AppError(code="OLLAMA_EMPTY_RESPONSE", message="Ollama gaf geen antwoord terug.", status_code=502)
|
||||
answer = self.ensure_estimate_disclosure(answer, metrics)
|
||||
return AssistantQueryResponse(
|
||||
answer=answer,
|
||||
model=model,
|
||||
|
||||
@@ -98,6 +98,51 @@ def test_geo_assistant_recognizes_dutch_historical_questions(question: str) -> N
|
||||
assert GeoAssistantService.history_requested(question) is True
|
||||
|
||||
|
||||
def test_geo_assistant_discloses_estimated_population_values() -> None:
|
||||
metrics = [
|
||||
AssistantContextMetric(
|
||||
theme="population",
|
||||
label="Geschatte bevolking",
|
||||
value=74_254,
|
||||
unit="personen",
|
||||
source="Statbel",
|
||||
dataset_id=uuid4(),
|
||||
is_estimate=True,
|
||||
)
|
||||
]
|
||||
|
||||
answer = GeoAssistantService.ensure_estimate_disclosure(
|
||||
"De bevolking bedraagt 74.254 personen.",
|
||||
metrics,
|
||||
)
|
||||
|
||||
assert answer.startswith(
|
||||
"Datakwaliteit: bevolkingswaarden in dit antwoord zijn schattingen volgens de bronmetadata, "
|
||||
"geen exacte tellingen."
|
||||
)
|
||||
|
||||
|
||||
def test_geo_assistant_does_not_add_irrelevant_estimate_disclosure() -> None:
|
||||
metrics = [
|
||||
AssistantContextMetric(
|
||||
theme="population",
|
||||
label="Geschatte bevolking",
|
||||
value=74_254,
|
||||
unit="personen",
|
||||
source="Statbel",
|
||||
dataset_id=uuid4(),
|
||||
is_estimate=True,
|
||||
)
|
||||
]
|
||||
|
||||
answer = GeoAssistantService.ensure_estimate_disclosure(
|
||||
"De bosoppervlakte bedraagt 3.626,56 hectare.",
|
||||
metrics,
|
||||
)
|
||||
|
||||
assert answer == "De bosoppervlakte bedraagt 3.626,56 hectare."
|
||||
|
||||
|
||||
def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch) -> None:
|
||||
service = GeoAssistantService(ollama_settings())
|
||||
project_id = uuid4()
|
||||
@@ -161,6 +206,7 @@ def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch
|
||||
assert "scope.label is het exact geanalyseerde gebied" in captured["payload"]["messages"][0]["content"]
|
||||
assert "noem de waarde verplicht een schatting" in captured["payload"]["messages"][0]["content"]
|
||||
assert "verzin geen oorzaak, voorspelling, verzadiging" in captured["payload"]["messages"][0]["content"]
|
||||
assert "bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend" in captured["payload"]["messages"][0]["content"]
|
||||
assert "zonder Markdown-symbolen" in captured["payload"]["messages"][0]["content"]
|
||||
assert "water_volume_available" in captured["payload"]["messages"][0]["content"]
|
||||
|
||||
|
||||
@@ -8417,6 +8417,11 @@ Validation evidence:
|
||||
was suggested. The system contract now mandates estimate wording, forbids
|
||||
causal/forecast claims not present in context, uses deterministic temperature
|
||||
zero and requests plain text. Full readiness remained green at 617 tests.
|
||||
- Assistant context now carries explicit measurement quality for current and
|
||||
historical values. A deterministic response guard prepends the estimate
|
||||
limitation whenever an answer discusses estimated population data, while the
|
||||
model prompt forbids independent averages, rates and derived trends. Focused
|
||||
tests and full readiness passed with 619 backend tests.
|
||||
|
||||
Known limitations:
|
||||
- Water volume remains unavailable until a governed depth/bathymetry source is
|
||||
|
||||
Reference in New Issue
Block a user