fix: enforce assistant data quality disclosures
This commit is contained in:
@@ -27,6 +27,9 @@
|
|||||||
- Tightened generated answers to descriptive source evidence: estimates must
|
- Tightened generated answers to descriptive source evidence: estimates must
|
||||||
remain labelled, unsupported causal/forecast claims are forbidden and plain
|
remain labelled, unsupported causal/forecast claims are forbidden and plain
|
||||||
text is requested for the existing chat renderer.
|
text is requested for the existing chat renderer.
|
||||||
|
- Added deterministic estimate disclosure for population answers and instructed
|
||||||
|
the local model to copy governed values literally instead of inventing
|
||||||
|
averages, rates or derived trends.
|
||||||
|
|
||||||
## Sprint 201 Semantic area-selection metrics (2026-07-15)
|
## Sprint 201 Semantic area-selection metrics (2026-07-15)
|
||||||
|
|
||||||
|
|||||||
@@ -37,12 +37,48 @@ class GeoAssistantService:
|
|||||||
"gedaald",
|
"gedaald",
|
||||||
"gestegen",
|
"gestegen",
|
||||||
)
|
)
|
||||||
|
ESTIMATE_TOPIC_TERMS = {
|
||||||
|
"population": ("bevolk", "inwoner"),
|
||||||
|
}
|
||||||
|
ESTIMATE_TOPIC_LABELS = {
|
||||||
|
"population": "bevolkingswaarden",
|
||||||
|
}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def history_requested(cls, question: str) -> bool:
|
def history_requested(cls, question: str) -> bool:
|
||||||
normalized = question.casefold()
|
normalized = question.casefold()
|
||||||
return any(keyword in normalized for keyword in cls.HISTORY_KEYWORDS)
|
return any(keyword in normalized for keyword in cls.HISTORY_KEYWORDS)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def ensure_estimate_disclosure(
|
||||||
|
cls,
|
||||||
|
answer: str,
|
||||||
|
metrics: list[AssistantContextMetric],
|
||||||
|
) -> str:
|
||||||
|
normalized = answer.casefold()
|
||||||
|
if "schat" in normalized:
|
||||||
|
return answer
|
||||||
|
disclosed_themes = {
|
||||||
|
metric.theme
|
||||||
|
for metric in metrics
|
||||||
|
if metric.is_estimate
|
||||||
|
and any(
|
||||||
|
term in normalized
|
||||||
|
for term in cls.ESTIMATE_TOPIC_TERMS.get(metric.theme, (metric.label.casefold(),))
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if not disclosed_themes:
|
||||||
|
return answer
|
||||||
|
labels = ", ".join(
|
||||||
|
cls.ESTIMATE_TOPIC_LABELS.get(theme, theme)
|
||||||
|
for theme in sorted(disclosed_themes)
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
f"Datakwaliteit: {labels} in dit antwoord zijn schattingen volgens de bronmetadata, "
|
||||||
|
"geen exacte tellingen.\n\n"
|
||||||
|
f"{answer}"
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self, settings: Settings | None = None):
|
def __init__(self, settings: Settings | None = None):
|
||||||
self.settings = settings or get_settings()
|
self.settings = settings or get_settings()
|
||||||
|
|
||||||
@@ -262,6 +298,9 @@ class GeoAssistantService:
|
|||||||
)
|
)
|
||||||
context_metrics.append(item)
|
context_metrics.append(item)
|
||||||
serialized_metrics.append(item.model_dump(mode="json"))
|
serialized_metrics.append(item.model_dump(mode="json"))
|
||||||
|
serialized_metrics[-1]["measurement_quality"] = (
|
||||||
|
"schatting" if item.is_estimate else "exact_binnen_bronrepresentatie"
|
||||||
|
)
|
||||||
source_dataset_ids.append(dataset.id)
|
source_dataset_ids.append(dataset.id)
|
||||||
current_context.append(
|
current_context.append(
|
||||||
{
|
{
|
||||||
@@ -307,6 +346,9 @@ class GeoAssistantService:
|
|||||||
"value": summary["metric_value"],
|
"value": summary["metric_value"],
|
||||||
"unit": summary["metric_unit"],
|
"unit": summary["metric_unit"],
|
||||||
"is_estimate": summary["is_estimate"],
|
"is_estimate": summary["is_estimate"],
|
||||||
|
"measurement_quality": (
|
||||||
|
"schatting" if summary["is_estimate"] else "exact_binnen_bronrepresentatie"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
if dataset.id not in source_dataset_ids:
|
if dataset.id not in source_dataset_ids:
|
||||||
@@ -356,6 +398,7 @@ class GeoAssistantService:
|
|||||||
"Als is_estimate true is, noem de waarde verplicht een schatting en nooit exact. "
|
"Als is_estimate true is, noem de waarde verplicht een schatting en nooit exact. "
|
||||||
"Objectaantallen zijn ondersteunend; geef betekenisvolle oppervlakte-, lengte- of bevolkingsmetriek voorrang. "
|
"Objectaantallen zijn ondersteunend; geef betekenisvolle oppervlakte-, lengte- of bevolkingsmetriek voorrang. "
|
||||||
"Beschrijf alleen waargenomen verschillen; verzin geen oorzaak, voorspelling, verzadiging of andere verklaring. "
|
"Beschrijf alleen waargenomen verschillen; verzin geen oorzaak, voorspelling, verzadiging of andere verklaring. "
|
||||||
|
"Neem waarden en jaren letterlijk over en bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend. "
|
||||||
"Gebruik platte tekst met korte alinea's en opsommingen, zonder Markdown-symbolen. "
|
"Gebruik platte tekst met korte alinea's en opsommingen, zonder Markdown-symbolen. "
|
||||||
"Bereken of suggereer nooit watervolume zonder gekoppelde diepte of bathymetrie. "
|
"Bereken of suggereer nooit watervolume zonder gekoppelde diepte of bathymetrie. "
|
||||||
"Als de gevraagde informatie niet in de context staat, zeg precies welke bron of meting ontbreekt. "
|
"Als de gevraagde informatie niet in de context staat, zeg precies welke bron of meting ontbreekt. "
|
||||||
@@ -393,6 +436,7 @@ class GeoAssistantService:
|
|||||||
answer = str(message.get("content") or "").strip()
|
answer = str(message.get("content") or "").strip()
|
||||||
if not answer:
|
if not answer:
|
||||||
raise AppError(code="OLLAMA_EMPTY_RESPONSE", message="Ollama gaf geen antwoord terug.", status_code=502)
|
raise AppError(code="OLLAMA_EMPTY_RESPONSE", message="Ollama gaf geen antwoord terug.", status_code=502)
|
||||||
|
answer = self.ensure_estimate_disclosure(answer, metrics)
|
||||||
return AssistantQueryResponse(
|
return AssistantQueryResponse(
|
||||||
answer=answer,
|
answer=answer,
|
||||||
model=model,
|
model=model,
|
||||||
|
|||||||
@@ -98,6 +98,51 @@ def test_geo_assistant_recognizes_dutch_historical_questions(question: str) -> N
|
|||||||
assert GeoAssistantService.history_requested(question) is True
|
assert GeoAssistantService.history_requested(question) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_geo_assistant_discloses_estimated_population_values() -> None:
|
||||||
|
metrics = [
|
||||||
|
AssistantContextMetric(
|
||||||
|
theme="population",
|
||||||
|
label="Geschatte bevolking",
|
||||||
|
value=74_254,
|
||||||
|
unit="personen",
|
||||||
|
source="Statbel",
|
||||||
|
dataset_id=uuid4(),
|
||||||
|
is_estimate=True,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
answer = GeoAssistantService.ensure_estimate_disclosure(
|
||||||
|
"De bevolking bedraagt 74.254 personen.",
|
||||||
|
metrics,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert answer.startswith(
|
||||||
|
"Datakwaliteit: bevolkingswaarden in dit antwoord zijn schattingen volgens de bronmetadata, "
|
||||||
|
"geen exacte tellingen."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_geo_assistant_does_not_add_irrelevant_estimate_disclosure() -> None:
|
||||||
|
metrics = [
|
||||||
|
AssistantContextMetric(
|
||||||
|
theme="population",
|
||||||
|
label="Geschatte bevolking",
|
||||||
|
value=74_254,
|
||||||
|
unit="personen",
|
||||||
|
source="Statbel",
|
||||||
|
dataset_id=uuid4(),
|
||||||
|
is_estimate=True,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
answer = GeoAssistantService.ensure_estimate_disclosure(
|
||||||
|
"De bosoppervlakte bedraagt 3.626,56 hectare.",
|
||||||
|
metrics,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert answer == "De bosoppervlakte bedraagt 3.626,56 hectare."
|
||||||
|
|
||||||
|
|
||||||
def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch) -> None:
|
def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch) -> None:
|
||||||
service = GeoAssistantService(ollama_settings())
|
service = GeoAssistantService(ollama_settings())
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
@@ -161,6 +206,7 @@ def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch
|
|||||||
assert "scope.label is het exact geanalyseerde gebied" in captured["payload"]["messages"][0]["content"]
|
assert "scope.label is het exact geanalyseerde gebied" in captured["payload"]["messages"][0]["content"]
|
||||||
assert "noem de waarde verplicht een schatting" in captured["payload"]["messages"][0]["content"]
|
assert "noem de waarde verplicht een schatting" in captured["payload"]["messages"][0]["content"]
|
||||||
assert "verzin geen oorzaak, voorspelling, verzadiging" in captured["payload"]["messages"][0]["content"]
|
assert "verzin geen oorzaak, voorspelling, verzadiging" in captured["payload"]["messages"][0]["content"]
|
||||||
|
assert "bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend" in captured["payload"]["messages"][0]["content"]
|
||||||
assert "zonder Markdown-symbolen" in captured["payload"]["messages"][0]["content"]
|
assert "zonder Markdown-symbolen" in captured["payload"]["messages"][0]["content"]
|
||||||
assert "water_volume_available" in captured["payload"]["messages"][0]["content"]
|
assert "water_volume_available" in captured["payload"]["messages"][0]["content"]
|
||||||
|
|
||||||
|
|||||||
@@ -8417,6 +8417,11 @@ Validation evidence:
|
|||||||
was suggested. The system contract now mandates estimate wording, forbids
|
was suggested. The system contract now mandates estimate wording, forbids
|
||||||
causal/forecast claims not present in context, uses deterministic temperature
|
causal/forecast claims not present in context, uses deterministic temperature
|
||||||
zero and requests plain text. Full readiness remained green at 617 tests.
|
zero and requests plain text. Full readiness remained green at 617 tests.
|
||||||
|
- Assistant context now carries explicit measurement quality for current and
|
||||||
|
historical values. A deterministic response guard prepends the estimate
|
||||||
|
limitation whenever an answer discusses estimated population data, while the
|
||||||
|
model prompt forbids independent averages, rates and derived trends. Focused
|
||||||
|
tests and full readiness passed with 619 backend tests.
|
||||||
|
|
||||||
Known limitations:
|
Known limitations:
|
||||||
- Water volume remains unavailable until a governed depth/bathymetry source is
|
- Water volume remains unavailable until a governed depth/bathymetry source is
|
||||||
|
|||||||
Reference in New Issue
Block a user