From c7faea430ffd77c06ba06efd236db848b297fcbe Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 07:51:23 +0200 Subject: [PATCH] fix: enforce assistant data quality disclosures --- CHANGELOG.md | 3 ++ backend/app/services/geo_assistant_service.py | 44 ++++++++++++++++++ ...t_sprint202_temporal_metrics_and_ollama.py | 46 +++++++++++++++++++ docs/CODEX_EXECUTION_LOG.md | 5 ++ 4 files changed, 98 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5791adc..9ba9deba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ - Tightened generated answers to descriptive source evidence: estimates must remain labelled, unsupported causal/forecast claims are forbidden and plain text is requested for the existing chat renderer. +- Added deterministic estimate disclosure for population answers and instructed + the local model to copy governed values literally instead of inventing + averages, rates or derived trends. ## Sprint 201 Semantic area-selection metrics (2026-07-15) diff --git a/backend/app/services/geo_assistant_service.py b/backend/app/services/geo_assistant_service.py index d2258007..4cc5f5ac 100644 --- a/backend/app/services/geo_assistant_service.py +++ b/backend/app/services/geo_assistant_service.py @@ -37,12 +37,48 @@ class GeoAssistantService: "gedaald", "gestegen", ) + ESTIMATE_TOPIC_TERMS = { + "population": ("bevolk", "inwoner"), + } + ESTIMATE_TOPIC_LABELS = { + "population": "bevolkingswaarden", + } @classmethod def history_requested(cls, question: str) -> bool: normalized = question.casefold() return any(keyword in normalized for keyword in cls.HISTORY_KEYWORDS) + @classmethod + def ensure_estimate_disclosure( + cls, + answer: str, + metrics: list[AssistantContextMetric], + ) -> str: + normalized = answer.casefold() + if "schat" in normalized: + return answer + disclosed_themes = { + metric.theme + for metric in metrics + if metric.is_estimate + and any( + term in normalized + for term in cls.ESTIMATE_TOPIC_TERMS.get(metric.theme, (metric.label.casefold(),)) + ) + } + if not disclosed_themes: + return answer + labels = ", ".join( + cls.ESTIMATE_TOPIC_LABELS.get(theme, theme) + for theme in sorted(disclosed_themes) + ) + return ( + f"Datakwaliteit: {labels} in dit antwoord zijn schattingen volgens de bronmetadata, " + "geen exacte tellingen.\n\n" + f"{answer}" + ) + def __init__(self, settings: Settings | None = None): self.settings = settings or get_settings() @@ -262,6 +298,9 @@ class GeoAssistantService: ) context_metrics.append(item) serialized_metrics.append(item.model_dump(mode="json")) + serialized_metrics[-1]["measurement_quality"] = ( + "schatting" if item.is_estimate else "exact_binnen_bronrepresentatie" + ) source_dataset_ids.append(dataset.id) current_context.append( { @@ -307,6 +346,9 @@ class GeoAssistantService: "value": summary["metric_value"], "unit": summary["metric_unit"], "is_estimate": summary["is_estimate"], + "measurement_quality": ( + "schatting" if summary["is_estimate"] else "exact_binnen_bronrepresentatie" + ), } ) if dataset.id not in source_dataset_ids: @@ -356,6 +398,7 @@ class GeoAssistantService: "Als is_estimate true is, noem de waarde verplicht een schatting en nooit exact. " "Objectaantallen zijn ondersteunend; geef betekenisvolle oppervlakte-, lengte- of bevolkingsmetriek voorrang. " "Beschrijf alleen waargenomen verschillen; verzin geen oorzaak, voorspelling, verzadiging of andere verklaring. " + "Neem waarden en jaren letterlijk over en bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend. " "Gebruik platte tekst met korte alinea's en opsommingen, zonder Markdown-symbolen. " "Bereken of suggereer nooit watervolume zonder gekoppelde diepte of bathymetrie. " "Als de gevraagde informatie niet in de context staat, zeg precies welke bron of meting ontbreekt. " @@ -393,6 +436,7 @@ class GeoAssistantService: answer = str(message.get("content") or "").strip() if not answer: raise AppError(code="OLLAMA_EMPTY_RESPONSE", message="Ollama gaf geen antwoord terug.", status_code=502) + answer = self.ensure_estimate_disclosure(answer, metrics) return AssistantQueryResponse( answer=answer, model=model, diff --git a/backend/tests/test_sprint202_temporal_metrics_and_ollama.py b/backend/tests/test_sprint202_temporal_metrics_and_ollama.py index 13c96bf7..00461600 100644 --- a/backend/tests/test_sprint202_temporal_metrics_and_ollama.py +++ b/backend/tests/test_sprint202_temporal_metrics_and_ollama.py @@ -98,6 +98,51 @@ def test_geo_assistant_recognizes_dutch_historical_questions(question: str) -> N assert GeoAssistantService.history_requested(question) is True +def test_geo_assistant_discloses_estimated_population_values() -> None: + metrics = [ + AssistantContextMetric( + theme="population", + label="Geschatte bevolking", + value=74_254, + unit="personen", + source="Statbel", + dataset_id=uuid4(), + is_estimate=True, + ) + ] + + answer = GeoAssistantService.ensure_estimate_disclosure( + "De bevolking bedraagt 74.254 personen.", + metrics, + ) + + assert answer.startswith( + "Datakwaliteit: bevolkingswaarden in dit antwoord zijn schattingen volgens de bronmetadata, " + "geen exacte tellingen." + ) + + +def test_geo_assistant_does_not_add_irrelevant_estimate_disclosure() -> None: + metrics = [ + AssistantContextMetric( + theme="population", + label="Geschatte bevolking", + value=74_254, + unit="personen", + source="Statbel", + dataset_id=uuid4(), + is_estimate=True, + ) + ] + + answer = GeoAssistantService.ensure_estimate_disclosure( + "De bosoppervlakte bedraagt 3.626,56 hectare.", + metrics, + ) + + assert answer == "De bosoppervlakte bedraagt 3.626,56 hectare." + + def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch) -> None: service = GeoAssistantService(ollama_settings()) project_id = uuid4() @@ -161,6 +206,7 @@ def test_geo_assistant_sends_grounded_context_without_thinking_trace(monkeypatch assert "scope.label is het exact geanalyseerde gebied" in captured["payload"]["messages"][0]["content"] assert "noem de waarde verplicht een schatting" in captured["payload"]["messages"][0]["content"] assert "verzin geen oorzaak, voorspelling, verzadiging" in captured["payload"]["messages"][0]["content"] + assert "bereken zelf geen gemiddelde, tempo, oorzaak of afgeleide trend" in captured["payload"]["messages"][0]["content"] assert "zonder Markdown-symbolen" in captured["payload"]["messages"][0]["content"] assert "water_volume_available" in captured["payload"]["messages"][0]["content"] diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index b4a516e8..67349025 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8417,6 +8417,11 @@ Validation evidence: was suggested. The system contract now mandates estimate wording, forbids causal/forecast claims not present in context, uses deterministic temperature zero and requests plain text. Full readiness remained green at 617 tests. +- Assistant context now carries explicit measurement quality for current and + historical values. A deterministic response guard prepends the estimate + limitation whenever an answer discusses estimated population data, while the + model prompt forbids independent averages, rates and derived trends. Focused + tests and full readiness passed with 619 backend tests. Known limitations: - Water volume remains unavailable until a governed depth/bathymetry source is