From e6539d17b6bab9cf76b0cb5ca3b5c487805f389d Mon Sep 17 00:00:00 2001 From: NuklearRabbit <145918611+NuklearRabbit@users.noreply.github.com> Date: Mon, 3 Aug 2026 21:54:15 +0200 Subject: [PATCH] fix: knowledge retrieval accuracy and remaining brand/PoC leaks in procedure docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix the demo knowledge provider's tokenizer: a plain [a-z0-9]+ regex silently dropped accented characters, splitting French words like "véhicule" into "v" + "hicule" and mangling retrieval for nearly every French query. Now matches the Latin-1 accented range too. - Reweight section scoring so the body match (the actual substance of a section) outranks a heading/title match (a shallow structural hint) rather than the reverse -- confirmed via the brief's exact validation question that the old weighting misranked the damage procedure behind a topically-adjacent document in all three languages (nl-BE: a checkout section; en-GB/fr-BE: the return procedure), purely because a generic word like "vehicle"/"voertuig" happened to sit in a heading/title. - Remove leftover "MobilityOps" and "PoC" mentions from 5 English and 4 NL/FR procedure documents -- knowledge-base prose is visible UI content and was missed by the earlier rebrand. - Add regression tests: the brief's exact NL/EN/FR damage question must ground on the damage procedure as the *primary* source (not just appear in the top 3), and no procedure file may contain "MobilityOps" or "PoC". 151 backend tests, Ruff, mypy green. Co-Authored-By: Claude Sonnet 5 --- backend/app/services/knowledge/demo.py | 25 +++++++++--- backend/tests/test_knowledge.py | 40 +++++++++++++++++++ .../procedures/en-GB/02-vehicle-return.md | 2 +- .../procedures/en-GB/03-damage-handling.md | 2 +- .../procedures/en-GB/05-cleaning-checklist.md | 2 +- .../procedures/en-GB/07-customer-documents.md | 2 +- .../procedures/en-GB/09-booking-conflicts.md | 2 +- .../procedures/fr-BE/03-damage-handling.md | 2 +- .../procedures/fr-BE/07-customer-documents.md | 2 +- .../procedures/nl-BE/03-damage-handling.md | 2 +- .../procedures/nl-BE/07-customer-documents.md | 2 +- 11 files changed, 69 insertions(+), 14 deletions(-) diff --git a/backend/app/services/knowledge/demo.py b/backend/app/services/knowledge/demo.py index 58e9433..0e4aa16 100644 --- a/backend/app/services/knowledge/demo.py +++ b/backend/app/services/knowledge/demo.py @@ -35,7 +35,11 @@ STOPWORDS_BY_LANGUAGE: dict[str, set[str]] = { }, } -_WORD_RE = re.compile(r"[a-z0-9]+") +# Includes the Latin-1 accented-letter range (à-ö, ø-ÿ) so French/Dutch words with +# diacritics (véhicule, réservation, geëscaleerd) tokenize as one word instead of +# splitting apart at the accented character -- a plain [a-z0-9]+ pattern silently +# drops every accent and fragments the word either side of it. +_WORD_RE = re.compile(r"[a-zà-öø-ÿ0-9]+") def _stem(word: str) -> str: @@ -218,17 +222,28 @@ class DemoKnowledgeProvider: def _score( self, query_tokens: set[str], section: ScoredSection, idf: dict[str, float] ) -> float: + # The section body is the strongest relevance signal -- it's the actual + # substance a heading or title can only hint at -- so a body match is weighted + # *above* heading/title matches, not below them. The previous 3x/2x/1x + # (heading/title/body) ordering let a single generic word in a heading (e.g. + # "vehicle", present in nearly every section) or a document's own title + # outrank a section whose body genuinely covers multiple, more distinctive + # query terms -- confirmed to misrank the brief's exact validation question in + # every one of the three languages (see docs/fleet-ops-correction/ + # current-gap-audit.md and i18n-inventory.md): nl-BE picked a checkout section + # over the damage procedure, en-GB and fr-BE picked the return procedure over + # the damage procedure, purely from heading/title overlap on common words. score = 0.0 for token in query_tokens: token_idf = idf.get(token, 0.0) if token_idf == 0.0: continue - if token in section.heading_tokens: + if token in section.body_tokens: score += 3 * token_idf - elif token in section.document.title_tokens: + elif token in section.heading_tokens: score += 2 * token_idf - elif token in section.body_tokens: - score += token_idf + elif token in section.document.title_tokens: + score += 1.5 * token_idf return score def ask( diff --git a/backend/tests/test_knowledge.py b/backend/tests/test_knowledge.py index ef8ebe3..30a899e 100644 --- a/backend/tests/test_knowledge.py +++ b/backend/tests/test_knowledge.py @@ -1,11 +1,51 @@ from __future__ import annotations +import re +from pathlib import Path + import httpx +from app.core.config import get_settings from app.services.knowledge.demo import DemoKnowledgeProvider from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider +def test_brief_exact_damage_question_in_all_three_languages(): + # The exact validation questions from docs/fleet-ops-correction/current-gap-audit.md + # -- each must ground on the damage procedure as its *primary* (top-ranked) source, + # not merely appear somewhere in the top-3, and the source/version/section/excerpt + # must all come from that same-language document (never an English fallback). + provider = DemoKnowledgeProvider() + cases = { + "nl-BE": "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?", + "en-GB": "What should I do when a vehicle returns with damage?", + "fr-BE": "Que dois-je faire lorsqu'un véhicule revient endommagé ?", + } + for language, question in cases.items(): + answer = provider.ask(question, f"test-brief-{language}", language) + assert answer.evidence_state == "grounded", language + assert answer.sources, language + assert answer.sources[0].document_id == "damage-procedure", ( + f"{language}: expected the damage procedure as the primary source, " + f"got {answer.sources[0].document_id!r}" + ) + assert answer.answer + assert answer.sources[0].excerpt + + +def test_knowledge_procedures_never_mention_mobilityops_or_poc(): + # Section 2 of docs/fleet-ops-correction/current-gap-audit.md: the visible brand + # name is exactly "Fleet Ops", and "PoC" must never appear in visible content -- + # including the demo knowledge base, not just the frontend. + procedures_dir = Path(get_settings().knowledge_dir) + offenders = [] + for path in sorted(procedures_dir.glob("*/*.md")): + text = path.read_text(encoding="utf-8") + if "MobilityOps" in text or re.search(r"\bPoC\b", text): + offenders.append(str(path)) + assert offenders == [] + + def test_s6_damage_question_is_grounded_with_expected_sources(): provider = DemoKnowledgeProvider() answer = provider.ask( diff --git a/knowledge/procedures/en-GB/02-vehicle-return.md b/knowledge/procedures/en-GB/02-vehicle-return.md index 199784a..dcf26c2 100644 --- a/knowledge/procedures/en-GB/02-vehicle-return.md +++ b/knowledge/procedures/en-GB/02-vehicle-return.md @@ -22,4 +22,4 @@ Damage or a technical warning requires a blocked state. A reached service thresh ## 4. Follow-up -The return automation may create a cleaning or maintenance follow-up, but MobilityOps remains the source of truth for the vehicle state. +The return automation may create a cleaning or maintenance follow-up, but Fleet Ops remains the source of truth for the vehicle state. diff --git a/knowledge/procedures/en-GB/03-damage-handling.md b/knowledge/procedures/en-GB/03-damage-handling.md index 3713b42..0eced1e 100644 --- a/knowledge/procedures/en-GB/03-damage-handling.md +++ b/knowledge/procedures/en-GB/03-damage-handling.md @@ -14,7 +14,7 @@ When a vehicle returns with visible or reported damage, mark damage in the retur ## 2. Evidence -Record at least four photographs in the source inspection system: full affected side, close view, wider context and vehicle registration. MobilityOps may store only the evidence reference in this PoC. +Record at least four photographs in the source inspection system: full affected side, close view, wider context and vehicle registration. Fleet Ops stores only the evidence reference in this demo environment. ## 3. Escalation diff --git a/knowledge/procedures/en-GB/05-cleaning-checklist.md b/knowledge/procedures/en-GB/05-cleaning-checklist.md index 733fac7..68255c8 100644 --- a/knowledge/procedures/en-GB/05-cleaning-checklist.md +++ b/knowledge/procedures/en-GB/05-cleaning-checklist.md @@ -18,4 +18,4 @@ Inspect windows, lights, tyres and bodywork while cleaning. Report newly observe ## 3. Completion -Cleaning completion alone does not make a blocked or maintenance vehicle available. MobilityOps derives availability from all active restrictions. +Cleaning completion alone does not make a blocked or maintenance vehicle available. Fleet Ops derives availability from all active restrictions. diff --git a/knowledge/procedures/en-GB/07-customer-documents.md b/knowledge/procedures/en-GB/07-customer-documents.md index 8653f78..c7fc3d0 100644 --- a/knowledge/procedures/en-GB/07-customer-documents.md +++ b/knowledge/procedures/en-GB/07-customer-documents.md @@ -14,7 +14,7 @@ The customer identity and valid driving licence must be verified in the source p ## 2. Data minimisation -MobilityOps records only completion status and source references for this PoC. It does not store identity-document images. +Fleet Ops records only completion status and source references for this demo environment. It does not store identity-document images. ## 3. Missing information diff --git a/knowledge/procedures/en-GB/09-booking-conflicts.md b/knowledge/procedures/en-GB/09-booking-conflicts.md index 7b5b020..0d77fe6 100644 --- a/knowledge/procedures/en-GB/09-booking-conflicts.md +++ b/knowledge/procedures/en-GB/09-booking-conflicts.md @@ -14,7 +14,7 @@ Two reserved or active bookings for the same vehicle may not overlap. Imported l ## 2. Resolution -Operations reviews both bookings, confirms source-system ownership and reassigns or cancels one booking in the authoritative booking system. MobilityOps records the reviewed outcome. +Operations reviews both bookings, confirms source-system ownership and reassigns or cancels one booking in the authoritative booking system. Fleet Ops records the reviewed outcome. ## 3. Safety diff --git a/knowledge/procedures/fr-BE/03-damage-handling.md b/knowledge/procedures/fr-BE/03-damage-handling.md index a524244..ca65194 100644 --- a/knowledge/procedures/fr-BE/03-damage-handling.md +++ b/knowledge/procedures/fr-BE/03-damage-handling.md @@ -14,7 +14,7 @@ Lorsqu'un véhicule revient avec des dommages visibles ou signalés, marquez le ## 2. Preuves -Enregistrez au moins quatre photographies dans le système d'inspection source : côté affecté en entier, gros plan, contexte plus large et plaque d'immatriculation. Fleet Ops ne peut stocker que la référence à cette preuve dans ce PoC. +Enregistrez au moins quatre photographies dans le système d'inspection source : côté affecté en entier, gros plan, contexte plus large et plaque d'immatriculation. Fleet Ops ne peut stocker que la référence à cette preuve dans cet environnement de démo. ## 3. Escalade diff --git a/knowledge/procedures/fr-BE/07-customer-documents.md b/knowledge/procedures/fr-BE/07-customer-documents.md index dfda656..641ccbb 100644 --- a/knowledge/procedures/fr-BE/07-customer-documents.md +++ b/knowledge/procedures/fr-BE/07-customer-documents.md @@ -14,7 +14,7 @@ L'identité du client et un permis de conduire valide doivent être vérifiés d ## 2. Minimisation des données -Fleet Ops n'enregistre dans ce PoC que le statut d'achèvement et les références sources. Aucune image de document d'identité n'est stockée. +Fleet Ops n'enregistre dans cet environnement de démo que le statut d'achèvement et les références sources. Aucune image de document d'identité n'est stockée. ## 3. Informations manquantes diff --git a/knowledge/procedures/nl-BE/03-damage-handling.md b/knowledge/procedures/nl-BE/03-damage-handling.md index 6e5e87c..f666818 100644 --- a/knowledge/procedures/nl-BE/03-damage-handling.md +++ b/knowledge/procedures/nl-BE/03-damage-handling.md @@ -14,7 +14,7 @@ Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schad ## 2. Bewijs -Registreer minstens vier foto's in het bron-inspectiesysteem: volledige aangetaste zijde, close-up, ruimere context en kentekenplaat. Fleet Ops mag in deze PoC enkel de verwijzing naar dat bewijs opslaan. +Registreer minstens vier foto's in het bron-inspectiesysteem: volledige aangetaste zijde, close-up, ruimere context en kentekenplaat. Fleet Ops mag in deze demo-omgeving enkel de verwijzing naar dat bewijs opslaan. ## 3. Escalatie diff --git a/knowledge/procedures/nl-BE/07-customer-documents.md b/knowledge/procedures/nl-BE/07-customer-documents.md index ffc2865..a7085e6 100644 --- a/knowledge/procedures/nl-BE/07-customer-documents.md +++ b/knowledge/procedures/nl-BE/07-customer-documents.md @@ -14,7 +14,7 @@ De identiteit van de klant en een geldig rijbewijs moeten geverifieerd zijn in h ## 2. Dataminimalisatie -Fleet Ops registreert in deze PoC enkel de afhandelingsstatus en bronverwijzingen. Er worden geen afbeeldingen van identiteitsdocumenten opgeslagen. +Fleet Ops registreert in deze demo-omgeving enkel de afhandelingsstatus en bronverwijzingen. Er worden geen afbeeldingen van identiteitsdocumenten opgeslagen. ## 3. Ontbrekende informatie