fix: knowledge retrieval accuracy and remaining brand/PoC leaks in procedure docs

- Fix the demo knowledge provider's tokenizer: a plain [a-z0-9]+ regex silently
  dropped accented characters, splitting French words like "véhicule" into "v" +
  "hicule" and mangling retrieval for nearly every French query. Now matches the
  Latin-1 accented range too.
- Reweight section scoring so the body match (the actual substance of a section)
  outranks a heading/title match (a shallow structural hint) rather than the reverse
  -- confirmed via the brief's exact validation question that the old weighting
  misranked the damage procedure behind a topically-adjacent document in all three
  languages (nl-BE: a checkout section; en-GB/fr-BE: the return procedure), purely
  because a generic word like "vehicle"/"voertuig" happened to sit in a heading/title.
- Remove leftover "MobilityOps" and "PoC" mentions from 5 English and 4 NL/FR
  procedure documents -- knowledge-base prose is visible UI content and was missed by
  the earlier rebrand.
- Add regression tests: the brief's exact NL/EN/FR damage question must ground on the
  damage procedure as the *primary* source (not just appear in the top 3), and no
  procedure file may contain "MobilityOps" or "PoC".

151 backend tests, Ruff, mypy green.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
NuklearRabbit
2026-08-03 21:54:15 +02:00
co-authored by Claude Sonnet 5
parent 6deb95524d
commit e6539d17b6
11 changed files with 69 additions and 14 deletions
+20 -5
View File
@@ -35,7 +35,11 @@ STOPWORDS_BY_LANGUAGE: dict[str, set[str]] = {
}, },
} }
_WORD_RE = re.compile(r"[a-z0-9]+") # Includes the Latin-1 accented-letter range (à-ö, ø-ÿ) so French/Dutch words with
# diacritics (véhicule, réservation, geëscaleerd) tokenize as one word instead of
# splitting apart at the accented character -- a plain [a-z0-9]+ pattern silently
# drops every accent and fragments the word either side of it.
_WORD_RE = re.compile(r"[a-zà-öø-ÿ0-9]+")
def _stem(word: str) -> str: def _stem(word: str) -> str:
@@ -218,17 +222,28 @@ class DemoKnowledgeProvider:
def _score( def _score(
self, query_tokens: set[str], section: ScoredSection, idf: dict[str, float] self, query_tokens: set[str], section: ScoredSection, idf: dict[str, float]
) -> float: ) -> float:
# The section body is the strongest relevance signal -- it's the actual
# substance a heading or title can only hint at -- so a body match is weighted
# *above* heading/title matches, not below them. The previous 3x/2x/1x
# (heading/title/body) ordering let a single generic word in a heading (e.g.
# "vehicle", present in nearly every section) or a document's own title
# outrank a section whose body genuinely covers multiple, more distinctive
# query terms -- confirmed to misrank the brief's exact validation question in
# every one of the three languages (see docs/fleet-ops-correction/
# current-gap-audit.md and i18n-inventory.md): nl-BE picked a checkout section
# over the damage procedure, en-GB and fr-BE picked the return procedure over
# the damage procedure, purely from heading/title overlap on common words.
score = 0.0 score = 0.0
for token in query_tokens: for token in query_tokens:
token_idf = idf.get(token, 0.0) token_idf = idf.get(token, 0.0)
if token_idf == 0.0: if token_idf == 0.0:
continue continue
if token in section.heading_tokens: if token in section.body_tokens:
score += 3 * token_idf score += 3 * token_idf
elif token in section.document.title_tokens: elif token in section.heading_tokens:
score += 2 * token_idf score += 2 * token_idf
elif token in section.body_tokens: elif token in section.document.title_tokens:
score += token_idf score += 1.5 * token_idf
return score return score
def ask( def ask(
+40
View File
@@ -1,11 +1,51 @@
from __future__ import annotations from __future__ import annotations
import re
from pathlib import Path
import httpx import httpx
from app.core.config import get_settings
from app.services.knowledge.demo import DemoKnowledgeProvider from app.services.knowledge.demo import DemoKnowledgeProvider
from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
def test_brief_exact_damage_question_in_all_three_languages():
# The exact validation questions from docs/fleet-ops-correction/current-gap-audit.md
# -- each must ground on the damage procedure as its *primary* (top-ranked) source,
# not merely appear somewhere in the top-3, and the source/version/section/excerpt
# must all come from that same-language document (never an English fallback).
provider = DemoKnowledgeProvider()
cases = {
"nl-BE": "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?",
"en-GB": "What should I do when a vehicle returns with damage?",
"fr-BE": "Que dois-je faire lorsqu'un véhicule revient endommagé ?",
}
for language, question in cases.items():
answer = provider.ask(question, f"test-brief-{language}", language)
assert answer.evidence_state == "grounded", language
assert answer.sources, language
assert answer.sources[0].document_id == "damage-procedure", (
f"{language}: expected the damage procedure as the primary source, "
f"got {answer.sources[0].document_id!r}"
)
assert answer.answer
assert answer.sources[0].excerpt
def test_knowledge_procedures_never_mention_mobilityops_or_poc():
# Section 2 of docs/fleet-ops-correction/current-gap-audit.md: the visible brand
# name is exactly "Fleet Ops", and "PoC" must never appear in visible content --
# including the demo knowledge base, not just the frontend.
procedures_dir = Path(get_settings().knowledge_dir)
offenders = []
for path in sorted(procedures_dir.glob("*/*.md")):
text = path.read_text(encoding="utf-8")
if "MobilityOps" in text or re.search(r"\bPoC\b", text):
offenders.append(str(path))
assert offenders == []
def test_s6_damage_question_is_grounded_with_expected_sources(): def test_s6_damage_question_is_grounded_with_expected_sources():
provider = DemoKnowledgeProvider() provider = DemoKnowledgeProvider()
answer = provider.ask( answer = provider.ask(
@@ -22,4 +22,4 @@ Damage or a technical warning requires a blocked state. A reached service thresh
## 4. Follow-up ## 4. Follow-up
The return automation may create a cleaning or maintenance follow-up, but MobilityOps remains the source of truth for the vehicle state. The return automation may create a cleaning or maintenance follow-up, but Fleet Ops remains the source of truth for the vehicle state.
@@ -14,7 +14,7 @@ When a vehicle returns with visible or reported damage, mark damage in the retur
## 2. Evidence ## 2. Evidence
Record at least four photographs in the source inspection system: full affected side, close view, wider context and vehicle registration. MobilityOps may store only the evidence reference in this PoC. Record at least four photographs in the source inspection system: full affected side, close view, wider context and vehicle registration. Fleet Ops stores only the evidence reference in this demo environment.
## 3. Escalation ## 3. Escalation
@@ -18,4 +18,4 @@ Inspect windows, lights, tyres and bodywork while cleaning. Report newly observe
## 3. Completion ## 3. Completion
Cleaning completion alone does not make a blocked or maintenance vehicle available. MobilityOps derives availability from all active restrictions. Cleaning completion alone does not make a blocked or maintenance vehicle available. Fleet Ops derives availability from all active restrictions.
@@ -14,7 +14,7 @@ The customer identity and valid driving licence must be verified in the source p
## 2. Data minimisation ## 2. Data minimisation
MobilityOps records only completion status and source references for this PoC. It does not store identity-document images. Fleet Ops records only completion status and source references for this demo environment. It does not store identity-document images.
## 3. Missing information ## 3. Missing information
@@ -14,7 +14,7 @@ Two reserved or active bookings for the same vehicle may not overlap. Imported l
## 2. Resolution ## 2. Resolution
Operations reviews both bookings, confirms source-system ownership and reassigns or cancels one booking in the authoritative booking system. MobilityOps records the reviewed outcome. Operations reviews both bookings, confirms source-system ownership and reassigns or cancels one booking in the authoritative booking system. Fleet Ops records the reviewed outcome.
## 3. Safety ## 3. Safety
@@ -14,7 +14,7 @@ Lorsqu'un véhicule revient avec des dommages visibles ou signalés, marquez le
## 2. Preuves ## 2. Preuves
Enregistrez au moins quatre photographies dans le système d'inspection source : côté affecté en entier, gros plan, contexte plus large et plaque d'immatriculation. Fleet Ops ne peut stocker que la référence à cette preuve dans ce PoC. Enregistrez au moins quatre photographies dans le système d'inspection source : côté affecté en entier, gros plan, contexte plus large et plaque d'immatriculation. Fleet Ops ne peut stocker que la référence à cette preuve dans cet environnement de démo.
## 3. Escalade ## 3. Escalade
@@ -14,7 +14,7 @@ L'identité du client et un permis de conduire valide doivent être vérifiés d
## 2. Minimisation des données ## 2. Minimisation des données
Fleet Ops n'enregistre dans ce PoC que le statut d'achèvement et les références sources. Aucune image de document d'identité n'est stockée. Fleet Ops n'enregistre dans cet environnement de démo que le statut d'achèvement et les références sources. Aucune image de document d'identité n'est stockée.
## 3. Informations manquantes ## 3. Informations manquantes
@@ -14,7 +14,7 @@ Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schad
## 2. Bewijs ## 2. Bewijs
Registreer minstens vier foto's in het bron-inspectiesysteem: volledige aangetaste zijde, close-up, ruimere context en kentekenplaat. Fleet Ops mag in deze PoC enkel de verwijzing naar dat bewijs opslaan. Registreer minstens vier foto's in het bron-inspectiesysteem: volledige aangetaste zijde, close-up, ruimere context en kentekenplaat. Fleet Ops mag in deze demo-omgeving enkel de verwijzing naar dat bewijs opslaan.
## 3. Escalatie ## 3. Escalatie
@@ -14,7 +14,7 @@ De identiteit van de klant en een geldig rijbewijs moeten geverifieerd zijn in h
## 2. Dataminimalisatie ## 2. Dataminimalisatie
Fleet Ops registreert in deze PoC enkel de afhandelingsstatus en bronverwijzingen. Er worden geen afbeeldingen van identiteitsdocumenten opgeslagen. Fleet Ops registreert in deze demo-omgeving enkel de afhandelingsstatus en bronverwijzingen. Er worden geen afbeeldingen van identiteitsdocumenten opgeslagen.
## 3. Ontbrekende informatie ## 3. Ontbrekende informatie