fix: knowledge retrieval accuracy and remaining brand/PoC leaks in procedure docs
- Fix the demo knowledge provider's tokenizer: a plain [a-z0-9]+ regex silently dropped accented characters, splitting French words like "véhicule" into "v" + "hicule" and mangling retrieval for nearly every French query. Now matches the Latin-1 accented range too. - Reweight section scoring so the body match (the actual substance of a section) outranks a heading/title match (a shallow structural hint) rather than the reverse -- confirmed via the brief's exact validation question that the old weighting misranked the damage procedure behind a topically-adjacent document in all three languages (nl-BE: a checkout section; en-GB/fr-BE: the return procedure), purely because a generic word like "vehicle"/"voertuig" happened to sit in a heading/title. - Remove leftover "MobilityOps" and "PoC" mentions from 5 English and 4 NL/FR procedure documents -- knowledge-base prose is visible UI content and was missed by the earlier rebrand. - Add regression tests: the brief's exact NL/EN/FR damage question must ground on the damage procedure as the *primary* source (not just appear in the top 3), and no procedure file may contain "MobilityOps" or "PoC". 151 backend tests, Ruff, mypy green. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
6deb95524d
commit
e6539d17b6
@@ -35,7 +35,11 @@ STOPWORDS_BY_LANGUAGE: dict[str, set[str]] = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_WORD_RE = re.compile(r"[a-z0-9]+")
|
# Includes the Latin-1 accented-letter range (à-ö, ø-ÿ) so French/Dutch words with
|
||||||
|
# diacritics (véhicule, réservation, geëscaleerd) tokenize as one word instead of
|
||||||
|
# splitting apart at the accented character -- a plain [a-z0-9]+ pattern silently
|
||||||
|
# drops every accent and fragments the word either side of it.
|
||||||
|
_WORD_RE = re.compile(r"[a-zà-öø-ÿ0-9]+")
|
||||||
|
|
||||||
|
|
||||||
def _stem(word: str) -> str:
|
def _stem(word: str) -> str:
|
||||||
@@ -218,17 +222,28 @@ class DemoKnowledgeProvider:
|
|||||||
def _score(
|
def _score(
|
||||||
self, query_tokens: set[str], section: ScoredSection, idf: dict[str, float]
|
self, query_tokens: set[str], section: ScoredSection, idf: dict[str, float]
|
||||||
) -> float:
|
) -> float:
|
||||||
|
# The section body is the strongest relevance signal -- it's the actual
|
||||||
|
# substance a heading or title can only hint at -- so a body match is weighted
|
||||||
|
# *above* heading/title matches, not below them. The previous 3x/2x/1x
|
||||||
|
# (heading/title/body) ordering let a single generic word in a heading (e.g.
|
||||||
|
# "vehicle", present in nearly every section) or a document's own title
|
||||||
|
# outrank a section whose body genuinely covers multiple, more distinctive
|
||||||
|
# query terms -- confirmed to misrank the brief's exact validation question in
|
||||||
|
# every one of the three languages (see docs/fleet-ops-correction/
|
||||||
|
# current-gap-audit.md and i18n-inventory.md): nl-BE picked a checkout section
|
||||||
|
# over the damage procedure, en-GB and fr-BE picked the return procedure over
|
||||||
|
# the damage procedure, purely from heading/title overlap on common words.
|
||||||
score = 0.0
|
score = 0.0
|
||||||
for token in query_tokens:
|
for token in query_tokens:
|
||||||
token_idf = idf.get(token, 0.0)
|
token_idf = idf.get(token, 0.0)
|
||||||
if token_idf == 0.0:
|
if token_idf == 0.0:
|
||||||
continue
|
continue
|
||||||
if token in section.heading_tokens:
|
if token in section.body_tokens:
|
||||||
score += 3 * token_idf
|
score += 3 * token_idf
|
||||||
elif token in section.document.title_tokens:
|
elif token in section.heading_tokens:
|
||||||
score += 2 * token_idf
|
score += 2 * token_idf
|
||||||
elif token in section.body_tokens:
|
elif token in section.document.title_tokens:
|
||||||
score += token_idf
|
score += 1.5 * token_idf
|
||||||
return score
|
return score
|
||||||
|
|
||||||
def ask(
|
def ask(
|
||||||
|
|||||||
@@ -1,11 +1,51 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
from app.services.knowledge.demo import DemoKnowledgeProvider
|
from app.services.knowledge.demo import DemoKnowledgeProvider
|
||||||
from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
|
from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
|
||||||
|
|
||||||
|
|
||||||
|
def test_brief_exact_damage_question_in_all_three_languages():
|
||||||
|
# The exact validation questions from docs/fleet-ops-correction/current-gap-audit.md
|
||||||
|
# -- each must ground on the damage procedure as its *primary* (top-ranked) source,
|
||||||
|
# not merely appear somewhere in the top-3, and the source/version/section/excerpt
|
||||||
|
# must all come from that same-language document (never an English fallback).
|
||||||
|
provider = DemoKnowledgeProvider()
|
||||||
|
cases = {
|
||||||
|
"nl-BE": "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?",
|
||||||
|
"en-GB": "What should I do when a vehicle returns with damage?",
|
||||||
|
"fr-BE": "Que dois-je faire lorsqu'un véhicule revient endommagé ?",
|
||||||
|
}
|
||||||
|
for language, question in cases.items():
|
||||||
|
answer = provider.ask(question, f"test-brief-{language}", language)
|
||||||
|
assert answer.evidence_state == "grounded", language
|
||||||
|
assert answer.sources, language
|
||||||
|
assert answer.sources[0].document_id == "damage-procedure", (
|
||||||
|
f"{language}: expected the damage procedure as the primary source, "
|
||||||
|
f"got {answer.sources[0].document_id!r}"
|
||||||
|
)
|
||||||
|
assert answer.answer
|
||||||
|
assert answer.sources[0].excerpt
|
||||||
|
|
||||||
|
|
||||||
|
def test_knowledge_procedures_never_mention_mobilityops_or_poc():
|
||||||
|
# Section 2 of docs/fleet-ops-correction/current-gap-audit.md: the visible brand
|
||||||
|
# name is exactly "Fleet Ops", and "PoC" must never appear in visible content --
|
||||||
|
# including the demo knowledge base, not just the frontend.
|
||||||
|
procedures_dir = Path(get_settings().knowledge_dir)
|
||||||
|
offenders = []
|
||||||
|
for path in sorted(procedures_dir.glob("*/*.md")):
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
if "MobilityOps" in text or re.search(r"\bPoC\b", text):
|
||||||
|
offenders.append(str(path))
|
||||||
|
assert offenders == []
|
||||||
|
|
||||||
|
|
||||||
def test_s6_damage_question_is_grounded_with_expected_sources():
|
def test_s6_damage_question_is_grounded_with_expected_sources():
|
||||||
provider = DemoKnowledgeProvider()
|
provider = DemoKnowledgeProvider()
|
||||||
answer = provider.ask(
|
answer = provider.ask(
|
||||||
|
|||||||
@@ -22,4 +22,4 @@ Damage or a technical warning requires a blocked state. A reached service thresh
|
|||||||
|
|
||||||
## 4. Follow-up
|
## 4. Follow-up
|
||||||
|
|
||||||
The return automation may create a cleaning or maintenance follow-up, but MobilityOps remains the source of truth for the vehicle state.
|
The return automation may create a cleaning or maintenance follow-up, but Fleet Ops remains the source of truth for the vehicle state.
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ When a vehicle returns with visible or reported damage, mark damage in the retur
|
|||||||
|
|
||||||
## 2. Evidence
|
## 2. Evidence
|
||||||
|
|
||||||
Record at least four photographs in the source inspection system: full affected side, close view, wider context and vehicle registration. MobilityOps may store only the evidence reference in this PoC.
|
Record at least four photographs in the source inspection system: full affected side, close view, wider context and vehicle registration. Fleet Ops stores only the evidence reference in this demo environment.
|
||||||
|
|
||||||
## 3. Escalation
|
## 3. Escalation
|
||||||
|
|
||||||
|
|||||||
@@ -18,4 +18,4 @@ Inspect windows, lights, tyres and bodywork while cleaning. Report newly observe
|
|||||||
|
|
||||||
## 3. Completion
|
## 3. Completion
|
||||||
|
|
||||||
Cleaning completion alone does not make a blocked or maintenance vehicle available. MobilityOps derives availability from all active restrictions.
|
Cleaning completion alone does not make a blocked or maintenance vehicle available. Fleet Ops derives availability from all active restrictions.
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ The customer identity and valid driving licence must be verified in the source p
|
|||||||
|
|
||||||
## 2. Data minimisation
|
## 2. Data minimisation
|
||||||
|
|
||||||
MobilityOps records only completion status and source references for this PoC. It does not store identity-document images.
|
Fleet Ops records only completion status and source references for this demo environment. It does not store identity-document images.
|
||||||
|
|
||||||
## 3. Missing information
|
## 3. Missing information
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Two reserved or active bookings for the same vehicle may not overlap. Imported l
|
|||||||
|
|
||||||
## 2. Resolution
|
## 2. Resolution
|
||||||
|
|
||||||
Operations reviews both bookings, confirms source-system ownership and reassigns or cancels one booking in the authoritative booking system. MobilityOps records the reviewed outcome.
|
Operations reviews both bookings, confirms source-system ownership and reassigns or cancels one booking in the authoritative booking system. Fleet Ops records the reviewed outcome.
|
||||||
|
|
||||||
## 3. Safety
|
## 3. Safety
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Lorsqu'un véhicule revient avec des dommages visibles ou signalés, marquez le
|
|||||||
|
|
||||||
## 2. Preuves
|
## 2. Preuves
|
||||||
|
|
||||||
Enregistrez au moins quatre photographies dans le système d'inspection source : côté affecté en entier, gros plan, contexte plus large et plaque d'immatriculation. Fleet Ops ne peut stocker que la référence à cette preuve dans ce PoC.
|
Enregistrez au moins quatre photographies dans le système d'inspection source : côté affecté en entier, gros plan, contexte plus large et plaque d'immatriculation. Fleet Ops ne peut stocker que la référence à cette preuve dans cet environnement de démo.
|
||||||
|
|
||||||
## 3. Escalade
|
## 3. Escalade
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ L'identité du client et un permis de conduire valide doivent être vérifiés d
|
|||||||
|
|
||||||
## 2. Minimisation des données
|
## 2. Minimisation des données
|
||||||
|
|
||||||
Fleet Ops n'enregistre dans ce PoC que le statut d'achèvement et les références sources. Aucune image de document d'identité n'est stockée.
|
Fleet Ops n'enregistre dans cet environnement de démo que le statut d'achèvement et les références sources. Aucune image de document d'identité n'est stockée.
|
||||||
|
|
||||||
## 3. Informations manquantes
|
## 3. Informations manquantes
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schad
|
|||||||
|
|
||||||
## 2. Bewijs
|
## 2. Bewijs
|
||||||
|
|
||||||
Registreer minstens vier foto's in het bron-inspectiesysteem: volledige aangetaste zijde, close-up, ruimere context en kentekenplaat. Fleet Ops mag in deze PoC enkel de verwijzing naar dat bewijs opslaan.
|
Registreer minstens vier foto's in het bron-inspectiesysteem: volledige aangetaste zijde, close-up, ruimere context en kentekenplaat. Fleet Ops mag in deze demo-omgeving enkel de verwijzing naar dat bewijs opslaan.
|
||||||
|
|
||||||
## 3. Escalatie
|
## 3. Escalatie
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ De identiteit van de klant en een geldig rijbewijs moeten geverifieerd zijn in h
|
|||||||
|
|
||||||
## 2. Dataminimalisatie
|
## 2. Dataminimalisatie
|
||||||
|
|
||||||
Fleet Ops registreert in deze PoC enkel de afhandelingsstatus en bronverwijzingen. Er worden geen afbeeldingen van identiteitsdocumenten opgeslagen.
|
Fleet Ops registreert in deze demo-omgeving enkel de afhandelingsstatus en bronverwijzingen. Er worden geen afbeeldingen van identiteitsdocumenten opgeslagen.
|
||||||
|
|
||||||
## 3. Ontbrekende informatie
|
## 3. Ontbrekende informatie
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user