fix: knowledge retrieval accuracy and remaining brand/PoC leaks in procedure docs
- Fix the demo knowledge provider's tokenizer: a plain [a-z0-9]+ regex silently dropped accented characters, splitting French words like "véhicule" into "v" + "hicule" and mangling retrieval for nearly every French query. Now matches the Latin-1 accented range too. - Reweight section scoring so the body match (the actual substance of a section) outranks a heading/title match (a shallow structural hint) rather than the reverse -- confirmed via the brief's exact validation question that the old weighting misranked the damage procedure behind a topically-adjacent document in all three languages (nl-BE: a checkout section; en-GB/fr-BE: the return procedure), purely because a generic word like "vehicle"/"voertuig" happened to sit in a heading/title. - Remove leftover "MobilityOps" and "PoC" mentions from 5 English and 4 NL/FR procedure documents -- knowledge-base prose is visible UI content and was missed by the earlier rebrand. - Add regression tests: the brief's exact NL/EN/FR damage question must ground on the damage procedure as the *primary* source (not just appear in the top 3), and no procedure file may contain "MobilityOps" or "PoC". 151 backend tests, Ruff, mypy green. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
6deb95524d
commit
e6539d17b6
@@ -1,11 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.services.knowledge.demo import DemoKnowledgeProvider
|
||||
from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
|
||||
|
||||
|
||||
def test_brief_exact_damage_question_in_all_three_languages():
|
||||
# The exact validation questions from docs/fleet-ops-correction/current-gap-audit.md
|
||||
# -- each must ground on the damage procedure as its *primary* (top-ranked) source,
|
||||
# not merely appear somewhere in the top-3, and the source/version/section/excerpt
|
||||
# must all come from that same-language document (never an English fallback).
|
||||
provider = DemoKnowledgeProvider()
|
||||
cases = {
|
||||
"nl-BE": "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?",
|
||||
"en-GB": "What should I do when a vehicle returns with damage?",
|
||||
"fr-BE": "Que dois-je faire lorsqu'un véhicule revient endommagé ?",
|
||||
}
|
||||
for language, question in cases.items():
|
||||
answer = provider.ask(question, f"test-brief-{language}", language)
|
||||
assert answer.evidence_state == "grounded", language
|
||||
assert answer.sources, language
|
||||
assert answer.sources[0].document_id == "damage-procedure", (
|
||||
f"{language}: expected the damage procedure as the primary source, "
|
||||
f"got {answer.sources[0].document_id!r}"
|
||||
)
|
||||
assert answer.answer
|
||||
assert answer.sources[0].excerpt
|
||||
|
||||
|
||||
def test_knowledge_procedures_never_mention_mobilityops_or_poc():
|
||||
# Section 2 of docs/fleet-ops-correction/current-gap-audit.md: the visible brand
|
||||
# name is exactly "Fleet Ops", and "PoC" must never appear in visible content --
|
||||
# including the demo knowledge base, not just the frontend.
|
||||
procedures_dir = Path(get_settings().knowledge_dir)
|
||||
offenders = []
|
||||
for path in sorted(procedures_dir.glob("*/*.md")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if "MobilityOps" in text or re.search(r"\bPoC\b", text):
|
||||
offenders.append(str(path))
|
||||
assert offenders == []
|
||||
|
||||
|
||||
def test_s6_damage_question_is_grounded_with_expected_sources():
|
||||
provider = DemoKnowledgeProvider()
|
||||
answer = provider.ask(
|
||||
|
||||
Reference in New Issue
Block a user