fix: knowledge retrieval accuracy and remaining brand/PoC leaks in procedure docs

- Fix the demo knowledge provider's tokenizer: a plain [a-z0-9]+ regex silently
  dropped accented characters, splitting French words like "véhicule" into "v" +
  "hicule" and mangling retrieval for nearly every French query. Now matches the
  Latin-1 accented range too.
- Reweight section scoring so the body match (the actual substance of a section)
  outranks a heading/title match (a shallow structural hint) rather than the reverse
  -- confirmed via the brief's exact validation question that the old weighting
  misranked the damage procedure behind a topically-adjacent document in all three
  languages (nl-BE: a checkout section; en-GB/fr-BE: the return procedure), purely
  because a generic word like "vehicle"/"voertuig" happened to sit in a heading/title.
- Remove leftover "MobilityOps" and "PoC" mentions from 5 English and 4 NL/FR
  procedure documents -- knowledge-base prose is visible UI content and was missed by
  the earlier rebrand.
- Add regression tests: the brief's exact NL/EN/FR damage question must ground on the
  damage procedure as the *primary* source (not just appear in the top 3), and no
  procedure file may contain "MobilityOps" or "PoC".

151 backend tests, Ruff, mypy green.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
NuklearRabbit
2026-08-03 21:54:15 +02:00
co-authored by Claude Sonnet 5
parent 6deb95524d
commit e6539d17b6
11 changed files with 69 additions and 14 deletions
+20 -5
View File
@@ -35,7 +35,11 @@ STOPWORDS_BY_LANGUAGE: dict[str, set[str]] = {
},
}
_WORD_RE = re.compile(r"[a-z0-9]+")
# Includes the Latin-1 accented-letter range (à-ö, ø-ÿ) so French/Dutch words with
# diacritics (véhicule, réservation, geëscaleerd) tokenize as one word instead of
# splitting apart at the accented character -- a plain [a-z0-9]+ pattern silently
# drops every accent and fragments the word either side of it.
_WORD_RE = re.compile(r"[a-zà-öø-ÿ0-9]+")
def _stem(word: str) -> str:
@@ -218,17 +222,28 @@ class DemoKnowledgeProvider:
def _score(
self, query_tokens: set[str], section: ScoredSection, idf: dict[str, float]
) -> float:
# The section body is the strongest relevance signal -- it's the actual
# substance a heading or title can only hint at -- so a body match is weighted
# *above* heading/title matches, not below them. The previous 3x/2x/1x
# (heading/title/body) ordering let a single generic word in a heading (e.g.
# "vehicle", present in nearly every section) or a document's own title
# outrank a section whose body genuinely covers multiple, more distinctive
# query terms -- confirmed to misrank the brief's exact validation question in
# every one of the three languages (see docs/fleet-ops-correction/
# current-gap-audit.md and i18n-inventory.md): nl-BE picked a checkout section
# over the damage procedure, en-GB and fr-BE picked the return procedure over
# the damage procedure, purely from heading/title overlap on common words.
score = 0.0
for token in query_tokens:
token_idf = idf.get(token, 0.0)
if token_idf == 0.0:
continue
if token in section.heading_tokens:
if token in section.body_tokens:
score += 3 * token_idf
elif token in section.document.title_tokens:
elif token in section.heading_tokens:
score += 2 * token_idf
elif token in section.body_tokens:
score += token_idf
elif token in section.document.title_tokens:
score += 1.5 * token_idf
return score
def ask(
+40
View File
@@ -1,11 +1,51 @@
from __future__ import annotations
import re
from pathlib import Path
import httpx
from app.core.config import get_settings
from app.services.knowledge.demo import DemoKnowledgeProvider
from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
def test_brief_exact_damage_question_in_all_three_languages():
# The exact validation questions from docs/fleet-ops-correction/current-gap-audit.md
# -- each must ground on the damage procedure as its *primary* (top-ranked) source,
# not merely appear somewhere in the top-3, and the source/version/section/excerpt
# must all come from that same-language document (never an English fallback).
provider = DemoKnowledgeProvider()
cases = {
"nl-BE": "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?",
"en-GB": "What should I do when a vehicle returns with damage?",
"fr-BE": "Que dois-je faire lorsqu'un véhicule revient endommagé ?",
}
for language, question in cases.items():
answer = provider.ask(question, f"test-brief-{language}", language)
assert answer.evidence_state == "grounded", language
assert answer.sources, language
assert answer.sources[0].document_id == "damage-procedure", (
f"{language}: expected the damage procedure as the primary source, "
f"got {answer.sources[0].document_id!r}"
)
assert answer.answer
assert answer.sources[0].excerpt
def test_knowledge_procedures_never_mention_mobilityops_or_poc():
# Section 2 of docs/fleet-ops-correction/current-gap-audit.md: the visible brand
# name is exactly "Fleet Ops", and "PoC" must never appear in visible content --
# including the demo knowledge base, not just the frontend.
procedures_dir = Path(get_settings().knowledge_dir)
offenders = []
for path in sorted(procedures_dir.glob("*/*.md")):
text = path.read_text(encoding="utf-8")
if "MobilityOps" in text or re.search(r"\bPoC\b", text):
offenders.append(str(path))
assert offenders == []
def test_s6_damage_question_is_grounded_with_expected_sources():
provider = DemoKnowledgeProvider()
answer = provider.ask(