Jens/MobilityOps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

M54: harden operations and demo resilience
MobilityOps acceptance / backend (push) Failing after 19s
Details
MobilityOps acceptance / frontend (push) Successful in 25s
Details
MobilityOps acceptance / e2e (push) Skipped
Details

Browse Source
This commit is contained in:
NuklearRabbit
2026-08-24 03:31:03 +02:00
parent b0706989db
commit 81e3fd63bd
101 changed files with 5641 additions and 828 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/knowledge/ragcore.py
+167 -45
View File
@@ -1,5 +1,8 @@
from __future__ import annotations
import hashlib
import re
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from threading import Lock
@@ -13,6 +16,10 @@ from app.services.knowledge import EvidenceState, GroundedAnswer, KnowledgeHealt
from app.services.knowledge.procedures import ProcedureDocument, iter_procedure_documents
_GROUNDED_ANSWERABILITY = {"answerable", "partially_answerable"}
_ANSWERABILITY_STATES = _GROUNDED_ANSWERABILITY | {
"not_answerable",
"conflicting_evidence",
}
# Mirrors DemoKnowledgeProvider's own extractive template in spirit: a real cited
# excerpt wrapped in a fixed sentence, never a generated summary. Used only as a
@@ -47,6 +54,11 @@ _DOMAIN_CONCEPTS: dict[str, tuple[str, ...]] = {
"availability": ("available", "availability", "beschikbaar", "disponible", "disponibilité"),
"technical": ("technical", "warning", "technisch", "waarschuwing", "technique", "alerte"),
}
_ANSWERABLE_INTENT_CONCEPTS = frozenset(_DOMAIN_CONCEPTS) - {"vehicle", "customer"}
def _normalize_evidence_text(value: str) -> str:
return " ".join(re.findall(r"\w+", value.casefold()))
def _question_concepts(question: str) -> set[str]:
@@ -93,7 +105,9 @@ def _rank_sources_for_concepts(sources: list[SourceCard], concepts: set[str]) ->
)
def _retrieval_score(result: dict) -> float | None:
def _retrieval_score(result: object) -> float | None:
if not isinstance(result, dict):
return None
scores = result.get("scores")
if not isinstance(scores, dict):
return None
@@ -124,23 +138,81 @@ class RAGcoreKnowledgeProvider:
excerpt RAGcore's own search actually found, wrapped in the same fixed citation
template `DemoKnowledgeProvider` uses -- never a fabricated summary.
Known gap, not fixable from this side: RAGcore's ingest pipeline currently tags every
chunk's `language` payload field as `"en"` regardless of actual document language (the
`/v1/uploads` contract has no per-file language field for a caller to set correctly).
Filtering search/answer requests by requested UI language would therefore silently
exclude genuinely-relevant nl-BE/fr-BE content, so this adapter deliberately does not
filter by language -- retrieval relies on the embedding model's cross-lingual matching.
Retrieval is scoped to the stable RAGcore ``source_id`` values owned by Fleet Ops for
the requested UI language. Returned internal document/version UUIDs are deliberately
not treated as Fleet Ops identifiers: every citation must instead prove the complete
managed-source chain (source id, URI, locator, checksum and extractive local text).
"""
name = "ragcore"
def __init__(self) -> None:
self._settings = get_settings()
self._managed_documents_by_source_id: dict[str, ProcedureDocument] = {}
for document in iter_procedure_documents(Path(self._settings.knowledge_dir)):
self._managed_documents_by_source_id[document.source_id] = document
self._verification_cache: dict[str, tuple[float, int]] = {}
self._verification_lock = Lock()
self._answers_circuit_lock = Lock()
self._answers_circuit_open_until = 0.0
def _managed_source(self, citation: object, language: str) -> SourceCard | None:
if not isinstance(citation, dict):
return None
source_id = citation.get("source_id")
if not isinstance(source_id, str):
return None
document = self._managed_documents_by_source_id.get(str(source_id))
if document is None or document.language != language:
return None
if citation.get("locator") != f"{document.document_id}.md":
return None
if citation.get("source_uri") != f"ragcore://source/{document.source_id}":
return None
for field_name in ("id", "document_id", "document_version_id"):
value = citation.get(field_name)
if not isinstance(value, str):
return None
try:
parsed = uuid.UUID(value)
except (ValueError, TypeError, AttributeError):
return None
if parsed.int == 0:
return None
if not isinstance(citation.get("title"), str):
return None
section = citation.get("section")
if section is not None and not isinstance(section, str):
return None
excerpt = citation.get("excerpt")
if not isinstance(excerpt, str) or not excerpt.strip():
return None
expected_hash = hashlib.sha256(excerpt.encode("utf-8")).hexdigest()
if citation.get("excerpt_sha256") != expected_hash:
return None
# RAGcore owns chunking, but the cited excerpt must still be extractive evidence
# from the authoritative local procedure. Token normalization tolerates Markdown
# punctuation/whitespace while rejecting provider text that was never uploaded.
normalized_excerpt = _normalize_evidence_text(excerpt)
if not normalized_excerpt or normalized_excerpt not in _normalize_evidence_text(
document.content
):
return None
return SourceCard(
document_id=document.document_id,
title=document.title,
version=document.version,
section=section or "",
excerpt=excerpt,
)
def _managed_source_ids(self, language: str) -> list[str]:
return [
document.source_id
for document in self._managed_documents_by_source_id.values()
if document.language == language
]
def _answers_circuit_is_open(self) -> bool:
with self._answers_circuit_lock:
return monotonic() < self._answers_circuit_open_until
@@ -284,7 +356,7 @@ class RAGcoreKnowledgeProvider:
if self._answers_circuit_is_open():
KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "circuit_open").inc()
else:
answered = self._ask_via_answers(question, correlation_id)
answered = self._ask_via_answers(question, correlation_id, language)
if answered is not None:
return answered
# /v1/answers itself is unavailable (non-2xx or unreachable) -- fall back to
@@ -295,7 +367,9 @@ class RAGcoreKnowledgeProvider:
# generation step.
return self._ask_via_search_fallback(question, correlation_id, language)
def _ask_via_answers(self, question: str, correlation_id: str) -> GroundedAnswer | None:
def _ask_via_answers(
self, question: str, correlation_id: str, language: str
) -> GroundedAnswer | None:
"""Returns None (not a GroundedAnswer) when /v1/answers itself is unavailable,
so the caller can fall back to search -- as opposed to a real 200 response
classifying the question as insufficiently answerable, which is a genuine,
@@ -307,6 +381,7 @@ class RAGcoreKnowledgeProvider:
json={
"query": question,
"requested_space_ids": [self._settings.ragcore_space_id],
"filters": {"source_ids": self._managed_source_ids(language)},
},
)
if response.status_code != 200:
@@ -320,25 +395,73 @@ class RAGcoreKnowledgeProvider:
return None
try:
citations = {c["id"]: c for c in body.get("citations", [])}
sources = [
SourceCard(
document_id=str(citation["document_id"]),
title=citation["title"],
version=str(citation["document_version_id"]),
section=citation.get("section") or "",
excerpt=citation["excerpt"],
)
for citation in citations.values()
if not isinstance(body, dict):
raise TypeError("answer response must be an object")
# Validate the provider's complete AnswerResponse contract before trusting
# generated prose. These IDs and claim bindings are the evidence that RAGcore
# ran its deterministic claim/citation validator; one unrelated but otherwise
# valid citation must never make arbitrary answer text appear grounded.
uuid.UUID(str(body["answer_id"]))
uuid.UUID(str(body["retrieval_run_id"]))
raw_citations = body.get("citations", [])
if not isinstance(raw_citations, list):
raise TypeError("citations must be a list")
citation_ids: set[uuid.UUID] = set()
for citation in raw_citations:
if not isinstance(citation, dict):
raise TypeError("citation must be an object")
citation_ids.add(uuid.UUID(str(citation["id"])))
raw_claims = body["claims"]
if not isinstance(raw_claims, list):
raise TypeError("claims must be a list")
claims_are_bound = bool(raw_claims)
answer_text = body.get("answer")
for claim in raw_claims:
if not isinstance(claim, dict):
raise TypeError("claim must be an object")
claim_text = claim.get("text")
claim_citation_ids = claim.get("citation_ids")
if (
not isinstance(claim_text, str)
or not claim_text.strip()
or not isinstance(answer_text, str)
or claim_text.strip() not in answer_text
or not isinstance(claim_citation_ids, list)
or not claim_citation_ids
):
claims_are_bound = False
continue
try:
bound_ids = {uuid.UUID(str(item)) for item in claim_citation_ids}
except (TypeError, ValueError):
claims_are_bound = False
continue
if not bound_ids.issubset(citation_ids):
claims_are_bound = False
mapped_sources = [
source
for citation in raw_citations
if (source := self._managed_source(citation, language)) is not None
]
sources = _deduplicate_sources(sources)
citations_are_managed = len(mapped_sources) == len(raw_citations)
sources = _deduplicate_sources(mapped_sources)
answerability = body.get("answerability", "not_answerable")
is_grounded = answerability in _GROUNDED_ANSWERABILITY and sources
if answerability not in _ANSWERABILITY_STATES:
raise TypeError("answerability is invalid")
if not isinstance(answer_text, str):
raise TypeError("answer must be a string")
is_grounded = (
answerability in _GROUNDED_ANSWERABILITY
and bool(sources)
and citations_are_managed
and claims_are_bound
and bool(answer_text.strip())
)
evidence_state: EvidenceState = "grounded" if is_grounded else "insufficient"
KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", evidence_state).inc()
self._close_answers_circuit()
return GroundedAnswer(
answer=body.get("answer", "") if evidence_state == "grounded" else "",
answer=answer_text if evidence_state == "grounded" else "",
evidence_state=evidence_state,
sources=sources if evidence_state == "grounded" else [],
provider=self.name,
@@ -366,6 +489,7 @@ class RAGcoreKnowledgeProvider:
json={
"query": question,
"requested_space_ids": [self._settings.ragcore_space_id],
"filters": {"source_ids": self._managed_source_ids(language)},
"max_results": 5,
},
)
@@ -378,24 +502,24 @@ class RAGcoreKnowledgeProvider:
return unavailable
try:
if not isinstance(body, dict):
raise TypeError("search response must be an object")
results = body.get("results", [])
if not isinstance(results, list):
raise TypeError("results must be a list")
sources = [
SourceCard(
document_id=str(result["citation"]["document_id"]),
title=result["citation"]["title"],
version=str(result["citation"]["document_version_id"]),
section=result["citation"].get("section") or "",
excerpt=result["citation"]["excerpt"],
)
source
for result in results
if isinstance(result, dict)
and (source := self._managed_source(result.get("citation"), language)) is not None
]
except (TypeError, KeyError, ValueError):
KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "malformed").inc()
return unavailable
for result in results:
if not isinstance(result, dict):
continue
score = _retrieval_score(result)
if score is not None:
KNOWLEDGE_RETRIEVAL_SCORE.observe(score)
@@ -403,15 +527,11 @@ class RAGcoreKnowledgeProvider:
all_sources = _deduplicate_sources(sources)
concepts = _question_concepts(question)
qualified_sources = [
SourceCard(
document_id=str(result["citation"]["document_id"]),
title=result["citation"]["title"],
version=str(result["citation"]["document_version_id"]),
section=result["citation"].get("section") or "",
excerpt=result["citation"]["excerpt"],
)
source
for result in results
if (_retrieval_score(result) or 0.0) >= self._settings.ragcore_min_search_score
if isinstance(result, dict)
and (_retrieval_score(result) or 0.0) >= self._settings.ragcore_min_search_score
and (source := self._managed_source(result.get("citation"), language)) is not None
]
sources = _rank_sources_for_concepts(_deduplicate_sources(qualified_sources), concepts)
if not all_sources:
@@ -424,15 +544,17 @@ class RAGcoreKnowledgeProvider:
correlation_id=correlation_id,
)
damage_evidence = any(
term
in (
f"{source.document_id} {source.title} {source.section} {source.excerpt}"
).casefold()
required_concepts = concepts & _ANSWERABLE_INTENT_CONCEPTS
evidence_text = " ".join(
f"{source.document_id} {source.title} {source.section} {source.excerpt}"
for source in sources
for term in _DOMAIN_CONCEPTS["damage"]
)
if not concepts or not sources or ("damage" in concepts and not damage_evidence):
).casefold()
covered_concepts = {
concept
for concept in required_concepts
if any(term in evidence_text for term in _DOMAIN_CONCEPTS[concept])
}
if not required_concepts or not sources or covered_concepts != required_concepts:
KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "insufficient").inc()
return GroundedAnswer(
answer="",