Jens/MobilityOps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

M54: harden operations and demo resilience
MobilityOps acceptance / backend (push) Failing after 19s
Details
MobilityOps acceptance / frontend (push) Successful in 25s
Details
MobilityOps acceptance / e2e (push) Skipped
Details

Browse Source
This commit is contained in:
NuklearRabbit
2026-08-24 03:31:03 +02:00
parent b0706989db
commit 81e3fd63bd
101 changed files with 5641 additions and 828 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_knowledge.py
+409 -70
View File
@@ -1,9 +1,12 @@
from __future__ import annotations
import hashlib
import re
import uuid
from pathlib import Path
import httpx
import pytest
from app.api.routers import knowledge as knowledge_router
from app.core.config import get_settings
@@ -244,11 +247,11 @@ def test_ragcore_status_preserves_stronger_verified_index_evidence(client, ops_c
class _FakeResponse:
def __init__(self, status_code: int, body: dict):
def __init__(self, status_code: int, body: object):
self.status_code = status_code
self._body = body
def json(self) -> dict:
def json(self) -> object:
return self._body
@@ -260,6 +263,7 @@ class _FakeClient:
post_responses=None,
raise_on=None,
get_handler=None,
requests=None,
):
self._get_response = get_response
self._post_response = post_response
@@ -270,6 +274,7 @@ class _FakeClient:
self._post_responses = post_responses or {}
self._raise_on = raise_on
self._get_handler = get_handler
self.requests = requests if requests is not None else []
def __enter__(self):
return self
@@ -289,6 +294,7 @@ class _FakeClient:
def post(self, path, json=None):
if self._raise_on == "post":
raise httpx.ConnectError("no ragcore in this environment")
self.requests.append((path, json))
return self._post_responses.get(path, self._post_response)
@@ -393,22 +399,107 @@ def test_ragcore_provider_health_degrades_on_connection_error(monkeypatch):
assert "unavailable" in health.detail.lower()
def _answers_body(**overrides) -> dict:
_EXCERPTS = {
("en-GB", "damage-procedure"): (
"When a vehicle returns with visible or reported damage, mark damage in the "
"return inspection, add a concise factual description and keep the vehicle "
"blocked."
),
("en-GB", "vehicle-return-procedure"): (
"Open the active booking and record the ending odometer, fuel level, cleanliness, "
"visible damage, technical warnings and relevant notes."
),
("en-GB", "cleaning-checklist"): (
"Cleaning completion alone does not make a blocked or maintenance vehicle "
"available. Fleet Ops derives availability from all active restrictions."
),
("en-GB", "maintenance-escalation"): (
"Escalate when a technical warning is reported, the service threshold is reached, "
"a safety-related defect is observed or an existing maintenance block remains "
"unresolved."
),
("nl-BE", "damage-procedure"): (
"Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schade "
"in de retourinspectie, voeg een beknopte feitelijke beschrijving toe en houd het "
"voertuig geblokkeerd."
),
("nl-BE", "vehicle-return-procedure"): (
"Open de actieve boeking en registreer de eindkilometerstand, het brandstofniveau, "
"de netheid, zichtbare schade, technische waarschuwingen en relevante notities."
),
}
def _managed_document(provider: RAGcoreKnowledgeProvider, document_id: str, language: str):
return next(
document
for document in provider._managed_documents_by_source_id.values()
if document.document_id == document_id and document.language == language
)
def _citation(
provider: RAGcoreKnowledgeProvider,
document_id: str,
*,
language: str = "en-GB",
section: str = "Procedure",
seed: str = "1",
) -> dict:
document = _managed_document(provider, document_id, language)
excerpt = _EXCERPTS[(language, document_id)]
return {
"id": str(uuid.uuid5(uuid.NAMESPACE_URL, f"citation:{language}:{document_id}:{seed}")),
# These are deliberately opaque RAGcore-owned UUIDs, not Fleet Ops' stable
# human-readable procedure id.
"document_id": str(
uuid.uuid5(uuid.NAMESPACE_URL, f"ragcore-document:{language}:{document_id}")
),
"document_version_id": str(
uuid.uuid5(
uuid.NAMESPACE_URL,
f"ragcore-document-version:{language}:{document_id}:{seed}",
)
),
"title": "untrusted-provider-title.md",
"section": section,
"excerpt": excerpt,
"excerpt_sha256": hashlib.sha256(excerpt.encode("utf-8")).hexdigest(),
"source_uri": f"ragcore://source/{document.source_id}",
"source_id": document.source_id,
"locator": f"{document.document_id}.md",
}
def _answers_body(provider: RAGcoreKnowledgeProvider, **overrides) -> dict:
citation = _citation(provider, "damage-procedure", section="Detection")
answer = "Report damage and route the vehicle to maintenance."
body = {
"answer": "Report damage and route the vehicle to maintenance.",
"answer_id": str(uuid.uuid4()),
"retrieval_run_id": str(uuid.uuid4()),
"answer": answer,
"answerability": "answerable",
"citations": [
{
"id": "cite-1",
"document_id": "doc-1",
"document_version_id": "version-1",
"title": "Damage handling procedure",
"section": "Detection",
"excerpt": "Inspect the vehicle for visible damage.",
}
],
"citations": [citation],
"claims": [{"text": answer, "citation_ids": [citation["id"]]}],
}
body.update(overrides)
if "claims" not in overrides:
effective_answer = body.get("answer")
effective_citations = body.get("citations")
body["claims"] = (
[
{
"text": effective_answer,
"citation_ids": [item["id"] for item in effective_citations],
}
]
if isinstance(effective_answer, str)
and effective_answer
and isinstance(effective_citations, list)
and effective_citations
and all(isinstance(item, dict) and "id" in item for item in effective_citations)
else []
)
return body
@@ -418,47 +509,178 @@ def test_ragcore_provider_grounded_answer_maps_citations_to_sources(monkeypatch)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body())),
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(provider))),
)
answer = provider.ask("What must I do about damage?", "test-correlation-grounded")
assert answer.evidence_state == "grounded"
assert answer.answer
assert len(answer.sources) == 1
source = answer.sources[0]
assert source.document_id == "doc-1"
assert source.document_id == "damage-procedure"
assert source.title == "Damage handling procedure"
assert source.version == "version-1"
assert source.version == "1.3"
assert source.section == "Detection"
assert source.excerpt
@pytest.mark.parametrize(
"mutate",
[
lambda body: body.pop("claims"),
lambda body: body.update(claims=[]),
lambda body: body["claims"][0].update(citation_ids=[str(uuid.uuid4())]),
lambda body: body.pop("answer_id"),
lambda body: body.update(retrieval_run_id="not-a-uuid"),
],
)
def test_ragcore_answer_requires_valid_claim_citation_contract(monkeypatch, mutate):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
body = _answers_body(provider)
mutate(body)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, body)),
)
answer = provider.ask("What must I do about damage?", "test-invalid-claim-contract")
assert answer.evidence_state != "grounded"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_requests_are_scoped_to_managed_sources_for_requested_language(
monkeypatch,
):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
requests = []
fake_client = _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
},
requests=requests,
)
monkeypatch.setattr(provider, "_client", lambda: fake_client)
answer = provider.ask(
"Wat is de procedure voor een voertuigretour?",
"language-filter",
"nl-BE",
)
assert answer.evidence_state == "grounded"
assert [path for path, _payload in requests] == ["/v1/answers", "/v1/search"]
expected_source_ids = provider._managed_source_ids("nl-BE")
assert len(expected_source_ids) == 11
assert all(uuid.UUID(source_id) for source_id in expected_source_ids)
for _path, payload in requests:
assert payload["filters"] == {"source_ids": expected_source_ids}
assert payload["requested_space_ids"] == ["space-1"]
def test_ragcore_sources_deduplicate_reuploaded_versions_and_cap_cards(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
citations = []
for index in range(5):
document_ids = [
"damage-procedure",
"damage-procedure",
"vehicle-return-procedure",
"cleaning-checklist",
"maintenance-escalation",
]
for index, document_id in enumerate(document_ids):
citations.append(
{
"id": f"cite-{index}",
"document_id": f"doc-{index}",
"document_version_id": f"version-{index}",
"title": "Damage procedure" if index < 2 else f"Procedure {index}",
"section": "Return",
"excerpt": (
f"Record visible damage before release, chunk {index}."
if index < 2
else f"Unique procedure evidence {index}."
),
}
_citation(
provider,
document_id,
section="Return",
seed=str(index),
)
)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(citations=citations))),
lambda: _FakeClient(
post_response=_FakeResponse(200, _answers_body(provider, citations=citations))
),
)
answer = provider.ask("What must I do about vehicle damage?", "dedupe-test")
assert len(answer.sources) == 3
assert sum(source.title == "Damage procedure" for source in answer.sources) == 1
assert sum(source.title == "Damage handling procedure" for source in answer.sources) == 1
def test_ragcore_answer_rejects_unknown_document_citation(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200,
_answers_body(
provider,
answer="A plausible-looking but unmanaged answer.",
citations=[
{
**_citation(provider, "damage-procedure"),
"source_id": str(uuid.uuid4()),
}
],
),
)
),
)
answer = provider.ask("What must I do about damage?", "unknown-answer-citation")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
@pytest.mark.parametrize(
("field_name", "bad_value"),
[
("id", "not-a-uuid"),
("document_id", "not-a-uuid"),
("document_version_id", "00000000-0000-0000-0000-000000000000"),
("source_id", "00000000-0000-4000-8000-000000000099"),
("source_uri", "ragcore://source/00000000-0000-4000-8000-000000000099"),
("locator", "another-procedure.md"),
("excerpt_sha256", "0" * 64),
("title", None),
("section", {"not": "a string"}),
],
)
def test_ragcore_rejects_broken_managed_source_provenance(field_name, bad_value):
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure")
citation[field_name] = bad_value
assert provider._managed_source(citation, "en-GB") is None
def test_ragcore_rejects_fabricated_excerpt_even_with_matching_hash():
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure")
fabricated = "Invent a repair price and promise it to the customer."
citation["excerpt"] = fabricated
citation["excerpt_sha256"] = hashlib.sha256(fabricated.encode("utf-8")).hexdigest()
assert provider._managed_source(citation, "en-GB") is None
def test_ragcore_never_relabels_an_english_source_as_dutch():
provider = RAGcoreKnowledgeProvider()
citation = _citation(provider, "damage-procedure", language="en-GB")
assert provider._managed_source(citation, "nl-BE") is None
def test_knowledge_feedback_is_audited_and_can_be_changed(ops_client):
@@ -503,6 +725,7 @@ def test_ragcore_provider_not_answerable_is_insufficient_and_never_fabricates(mo
post_response=_FakeResponse(
200,
_answers_body(
provider,
answer="This should never be shown.",
answerability="not_answerable",
citations=[],
@@ -524,7 +747,8 @@ def test_ragcore_provider_answerable_without_citations_is_insufficient(monkeypat
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200, _answers_body(answerability="answerable", citations=[])
200,
_answers_body(provider, answerability="answerable", citations=[]),
)
),
)
@@ -565,19 +789,66 @@ def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch):
assert answer.evidence_state == "unavailable"
def _search_body(**overrides) -> dict:
def test_ragcore_provider_malformed_top_level_bodies_do_not_escape(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(200, ["not", "an", "object"]),
"/v1/search": _FakeResponse(200, None),
}
),
)
answer = provider.ask("Anything?", "test-correlation-malformed-top-level")
assert answer.evidence_state == "unavailable"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_answer_with_malformed_citation_is_insufficient(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_response=_FakeResponse(
200,
_answers_body(provider, citations=[None]),
)
),
)
answer = provider.ask("What must I do about damage?", "malformed-answer-citation")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def _search_body(
provider: RAGcoreKnowledgeProvider,
language: str = "en-GB",
**overrides,
) -> dict:
body = {
"retrieval_run_id": str(uuid.uuid4()),
"effective_space_ids": ["00000000-0000-4000-8000-000000000001"],
"results": [
{
"chunk_id": "chunk-1",
"citation": {
"id": "cite-1",
"document_id": "doc-1",
"document_version_id": "version-1",
"title": "Vehicle return procedure",
"section": "Return",
"excerpt": "Register the return odometer reading before releasing the vehicle.",
},
"chunk_id": str(uuid.uuid4()),
"text": _EXCERPTS[(language, "vehicle-return-procedure")],
"citation": _citation(
provider,
"vehicle-return-procedure",
language=language,
section="Return",
),
"rank": 1,
"scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None},
}
@@ -588,6 +859,31 @@ def _search_body(**overrides) -> dict:
return body
def test_ragcore_search_ignores_malformed_result_and_citation_bodies(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
malformed_search = _search_body(
provider,
results=[None, {"citation": ["not-an-object"], "scores": "not-an-object"}],
)
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, malformed_search),
}
),
)
answer = provider.ask("What is the vehicle return procedure?", "malformed-search-items")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch):
"""/v1/answers itself failing (a real RAGcore-side outage in its generation step,
not a real 'insufficient evidence' classification) must not silently degrade
@@ -601,26 +897,24 @@ def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypa
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider)),
}
),
)
answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback")
assert answer.evidence_state == "grounded"
assert "Register the return odometer reading" in answer.answer
assert "Open the active booking and record the ending odometer" in answer.answer
assert "Vehicle return procedure" in answer.answer
assert len(answer.sources) == 1
assert answer.sources[0].title == "Vehicle return procedure"
assert answer.sources[0].excerpt == (
"Register the return odometer reading before releasing the vehicle."
)
assert answer.sources[0].excerpt == _EXCERPTS[("en-GB", "vehicle-return-procedure")]
def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0)
search_response = _FakeResponse(200, _search_body())
search_response = _FakeResponse(200, _search_body(provider))
monkeypatch.setattr(
provider,
"_client",
@@ -632,8 +926,7 @@ def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
),
)
assert (
provider.ask("What is the vehicle return procedure?", "first").evidence_state
== "grounded"
provider.ask("What is the vehicle return procedure?", "first").evidence_state == "grounded"
)
monkeypatch.setattr(
@@ -654,7 +947,7 @@ def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
}
),
)
@@ -676,7 +969,7 @@ def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkey
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body(results=[])),
"/v1/search": _FakeResponse(200, _search_body(provider, results=[])),
}
),
)
@@ -695,7 +988,7 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, _search_body()),
"/v1/search": _FakeResponse(200, _search_body(provider)),
}
),
)
@@ -707,19 +1000,66 @@ def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
assert answer.answer == ""
def test_ragcore_search_fallback_rejects_unknown_document_citation(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body(provider)
search["results"][0]["citation"]["source_id"] = str(uuid.uuid4())
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, search),
}
),
)
answer = provider.ask(
"What is the vehicle return procedure?",
"unknown-search-citation",
)
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []
def test_ragcore_search_fallback_rejects_vehicle_colour_despite_high_score(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body(provider)
search["results"][0]["scores"]["fused"] = 1.0
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {}),
"/v1/search": _FakeResponse(200, search),
}
),
)
answer = provider.ask("What colour is this vehicle?", "vehicle-colour")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body()
search = _search_body(provider, "nl-BE", results=[])
search["results"].append(
{
"citation": {
"document_id": "damage-procedure",
"document_version_id": "version-2",
"title": "damage-procedure.md",
"section": "Damage",
"excerpt": "Record damage and keep the vehicle blocked.",
},
"citation": _citation(
provider,
"damage-procedure",
language="nl-BE",
section="Damage",
),
"rank": 2,
"scores": {"fused": 0.01},
}
@@ -743,16 +1083,15 @@ def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
def test_ragcore_search_fallback_accepts_top_ranked_ragcore_damage_evidence(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
search = _search_body()
search = _search_body(provider, "nl-BE", results=[])
search["results"].append(
{
"citation": {
"document_id": "damage-procedure",
"document_version_id": "version-2",
"title": "damage-procedure.md",
"section": "Damage",
"excerpt": "Record damage and keep the vehicle blocked.",
},
"citation": _citation(
provider,
"damage-procedure",
language="nl-BE",
section="Damage",
),
"rank": 1,
# RAGcore uses reciprocal-rank fusion; a genuine rank-one result is about
# 1 / (60 + 1), not a normalized 0..1 relevance score.