1114 lines
40 KiB
Python
1114 lines
40 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import re
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from app.api.routers import knowledge as knowledge_router
|
|
from app.core.config import get_settings
|
|
from app.core.ratelimit import SlidingWindowLimiter
|
|
from app.services.knowledge import KnowledgeHealth
|
|
from app.services.knowledge.demo import DemoKnowledgeProvider
|
|
from app.services.knowledge.procedures import iter_procedure_documents
|
|
from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
|
|
|
|
|
|
def test_brief_exact_damage_question_in_all_three_languages():
|
|
# The exact validation questions from docs/fleet-ops-correction/current-gap-audit.md
|
|
# -- each must ground on the damage procedure as its *primary* (top-ranked) source,
|
|
# not merely appear somewhere in the top-3, and the source/version/section/excerpt
|
|
# must all come from that same-language document (never an English fallback).
|
|
provider = DemoKnowledgeProvider()
|
|
cases = {
|
|
"nl-BE": "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?",
|
|
"en-GB": "What should I do when a vehicle returns with damage?",
|
|
"fr-BE": "Que dois-je faire lorsqu'un véhicule revient endommagé ?",
|
|
}
|
|
for language, question in cases.items():
|
|
answer = provider.ask(question, f"test-brief-{language}", language)
|
|
assert answer.evidence_state == "grounded", language
|
|
assert answer.sources, language
|
|
assert answer.sources[0].document_id == "damage-procedure", (
|
|
f"{language}: expected the damage procedure as the primary source, "
|
|
f"got {answer.sources[0].document_id!r}"
|
|
)
|
|
assert answer.answer
|
|
assert answer.sources[0].excerpt
|
|
|
|
|
|
def test_knowledge_procedures_never_mention_mobilityops_or_poc():
|
|
# Section 2 of docs/fleet-ops-correction/current-gap-audit.md: the visible brand
|
|
# name is exactly "Fleet Ops", and "PoC" must never appear in visible content --
|
|
# including the demo knowledge base, not just the frontend.
|
|
procedures_dir = Path(get_settings().knowledge_dir)
|
|
offenders = []
|
|
for path in sorted(procedures_dir.glob("*/*.md")):
|
|
text = path.read_text(encoding="utf-8")
|
|
if "MobilityOps" in text or re.search(r"\bPoC\b", text):
|
|
offenders.append(str(path))
|
|
assert offenders == []
|
|
|
|
|
|
def test_s6_damage_question_is_grounded_with_expected_sources():
|
|
provider = DemoKnowledgeProvider()
|
|
answer = provider.ask(
|
|
"What must I do when a vehicle returns with damage?", "test-correlation-1"
|
|
)
|
|
assert answer.evidence_state == "grounded"
|
|
document_ids = {s.document_id for s in answer.sources}
|
|
assert "damage-procedure" in document_ids
|
|
assert "vehicle-return-procedure" in document_ids
|
|
assert answer.answer # never empty for a grounded answer
|
|
for source in answer.sources:
|
|
assert source.excerpt
|
|
|
|
|
|
def test_unrelated_question_is_insufficient():
|
|
provider = DemoKnowledgeProvider()
|
|
answer = provider.ask("What is the capital of France?", "test-correlation-2")
|
|
assert answer.evidence_state == "insufficient"
|
|
assert "France" not in answer.answer # never fabricates an answer beyond the procedures
|
|
|
|
|
|
def test_demo_provider_health_reports_document_count():
|
|
provider = DemoKnowledgeProvider()
|
|
health = provider.health()
|
|
assert health.provider == "demo"
|
|
assert health.available is True
|
|
assert health.document_count == 11
|
|
assert health.source_document_count == 11
|
|
assert health.statistics_state == "verified"
|
|
|
|
|
|
def test_demo_provider_health_reports_document_count_per_language():
|
|
provider = DemoKnowledgeProvider()
|
|
for language in ("nl-BE", "en-GB", "fr-BE"):
|
|
assert provider.health(language).document_count == 11
|
|
|
|
|
|
def test_demo_provider_grounds_damage_question_in_dutch():
|
|
provider = DemoKnowledgeProvider()
|
|
answer = provider.ask(
|
|
"Wat moet ik doen als een voertuig terugkomt met schade?",
|
|
"test-correlation-nl",
|
|
"nl-BE",
|
|
)
|
|
assert answer.evidence_state == "grounded"
|
|
document_ids = {s.document_id for s in answer.sources}
|
|
assert "damage-procedure" in document_ids
|
|
|
|
|
|
def test_demo_provider_grounds_damage_question_in_french():
|
|
provider = DemoKnowledgeProvider()
|
|
answer = provider.ask(
|
|
"Que dois-je faire quand un véhicule revient avec des dommages ?",
|
|
"test-correlation-fr",
|
|
"fr-BE",
|
|
)
|
|
assert answer.evidence_state == "grounded"
|
|
document_ids = {s.document_id for s in answer.sources}
|
|
assert "damage-procedure" in document_ids
|
|
|
|
|
|
def test_demo_provider_insufficient_evidence_message_is_localized():
|
|
provider = DemoKnowledgeProvider()
|
|
nl_answer = provider.ask("Wat is de hoofdstad van Frankrijk?", "test-correlation-nl-2", "nl-BE")
|
|
fr_answer = provider.ask(
|
|
"Quelle est la capitale de la France ?", "test-correlation-fr-2", "fr-BE"
|
|
)
|
|
assert nl_answer.evidence_state == "insufficient"
|
|
assert fr_answer.evidence_state == "insufficient"
|
|
assert nl_answer.answer != fr_answer.answer
|
|
assert "France" not in nl_answer.answer
|
|
assert "France" not in fr_answer.answer
|
|
|
|
|
|
def test_ask_question_endpoint_grounded(ops_client):
|
|
response = ops_client.post(
|
|
"/api/v1/knowledge/questions",
|
|
json={"question": "What must I do when a vehicle returns with damage?"},
|
|
)
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert body["evidence_state"] == "grounded"
|
|
assert body["provider"] == "demo"
|
|
assert len(body["sources"]) > 0
|
|
|
|
|
|
def test_ask_question_requires_authentication(client):
|
|
response = client.post("/api/v1/knowledge/questions", json={"question": "Anything?"})
|
|
assert response.status_code == 401
|
|
|
|
|
|
def test_knowledge_questions_are_bounded_per_session_and_client(ops_client, monkeypatch):
|
|
limiter = SlidingWindowLimiter(max_requests=1, window_seconds=60)
|
|
monkeypatch.setattr(knowledge_router, "_question_limiter", limiter)
|
|
body = {"question": "What is the vehicle return procedure?"}
|
|
assert ops_client.post("/api/v1/knowledge/questions", json=body).status_code == 200
|
|
blocked = ops_client.post("/api/v1/knowledge/questions", json=body)
|
|
assert blocked.status_code == 429
|
|
assert int(blocked.headers["retry-after"]) >= 1
|
|
|
|
|
|
def test_ask_question_is_audited_without_leaking_full_text(ops_client):
|
|
ops_client.post(
|
|
"/api/v1/knowledge/questions",
|
|
json={"question": "What must I do when a vehicle returns with damage?"},
|
|
)
|
|
events = ops_client.get("/api/v1/audit", params={"action": "knowledge_question_asked"}).json()
|
|
assert len(events) >= 1
|
|
metadata = events[0]["metadata"]
|
|
assert "evidence_state" in metadata
|
|
assert "source_ids" in metadata
|
|
assert "question" not in metadata
|
|
|
|
|
|
def test_knowledge_status_endpoint(ops_client):
|
|
response = ops_client.get("/api/v1/knowledge/status")
|
|
assert response.status_code == 200
|
|
assert response.json()["provider"] == "demo"
|
|
|
|
|
|
def test_ragcore_status_separates_sync_report_from_unverifiable_index(
|
|
client, ops_client, monkeypatch
|
|
):
|
|
class FakeRagcoreProvider:
|
|
def health(self, language="en-GB"):
|
|
return KnowledgeHealth(
|
|
provider="ragcore",
|
|
available=True,
|
|
detail="ready",
|
|
tenant="fleet-ops",
|
|
workspace="operations",
|
|
collection="internal-procedures",
|
|
document_count=None,
|
|
source_document_count=11,
|
|
reported_synced_document_count=None,
|
|
reported_failed_document_count=None,
|
|
last_sync_at=None,
|
|
statistics_state="not_reported",
|
|
)
|
|
|
|
monkeypatch.setattr(knowledge_router, "get_knowledge_provider", FakeRagcoreProvider)
|
|
settings = get_settings()
|
|
sync = client.post(
|
|
"/api/v1/integrations/n8n/procedures-sync-result",
|
|
json={"execution_id": "rag-statistics-test", "synced": 32, "failed": 1},
|
|
headers={"X-Service-Token": settings.n8n_callback_token},
|
|
)
|
|
assert sync.status_code == 200
|
|
|
|
response = ops_client.get("/api/v1/knowledge/status?language=nl-BE")
|
|
assert response.status_code == 200
|
|
status = response.json()
|
|
assert status["document_count"] is None
|
|
assert status["source_document_count"] == 11
|
|
assert status["reported_synced_document_count"] == 32
|
|
assert status["reported_failed_document_count"] == 1
|
|
assert status["last_sync_at"] is not None
|
|
assert status["statistics_state"] == "sync_reported"
|
|
|
|
|
|
def test_ragcore_status_preserves_stronger_verified_index_evidence(client, ops_client, monkeypatch):
|
|
class VerifiedRagcoreProvider:
|
|
def health(self, language="en-GB"):
|
|
return KnowledgeHealth(
|
|
provider="ragcore",
|
|
available=True,
|
|
detail="ready and verified",
|
|
tenant="fleet-ops",
|
|
workspace="operations",
|
|
collection="internal-procedures",
|
|
document_count=11,
|
|
source_document_count=11,
|
|
reported_synced_document_count=None,
|
|
reported_failed_document_count=None,
|
|
last_sync_at=None,
|
|
statistics_state="verified",
|
|
)
|
|
|
|
monkeypatch.setattr(knowledge_router, "get_knowledge_provider", VerifiedRagcoreProvider)
|
|
settings = get_settings()
|
|
sync = client.post(
|
|
"/api/v1/integrations/n8n/procedures-sync-result",
|
|
json={"execution_id": "rag-verified-statistics-test", "synced": 33, "failed": 0},
|
|
headers={"X-Service-Token": settings.n8n_callback_token},
|
|
)
|
|
assert sync.status_code == 200
|
|
|
|
status = ops_client.get("/api/v1/knowledge/status?language=nl-BE").json()
|
|
assert status["document_count"] == 11
|
|
assert status["reported_synced_document_count"] == 33
|
|
assert status["statistics_state"] == "verified"
|
|
|
|
|
|
class _FakeResponse:
|
|
def __init__(self, status_code: int, body: object):
|
|
self.status_code = status_code
|
|
self._body = body
|
|
|
|
def json(self) -> object:
|
|
return self._body
|
|
|
|
|
|
class _FakeClient:
|
|
def __init__(
|
|
self,
|
|
get_response=None,
|
|
post_response=None,
|
|
post_responses=None,
|
|
raise_on=None,
|
|
get_handler=None,
|
|
requests=None,
|
|
):
|
|
self._get_response = get_response
|
|
self._post_response = post_response
|
|
# Maps a path (e.g. "/v1/search") to its own response, for tests that need
|
|
# /v1/answers and /v1/search to behave differently in the same call. Falls back
|
|
# to the single post_response when a path has no specific entry, so every
|
|
# existing single-endpoint test keeps working unchanged.
|
|
self._post_responses = post_responses or {}
|
|
self._raise_on = raise_on
|
|
self._get_handler = get_handler
|
|
self.requests = requests if requests is not None else []
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
def get(self, path, params=None, timeout=None):
|
|
if self._raise_on == "get":
|
|
raise httpx.ConnectError("no ragcore in this environment")
|
|
if self._get_handler is not None:
|
|
return self._get_handler(path, params, timeout)
|
|
if path == "/v1/documents":
|
|
return _FakeResponse(503, {})
|
|
return self._get_response
|
|
|
|
def post(self, path, json=None):
|
|
if self._raise_on == "post":
|
|
raise httpx.ConnectError("no ragcore in this environment")
|
|
self.requests.append((path, json))
|
|
return self._post_responses.get(path, self._post_response)
|
|
|
|
|
|
def test_ragcore_provider_degrades_to_unavailable(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(provider, "_client", lambda: _FakeClient(raise_on="post"))
|
|
answer = provider.ask("Anything?", "test-correlation-3")
|
|
assert answer.evidence_state == "unavailable"
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_provider_without_configured_space_is_unavailable_without_a_network_call(
|
|
monkeypatch,
|
|
):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "")
|
|
|
|
def fail_if_called():
|
|
raise AssertionError("should not call RAGcore without a configured space id")
|
|
|
|
monkeypatch.setattr(provider, "_client", fail_if_called)
|
|
answer = provider.ask("Anything?", "test-correlation-no-space")
|
|
assert answer.evidence_state == "unavailable"
|
|
|
|
|
|
def test_ragcore_provider_health_reports_ready_status(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(get_response=_FakeResponse(200, {"status": "ok"})),
|
|
)
|
|
health = provider.health()
|
|
assert health.provider == "ragcore"
|
|
assert health.available is True
|
|
assert health.document_count is None
|
|
assert health.source_document_count == 11
|
|
assert health.statistics_state == "not_reported"
|
|
|
|
|
|
def test_ragcore_provider_verifies_published_documents_and_caches_count(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
procedure_documents = [
|
|
document
|
|
for document in iter_procedure_documents(Path(provider._settings.knowledge_dir))
|
|
if document.language == "en-GB"
|
|
]
|
|
documents = {document.source_id: document for document in procedure_documents}
|
|
lookups: list[str] = []
|
|
|
|
def get_handler(path, params, _timeout):
|
|
if path == "/health/ready":
|
|
return _FakeResponse(200, {"status": "ok"})
|
|
document = documents[params["source_id"]]
|
|
lookups.append(document.source_id)
|
|
return _FakeResponse(
|
|
200,
|
|
{
|
|
"items": [
|
|
{
|
|
"space_id": "space-1",
|
|
"source_id": document.source_id,
|
|
"external_id": f"{document.document_id}.md",
|
|
"status": "active",
|
|
"active_version": {
|
|
"status": "published",
|
|
"content_sha256": "provider-canonical-hash",
|
|
},
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
monkeypatch.setattr(provider, "_client", lambda: _FakeClient(get_handler=get_handler))
|
|
|
|
first = provider.health("en-GB")
|
|
second = provider.health("en-GB")
|
|
assert first.document_count == 11
|
|
assert first.statistics_state == "verified"
|
|
assert second.document_count == 11
|
|
assert len(lookups) == 11
|
|
|
|
|
|
def test_ragcore_provider_health_reports_degraded_status(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(get_response=_FakeResponse(200, {"status": "degraded"})),
|
|
)
|
|
health = provider.health()
|
|
assert health.available is False
|
|
|
|
|
|
def test_ragcore_provider_health_degrades_on_connection_error(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider, "_client", lambda: _FakeClient(raise_on="get"))
|
|
health = provider.health()
|
|
assert health.available is False
|
|
assert "unavailable" in health.detail.lower()
|
|
|
|
|
|
_EXCERPTS = {
|
|
("en-GB", "damage-procedure"): (
|
|
"When a vehicle returns with visible or reported damage, mark damage in the "
|
|
"return inspection, add a concise factual description and keep the vehicle "
|
|
"blocked."
|
|
),
|
|
("en-GB", "vehicle-return-procedure"): (
|
|
"Open the active booking and record the ending odometer, fuel level, cleanliness, "
|
|
"visible damage, technical warnings and relevant notes."
|
|
),
|
|
("en-GB", "cleaning-checklist"): (
|
|
"Cleaning completion alone does not make a blocked or maintenance vehicle "
|
|
"available. Fleet Ops derives availability from all active restrictions."
|
|
),
|
|
("en-GB", "maintenance-escalation"): (
|
|
"Escalate when a technical warning is reported, the service threshold is reached, "
|
|
"a safety-related defect is observed or an existing maintenance block remains "
|
|
"unresolved."
|
|
),
|
|
("nl-BE", "damage-procedure"): (
|
|
"Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schade "
|
|
"in de retourinspectie, voeg een beknopte feitelijke beschrijving toe en houd het "
|
|
"voertuig geblokkeerd."
|
|
),
|
|
("nl-BE", "vehicle-return-procedure"): (
|
|
"Open de actieve boeking en registreer de eindkilometerstand, het brandstofniveau, "
|
|
"de netheid, zichtbare schade, technische waarschuwingen en relevante notities."
|
|
),
|
|
}
|
|
|
|
|
|
def _managed_document(provider: RAGcoreKnowledgeProvider, document_id: str, language: str):
|
|
return next(
|
|
document
|
|
for document in provider._managed_documents_by_source_id.values()
|
|
if document.document_id == document_id and document.language == language
|
|
)
|
|
|
|
|
|
def _citation(
|
|
provider: RAGcoreKnowledgeProvider,
|
|
document_id: str,
|
|
*,
|
|
language: str = "en-GB",
|
|
section: str = "Procedure",
|
|
seed: str = "1",
|
|
) -> dict:
|
|
document = _managed_document(provider, document_id, language)
|
|
excerpt = _EXCERPTS[(language, document_id)]
|
|
return {
|
|
"id": str(uuid.uuid5(uuid.NAMESPACE_URL, f"citation:{language}:{document_id}:{seed}")),
|
|
# These are deliberately opaque RAGcore-owned UUIDs, not Fleet Ops' stable
|
|
# human-readable procedure id.
|
|
"document_id": str(
|
|
uuid.uuid5(uuid.NAMESPACE_URL, f"ragcore-document:{language}:{document_id}")
|
|
),
|
|
"document_version_id": str(
|
|
uuid.uuid5(
|
|
uuid.NAMESPACE_URL,
|
|
f"ragcore-document-version:{language}:{document_id}:{seed}",
|
|
)
|
|
),
|
|
"title": "untrusted-provider-title.md",
|
|
"section": section,
|
|
"excerpt": excerpt,
|
|
"excerpt_sha256": hashlib.sha256(excerpt.encode("utf-8")).hexdigest(),
|
|
"source_uri": f"ragcore://source/{document.source_id}",
|
|
"source_id": document.source_id,
|
|
"locator": f"{document.document_id}.md",
|
|
}
|
|
|
|
|
|
def _answers_body(provider: RAGcoreKnowledgeProvider, **overrides) -> dict:
|
|
citation = _citation(provider, "damage-procedure", section="Detection")
|
|
answer = "Report damage and route the vehicle to maintenance."
|
|
body = {
|
|
"answer_id": str(uuid.uuid4()),
|
|
"retrieval_run_id": str(uuid.uuid4()),
|
|
"answer": answer,
|
|
"answerability": "answerable",
|
|
"citations": [citation],
|
|
"claims": [{"text": answer, "citation_ids": [citation["id"]]}],
|
|
}
|
|
body.update(overrides)
|
|
if "claims" not in overrides:
|
|
effective_answer = body.get("answer")
|
|
effective_citations = body.get("citations")
|
|
body["claims"] = (
|
|
[
|
|
{
|
|
"text": effective_answer,
|
|
"citation_ids": [item["id"] for item in effective_citations],
|
|
}
|
|
]
|
|
if isinstance(effective_answer, str)
|
|
and effective_answer
|
|
and isinstance(effective_citations, list)
|
|
and effective_citations
|
|
and all(isinstance(item, dict) and "id" in item for item in effective_citations)
|
|
else []
|
|
)
|
|
return body
|
|
|
|
|
|
def test_ragcore_provider_grounded_answer_maps_citations_to_sources(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(provider))),
|
|
)
|
|
answer = provider.ask("What must I do about damage?", "test-correlation-grounded")
|
|
assert answer.evidence_state == "grounded"
|
|
assert answer.answer
|
|
assert len(answer.sources) == 1
|
|
source = answer.sources[0]
|
|
assert source.document_id == "damage-procedure"
|
|
assert source.title == "Damage handling procedure"
|
|
assert source.version == "1.3"
|
|
assert source.section == "Detection"
|
|
assert source.excerpt
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutate",
|
|
[
|
|
lambda body: body.pop("claims"),
|
|
lambda body: body.update(claims=[]),
|
|
lambda body: body["claims"][0].update(citation_ids=[str(uuid.uuid4())]),
|
|
lambda body: body.pop("answer_id"),
|
|
lambda body: body.update(retrieval_run_id="not-a-uuid"),
|
|
],
|
|
)
|
|
def test_ragcore_answer_requires_valid_claim_citation_contract(monkeypatch, mutate):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
body = _answers_body(provider)
|
|
mutate(body)
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(post_response=_FakeResponse(200, body)),
|
|
)
|
|
|
|
answer = provider.ask("What must I do about damage?", "test-invalid-claim-contract")
|
|
|
|
assert answer.evidence_state != "grounded"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_requests_are_scoped_to_managed_sources_for_requested_language(
|
|
monkeypatch,
|
|
):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
requests = []
|
|
fake_client = _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
|
|
},
|
|
requests=requests,
|
|
)
|
|
monkeypatch.setattr(provider, "_client", lambda: fake_client)
|
|
|
|
answer = provider.ask(
|
|
"Wat is de procedure voor een voertuigretour?",
|
|
"language-filter",
|
|
"nl-BE",
|
|
)
|
|
|
|
assert answer.evidence_state == "grounded"
|
|
assert [path for path, _payload in requests] == ["/v1/answers", "/v1/search"]
|
|
expected_source_ids = provider._managed_source_ids("nl-BE")
|
|
assert len(expected_source_ids) == 11
|
|
assert all(uuid.UUID(source_id) for source_id in expected_source_ids)
|
|
for _path, payload in requests:
|
|
assert payload["filters"] == {"source_ids": expected_source_ids}
|
|
assert payload["requested_space_ids"] == ["space-1"]
|
|
|
|
|
|
def test_ragcore_sources_deduplicate_reuploaded_versions_and_cap_cards(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
citations = []
|
|
document_ids = [
|
|
"damage-procedure",
|
|
"damage-procedure",
|
|
"vehicle-return-procedure",
|
|
"cleaning-checklist",
|
|
"maintenance-escalation",
|
|
]
|
|
for index, document_id in enumerate(document_ids):
|
|
citations.append(
|
|
_citation(
|
|
provider,
|
|
document_id,
|
|
section="Return",
|
|
seed=str(index),
|
|
)
|
|
)
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_response=_FakeResponse(200, _answers_body(provider, citations=citations))
|
|
),
|
|
)
|
|
answer = provider.ask("What must I do about vehicle damage?", "dedupe-test")
|
|
assert len(answer.sources) == 3
|
|
assert sum(source.title == "Damage handling procedure" for source in answer.sources) == 1
|
|
|
|
|
|
def test_ragcore_answer_rejects_unknown_document_citation(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_response=_FakeResponse(
|
|
200,
|
|
_answers_body(
|
|
provider,
|
|
answer="A plausible-looking but unmanaged answer.",
|
|
citations=[
|
|
{
|
|
**_citation(provider, "damage-procedure"),
|
|
"source_id": str(uuid.uuid4()),
|
|
}
|
|
],
|
|
),
|
|
)
|
|
),
|
|
)
|
|
|
|
answer = provider.ask("What must I do about damage?", "unknown-answer-citation")
|
|
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field_name", "bad_value"),
|
|
[
|
|
("id", "not-a-uuid"),
|
|
("document_id", "not-a-uuid"),
|
|
("document_version_id", "00000000-0000-0000-0000-000000000000"),
|
|
("source_id", "00000000-0000-4000-8000-000000000099"),
|
|
("source_uri", "ragcore://source/00000000-0000-4000-8000-000000000099"),
|
|
("locator", "another-procedure.md"),
|
|
("excerpt_sha256", "0" * 64),
|
|
("title", None),
|
|
("section", {"not": "a string"}),
|
|
],
|
|
)
|
|
def test_ragcore_rejects_broken_managed_source_provenance(field_name, bad_value):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
citation = _citation(provider, "damage-procedure")
|
|
citation[field_name] = bad_value
|
|
|
|
assert provider._managed_source(citation, "en-GB") is None
|
|
|
|
|
|
def test_ragcore_rejects_fabricated_excerpt_even_with_matching_hash():
|
|
provider = RAGcoreKnowledgeProvider()
|
|
citation = _citation(provider, "damage-procedure")
|
|
fabricated = "Invent a repair price and promise it to the customer."
|
|
citation["excerpt"] = fabricated
|
|
citation["excerpt_sha256"] = hashlib.sha256(fabricated.encode("utf-8")).hexdigest()
|
|
|
|
assert provider._managed_source(citation, "en-GB") is None
|
|
|
|
|
|
def test_ragcore_never_relabels_an_english_source_as_dutch():
|
|
provider = RAGcoreKnowledgeProvider()
|
|
citation = _citation(provider, "damage-procedure", language="en-GB")
|
|
|
|
assert provider._managed_source(citation, "nl-BE") is None
|
|
|
|
|
|
def test_knowledge_feedback_is_audited_and_can_be_changed(ops_client):
|
|
answer = ops_client.post(
|
|
"/api/v1/knowledge/questions",
|
|
json={"question": "What must I do when a vehicle returns with damage?"},
|
|
).json()
|
|
payload = {"correlation_id": answer["correlation_id"], "helpful": True}
|
|
assert ops_client.post("/api/v1/knowledge/feedback", json=payload).status_code == 200
|
|
payload["helpful"] = False
|
|
assert ops_client.post("/api/v1/knowledge/feedback", json=payload).status_code == 200
|
|
|
|
events = ops_client.get(
|
|
"/api/v1/audit", params={"action": "knowledge_feedback_recorded"}
|
|
).json()
|
|
matching = [e for e in events if e["correlation_id"] == answer["correlation_id"]]
|
|
assert len(matching) == 1
|
|
assert matching[0]["metadata"]["helpful"] is False
|
|
|
|
|
|
def test_knowledge_feedback_cannot_target_another_users_exchange(client):
|
|
assert client.post("/api/v1/demo/login", json={"role": "rental_employee"}).status_code == 200
|
|
answer = client.post(
|
|
"/api/v1/knowledge/questions",
|
|
json={"question": "How do I register a vehicle return?"},
|
|
).json()
|
|
assert client.post("/api/v1/demo/login", json={"role": "operations_manager"}).status_code == 200
|
|
response = client.post(
|
|
"/api/v1/knowledge/feedback",
|
|
json={"correlation_id": answer["correlation_id"], "helpful": True},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
|
|
def test_ragcore_provider_not_answerable_is_insufficient_and_never_fabricates(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_response=_FakeResponse(
|
|
200,
|
|
_answers_body(
|
|
provider,
|
|
answer="This should never be shown.",
|
|
answerability="not_answerable",
|
|
citations=[],
|
|
),
|
|
)
|
|
),
|
|
)
|
|
answer = provider.ask("Unrelated question?", "test-correlation-insufficient")
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_provider_answerable_without_citations_is_insufficient(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_response=_FakeResponse(
|
|
200,
|
|
_answers_body(provider, answerability="answerable", citations=[]),
|
|
)
|
|
),
|
|
)
|
|
answer = provider.ask("What must I do about damage?", "test-correlation-no-citations")
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_provider_non_200_response_is_unavailable(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(post_response=_FakeResponse(401, {"code": "AUTHENTICATION_REQUIRED"})),
|
|
)
|
|
answer = provider.ask("Anything?", "test-correlation-401")
|
|
assert answer.evidence_state == "unavailable"
|
|
|
|
|
|
def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
# A malformed /v1/answers body triggers the same search fallback a real outage
|
|
# would, so the fallback's own /v1/search response must also be malformed here to
|
|
# exercise "the whole backend is misbehaving, not just one endpoint" honestly.
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(200, {"citations": "not-a-list"}),
|
|
"/v1/search": _FakeResponse(200, {"results": "not-a-list"}),
|
|
}
|
|
),
|
|
)
|
|
answer = provider.ask("Anything?", "test-correlation-malformed")
|
|
assert answer.evidence_state == "unavailable"
|
|
|
|
|
|
def test_ragcore_provider_malformed_top_level_bodies_do_not_escape(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(200, ["not", "an", "object"]),
|
|
"/v1/search": _FakeResponse(200, None),
|
|
}
|
|
),
|
|
)
|
|
|
|
answer = provider.ask("Anything?", "test-correlation-malformed-top-level")
|
|
|
|
assert answer.evidence_state == "unavailable"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_answer_with_malformed_citation_is_insufficient(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_response=_FakeResponse(
|
|
200,
|
|
_answers_body(provider, citations=[None]),
|
|
)
|
|
),
|
|
)
|
|
|
|
answer = provider.ask("What must I do about damage?", "malformed-answer-citation")
|
|
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
def _search_body(
|
|
provider: RAGcoreKnowledgeProvider,
|
|
language: str = "en-GB",
|
|
**overrides,
|
|
) -> dict:
|
|
body = {
|
|
"retrieval_run_id": str(uuid.uuid4()),
|
|
"effective_space_ids": ["00000000-0000-4000-8000-000000000001"],
|
|
"results": [
|
|
{
|
|
"chunk_id": str(uuid.uuid4()),
|
|
"text": _EXCERPTS[(language, "vehicle-return-procedure")],
|
|
"citation": _citation(
|
|
provider,
|
|
"vehicle-return-procedure",
|
|
language=language,
|
|
section="Return",
|
|
),
|
|
"rank": 1,
|
|
"scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None},
|
|
}
|
|
],
|
|
"degraded": False,
|
|
}
|
|
body.update(overrides)
|
|
return body
|
|
|
|
|
|
def test_ragcore_search_ignores_malformed_result_and_citation_bodies(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
malformed_search = _search_body(
|
|
provider,
|
|
results=[None, {"citation": ["not-an-object"], "scores": "not-an-object"}],
|
|
)
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": _FakeResponse(200, malformed_search),
|
|
}
|
|
),
|
|
)
|
|
|
|
answer = provider.ask("What is the vehicle return procedure?", "malformed-search-items")
|
|
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch):
|
|
"""/v1/answers itself failing (a real RAGcore-side outage in its generation step,
|
|
not a real 'insufficient evidence' classification) must not silently degrade
|
|
straight to 'unavailable' when RAGcore's own retrieval still works -- it should show
|
|
the real, cited excerpt search actually found instead."""
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
|
"/v1/search": _FakeResponse(200, _search_body(provider)),
|
|
}
|
|
),
|
|
)
|
|
answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback")
|
|
assert answer.evidence_state == "grounded"
|
|
assert "Open the active booking and record the ending odometer" in answer.answer
|
|
assert "Vehicle return procedure" in answer.answer
|
|
assert len(answer.sources) == 1
|
|
assert answer.sources[0].title == "Vehicle return procedure"
|
|
assert answer.sources[0].excerpt == _EXCERPTS[("en-GB", "vehicle-return-procedure")]
|
|
|
|
|
|
def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0)
|
|
search_response = _FakeResponse(200, _search_body(provider))
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": search_response,
|
|
}
|
|
),
|
|
)
|
|
assert (
|
|
provider.ask("What is the vehicle return procedure?", "first").evidence_state == "grounded"
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_ask_via_answers",
|
|
lambda *_args: (_ for _ in ()).throw(AssertionError("open circuit called answers")),
|
|
)
|
|
second = provider.ask("What is the vehicle return procedure?", "second")
|
|
assert second.evidence_state == "grounded"
|
|
|
|
|
|
def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
|
"/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")),
|
|
}
|
|
),
|
|
)
|
|
answer = provider.ask(
|
|
"Wat is de procedure voor een voertuigretour?",
|
|
"test-correlation-fallback-nl",
|
|
language="nl-BE",
|
|
)
|
|
assert answer.evidence_state == "grounded"
|
|
assert answer.answer.startswith("Volgens ")
|
|
|
|
|
|
def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
|
"/v1/search": _FakeResponse(200, _search_body(provider, results=[])),
|
|
}
|
|
),
|
|
)
|
|
answer = provider.ask("Unrelated question?", "test-correlation-fallback-empty")
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": _FakeResponse(200, _search_body(provider)),
|
|
}
|
|
),
|
|
)
|
|
answer = provider.ask(
|
|
"Who won the football world cup in 1998?",
|
|
"test-correlation-out-of-domain",
|
|
)
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
|
|
|
|
def test_ragcore_search_fallback_rejects_unknown_document_citation(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
search = _search_body(provider)
|
|
search["results"][0]["citation"]["source_id"] = str(uuid.uuid4())
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": _FakeResponse(200, search),
|
|
}
|
|
),
|
|
)
|
|
|
|
answer = provider.ask(
|
|
"What is the vehicle return procedure?",
|
|
"unknown-search-citation",
|
|
)
|
|
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
assert answer.sources == []
|
|
|
|
|
|
def test_ragcore_search_fallback_rejects_vehicle_colour_despite_high_score(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
search = _search_body(provider)
|
|
search["results"][0]["scores"]["fused"] = 1.0
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": _FakeResponse(200, search),
|
|
}
|
|
),
|
|
)
|
|
|
|
answer = provider.ask("What colour is this vehicle?", "vehicle-colour")
|
|
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
|
|
|
|
def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
search = _search_body(provider, "nl-BE", results=[])
|
|
search["results"].append(
|
|
{
|
|
"citation": _citation(
|
|
provider,
|
|
"damage-procedure",
|
|
language="nl-BE",
|
|
section="Damage",
|
|
),
|
|
"rank": 2,
|
|
"scores": {"fused": 0.01},
|
|
}
|
|
)
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": _FakeResponse(200, search),
|
|
}
|
|
),
|
|
)
|
|
answer = provider.ask("Wat moet ik doen bij schade?", "test-correlation-damage", "nl-BE")
|
|
assert answer.evidence_state == "insufficient"
|
|
assert answer.answer == ""
|
|
assert any(source.document_id == "damage-procedure" for source in answer.sources)
|
|
|
|
|
|
def test_ragcore_search_fallback_accepts_top_ranked_ragcore_damage_evidence(monkeypatch):
|
|
provider = RAGcoreKnowledgeProvider()
|
|
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
|
search = _search_body(provider, "nl-BE", results=[])
|
|
search["results"].append(
|
|
{
|
|
"citation": _citation(
|
|
provider,
|
|
"damage-procedure",
|
|
language="nl-BE",
|
|
section="Damage",
|
|
),
|
|
"rank": 1,
|
|
# RAGcore uses reciprocal-rank fusion; a genuine rank-one result is about
|
|
# 1 / (60 + 1), not a normalized 0..1 relevance score.
|
|
"scores": {"fused": 0.01639344262295082},
|
|
}
|
|
)
|
|
monkeypatch.setattr(
|
|
provider,
|
|
"_client",
|
|
lambda: _FakeClient(
|
|
post_responses={
|
|
"/v1/answers": _FakeResponse(503, {}),
|
|
"/v1/search": _FakeResponse(200, search),
|
|
}
|
|
),
|
|
)
|
|
answer = provider.ask("Wat moet ik doen bij schade?", "strong-damage", "nl-BE")
|
|
assert answer.evidence_state == "grounded"
|
|
assert answer.sources[0].document_id == "damage-procedure"
|