from __future__ import annotations import hashlib import re import uuid from pathlib import Path import httpx import pytest from app.api.routers import knowledge as knowledge_router from app.core.config import get_settings from app.core.ratelimit import SlidingWindowLimiter from app.services.knowledge import KnowledgeHealth from app.services.knowledge.demo import DemoKnowledgeProvider from app.services.knowledge.procedures import iter_procedure_documents from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider def test_brief_exact_damage_question_in_all_three_languages(): # The exact validation questions from docs/fleet-ops-correction/current-gap-audit.md # -- each must ground on the damage procedure as its *primary* (top-ranked) source, # not merely appear somewhere in the top-3, and the source/version/section/excerpt # must all come from that same-language document (never an English fallback). provider = DemoKnowledgeProvider() cases = { "nl-BE": "Wat moet ik doen wanneer een voertuig beschadigd terugkomt?", "en-GB": "What should I do when a vehicle returns with damage?", "fr-BE": "Que dois-je faire lorsqu'un véhicule revient endommagé ?", } for language, question in cases.items(): answer = provider.ask(question, f"test-brief-{language}", language) assert answer.evidence_state == "grounded", language assert answer.sources, language assert answer.sources[0].document_id == "damage-procedure", ( f"{language}: expected the damage procedure as the primary source, " f"got {answer.sources[0].document_id!r}" ) assert answer.answer assert answer.sources[0].excerpt def test_knowledge_procedures_never_mention_mobilityops_or_poc(): # Section 2 of docs/fleet-ops-correction/current-gap-audit.md: the visible brand # name is exactly "Fleet Ops", and "PoC" must never appear in visible content -- # including the demo knowledge base, not just the frontend. procedures_dir = Path(get_settings().knowledge_dir) offenders = [] for path in sorted(procedures_dir.glob("*/*.md")): text = path.read_text(encoding="utf-8") if "MobilityOps" in text or re.search(r"\bPoC\b", text): offenders.append(str(path)) assert offenders == [] def test_s6_damage_question_is_grounded_with_expected_sources(): provider = DemoKnowledgeProvider() answer = provider.ask( "What must I do when a vehicle returns with damage?", "test-correlation-1" ) assert answer.evidence_state == "grounded" document_ids = {s.document_id for s in answer.sources} assert "damage-procedure" in document_ids assert "vehicle-return-procedure" in document_ids assert answer.answer # never empty for a grounded answer for source in answer.sources: assert source.excerpt def test_unrelated_question_is_insufficient(): provider = DemoKnowledgeProvider() answer = provider.ask("What is the capital of France?", "test-correlation-2") assert answer.evidence_state == "insufficient" assert "France" not in answer.answer # never fabricates an answer beyond the procedures def test_demo_provider_health_reports_document_count(): provider = DemoKnowledgeProvider() health = provider.health() assert health.provider == "demo" assert health.available is True assert health.document_count == 11 assert health.source_document_count == 11 assert health.statistics_state == "verified" def test_demo_provider_health_reports_document_count_per_language(): provider = DemoKnowledgeProvider() for language in ("nl-BE", "en-GB", "fr-BE"): assert provider.health(language).document_count == 11 def test_demo_provider_grounds_damage_question_in_dutch(): provider = DemoKnowledgeProvider() answer = provider.ask( "Wat moet ik doen als een voertuig terugkomt met schade?", "test-correlation-nl", "nl-BE", ) assert answer.evidence_state == "grounded" document_ids = {s.document_id for s in answer.sources} assert "damage-procedure" in document_ids def test_demo_provider_grounds_damage_question_in_french(): provider = DemoKnowledgeProvider() answer = provider.ask( "Que dois-je faire quand un véhicule revient avec des dommages ?", "test-correlation-fr", "fr-BE", ) assert answer.evidence_state == "grounded" document_ids = {s.document_id for s in answer.sources} assert "damage-procedure" in document_ids def test_demo_provider_insufficient_evidence_message_is_localized(): provider = DemoKnowledgeProvider() nl_answer = provider.ask("Wat is de hoofdstad van Frankrijk?", "test-correlation-nl-2", "nl-BE") fr_answer = provider.ask( "Quelle est la capitale de la France ?", "test-correlation-fr-2", "fr-BE" ) assert nl_answer.evidence_state == "insufficient" assert fr_answer.evidence_state == "insufficient" assert nl_answer.answer != fr_answer.answer assert "France" not in nl_answer.answer assert "France" not in fr_answer.answer def test_ask_question_endpoint_grounded(ops_client): response = ops_client.post( "/api/v1/knowledge/questions", json={"question": "What must I do when a vehicle returns with damage?"}, ) assert response.status_code == 200 body = response.json() assert body["evidence_state"] == "grounded" assert body["provider"] == "demo" assert len(body["sources"]) > 0 def test_ask_question_requires_authentication(client): response = client.post("/api/v1/knowledge/questions", json={"question": "Anything?"}) assert response.status_code == 401 def test_knowledge_questions_are_bounded_per_session_and_client(ops_client, monkeypatch): limiter = SlidingWindowLimiter(max_requests=1, window_seconds=60) monkeypatch.setattr(knowledge_router, "_question_limiter", limiter) body = {"question": "What is the vehicle return procedure?"} assert ops_client.post("/api/v1/knowledge/questions", json=body).status_code == 200 blocked = ops_client.post("/api/v1/knowledge/questions", json=body) assert blocked.status_code == 429 assert int(blocked.headers["retry-after"]) >= 1 def test_ask_question_is_audited_without_leaking_full_text(ops_client): ops_client.post( "/api/v1/knowledge/questions", json={"question": "What must I do when a vehicle returns with damage?"}, ) events = ops_client.get("/api/v1/audit", params={"action": "knowledge_question_asked"}).json() assert len(events) >= 1 metadata = events[0]["metadata"] assert "evidence_state" in metadata assert "source_ids" in metadata assert "question" not in metadata def test_knowledge_status_endpoint(ops_client): response = ops_client.get("/api/v1/knowledge/status") assert response.status_code == 200 assert response.json()["provider"] == "demo" def test_ragcore_status_separates_sync_report_from_unverifiable_index( client, ops_client, monkeypatch ): class FakeRagcoreProvider: def health(self, language="en-GB"): return KnowledgeHealth( provider="ragcore", available=True, detail="ready", tenant="fleet-ops", workspace="operations", collection="internal-procedures", document_count=None, source_document_count=11, reported_synced_document_count=None, reported_failed_document_count=None, last_sync_at=None, statistics_state="not_reported", ) monkeypatch.setattr(knowledge_router, "get_knowledge_provider", FakeRagcoreProvider) settings = get_settings() sync = client.post( "/api/v1/integrations/n8n/procedures-sync-result", json={"execution_id": "rag-statistics-test", "synced": 32, "failed": 1}, headers={"X-Service-Token": settings.n8n_callback_token}, ) assert sync.status_code == 200 response = ops_client.get("/api/v1/knowledge/status?language=nl-BE") assert response.status_code == 200 status = response.json() assert status["document_count"] is None assert status["source_document_count"] == 11 assert status["reported_synced_document_count"] == 32 assert status["reported_failed_document_count"] == 1 assert status["last_sync_at"] is not None assert status["statistics_state"] == "sync_reported" def test_ragcore_status_preserves_stronger_verified_index_evidence(client, ops_client, monkeypatch): class VerifiedRagcoreProvider: def health(self, language="en-GB"): return KnowledgeHealth( provider="ragcore", available=True, detail="ready and verified", tenant="fleet-ops", workspace="operations", collection="internal-procedures", document_count=11, source_document_count=11, reported_synced_document_count=None, reported_failed_document_count=None, last_sync_at=None, statistics_state="verified", ) monkeypatch.setattr(knowledge_router, "get_knowledge_provider", VerifiedRagcoreProvider) settings = get_settings() sync = client.post( "/api/v1/integrations/n8n/procedures-sync-result", json={"execution_id": "rag-verified-statistics-test", "synced": 33, "failed": 0}, headers={"X-Service-Token": settings.n8n_callback_token}, ) assert sync.status_code == 200 status = ops_client.get("/api/v1/knowledge/status?language=nl-BE").json() assert status["document_count"] == 11 assert status["reported_synced_document_count"] == 33 assert status["statistics_state"] == "verified" class _FakeResponse: def __init__(self, status_code: int, body: object): self.status_code = status_code self._body = body def json(self) -> object: return self._body class _FakeClient: def __init__( self, get_response=None, post_response=None, post_responses=None, raise_on=None, get_handler=None, requests=None, ): self._get_response = get_response self._post_response = post_response # Maps a path (e.g. "/v1/search") to its own response, for tests that need # /v1/answers and /v1/search to behave differently in the same call. Falls back # to the single post_response when a path has no specific entry, so every # existing single-endpoint test keeps working unchanged. self._post_responses = post_responses or {} self._raise_on = raise_on self._get_handler = get_handler self.requests = requests if requests is not None else [] def __enter__(self): return self def __exit__(self, *args): return False def get(self, path, params=None, timeout=None): if self._raise_on == "get": raise httpx.ConnectError("no ragcore in this environment") if self._get_handler is not None: return self._get_handler(path, params, timeout) if path == "/v1/documents": return _FakeResponse(503, {}) return self._get_response def post(self, path, json=None): if self._raise_on == "post": raise httpx.ConnectError("no ragcore in this environment") self.requests.append((path, json)) return self._post_responses.get(path, self._post_response) def test_ragcore_provider_degrades_to_unavailable(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr(provider, "_client", lambda: _FakeClient(raise_on="post")) answer = provider.ask("Anything?", "test-correlation-3") assert answer.evidence_state == "unavailable" assert answer.sources == [] def test_ragcore_provider_without_configured_space_is_unavailable_without_a_network_call( monkeypatch, ): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "") def fail_if_called(): raise AssertionError("should not call RAGcore without a configured space id") monkeypatch.setattr(provider, "_client", fail_if_called) answer = provider.ask("Anything?", "test-correlation-no-space") assert answer.evidence_state == "unavailable" def test_ragcore_provider_health_reports_ready_status(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr( provider, "_client", lambda: _FakeClient(get_response=_FakeResponse(200, {"status": "ok"})), ) health = provider.health() assert health.provider == "ragcore" assert health.available is True assert health.document_count is None assert health.source_document_count == 11 assert health.statistics_state == "not_reported" def test_ragcore_provider_verifies_published_documents_and_caches_count(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") procedure_documents = [ document for document in iter_procedure_documents(Path(provider._settings.knowledge_dir)) if document.language == "en-GB" ] documents = {document.source_id: document for document in procedure_documents} lookups: list[str] = [] def get_handler(path, params, _timeout): if path == "/health/ready": return _FakeResponse(200, {"status": "ok"}) document = documents[params["source_id"]] lookups.append(document.source_id) return _FakeResponse( 200, { "items": [ { "space_id": "space-1", "source_id": document.source_id, "external_id": f"{document.document_id}.md", "status": "active", "active_version": { "status": "published", "content_sha256": "provider-canonical-hash", }, } ] }, ) monkeypatch.setattr(provider, "_client", lambda: _FakeClient(get_handler=get_handler)) first = provider.health("en-GB") second = provider.health("en-GB") assert first.document_count == 11 assert first.statistics_state == "verified" assert second.document_count == 11 assert len(lookups) == 11 def test_ragcore_provider_health_reports_degraded_status(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr( provider, "_client", lambda: _FakeClient(get_response=_FakeResponse(200, {"status": "degraded"})), ) health = provider.health() assert health.available is False def test_ragcore_provider_health_degrades_on_connection_error(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider, "_client", lambda: _FakeClient(raise_on="get")) health = provider.health() assert health.available is False assert "unavailable" in health.detail.lower() _EXCERPTS = { ("en-GB", "damage-procedure"): ( "When a vehicle returns with visible or reported damage, mark damage in the " "return inspection, add a concise factual description and keep the vehicle " "blocked." ), ("en-GB", "vehicle-return-procedure"): ( "Open the active booking and record the ending odometer, fuel level, cleanliness, " "visible damage, technical warnings and relevant notes." ), ("en-GB", "cleaning-checklist"): ( "Cleaning completion alone does not make a blocked or maintenance vehicle " "available. Fleet Ops derives availability from all active restrictions." ), ("en-GB", "maintenance-escalation"): ( "Escalate when a technical warning is reported, the service threshold is reached, " "a safety-related defect is observed or an existing maintenance block remains " "unresolved." ), ("nl-BE", "damage-procedure"): ( "Wanneer een voertuig terugkomt met zichtbare of gemelde schade, markeer de schade " "in de retourinspectie, voeg een beknopte feitelijke beschrijving toe en houd het " "voertuig geblokkeerd." ), ("nl-BE", "vehicle-return-procedure"): ( "Open de actieve boeking en registreer de eindkilometerstand, het brandstofniveau, " "de netheid, zichtbare schade, technische waarschuwingen en relevante notities." ), } def _managed_document(provider: RAGcoreKnowledgeProvider, document_id: str, language: str): return next( document for document in provider._managed_documents_by_source_id.values() if document.document_id == document_id and document.language == language ) def _citation( provider: RAGcoreKnowledgeProvider, document_id: str, *, language: str = "en-GB", section: str = "Procedure", seed: str = "1", ) -> dict: document = _managed_document(provider, document_id, language) excerpt = _EXCERPTS[(language, document_id)] return { "id": str(uuid.uuid5(uuid.NAMESPACE_URL, f"citation:{language}:{document_id}:{seed}")), # These are deliberately opaque RAGcore-owned UUIDs, not Fleet Ops' stable # human-readable procedure id. "document_id": str( uuid.uuid5(uuid.NAMESPACE_URL, f"ragcore-document:{language}:{document_id}") ), "document_version_id": str( uuid.uuid5( uuid.NAMESPACE_URL, f"ragcore-document-version:{language}:{document_id}:{seed}", ) ), "title": "untrusted-provider-title.md", "section": section, "excerpt": excerpt, "excerpt_sha256": hashlib.sha256(excerpt.encode("utf-8")).hexdigest(), "source_uri": f"ragcore://source/{document.source_id}", "source_id": document.source_id, "locator": f"{document.document_id}.md", } def _answers_body(provider: RAGcoreKnowledgeProvider, **overrides) -> dict: citation = _citation(provider, "damage-procedure", section="Detection") answer = "Report damage and route the vehicle to maintenance." body = { "answer_id": str(uuid.uuid4()), "retrieval_run_id": str(uuid.uuid4()), "answer": answer, "answerability": "answerable", "citations": [citation], "claims": [{"text": answer, "citation_ids": [citation["id"]]}], } body.update(overrides) if "claims" not in overrides: effective_answer = body.get("answer") effective_citations = body.get("citations") body["claims"] = ( [ { "text": effective_answer, "citation_ids": [item["id"] for item in effective_citations], } ] if isinstance(effective_answer, str) and effective_answer and isinstance(effective_citations, list) and effective_citations and all(isinstance(item, dict) and "id" in item for item in effective_citations) else [] ) return body def test_ragcore_provider_grounded_answer_maps_citations_to_sources(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient(post_response=_FakeResponse(200, _answers_body(provider))), ) answer = provider.ask("What must I do about damage?", "test-correlation-grounded") assert answer.evidence_state == "grounded" assert answer.answer assert len(answer.sources) == 1 source = answer.sources[0] assert source.document_id == "damage-procedure" assert source.title == "Damage handling procedure" assert source.version == "1.3" assert source.section == "Detection" assert source.excerpt @pytest.mark.parametrize( "mutate", [ lambda body: body.pop("claims"), lambda body: body.update(claims=[]), lambda body: body["claims"][0].update(citation_ids=[str(uuid.uuid4())]), lambda body: body.pop("answer_id"), lambda body: body.update(retrieval_run_id="not-a-uuid"), ], ) def test_ragcore_answer_requires_valid_claim_citation_contract(monkeypatch, mutate): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") body = _answers_body(provider) mutate(body) monkeypatch.setattr( provider, "_client", lambda: _FakeClient(post_response=_FakeResponse(200, body)), ) answer = provider.ask("What must I do about damage?", "test-invalid-claim-contract") assert answer.evidence_state != "grounded" assert answer.answer == "" assert answer.sources == [] def test_ragcore_requests_are_scoped_to_managed_sources_for_requested_language( monkeypatch, ): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") requests = [] fake_client = _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")), }, requests=requests, ) monkeypatch.setattr(provider, "_client", lambda: fake_client) answer = provider.ask( "Wat is de procedure voor een voertuigretour?", "language-filter", "nl-BE", ) assert answer.evidence_state == "grounded" assert [path for path, _payload in requests] == ["/v1/answers", "/v1/search"] expected_source_ids = provider._managed_source_ids("nl-BE") assert len(expected_source_ids) == 11 assert all(uuid.UUID(source_id) for source_id in expected_source_ids) for _path, payload in requests: assert payload["filters"] == {"source_ids": expected_source_ids} assert payload["requested_space_ids"] == ["space-1"] def test_ragcore_sources_deduplicate_reuploaded_versions_and_cap_cards(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") citations = [] document_ids = [ "damage-procedure", "damage-procedure", "vehicle-return-procedure", "cleaning-checklist", "maintenance-escalation", ] for index, document_id in enumerate(document_ids): citations.append( _citation( provider, document_id, section="Return", seed=str(index), ) ) monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_response=_FakeResponse(200, _answers_body(provider, citations=citations)) ), ) answer = provider.ask("What must I do about vehicle damage?", "dedupe-test") assert len(answer.sources) == 3 assert sum(source.title == "Damage handling procedure" for source in answer.sources) == 1 def test_ragcore_answer_rejects_unknown_document_citation(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_response=_FakeResponse( 200, _answers_body( provider, answer="A plausible-looking but unmanaged answer.", citations=[ { **_citation(provider, "damage-procedure"), "source_id": str(uuid.uuid4()), } ], ), ) ), ) answer = provider.ask("What must I do about damage?", "unknown-answer-citation") assert answer.evidence_state == "insufficient" assert answer.answer == "" assert answer.sources == [] @pytest.mark.parametrize( ("field_name", "bad_value"), [ ("id", "not-a-uuid"), ("document_id", "not-a-uuid"), ("document_version_id", "00000000-0000-0000-0000-000000000000"), ("source_id", "00000000-0000-4000-8000-000000000099"), ("source_uri", "ragcore://source/00000000-0000-4000-8000-000000000099"), ("locator", "another-procedure.md"), ("excerpt_sha256", "0" * 64), ("title", None), ("section", {"not": "a string"}), ], ) def test_ragcore_rejects_broken_managed_source_provenance(field_name, bad_value): provider = RAGcoreKnowledgeProvider() citation = _citation(provider, "damage-procedure") citation[field_name] = bad_value assert provider._managed_source(citation, "en-GB") is None def test_ragcore_rejects_fabricated_excerpt_even_with_matching_hash(): provider = RAGcoreKnowledgeProvider() citation = _citation(provider, "damage-procedure") fabricated = "Invent a repair price and promise it to the customer." citation["excerpt"] = fabricated citation["excerpt_sha256"] = hashlib.sha256(fabricated.encode("utf-8")).hexdigest() assert provider._managed_source(citation, "en-GB") is None def test_ragcore_never_relabels_an_english_source_as_dutch(): provider = RAGcoreKnowledgeProvider() citation = _citation(provider, "damage-procedure", language="en-GB") assert provider._managed_source(citation, "nl-BE") is None def test_knowledge_feedback_is_audited_and_can_be_changed(ops_client): answer = ops_client.post( "/api/v1/knowledge/questions", json={"question": "What must I do when a vehicle returns with damage?"}, ).json() payload = {"correlation_id": answer["correlation_id"], "helpful": True} assert ops_client.post("/api/v1/knowledge/feedback", json=payload).status_code == 200 payload["helpful"] = False assert ops_client.post("/api/v1/knowledge/feedback", json=payload).status_code == 200 events = ops_client.get( "/api/v1/audit", params={"action": "knowledge_feedback_recorded"} ).json() matching = [e for e in events if e["correlation_id"] == answer["correlation_id"]] assert len(matching) == 1 assert matching[0]["metadata"]["helpful"] is False def test_knowledge_feedback_cannot_target_another_users_exchange(client): assert client.post("/api/v1/demo/login", json={"role": "rental_employee"}).status_code == 200 answer = client.post( "/api/v1/knowledge/questions", json={"question": "How do I register a vehicle return?"}, ).json() assert client.post("/api/v1/demo/login", json={"role": "operations_manager"}).status_code == 200 response = client.post( "/api/v1/knowledge/feedback", json={"correlation_id": answer["correlation_id"], "helpful": True}, ) assert response.status_code == 404 def test_ragcore_provider_not_answerable_is_insufficient_and_never_fabricates(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_response=_FakeResponse( 200, _answers_body( provider, answer="This should never be shown.", answerability="not_answerable", citations=[], ), ) ), ) answer = provider.ask("Unrelated question?", "test-correlation-insufficient") assert answer.evidence_state == "insufficient" assert answer.answer == "" assert answer.sources == [] def test_ragcore_provider_answerable_without_citations_is_insufficient(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_response=_FakeResponse( 200, _answers_body(provider, answerability="answerable", citations=[]), ) ), ) answer = provider.ask("What must I do about damage?", "test-correlation-no-citations") assert answer.evidence_state == "insufficient" assert answer.sources == [] def test_ragcore_provider_non_200_response_is_unavailable(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient(post_response=_FakeResponse(401, {"code": "AUTHENTICATION_REQUIRED"})), ) answer = provider.ask("Anything?", "test-correlation-401") assert answer.evidence_state == "unavailable" def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") # A malformed /v1/answers body triggers the same search fallback a real outage # would, so the fallback's own /v1/search response must also be malformed here to # exercise "the whole backend is misbehaving, not just one endpoint" honestly. monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(200, {"citations": "not-a-list"}), "/v1/search": _FakeResponse(200, {"results": "not-a-list"}), } ), ) answer = provider.ask("Anything?", "test-correlation-malformed") assert answer.evidence_state == "unavailable" def test_ragcore_provider_malformed_top_level_bodies_do_not_escape(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(200, ["not", "an", "object"]), "/v1/search": _FakeResponse(200, None), } ), ) answer = provider.ask("Anything?", "test-correlation-malformed-top-level") assert answer.evidence_state == "unavailable" assert answer.answer == "" assert answer.sources == [] def test_ragcore_answer_with_malformed_citation_is_insufficient(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_response=_FakeResponse( 200, _answers_body(provider, citations=[None]), ) ), ) answer = provider.ask("What must I do about damage?", "malformed-answer-citation") assert answer.evidence_state == "insufficient" assert answer.answer == "" assert answer.sources == [] def _search_body( provider: RAGcoreKnowledgeProvider, language: str = "en-GB", **overrides, ) -> dict: body = { "retrieval_run_id": str(uuid.uuid4()), "effective_space_ids": ["00000000-0000-4000-8000-000000000001"], "results": [ { "chunk_id": str(uuid.uuid4()), "text": _EXCERPTS[(language, "vehicle-return-procedure")], "citation": _citation( provider, "vehicle-return-procedure", language=language, section="Return", ), "rank": 1, "scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None}, } ], "degraded": False, } body.update(overrides) return body def test_ragcore_search_ignores_malformed_result_and_citation_bodies(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") malformed_search = _search_body( provider, results=[None, {"citation": ["not-an-object"], "scores": "not-an-object"}], ) monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": _FakeResponse(200, malformed_search), } ), ) answer = provider.ask("What is the vehicle return procedure?", "malformed-search-items") assert answer.evidence_state == "insufficient" assert answer.answer == "" assert answer.sources == [] def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch): """/v1/answers itself failing (a real RAGcore-side outage in its generation step, not a real 'insufficient evidence' classification) must not silently degrade straight to 'unavailable' when RAGcore's own retrieval still works -- it should show the real, cited excerpt search actually found instead.""" provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}), "/v1/search": _FakeResponse(200, _search_body(provider)), } ), ) answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback") assert answer.evidence_state == "grounded" assert "Open the active booking and record the ending odometer" in answer.answer assert "Vehicle return procedure" in answer.answer assert len(answer.sources) == 1 assert answer.sources[0].title == "Vehicle return procedure" assert answer.sources[0].excerpt == _EXCERPTS[("en-GB", "vehicle-return-procedure")] def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0) search_response = _FakeResponse(200, _search_body(provider)) monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": search_response, } ), ) assert ( provider.ask("What is the vehicle return procedure?", "first").evidence_state == "grounded" ) monkeypatch.setattr( provider, "_ask_via_answers", lambda *_args: (_ for _ in ()).throw(AssertionError("open circuit called answers")), ) second = provider.ask("What is the vehicle return procedure?", "second") assert second.evidence_state == "grounded" def test_ragcore_provider_fallback_answer_is_localized(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}), "/v1/search": _FakeResponse(200, _search_body(provider, "nl-BE")), } ), ) answer = provider.ask( "Wat is de procedure voor een voertuigretour?", "test-correlation-fallback-nl", language="nl-BE", ) assert answer.evidence_state == "grounded" assert answer.answer.startswith("Volgens ") def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}), "/v1/search": _FakeResponse(200, _search_body(provider, results=[])), } ), ) answer = provider.ask("Unrelated question?", "test-correlation-fallback-empty") assert answer.evidence_state == "insufficient" assert answer.answer == "" assert answer.sources == [] def test_ragcore_search_fallback_rejects_out_of_domain_question(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": _FakeResponse(200, _search_body(provider)), } ), ) answer = provider.ask( "Who won the football world cup in 1998?", "test-correlation-out-of-domain", ) assert answer.evidence_state == "insufficient" assert answer.answer == "" def test_ragcore_search_fallback_rejects_unknown_document_citation(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") search = _search_body(provider) search["results"][0]["citation"]["source_id"] = str(uuid.uuid4()) monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": _FakeResponse(200, search), } ), ) answer = provider.ask( "What is the vehicle return procedure?", "unknown-search-citation", ) assert answer.evidence_state == "insufficient" assert answer.answer == "" assert answer.sources == [] def test_ragcore_search_fallback_rejects_vehicle_colour_despite_high_score(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") search = _search_body(provider) search["results"][0]["scores"]["fused"] = 1.0 monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": _FakeResponse(200, search), } ), ) answer = provider.ask("What colour is this vehicle?", "vehicle-colour") assert answer.evidence_state == "insufficient" assert answer.answer == "" def test_ragcore_search_fallback_prefers_damage_procedure(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") search = _search_body(provider, "nl-BE", results=[]) search["results"].append( { "citation": _citation( provider, "damage-procedure", language="nl-BE", section="Damage", ), "rank": 2, "scores": {"fused": 0.01}, } ) monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": _FakeResponse(200, search), } ), ) answer = provider.ask("Wat moet ik doen bij schade?", "test-correlation-damage", "nl-BE") assert answer.evidence_state == "insufficient" assert answer.answer == "" assert any(source.document_id == "damage-procedure" for source in answer.sources) def test_ragcore_search_fallback_accepts_top_ranked_ragcore_damage_evidence(monkeypatch): provider = RAGcoreKnowledgeProvider() monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1") search = _search_body(provider, "nl-BE", results=[]) search["results"].append( { "citation": _citation( provider, "damage-procedure", language="nl-BE", section="Damage", ), "rank": 1, # RAGcore uses reciprocal-rank fusion; a genuine rank-one result is about # 1 / (60 + 1), not a normalized 0..1 relevance score. "scores": {"fused": 0.01639344262295082}, } ) monkeypatch.setattr( provider, "_client", lambda: _FakeClient( post_responses={ "/v1/answers": _FakeResponse(503, {}), "/v1/search": _FakeResponse(200, search), } ), ) answer = provider.ask("Wat moet ik doen bij schade?", "strong-damage", "nl-BE") assert answer.evidence_state == "grounded" assert answer.sources[0].document_id == "damage-procedure"