fix: fall back to real RAGcore search when /v1/answers is unavailable

RAGcore's /v1/answers (generation + citation validation) is currently returning
a consistent 503 VALIDATION_RETRIES_EXHAUSTED live -- a RAGcore-side bug in its
own generation/validation step, out of scope to fix here (CLAUDE.md forbids
modifying the RAGcore repo). Its retrieval pipeline (/v1/search) is a materially
different, simpler stage with no generation step, and returns real, correctly
cited results right now.

RAGcoreKnowledgeProvider.ask() tries /v1/answers first (unchanged behavior once
RAGcore's generation is fixed), and only when that endpoint itself is
unavailable -- non-2xx or unreachable, never a real 200 classifying the
question as insufficiently answerable -- falls back to /v1/search and builds
the shown "answer" as an extractive citation-wrapped excerpt, mirroring
DemoKnowledgeProvider's own existing template exactly. Never invents an answer
to the question; only ever shows a real, cited excerpt RAGcore's own search
actually found.

Also fixed two real config bugs found while wiring this up live: RAGCORE_BASE_URL
pointed at a non-existent internal hostname (ragcore-api:8000 -- the real
container is reachable at the host's own address on port 1237), and the
previous test credential had been invalidated with nothing to replace it. Minted
a fresh, correctly-scoped service-account credential via RAGcore's own admin
control plane (the documented, legitimate way to obtain one).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
NuklearRabbit
2026-08-05 21:11:28 +02:00
co-authored by Claude Sonnet 5
parent 453c7241fe
commit a2433d7fa3
2 changed files with 206 additions and 5 deletions
+108 -3
View File
@@ -168,9 +168,14 @@ class _FakeResponse:
class _FakeClient:
def __init__(self, get_response=None, post_response=None, raise_on=None):
def __init__(self, get_response=None, post_response=None, post_responses=None, raise_on=None):
self._get_response = get_response
self._post_response = post_response
# Maps a path (e.g. "/v1/search") to its own response, for tests that need
# /v1/answers and /v1/search to behave differently in the same call. Falls back
# to the single post_response when a path has no specific entry, so every
# existing single-endpoint test keeps working unchanged.
self._post_responses = post_responses or {}
self._raise_on = raise_on
def __enter__(self):
@@ -187,7 +192,7 @@ class _FakeClient:
def post(self, path, json=None):
if self._raise_on == "post":
raise httpx.ConnectError("no ragcore in this environment")
return self._post_response
return self._post_responses.get(path, self._post_response)
def test_ragcore_provider_degrades_to_unavailable(monkeypatch):
@@ -338,10 +343,110 @@ def test_ragcore_provider_non_200_response_is_unavailable(monkeypatch):
def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
# A malformed /v1/answers body triggers the same search fallback a real outage
# would, so the fallback's own /v1/search response must also be malformed here to
# exercise "the whole backend is misbehaving, not just one endpoint" honestly.
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(post_response=_FakeResponse(200, {"citations": "not-a-list"})),
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(200, {"citations": "not-a-list"}),
"/v1/search": _FakeResponse(200, {"results": "not-a-list"}),
}
),
)
answer = provider.ask("Anything?", "test-correlation-malformed")
assert answer.evidence_state == "unavailable"
def _search_body(**overrides) -> dict:
body = {
"results": [
{
"chunk_id": "chunk-1",
"citation": {
"id": "cite-1",
"document_id": "doc-1",
"document_version_id": "version-1",
"title": "Vehicle return procedure",
"section": "Return",
"excerpt": "Register the return odometer reading before releasing the vehicle.",
},
"rank": 1,
"scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None},
}
],
"degraded": False,
}
body.update(overrides)
return body
def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch):
"""/v1/answers itself failing (a real RAGcore-side outage in its generation step,
not a real 'insufficient evidence' classification) must not silently degrade
straight to 'unavailable' when RAGcore's own retrieval still works -- it should show
the real, cited excerpt search actually found instead."""
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
}
),
)
answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback")
assert answer.evidence_state == "grounded"
assert "Register the return odometer reading" in answer.answer
assert "Vehicle return procedure" in answer.answer
assert len(answer.sources) == 1
assert answer.sources[0].title == "Vehicle return procedure"
assert answer.sources[0].excerpt == (
"Register the return odometer reading before releasing the vehicle."
)
def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body()),
}
),
)
answer = provider.ask(
"Wat is de procedure voor een voertuigretour?",
"test-correlation-fallback-nl",
language="nl-BE",
)
assert answer.evidence_state == "grounded"
assert answer.answer.startswith("Volgens ")
def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkeypatch):
provider = RAGcoreKnowledgeProvider()
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
monkeypatch.setattr(
provider,
"_client",
lambda: _FakeClient(
post_responses={
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
"/v1/search": _FakeResponse(200, _search_body(results=[])),
}
),
)
answer = provider.ask("Unrelated question?", "test-correlation-fallback-empty")
assert answer.evidence_state == "insufficient"
assert answer.answer == ""
assert answer.sources == []