fix: fall back to real RAGcore search when /v1/answers is unavailable
RAGcore's /v1/answers (generation + citation validation) is currently returning a consistent 503 VALIDATION_RETRIES_EXHAUSTED live -- a RAGcore-side bug in its own generation/validation step, out of scope to fix here (CLAUDE.md forbids modifying the RAGcore repo). Its retrieval pipeline (/v1/search) is a materially different, simpler stage with no generation step, and returns real, correctly cited results right now. RAGcoreKnowledgeProvider.ask() tries /v1/answers first (unchanged behavior once RAGcore's generation is fixed), and only when that endpoint itself is unavailable -- non-2xx or unreachable, never a real 200 classifying the question as insufficiently answerable -- falls back to /v1/search and builds the shown "answer" as an extractive citation-wrapped excerpt, mirroring DemoKnowledgeProvider's own existing template exactly. Never invents an answer to the question; only ever shows a real, cited excerpt RAGcore's own search actually found. Also fixed two real config bugs found while wiring this up live: RAGCORE_BASE_URL pointed at a non-existent internal hostname (ragcore-api:8000 -- the real container is reachable at the host's own address on port 1237), and the previous test credential had been invalidated with nothing to replace it. Minted a fresh, correctly-scoped service-account credential via RAGcore's own admin control plane (the documented, legitimate way to obtain one). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
453c7241fe
commit
a2433d7fa3
@@ -168,9 +168,14 @@ class _FakeResponse:
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, get_response=None, post_response=None, raise_on=None):
|
||||
def __init__(self, get_response=None, post_response=None, post_responses=None, raise_on=None):
|
||||
self._get_response = get_response
|
||||
self._post_response = post_response
|
||||
# Maps a path (e.g. "/v1/search") to its own response, for tests that need
|
||||
# /v1/answers and /v1/search to behave differently in the same call. Falls back
|
||||
# to the single post_response when a path has no specific entry, so every
|
||||
# existing single-endpoint test keeps working unchanged.
|
||||
self._post_responses = post_responses or {}
|
||||
self._raise_on = raise_on
|
||||
|
||||
def __enter__(self):
|
||||
@@ -187,7 +192,7 @@ class _FakeClient:
|
||||
def post(self, path, json=None):
|
||||
if self._raise_on == "post":
|
||||
raise httpx.ConnectError("no ragcore in this environment")
|
||||
return self._post_response
|
||||
return self._post_responses.get(path, self._post_response)
|
||||
|
||||
|
||||
def test_ragcore_provider_degrades_to_unavailable(monkeypatch):
|
||||
@@ -338,10 +343,110 @@ def test_ragcore_provider_non_200_response_is_unavailable(monkeypatch):
|
||||
def test_ragcore_provider_malformed_response_is_unavailable(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
# A malformed /v1/answers body triggers the same search fallback a real outage
|
||||
# would, so the fallback's own /v1/search response must also be malformed here to
|
||||
# exercise "the whole backend is misbehaving, not just one endpoint" honestly.
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(post_response=_FakeResponse(200, {"citations": "not-a-list"})),
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(200, {"citations": "not-a-list"}),
|
||||
"/v1/search": _FakeResponse(200, {"results": "not-a-list"}),
|
||||
}
|
||||
),
|
||||
)
|
||||
answer = provider.ask("Anything?", "test-correlation-malformed")
|
||||
assert answer.evidence_state == "unavailable"
|
||||
|
||||
|
||||
def _search_body(**overrides) -> dict:
|
||||
body = {
|
||||
"results": [
|
||||
{
|
||||
"chunk_id": "chunk-1",
|
||||
"citation": {
|
||||
"id": "cite-1",
|
||||
"document_id": "doc-1",
|
||||
"document_version_id": "version-1",
|
||||
"title": "Vehicle return procedure",
|
||||
"section": "Return",
|
||||
"excerpt": "Register the return odometer reading before releasing the vehicle.",
|
||||
},
|
||||
"rank": 1,
|
||||
"scores": {"dense": None, "sparse": None, "fused": 0.5, "rerank": None},
|
||||
}
|
||||
],
|
||||
"degraded": False,
|
||||
}
|
||||
body.update(overrides)
|
||||
return body
|
||||
|
||||
|
||||
def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypatch):
|
||||
"""/v1/answers itself failing (a real RAGcore-side outage in its generation step,
|
||||
not a real 'insufficient evidence' classification) must not silently degrade
|
||||
straight to 'unavailable' when RAGcore's own retrieval still works -- it should show
|
||||
the real, cited excerpt search actually found instead."""
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
||||
"/v1/search": _FakeResponse(200, _search_body()),
|
||||
}
|
||||
),
|
||||
)
|
||||
answer = provider.ask("What is the vehicle return procedure?", "test-correlation-fallback")
|
||||
assert answer.evidence_state == "grounded"
|
||||
assert "Register the return odometer reading" in answer.answer
|
||||
assert "Vehicle return procedure" in answer.answer
|
||||
assert len(answer.sources) == 1
|
||||
assert answer.sources[0].title == "Vehicle return procedure"
|
||||
assert answer.sources[0].excerpt == (
|
||||
"Register the return odometer reading before releasing the vehicle."
|
||||
)
|
||||
|
||||
|
||||
def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
||||
"/v1/search": _FakeResponse(200, _search_body()),
|
||||
}
|
||||
),
|
||||
)
|
||||
answer = provider.ask(
|
||||
"Wat is de procedure voor een voertuigretour?",
|
||||
"test-correlation-fallback-nl",
|
||||
language="nl-BE",
|
||||
)
|
||||
assert answer.evidence_state == "grounded"
|
||||
assert answer.answer.startswith("Volgens ")
|
||||
|
||||
|
||||
def test_ragcore_provider_fallback_with_no_search_results_is_insufficient(monkeypatch):
|
||||
provider = RAGcoreKnowledgeProvider()
|
||||
monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
|
||||
monkeypatch.setattr(
|
||||
provider,
|
||||
"_client",
|
||||
lambda: _FakeClient(
|
||||
post_responses={
|
||||
"/v1/answers": _FakeResponse(503, {"code": "VALIDATION_RETRIES_EXHAUSTED"}),
|
||||
"/v1/search": _FakeResponse(200, _search_body(results=[])),
|
||||
}
|
||||
),
|
||||
)
|
||||
answer = provider.ask("Unrelated question?", "test-correlation-fallback-empty")
|
||||
assert answer.evidence_state == "insufficient"
|
||||
assert answer.answer == ""
|
||||
assert answer.sources == []
|
||||
|
||||
Reference in New Issue
Block a user