M44: harden release integrity and assurance

backend/Dockerfile

@@ -1,4 +1,10 @@
 FROM python:3.12-slim-bookworm@sha256:a116514e19457bcb7af7efe9c3dd0b9b71e85b317694e7882a1c52aa15a78134 AS runtime-base
+ARG VCS_REF=development
+ARG BUILD_DATE=unknown
+LABEL org.opencontainers.image.title="Fleet Ops API" \
+      org.opencontainers.image.revision="$VCS_REF" \
+      org.opencontainers.image.created="$BUILD_DATE" \
+      org.opencontainers.image.source="https://fleetops.itworx.tech"
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
 WORKDIR /app
 COPY backend/requirements-prod.lock ./

backend/alembic.ini

@@ -1,7 +1,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-version_path_separator = os
+path_separator = os
 
 [loggers]
 keys = root,sqlalchemy,alembic

backend/app/core/config.py

@@ -23,6 +23,7 @@ class Settings(BaseSettings):
     ragcore_api_token: str = ""
     ragcore_space_id: str = ""
     ragcore_http_timeout_seconds: float = 5.0
+    ragcore_answers_circuit_breaker_seconds: float = 60.0
     # Search fallback is only labelled grounded above this explicit retrieval threshold.
     # RAGcore's fused score is reciprocal-rank based (top ranks are ~1/61), so this
     # accepts only leading results while still rejecting absent and low-ranked evidence.

backend/app/core/observability.py

@@ -36,6 +36,16 @@ DATABASE_READY = Gauge(
     "mobilityops_database_ready",
     "Whether the canonical PostgreSQL database answered the most recent readiness probe.",
 )
+KNOWLEDGE_PROVIDER_REQUESTS = Counter(
+    "mobilityops_knowledge_provider_requests_total",
+    "RAGcore adapter requests by stage and outcome.",
+    ("stage", "outcome"),
+)
+KNOWLEDGE_RETRIEVAL_SCORE = Histogram(
+    "mobilityops_knowledge_retrieval_score",
+    "Observed RAGcore fused/rerank retrieval scores.",
+    buckets=(0.005, 0.01, 0.015, 0.016, 0.0162, 0.0164, 0.02, 0.05, 0.1, 0.5, 1.0),
+)
 
 
 class JsonFormatter(logging.Formatter):

backend/app/main.py

@@ -59,6 +59,11 @@ production = settings.mobilityops_env.lower() == "production"
 app = FastAPI(
     title=f"{PRODUCT_NAME} API",
     version="0.1.0",
+    description=(
+        "Generated contract for Fleet Ops. The visible product name is Fleet Ops; "
+        "MobilityOps remains the technical repository and service identifier."
+    ),
+    servers=[{"url": "http://localhost:8128"}],
     lifespan=lifespan,
     docs_url=None if production else "/docs",
     redoc_url=None if production else "/redoc",

backend/app/services/knowledge/ragcore.py

@@ -8,6 +8,7 @@ from time import monotonic
 import httpx
 
 from app.core.config import get_settings
+from app.core.observability import KNOWLEDGE_PROVIDER_REQUESTS, KNOWLEDGE_RETRIEVAL_SCORE
 from app.services.knowledge import EvidenceState, GroundedAnswer, KnowledgeHealth, SourceCard
 from app.services.knowledge.procedures import ProcedureDocument, iter_procedure_documents
 
@@ -137,6 +138,22 @@ class RAGcoreKnowledgeProvider:
         self._settings = get_settings()
         self._verification_cache: dict[str, tuple[float, int]] = {}
         self._verification_lock = Lock()
+        self._answers_circuit_lock = Lock()
+        self._answers_circuit_open_until = 0.0
+
+    def _answers_circuit_is_open(self) -> bool:
+        with self._answers_circuit_lock:
+            return monotonic() < self._answers_circuit_open_until
+
+    def _open_answers_circuit(self) -> None:
+        with self._answers_circuit_lock:
+            self._answers_circuit_open_until = monotonic() + max(
+                0.0, self._settings.ragcore_answers_circuit_breaker_seconds
+            )
+
+    def _close_answers_circuit(self) -> None:
+        with self._answers_circuit_lock:
+            self._answers_circuit_open_until = 0.0
 
     def _client(self) -> httpx.Client:
         headers = {}
@@ -264,9 +281,12 @@ class RAGcoreKnowledgeProvider:
         if not self._settings.ragcore_space_id:
             return unavailable
 
-        answered = self._ask_via_answers(question, correlation_id)
-        if answered is not None:
-            return answered
+        if self._answers_circuit_is_open():
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "circuit_open").inc()
+        else:
+            answered = self._ask_via_answers(question, correlation_id)
+            if answered is not None:
+                return answered
         # /v1/answers itself is unavailable (non-2xx or unreachable) -- fall back to
         # real retrieval rather than degrading straight to "unavailable". This never
         # fabricates an answer to the question: it only ever shows an actually-cited
@@ -290,9 +310,13 @@ class RAGcoreKnowledgeProvider:
                     },
                 )
                 if response.status_code != 200:
+                    KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "non_2xx").inc()
+                    self._open_answers_circuit()
                     return None
                 body = response.json()
         except (httpx.HTTPError, ValueError):
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "error").inc()
+            self._open_answers_circuit()
             return None
 
         try:
@@ -311,6 +335,8 @@ class RAGcoreKnowledgeProvider:
             answerability = body.get("answerability", "not_answerable")
             is_grounded = answerability in _GROUNDED_ANSWERABILITY and sources
             evidence_state: EvidenceState = "grounded" if is_grounded else "insufficient"
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", evidence_state).inc()
+            self._close_answers_circuit()
             return GroundedAnswer(
                 answer=body.get("answer", "") if evidence_state == "grounded" else "",
                 evidence_state=evidence_state,
@@ -319,6 +345,8 @@ class RAGcoreKnowledgeProvider:
                 correlation_id=correlation_id,
             )
         except (TypeError, KeyError, ValueError):
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("answers", "malformed").inc()
+            self._open_answers_circuit()
             return None
 
     def _ask_via_search_fallback(
@@ -342,9 +370,11 @@ class RAGcoreKnowledgeProvider:
                     },
                 )
                 if response.status_code != 200:
+                    KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "non_2xx").inc()
                     return unavailable
                 body = response.json()
         except (httpx.HTTPError, ValueError):
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "error").inc()
             return unavailable
 
         try:
@@ -362,8 +392,14 @@ class RAGcoreKnowledgeProvider:
                 for result in results
             ]
         except (TypeError, KeyError, ValueError):
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "malformed").inc()
             return unavailable
 
+        for result in results:
+            score = _retrieval_score(result)
+            if score is not None:
+                KNOWLEDGE_RETRIEVAL_SCORE.observe(score)
+
         all_sources = _deduplicate_sources(sources)
         concepts = _question_concepts(question)
         qualified_sources = [
@@ -379,6 +415,7 @@ class RAGcoreKnowledgeProvider:
         ]
         sources = _rank_sources_for_concepts(_deduplicate_sources(qualified_sources), concepts)
         if not all_sources:
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "insufficient").inc()
             return GroundedAnswer(
                 answer="",
                 evidence_state="insufficient",
@@ -396,6 +433,7 @@ class RAGcoreKnowledgeProvider:
             for term in _DOMAIN_CONCEPTS["damage"]
         )
         if not concepts or not sources or ("damage" in concepts and not damage_evidence):
+            KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "insufficient").inc()
             return GroundedAnswer(
                 answer="",
                 evidence_state="insufficient",
@@ -407,6 +445,7 @@ class RAGcoreKnowledgeProvider:
         template = _LEAD_ANSWER_TEMPLATE.get(language, _LEAD_ANSWER_TEMPLATE[_DEFAULT_LANGUAGE])
         lead = sources[0]
         answer = template.format(title=lead.title, excerpt=lead.excerpt)
+        KNOWLEDGE_PROVIDER_REQUESTS.labels("search", "grounded").inc()
         return GroundedAnswer(
             answer=answer,
             evidence_state="grounded",

backend/tests/test_knowledge.py

@@ -616,6 +616,35 @@ def test_ragcore_provider_falls_back_to_search_when_answers_unavailable(monkeypa
     )
 
 
+def test_ragcore_answers_circuit_skips_repeated_generation_failure(monkeypatch):
+    provider = RAGcoreKnowledgeProvider()
+    monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")
+    monkeypatch.setattr(provider._settings, "ragcore_answers_circuit_breaker_seconds", 60.0)
+    search_response = _FakeResponse(200, _search_body())
+    monkeypatch.setattr(
+        provider,
+        "_client",
+        lambda: _FakeClient(
+            post_responses={
+                "/v1/answers": _FakeResponse(503, {}),
+                "/v1/search": search_response,
+            }
+        ),
+    )
+    assert (
+        provider.ask("What is the vehicle return procedure?", "first").evidence_state
+        == "grounded"
+    )
+
+    monkeypatch.setattr(
+        provider,
+        "_ask_via_answers",
+        lambda *_args: (_ for _ in ()).throw(AssertionError("open circuit called answers")),
+    )
+    second = provider.ask("What is the vehicle return procedure?", "second")
+    assert second.evidence_state == "grounded"
+
+
 def test_ragcore_provider_fallback_answer_is_localized(monkeypatch):
     provider = RAGcoreKnowledgeProvider()
     monkeypatch.setattr(provider._settings, "ragcore_space_id", "space-1")

backend/tests/test_observability.py

@@ -29,6 +29,8 @@ def test_metrics_expose_http_database_and_outbox_state(client):
     assert "mobilityops_http_requests_total" in response.text
     assert "mobilityops_database_ready 1.0" in response.text
     assert 'mobilityops_outbox_events{scenario="synthetic",status="failed"}' in response.text
+    assert "mobilityops_knowledge_provider_requests_total" in response.text
+    assert "mobilityops_knowledge_retrieval_score" in response.text
 
 
 def test_metrics_token_is_enforced_when_configured(client, monkeypatch):