M31: verify RAG inventory and polish attention queue

backend/app/services/knowledge/ragcore.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
+from threading import Lock
+from time import monotonic
 
 import httpx
 
 from app.core.config import get_settings
 from app.services.knowledge import EvidenceState, GroundedAnswer, KnowledgeHealth, SourceCard
-from app.services.knowledge.procedures import iter_procedure_documents
+from app.services.knowledge.procedures import ProcedureDocument, iter_procedure_documents
 
 _GROUNDED_ANSWERABILITY = {"answerable", "partially_answerable"}
 
@@ -24,6 +27,9 @@ _LEAD_ANSWER_TEMPLATE = {
 }
 _DEFAULT_LANGUAGE = "en-GB"
 _MAX_SOURCE_CARDS = 3
+_INDEX_LOOKUP_TIMEOUT_SECONDS = 2.0
+_INDEX_VERIFICATION_TTL_SECONDS = 300.0
+_INDEX_VERIFICATION_WORKERS = 6
 
 _DOMAIN_CONCEPTS: dict[str, tuple[str, ...]] = {
     "damage": ("damage", "damaged", "schade", "beschadigd", "dommage", "endommagé"),
@@ -118,6 +124,8 @@ class RAGcoreKnowledgeProvider:
 
     def __init__(self) -> None:
         self._settings = get_settings()
+        self._verification_cache: dict[str, tuple[float, int]] = {}
+        self._verification_lock = Lock()
 
     def _client(self) -> httpx.Client:
         headers = {}
@@ -130,6 +138,12 @@ class RAGcoreKnowledgeProvider:
         )
 
     def health(self, language: str = "en-GB") -> KnowledgeHealth:
+        documents = [
+            document
+            for document in iter_procedure_documents(Path(self._settings.knowledge_dir))
+            if document.language == language
+        ]
+        verified_document_count: int | None = None
         try:
             with self._client() as client:
                 response = client.get("/health/ready")
@@ -140,13 +154,20 @@ class RAGcoreKnowledgeProvider:
                     if available
                     else f"RAGcore degraded: {body.get('status', 'unknown')}"
                 )
+                if available:
+                    verified_document_count = self._verify_indexed_documents(
+                        client, language, documents
+                    )
+                    if verified_document_count is None:
+                        detail += " Index verification is temporarily unavailable."
+                    else:
+                        detail += (
+                            f" {verified_document_count}/{len(documents)} managed sources have "
+                            "an active published version with the expected content hash."
+                        )
         except (httpx.HTTPError, ValueError) as exc:
             available = False
             detail = f"RAGcore unavailable: {type(exc).__name__}: {exc}"
-        source_document_count = sum(
-            document.language == language
-            for document in iter_procedure_documents(Path(self._settings.knowledge_dir))
-        )
         return KnowledgeHealth(
             provider=self.name,
             available=available,
@@ -154,16 +175,70 @@ class RAGcoreKnowledgeProvider:
             tenant=self._settings.ragcore_tenant,
             workspace=self._settings.ragcore_workspace,
             collection=self._settings.ragcore_collection,
-            # RAGcore's retrieval API has no corpus-size endpoint. Unknown is explicit
-            # so the UI never turns this into the misleading claim "0 procedures".
-            document_count=None,
-            source_document_count=source_document_count,
+            # RAGcore deliberately has no browse/count endpoint. Fleet Ops instead
+            # verifies each managed source through its exact identity lookup and only
+            # counts an active published version whose content hash still matches.
+            document_count=verified_document_count,
+            source_document_count=len(documents),
             reported_synced_document_count=None,
             reported_failed_document_count=None,
             last_sync_at=None,
-            statistics_state="not_reported",
+            statistics_state=(
+                "verified" if verified_document_count is not None else "not_reported"
+            ),
         )
 
+    def _verify_indexed_documents(
+        self, client: httpx.Client, language: str, documents: list[ProcedureDocument]
+    ) -> int | None:
+        if not self._settings.ragcore_space_id or not documents:
+            return None
+
+        now = monotonic()
+        with self._verification_lock:
+            cached = self._verification_cache.get(language)
+            if cached is not None and now - cached[0] < _INDEX_VERIFICATION_TTL_SECONDS:
+                return cached[1]
+
+        def is_verified(document: ProcedureDocument) -> bool:
+            response = client.get(
+                "/v1/documents",
+                params={
+                    "source_id": document.source_id,
+                    "external_id": f"{document.document_id}.md",
+                },
+                timeout=_INDEX_LOOKUP_TIMEOUT_SECONDS,
+            )
+            if response.status_code != 200:
+                raise RuntimeError("RAGcore document verification failed")
+            body = response.json()
+            items = body.get("items") if isinstance(body, dict) else None
+            if not isinstance(items, list) or len(items) != 1 or not isinstance(items[0], dict):
+                return False
+            item = items[0]
+            active_version = item.get("active_version")
+            return bool(
+                item.get("space_id") == self._settings.ragcore_space_id
+                and item.get("source_id") == document.source_id
+                and item.get("external_id") == f"{document.document_id}.md"
+                and item.get("status") == "active"
+                and isinstance(active_version, dict)
+                and active_version.get("status") == "published"
+                and active_version.get("content_sha256") == document.content_hash
+            )
+
+        try:
+            with ThreadPoolExecutor(
+                max_workers=min(_INDEX_VERIFICATION_WORKERS, len(documents))
+            ) as executor:
+                verified_count = sum(executor.map(is_verified, documents))
+        except (httpx.HTTPError, RuntimeError, TypeError, ValueError):
+            return None
+
+        with self._verification_lock:
+            self._verification_cache[language] = (monotonic(), verified_count)
+        return verified_count
+
     def ask(self, question: str, correlation_id: str, language: str = "en-GB") -> GroundedAnswer:
         unavailable = GroundedAnswer(
             answer="",