M5: implement RAGcore knowledge integration

KnowledgeProvider protocol with a deterministic TF-IDF-weighted extractive demo provider (never generative, always cites real excerpts) and a RAGcore HTTP adapter that degrades cleanly to unavailable. Knowledge nav + chat-style Q&A UI with source cards and honest grounded/insufficient/unavailable states. 57 backend tests passing, ruff clean. Fixed a real relevance bug (generic terms like "vehicle" crowding out distinctive matches) via IDF weighting, found by testing the actual S6 scenario. Verified end-to-end in the browser: grounded damage question cites both expected procedures; unrelated question honestly returns insufficient evidence with no fabrication.

backend/app/api/routers/knowledge.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import uuid
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_current_user, get_db
+from app.schemas import CurrentUser
+from app.services.audit import record_audit_event
+from app.services.knowledge import GroundedAnswer, KnowledgeHealth, get_knowledge_provider
+
+router = APIRouter(prefix="/api/v1/knowledge", tags=["knowledge"])
+
+
+class AskQuestionRequest(BaseModel):
+    question: str = Field(min_length=3, max_length=1000)
+
+
+@router.post("/questions", response_model=GroundedAnswer)
+def ask_question(
+    body: AskQuestionRequest,
+    db: Session = Depends(get_db),
+    user: CurrentUser = Depends(get_current_user),
+) -> GroundedAnswer:
+    correlation_id = str(uuid.uuid4())
+    provider = get_knowledge_provider()
+    answer = provider.ask(body.question, correlation_id)
+
+    record_audit_event(
+        db,
+        actor_type="user",
+        actor_label=user.display_name,
+        action="knowledge_question_asked",
+        entity_type="knowledge",
+        correlation_id=uuid.UUID(correlation_id),
+        metadata={
+            "evidence_state": answer.evidence_state,
+            "provider": answer.provider,
+            "source_ids": [s.document_id for s in answer.sources],
+            "question_length": len(body.question),
+        },
+    )
+    db.commit()
+    return answer
+
+
+@router.get("/status", response_model=KnowledgeHealth)
+def knowledge_status(_user: CurrentUser = Depends(get_current_user)) -> KnowledgeHealth:
+    return get_knowledge_provider().health()

backend/app/core/config.py

@@ -14,6 +14,8 @@ class Settings(BaseSettings):
     ragcore_tenant: str = "northstar-mobility-demo"
     ragcore_workspace: str = "mobilityops"
     ragcore_collection: str = "internal-procedures"
+    ragcore_api_token: str = ""
+    ragcore_http_timeout_seconds: float = 5.0
     n8n_webhook_url: str = "http://n8n:5678/webhook/mobilityops-return"
     n8n_callback_token: str = "replace-me-n8n-callback-token"
     n8n_dispatch_enabled: bool = True
@@ -24,6 +26,7 @@ class Settings(BaseSettings):
     session_cookie_name: str = "mobilityops_session"
     session_ttl_seconds: int = 60 * 60 * 8
     seed_dir: str = "/app/seed"
+    knowledge_dir: str = "/app/knowledge/procedures"
     cors_allow_origins: str = "http://localhost:1228"
     demo_today: str = "2026-08-01"
 

backend/app/main.py

@@ -12,6 +12,7 @@ from app.api.routers import (
     data_quality,
     demo,
     integrations,
+    knowledge,
     vehicles,
     workflows,
 )
@@ -84,3 +85,4 @@ app.include_router(audit.router)
 app.include_router(data_quality.router)
 app.include_router(workflows.router)
 app.include_router(integrations.router)
+app.include_router(knowledge.router)

backend/app/services/knowledge/__init__.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Literal, Protocol
+
+from pydantic import BaseModel
+
+from app.core.config import get_settings
+
+EvidenceState = Literal["grounded", "insufficient", "unavailable"]
+
+
+class SourceCard(BaseModel):
+    document_id: str
+    title: str
+    version: str
+    section: str
+    excerpt: str
+
+
+class GroundedAnswer(BaseModel):
+    answer: str
+    evidence_state: EvidenceState
+    sources: list[SourceCard]
+    provider: str
+    correlation_id: str
+
+
+class KnowledgeHealth(BaseModel):
+    provider: str
+    available: bool
+    detail: str
+    tenant: str
+    workspace: str
+    collection: str
+    document_count: int
+
+
+class KnowledgeProvider(Protocol):
+    name: str
+
+    def health(self) -> KnowledgeHealth: ...
+
+    def ask(self, question: str, correlation_id: str) -> GroundedAnswer: ...
+
+
+@lru_cache
+def get_knowledge_provider() -> KnowledgeProvider:
+    settings = get_settings()
+    if settings.knowledge_provider == "ragcore":
+        from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
+
+        return RAGcoreKnowledgeProvider()
+    from app.services.knowledge.demo import DemoKnowledgeProvider
+
+    return DemoKnowledgeProvider()

backend/app/services/knowledge/demo.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from app.core.config import get_settings
+from app.services.knowledge import GroundedAnswer, KnowledgeHealth, SourceCard
+
+STOPWORDS = {
+    "a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
+    "to", "of", "in", "on", "at", "for", "and", "or", "but", "if", "then",
+    "do", "does", "did", "must", "may", "can", "could", "should", "would",
+    "i", "you", "it", "we", "they", "my", "your", "what", "when", "how",
+    "with", "without", "this", "that", "these", "those", "not", "no",
+}
+
+_WORD_RE = re.compile(r"[a-z0-9]+")
+
+
+def _stem(word: str) -> str:
+    # Deterministic, intentionally crude suffix stripping — good enough to match "returns"
+    # with "return" or "damaged" with "damage" without pulling in a stemming dependency.
+    for suffix in ("ing", "edly", "ed", "es", "s"):
+        if len(word) > len(suffix) + 2 and word.endswith(suffix):
+            return word[: -len(suffix)]
+    return word
+
+
+def _tokenize(text: str) -> set[str]:
+    words = _WORD_RE.findall(text.lower())
+    return {_stem(w) for w in words if w not in STOPWORDS and len(w) > 2}
+
+
+@dataclass
+class Document:
+    document_id: str
+    title: str
+    version: str
+    title_tokens: set[str] = field(default_factory=set)
+
+
+@dataclass
+class ScoredSection:
+    document: Document
+    heading: str
+    text: str
+    heading_tokens: set[str]
+    body_tokens: set[str]
+
+
+def _parse_frontmatter(raw: str) -> tuple[dict[str, str], str]:
+    if not raw.startswith("---"):
+        return {}, raw
+    end = raw.find("\n---", 3)
+    if end == -1:
+        return {}, raw
+    block = raw[3:end].strip()
+    body = raw[end + 4 :].lstrip("\n")
+    meta: dict[str, str] = {}
+    for line in block.splitlines():
+        if ":" not in line:
+            continue
+        key, _, value = line.partition(":")
+        meta[key.strip()] = value.strip().strip('"')
+    return meta, body
+
+
+def _split_sections(body: str) -> list[tuple[str, str]]:
+    sections: list[tuple[str, str]] = []
+    current_heading = "Overview"
+    current_lines: list[str] = []
+    for line in body.splitlines():
+        if line.startswith("## "):
+            if current_lines:
+                sections.append((current_heading, "\n".join(current_lines).strip()))
+            current_heading = line[3:].strip()
+            current_lines = []
+        elif line.startswith("# "):
+            continue
+        else:
+            current_lines.append(line)
+    if current_lines:
+        sections.append((current_heading, "\n".join(current_lines).strip()))
+    return sections
+
+
+def _load_sections(procedures_dir: Path) -> list[ScoredSection]:
+    sections: list[ScoredSection] = []
+    for path in sorted(procedures_dir.glob("*.md")):
+        raw = path.read_text(encoding="utf-8")
+        meta, body = _parse_frontmatter(raw)
+        title = meta.get("title", path.stem)
+        doc = Document(
+            document_id=meta.get("document_id", path.stem),
+            title=title,
+            version=meta.get("version", "1.0"),
+            title_tokens=_tokenize(title),
+        )
+        for heading, text in _split_sections(body):
+            sections.append(
+                ScoredSection(
+                    document=doc,
+                    heading=heading,
+                    text=text,
+                    heading_tokens=_tokenize(heading),
+                    body_tokens=_tokenize(text),
+                )
+            )
+    return sections
+
+
+class DemoKnowledgeProvider:
+    """Deterministic extractive retrieval over the local procedure Markdown files.
+
+    Not a generative model: it scores sections with TF-IDF-weighted keyword overlap
+    (downweighting terms common across the whole corpus, like "vehicle", in favor of
+    distinctive ones, like "damage") and returns real excerpts, never invented text.
+    """
+
+    name = "demo"
+
+    def __init__(self) -> None:
+        settings = get_settings()
+        self._settings = settings
+        self._procedures_dir = Path(settings.knowledge_dir)
+        self._sections = _load_sections(self._procedures_dir)
+        self._document_count = len({s.document.document_id for s in self._sections})
+        self._idf = self._build_idf(self._sections)
+
+    @staticmethod
+    def _build_idf(sections: list[ScoredSection]) -> dict[str, float]:
+        n = len(sections) or 1
+        doc_freq: dict[str, int] = {}
+        for section in sections:
+            doc = section.document
+            all_tokens = doc.title_tokens | section.heading_tokens | section.body_tokens
+            for token in all_tokens:
+                doc_freq[token] = doc_freq.get(token, 0) + 1
+        return {token: math.log((n + 1) / (df + 1)) + 1 for token, df in doc_freq.items()}
+
+    def health(self) -> KnowledgeHealth:
+        return KnowledgeHealth(
+            provider=self.name,
+            available=True,
+            detail="Deterministic keyword-matching demo provider; no external service.",
+            tenant=self._settings.ragcore_tenant,
+            workspace=self._settings.ragcore_workspace,
+            collection=self._settings.ragcore_collection,
+            document_count=self._document_count,
+        )
+
+    def _score(self, query_tokens: set[str], section: ScoredSection) -> float:
+        score = 0.0
+        for token in query_tokens:
+            idf = self._idf.get(token, 0.0)
+            if idf == 0.0:
+                continue
+            if token in section.heading_tokens:
+                score += 3 * idf
+            elif token in section.document.title_tokens:
+                score += 2 * idf
+            elif token in section.body_tokens:
+                score += idf
+        return score
+
+    def ask(self, question: str, correlation_id: str) -> GroundedAnswer:
+        query_tokens = _tokenize(question)
+        scored = [
+            (self._score(query_tokens, section), section)
+            for section in self._sections
+        ]
+        scored = [(score, section) for score, section in scored if score > 0]
+        scored.sort(key=lambda item: item[0], reverse=True)
+        top = scored[:3]
+
+        if not top:
+            return GroundedAnswer(
+                answer="No matching procedure was found for this question.",
+                evidence_state="insufficient",
+                sources=[],
+                provider=self.name,
+                correlation_id=correlation_id,
+            )
+
+        sources = [
+            SourceCard(
+                document_id=section.document.document_id,
+                title=section.document.title,
+                version=section.document.version,
+                section=section.heading,
+                excerpt=(section.text[:400] + "…") if len(section.text) > 400 else section.text,
+            )
+            for _, section in top
+        ]
+
+        if top[0][0] < 3:
+            return GroundedAnswer(
+                answer=(
+                    "The available procedures do not clearly answer this question. "
+                    "The closest matches are included below for review."
+                ),
+                evidence_state="insufficient",
+                sources=sources,
+                provider=self.name,
+                correlation_id=correlation_id,
+            )
+
+        lead_section = top[0][1]
+        answer = (
+            f'Per "{lead_section.document.title}" (v{lead_section.document.version}), '
+            f'section "{lead_section.heading}": {lead_section.text.splitlines()[0][:300]}'
+        )
+        return GroundedAnswer(
+            answer=answer,
+            evidence_state="grounded",
+            sources=sources,
+            provider=self.name,
+            correlation_id=correlation_id,
+        )

backend/app/services/knowledge/ragcore.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import httpx
+
+from app.core.config import get_settings
+from app.services.knowledge import GroundedAnswer, KnowledgeHealth, SourceCard
+
+
+class RAGcoreKnowledgeProvider:
+    """Adapter for the central RAGcore service.
+
+    RAGcore is built and owned separately (see contracts/ragcore-contract-assumptions.md).
+    No live RAGcore instance was reachable during this build, so the exact request/response
+    shape below is a best-effort guess at a REST contract; any failure (connection, timeout,
+    malformed response) degrades to `unavailable` rather than raising, per the architecture's
+    reliability boundary: RAGcore failure disables knowledge answers only, never the rest of
+    the app, and never fabricates an answer.
+    """
+
+    name = "ragcore"
+
+    def __init__(self) -> None:
+        self._settings = get_settings()
+
+    def _client(self) -> httpx.Client:
+        headers = {}
+        if self._settings.ragcore_api_token:
+            headers["Authorization"] = f"Bearer {self._settings.ragcore_api_token}"
+        return httpx.Client(
+            base_url=self._settings.ragcore_base_url,
+            headers=headers,
+            timeout=self._settings.ragcore_http_timeout_seconds,
+        )
+
+    def health(self) -> KnowledgeHealth:
+        try:
+            with self._client() as client:
+                response = client.get("/health")
+                response.raise_for_status()
+                available = True
+                detail = "RAGcore reachable."
+        except httpx.HTTPError as exc:
+            available = False
+            detail = f"RAGcore unavailable: {type(exc).__name__}: {exc}"
+        return KnowledgeHealth(
+            provider=self.name,
+            available=available,
+            detail=detail,
+            tenant=self._settings.ragcore_tenant,
+            workspace=self._settings.ragcore_workspace,
+            collection=self._settings.ragcore_collection,
+            document_count=0,
+        )
+
+    def ask(self, question: str, correlation_id: str) -> GroundedAnswer:
+        try:
+            with self._client() as client:
+                response = client.post(
+                    "/api/v1/ask",
+                    json={
+                        "tenant": self._settings.ragcore_tenant,
+                        "workspace": self._settings.ragcore_workspace,
+                        "collection": self._settings.ragcore_collection,
+                        "question": question,
+                        "correlation_id": correlation_id,
+                    },
+                )
+                response.raise_for_status()
+                body = response.json()
+        except (httpx.HTTPError, ValueError):
+            return GroundedAnswer(
+                answer="",
+                evidence_state="unavailable",
+                sources=[],
+                provider=self.name,
+                correlation_id=correlation_id,
+            )
+
+        try:
+            sources = [SourceCard(**s) for s in body.get("sources", [])]
+            evidence_state = body.get("evidence_state", "insufficient")
+            if evidence_state not in ("grounded", "insufficient", "unavailable"):
+                evidence_state = "insufficient"
+            return GroundedAnswer(
+                answer=body.get("answer", ""),
+                evidence_state=evidence_state,
+                sources=sources,
+                provider=self.name,
+                correlation_id=correlation_id,
+            )
+        except (TypeError, ValueError):
+            return GroundedAnswer(
+                answer="",
+                evidence_state="unavailable",
+                sources=[],
+                provider=self.name,
+                correlation_id=correlation_id,
+            )