M25: expose provenance-aware knowledge statistics

This commit is contained in:
NuklearRabbit
2026-08-10 16:04:39 +02:00
parent 0935901f11
commit 90cc3cf378
22 changed files with 238 additions and 10 deletions
+20
View File
@@ -2574,3 +2574,23 @@ evidence yet."
to stable callback/effect dependencies and a lazy route chunk; TypeScript, lint and
production build pass. Exact next action: extend RAGcore corpus statistics and health
evidence, then run complete acceptance and deploy all production-readiness milestones.
## M25 — provenance-aware knowledge statistics (2026-08-10)
- Expanded knowledge health with separate counts for authoritative local source
documents, the latest persisted n8n sync report and documents independently verified
as indexed. The API includes failed sync count, report time and an explicit statistics
provenance state.
- Confirmed against RAGcore's checked-in OpenAPI and route implementation that the
service intentionally exposes identity-based single-document lookup but no corpus-size
or space-browse endpoint. Fleet Ops therefore keeps `document_count=null` for RAGcore
and never mislabels an accepted upload as proven indexing/publication.
- Knowledge, dashboard and Integration Management now present the available source and
sync evidence in all three locales. The deterministic demo provider continues to
report its directly verified per-language corpus count.
- Regenerated the Fleet Ops OpenAPI contract and documented the provenance rules.
- Evidence: focused knowledge/demo suite **36 passed with zero warnings** from a rebuilt
image; ruff/mypy passed; frontend TypeScript lint and production build passed. The
prior Starlette/httpx warning is confirmed absent in the rebuilt environment.
- Exact next action: run complete clean acceptance, push all five milestone commits,
create a verified live backup, redeploy and execute live browser acceptance.
+25 -1
View File
@@ -100,6 +100,30 @@ def record_feedback(
@router.get("/status", response_model=KnowledgeHealth)
def knowledge_status(
language: SupportedLanguage = "en-GB",
db: Session = Depends(get_db),
_user: CurrentUser = Depends(get_current_user),
) -> KnowledgeHealth:
return get_knowledge_provider().health(language)
health = get_knowledge_provider().health(language)
if health.provider != "ragcore":
return health
latest_sync = db.scalar(
select(AuditEvent)
.where(AuditEvent.action == "n8n_procedures_synced")
.order_by(AuditEvent.occurred_at.desc())
.limit(1)
)
if latest_sync is None:
return health
reported = latest_sync.after_json or {}
synced = reported.get("synced")
failed = reported.get("failed")
return health.model_copy(
update={
"reported_synced_document_count": synced if isinstance(synced, int) else None,
"reported_failed_document_count": failed if isinstance(failed, int) else None,
"last_sync_at": latest_sync.occurred_at,
"statistics_state": "sync_reported",
}
)
@@ -1,5 +1,6 @@
from __future__ import annotations
from datetime import datetime
from functools import lru_cache
from typing import Literal, Protocol
@@ -8,6 +9,7 @@ from pydantic import BaseModel
from app.core.config import get_settings
EvidenceState = Literal["grounded", "insufficient", "unavailable"]
KnowledgeStatisticsState = Literal["verified", "sync_reported", "not_reported"]
class SourceCard(BaseModel):
@@ -36,6 +38,11 @@ class KnowledgeHealth(BaseModel):
# A provider may be healthy without exposing a corpus-size endpoint. `None` means
# unknown, never "zero procedures".
document_count: int | None
source_document_count: int
reported_synced_document_count: int | None
reported_failed_document_count: int | None
last_sync_at: datetime | None
statistics_state: KnowledgeStatisticsState
class KnowledgeProvider(Protocol):
+5
View File
@@ -322,6 +322,11 @@ class DemoKnowledgeProvider:
workspace=self._settings.ragcore_workspace,
collection=self._settings.ragcore_collection,
document_count=self._document_count_by_language[language],
source_document_count=self._document_count_by_language[language],
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="verified",
)
def _score(
+12
View File
@@ -1,9 +1,12 @@
from __future__ import annotations
from pathlib import Path
import httpx
from app.core.config import get_settings
from app.services.knowledge import EvidenceState, GroundedAnswer, KnowledgeHealth, SourceCard
from app.services.knowledge.procedures import iter_procedure_documents
_GROUNDED_ANSWERABILITY = {"answerable", "partially_answerable"}
@@ -140,6 +143,10 @@ class RAGcoreKnowledgeProvider:
except (httpx.HTTPError, ValueError) as exc:
available = False
detail = f"RAGcore unavailable: {type(exc).__name__}: {exc}"
source_document_count = sum(
document.language == language
for document in iter_procedure_documents(Path(self._settings.knowledge_dir))
)
return KnowledgeHealth(
provider=self.name,
available=available,
@@ -150,6 +157,11 @@ class RAGcoreKnowledgeProvider:
# RAGcore's retrieval API has no corpus-size endpoint. Unknown is explicit
# so the UI never turns this into the misleading claim "0 procedures".
document_count=None,
source_document_count=source_document_count,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="not_reported",
)
def ask(self, question: str, correlation_id: str, language: str = "en-GB") -> GroundedAnswer:
+47
View File
@@ -5,7 +5,9 @@ from pathlib import Path
import httpx
from app.api.routers import knowledge as knowledge_router
from app.core.config import get_settings
from app.services.knowledge import KnowledgeHealth
from app.services.knowledge.demo import DemoKnowledgeProvider
from app.services.knowledge.ragcore import RAGcoreKnowledgeProvider
@@ -73,6 +75,8 @@ def test_demo_provider_health_reports_document_count():
assert health.provider == "demo"
assert health.available is True
assert health.document_count == 11
assert health.source_document_count == 11
assert health.statistics_state == "verified"
def test_demo_provider_health_reports_document_count_per_language():
@@ -154,6 +158,46 @@ def test_knowledge_status_endpoint(ops_client):
assert response.json()["provider"] == "demo"
def test_ragcore_status_separates_sync_report_from_unverifiable_index(
client, ops_client, monkeypatch
):
class FakeRagcoreProvider:
def health(self, language="en-GB"):
return KnowledgeHealth(
provider="ragcore",
available=True,
detail="ready",
tenant="fleet-ops",
workspace="operations",
collection="internal-procedures",
document_count=None,
source_document_count=11,
reported_synced_document_count=None,
reported_failed_document_count=None,
last_sync_at=None,
statistics_state="not_reported",
)
monkeypatch.setattr(knowledge_router, "get_knowledge_provider", FakeRagcoreProvider)
settings = get_settings()
sync = client.post(
"/api/v1/integrations/n8n/procedures-sync-result",
json={"execution_id": "rag-statistics-test", "synced": 32, "failed": 1},
headers={"X-Service-Token": settings.n8n_callback_token},
)
assert sync.status_code == 200
response = ops_client.get("/api/v1/knowledge/status?language=nl-BE")
assert response.status_code == 200
status = response.json()
assert status["document_count"] is None
assert status["source_document_count"] == 11
assert status["reported_synced_document_count"] == 32
assert status["reported_failed_document_count"] == 1
assert status["last_sync_at"] is not None
assert status["statistics_state"] == "sync_reported"
class _FakeResponse:
def __init__(self, status_code: int, body: dict):
self.status_code = status_code
@@ -224,6 +268,9 @@ def test_ragcore_provider_health_reports_ready_status(monkeypatch):
health = provider.health()
assert health.provider == "ragcore"
assert health.available is True
assert health.document_count is None
assert health.source_document_count == 11
assert health.statistics_state == "not_reported"
def test_ragcore_provider_health_reports_degraded_status(monkeypatch):
+31
View File
@@ -3332,6 +3332,32 @@ components:
- type: integer
- type: 'null'
title: Document Count
source_document_count:
type: integer
title: Source Document Count
reported_synced_document_count:
anyOf:
- type: integer
- type: 'null'
title: Reported Synced Document Count
reported_failed_document_count:
anyOf:
- type: integer
- type: 'null'
title: Reported Failed Document Count
last_sync_at:
anyOf:
- type: string
format: date-time
- type: 'null'
title: Last Sync At
statistics_state:
type: string
enum:
- verified
- sync_reported
- not_reported
title: Statistics State
type: object
required:
- provider
@@ -3341,6 +3367,11 @@ components:
- workspace
- collection
- document_count
- source_document_count
- reported_synced_document_count
- reported_failed_document_count
- last_sync_at
- statistics_state
title: KnowledgeHealth
MaintenanceOut:
properties:
+18 -1
View File
@@ -14,7 +14,24 @@ These values are configurable.
## Source documents
The ten Markdown files under `knowledge/procedures/` are authoritative PoC sources. Keep their IDs, versions and effective dates as metadata.
The versioned Markdown files under `knowledge/procedures/` (currently eleven per supported
language) are the authoritative sources. Keep their IDs, versions and effective dates as
metadata.
## Index statistics and provenance
`GET /api/v1/knowledge/status` keeps three different measurements separate:
- `source_document_count`: authoritative procedure files available to Fleet Ops for the requested language;
- `reported_synced_document_count`, `reported_failed_document_count` and `last_sync_at`: the latest persisted result reported by the central n8n synchronization workflow;
- `document_count`: documents independently verified as indexed by the active provider.
The current RAGcore contract deliberately has no corpus-size or space-browse endpoint. For
the RAGcore provider, `document_count` therefore remains `null`; a successful upload report
is never relabelled as proof that indexing and publishing completed. The deterministic demo
provider can verify its in-memory corpus and reports `statistics_state=verified`. RAGcore
reports `sync_reported` only when a persisted workflow callback exists, otherwise
`not_reported`.
## Required adapter interface
+5
View File
@@ -333,6 +333,11 @@ export interface KnowledgeHealth {
workspace: string;
collection: string;
document_count: number | null;
source_document_count: number;
reported_synced_document_count: number | null;
reported_failed_document_count: number | null;
last_sync_at: string | null;
statistics_state: "verified" | "sync_reported" | "not_reported";
}
export interface N8nWorkflowEvidence {
@@ -69,7 +69,7 @@
"n8nNoEvidence": "No workflow evidence recorded",
"knowledgeTitle": "Knowledge assistant",
"knowledgeSummary": "{{count}} procedures indexed",
"knowledgeIndexUnknown": "Knowledge source available · index size unknown",
"knowledgeIndexUnknown": "{{count}} knowledge sources · index size not verifiable",
"knowledgeUnavailable": "Health check unavailable",
"mcpTitle": "MCP Hub",
"mcpEnabled": "Registration enabled",
@@ -12,7 +12,8 @@
"knowledgeTitle": "Knowledge assistant",
"knowledgeSummaryDemo": "Demo knowledge base · {{count}} procedures indexed in {{collection}}.",
"knowledgeSummaryRagcore": "RAGcore · {{count}} procedures indexed in {{collection}}.",
"knowledgeSummaryIndexUnknown": "Knowledge source available · index size is unavailable for {{collection}}.",
"knowledgeSummaryIndexUnknown": "RAGcore reachable · {{sourceCount}} source documents for this language · index size not verifiable for {{collection}}.",
"knowledgeSummarySyncReported": "RAGcore reachable · {{sourceCount}} sources for this language · latest sync reported {{synced}} processed, {{failed}} failed at {{when}} · index size not verifiable for {{collection}}.",
"knowledgeUnavailable": "Health evidence is currently unavailable.",
"gatewayKicker": "Tool gateway",
"mcpTitle": "MCP Hub",
@@ -12,6 +12,13 @@
"statusUnavailable": "Unavailable",
"proceduresIndexed": "{{count}} procedures indexed",
"proceduresIndexUnknown": "Index size is not available through this connection",
"statistics": {
"sourceDocuments": "source documents in this language",
"reportedSynced": "sync reported by n8n",
"reportedFailed": "sync failures reported",
"lastSync": "Latest sync report: {{when}}. RAGcore does not expose a verifiable index size, so sync counts are not presented as indexed documents.",
"noSyncReport": "No sync report has been received yet. RAGcore does not expose a verifiable index size."
},
"providerNote": "This demo answers from a small, fixed set of indexed procedures — not a live RAGcore connection. A live RAGcore backend will later take over the same interface without changing how this page works.",
"askHeading": "Ask a procedure question",
"askSubheading": "Retrieval → evidence check → grounded answer",
@@ -69,7 +69,7 @@
"n8nNoEvidence": "Aucune preuve d'automatisation enregistrée",
"knowledgeTitle": "Assistant de connaissances",
"knowledgeSummary": "{{count}} procédures indexées",
"knowledgeIndexUnknown": "Source de connaissances disponible · taille dindex inconnue",
"knowledgeIndexUnknown": "{{count}} sources de connaissances · taille dindex non vérifiable",
"knowledgeUnavailable": "Vérification de santé indisponible",
"mcpTitle": "MCP Hub",
"mcpEnabled": "Enregistrement activé",
@@ -12,7 +12,8 @@
"knowledgeTitle": "Assistant de connaissances",
"knowledgeSummaryDemo": "Base de connaissances de démo · {{count}} procédures indexées dans {{collection}}.",
"knowledgeSummaryRagcore": "RAGcore · {{count}} procédures indexées dans {{collection}}.",
"knowledgeSummaryIndexUnknown": "Source de connaissances disponible · taille dindex indisponible pour {{collection}}.",
"knowledgeSummaryIndexUnknown": "RAGcore accessible · {{sourceCount}} documents sources dans cette langue · taille dindex non vérifiable pour {{collection}}.",
"knowledgeSummarySyncReported": "RAGcore accessible · {{sourceCount}} sources dans cette langue · dernier rapport : {{synced}} traités, {{failed}} échoués à {{when}} · taille dindex non vérifiable pour {{collection}}.",
"knowledgeUnavailable": "Preuves de santé actuellement indisponibles.",
"gatewayKicker": "Passerelle d'outils",
"mcpTitle": "MCP Hub",
@@ -12,6 +12,13 @@
"statusUnavailable": "Indisponible",
"proceduresIndexed": "{{count}} procédures indexées",
"proceduresIndexUnknown": "La taille de lindex nest pas disponible via cette connexion",
"statistics": {
"sourceDocuments": "documents sources dans cette langue",
"reportedSynced": "synchronisation signalée par n8n",
"reportedFailed": "échecs de synchronisation signalés",
"lastSync": "Dernier rapport de synchronisation : {{when}}. RAGcore nexpose pas de taille dindex vérifiable ; les nombres synchronisés ne sont donc pas présentés comme des documents indexés.",
"noSyncReport": "Aucun rapport de synchronisation reçu. RAGcore nexpose pas de taille dindex vérifiable."
},
"providerNote": "Cette démo répond à partir d'un petit ensemble fixe de procédures indexées — pas d'une connexion RAGcore en direct. Un backend RAGcore en direct reprendra plus tard la même interface sans changer le fonctionnement de cette page.",
"askHeading": "Poser une question de procédure",
"askSubheading": "Recherche → vérification des preuves → réponse étayée",
@@ -69,7 +69,7 @@
"n8nNoEvidence": "Geen automatiseringsevidentie geregistreerd",
"knowledgeTitle": "Kennisassistent",
"knowledgeSummary": "{{count}} procedures geïndexeerd",
"knowledgeIndexUnknown": "Kennisbron bereikbaar · indexomvang onbekend",
"knowledgeIndexUnknown": "{{count}} kennisbronnen · indexomvang niet verifieerbaar",
"knowledgeUnavailable": "Statuscontrole niet beschikbaar",
"mcpTitle": "MCP Hub",
"mcpEnabled": "Registratie ingeschakeld",
@@ -12,7 +12,8 @@
"knowledgeTitle": "Kennisassistent",
"knowledgeSummaryDemo": "Demokennisbank · {{count}} procedures geïndexeerd in {{collection}}.",
"knowledgeSummaryRagcore": "RAGcore · {{count}} procedures geïndexeerd in {{collection}}.",
"knowledgeSummaryIndexUnknown": "Kennisbron bereikbaar · indexomvang niet beschikbaar voor {{collection}}.",
"knowledgeSummaryIndexUnknown": "RAGcore bereikbaar · {{sourceCount}} brondocumenten voor deze taal · indexomvang niet verifieerbaar voor {{collection}}.",
"knowledgeSummarySyncReported": "RAGcore bereikbaar · {{sourceCount}} bronnen voor deze taal · laatste syncrapport {{synced}} verwerkt, {{failed}} mislukt op {{when}} · indexomvang niet verifieerbaar voor {{collection}}.",
"knowledgeUnavailable": "Statusevidentie momenteel niet beschikbaar.",
"gatewayKicker": "Tool-gateway",
"mcpTitle": "MCP Hub",
@@ -12,6 +12,13 @@
"statusUnavailable": "Niet beschikbaar",
"proceduresIndexed": "{{count}} procedures geïndexeerd",
"proceduresIndexUnknown": "Indexomvang niet beschikbaar via deze koppeling",
"statistics": {
"sourceDocuments": "brondocumenten in deze taal",
"reportedSynced": "sync door n8n gerapporteerd",
"reportedFailed": "syncfouten gerapporteerd",
"lastSync": "Laatste syncrapport: {{when}}. RAGcore stelt geen verifieerbare indexomvang beschikbaar; syncaantallen worden daarom niet als geïndexeerde documenten voorgesteld.",
"noSyncReport": "Nog geen syncrapport ontvangen. RAGcore stelt geen verifieerbare indexomvang beschikbaar."
},
"providerNote": "Deze demo beantwoordt vanuit een kleine, vaste set geïndexeerde procedures — geen live RAGcore-koppeling. Een live RAGcore-backend zal later dezelfde interface overnemen, zonder dat deze pagina verandert.",
"askHeading": "Stel een procedurevraag",
"askSubheading": "Ophalen → evidentiecontrole → onderbouwd antwoord",
+12 -1
View File
@@ -215,7 +215,18 @@ export function Automation() {
? t("cards.checking")
: knowledge
? knowledge.document_count === null
? t("cards.knowledgeSummaryIndexUnknown", { collection: knowledge.collection })
? knowledge.statistics_state === "sync_reported"
? t("cards.knowledgeSummarySyncReported", {
collection: knowledge.collection,
sourceCount: knowledge.source_document_count,
synced: knowledge.reported_synced_document_count,
failed: knowledge.reported_failed_document_count,
when: knowledge.last_sync_at ? formatDateTime(knowledge.last_sync_at) : "—",
})
: t("cards.knowledgeSummaryIndexUnknown", {
collection: knowledge.collection,
sourceCount: knowledge.source_document_count,
})
: t(knowledge.provider === "ragcore" ? "cards.knowledgeSummaryRagcore" : "cards.knowledgeSummaryDemo", {
count: knowledge.document_count,
collection: knowledge.collection,
+1 -1
View File
@@ -245,7 +245,7 @@ export function Dashboard() {
<IntegrationMark kind="rag" />
<div>
<strong>{t("integrationPulse.knowledgeTitle")}</strong>
<span>{knowledge ? (knowledge.document_count === null ? t("integrationPulse.knowledgeIndexUnknown") : t("integrationPulse.knowledgeSummary", { count: knowledge.document_count })) : t("integrationPulse.knowledgeUnavailable")}</span>
<span>{knowledge ? (knowledge.document_count === null ? t("integrationPulse.knowledgeIndexUnknown", { count: knowledge.source_document_count }) : t("integrationPulse.knowledgeSummary", { count: knowledge.document_count })) : t("integrationPulse.knowledgeUnavailable")}</span>
</div>
<StatusBadge
status={knowledge?.available ? "available" : "unavailable"}
+20
View File
@@ -5,6 +5,7 @@ import { describeApiError, type ApiErrorInfo } from "../api/errorMessages";
import type { GroundedAnswer, KnowledgeHealth } from "../api/types";
import { Icon } from "../components/Icons";
import { ApiErrorNotice, PageHeader } from "../components/PageChrome";
import { useLocaleFormat } from "../i18n/format";
import { PRODUCT_NAME } from "../product";
interface Exchange {
@@ -21,6 +22,7 @@ function isOpaqueVersion(value: string): boolean {
export function Knowledge() {
const { t, i18n } = useTranslation(["knowledge", "errors"]);
const { formatDateTime } = useLocaleFormat();
const [status, setStatus] = useState<KnowledgeHealth | null>(null);
const [statusSettled, setStatusSettled] = useState(false);
const [question, setQuestion] = useState("");
@@ -138,6 +140,24 @@ export function Knowledge() {
{statusSettled && status?.provider === "demo" && (
<p className="knowledge-provider-note"><Icon name="shield" /> {t("providerNote")}</p>
)}
{statusSettled && status?.provider === "ragcore" && (
<div className="knowledge-index-evidence" role="status">
<div><strong>{status.source_document_count}</strong><span>{t("statistics.sourceDocuments")}</span></div>
<div>
<strong>{status.reported_synced_document_count ?? "—"}</strong>
<span>{t("statistics.reportedSynced")}</span>
</div>
<div>
<strong>{status.reported_failed_document_count ?? "—"}</strong>
<span>{t("statistics.reportedFailed")}</span>
</div>
<p>
{status.last_sync_at
? t("statistics.lastSync", { when: formatDateTime(status.last_sync_at) })
: t("statistics.noSyncReport")}
</p>
</div>
)}
<form className="panel knowledge-form" onSubmit={handleSubmit} aria-labelledby="ask-heading">
<div className="ask-heading">
+5
View File
@@ -456,6 +456,11 @@ details summary { cursor: pointer; color: var(--teal-dark); }.data-table details
.knowledge-provider-note { display: flex; align-items: flex-start; gap: 8px; margin: -6px 0 18px; padding: 10px 14px; color: var(--ink-soft); background: var(--info-pale); border: 1px solid #cfe3ee; border-radius: var(--radius); font-size: .72rem; line-height: 1.5; }
.knowledge-provider-note svg { width: 15px; flex-shrink: 0; margin-top: 1px; color: var(--info); }
.knowledge-index-evidence { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 1px; margin: -6px 0 18px; overflow: hidden; background: var(--line); border: 1px solid var(--line); border-radius: var(--radius); }
.knowledge-index-evidence > div { display: grid; gap: 3px; padding: 11px 14px; background: white; }
.knowledge-index-evidence strong { font-size: .86rem; }
.knowledge-index-evidence span, .knowledge-index-evidence p { color: var(--muted); font-size: .66rem; line-height: 1.45; }
.knowledge-index-evidence p { grid-column: 1 / -1; margin: 0; padding: 9px 14px; background: var(--surface-subtle); }
.knowledge-suggestions { display: flex; flex-wrap: wrap; align-items: center; gap: 7px; margin-top: 12px; }
.knowledge-suggestions > span { color: var(--muted); font-size: .68rem; font-weight: 700; }
.suggestion-chip { padding: 6px 11px; color: var(--teal-dark); background: var(--teal-pale); border: 1px solid #bfe6df; border-radius: 999px; font-size: .68rem; font-weight: 600; cursor: pointer; }