Files
ServiceMap-Public/backend/tests/test_v163_health_performance.py
NuklearRabbit 41a8eee259 ServiceMap: update dependencies and backend/app (+5 areas)
Public changes compared with the previous published export:
16 files changed.

Affected areas:
- backend/app: 3 files
- dependencies: 5 files
- deployment and CI: 2 files
- documentation: 2 files
- frontend/src: 1 file
- project files: 1 file
- tests: 2 files

Changed files:
- Updated: ".env.example"
- Updated: "Dockerfile.unraid"
- Updated: "README.md"
- Updated: "backend/app/incidents.py"
- Updated: "backend/app/main.py"
- Updated: "backend/app/store.py"
- Updated: "backend/pyproject.toml"
- Updated: "backend/tests/test_docker_provider.py"
- Updated: "backend/tests/test_v163_health_performance.py"
- Updated: "docker-compose.broker.yml"
- Updated: "docs/ARCHITECTURE.md"
- Updated: "frontend/package-lock.json"
- Updated: "frontend/package.json"
- Updated: "frontend/src/App.tsx"
- Updated: "package-lock.json"
- Updated: "package.json"

Export provenance/manifests refreshed; excluded from the change summary.

Publish curated ServiceMap source from 8c297f756ff84aaa917ae9c48535e835f8c09ec7
2026-11-20 16:01:36 +01:00

399 lines
15 KiB
Python

"""v1.6.3 regression guards for health-evidence reads and graph resolution.
Production profiling showed 78% of server CPU in a full ``health_evidence``
scan, issued once per node of a recursive, un-memoised health walk. These
tests pin the indexed query shape, equivalence of the bulk and single reads,
the batched graph result and the bounded query count.
"""
from __future__ import annotations
import os
import random
import sqlite3
from datetime import UTC, datetime, timedelta
from pathlib import Path
from statistics import quantiles
from time import perf_counter
from fastapi.testclient import TestClient
from app import main as main_module
from app.discovery_store import DiscoveryStore
from app.health import SEVERITY, status_maps, topology_health
from app.incidents import incident_overview, sync_incidents
from app.models import (
ConfigService,
EvidenceProvider,
EvidenceScope,
HealthEvidence,
HealthStatus,
ServiceCreate,
ServiceKind,
)
from app.store import Store
def _service(store: Store, name: str, depends_on: list[str] | None = None):
return store.create_service(
ServiceCreate(
name=name,
kind=ServiceKind.HTTP,
url=f"https://{name.lower()}.test",
depends_on=depends_on or [],
)
)
def _component(
evidence_id: str,
service_id: str,
status: HealthStatus,
*,
at: datetime | None = None,
) -> HealthEvidence:
now = at or datetime.now(UTC)
return HealthEvidence(
id=evidence_id,
subject_type="service",
subject_id=service_id,
service_id=service_id,
provider=EvidenceProvider.DOCKER,
provider_ref=service_id,
status=status,
scope=EvidenceScope.COMPONENT,
confidence=0.95,
observed_at=now,
valid_until=datetime.now(UTC) + timedelta(minutes=5),
received_at=now,
summary="Docker healthcheck",
)
def _application(evidence_id: str, status: HealthStatus) -> HealthEvidence:
now = datetime.now(UTC)
return HealthEvidence(
id=evidence_id,
subject_type="application",
subject_id="appops:northstar",
provider=EvidenceProvider.APPOPS,
provider_ref="northstar",
status=status,
scope=EvidenceScope.APPLICATION,
confidence=0.9,
observed_at=now,
valid_until=now + timedelta(minutes=5),
received_at=now,
summary="AppOps aggregate",
)
def _plan(store: Store, sql: str, params: list[object]) -> list[str]:
with sqlite3.connect(store.path) as db:
return [row[3] for row in db.execute(f"EXPLAIN QUERY PLAN {sql}", params)]
def test_evidence_reads_use_indexes_instead_of_full_scans(tmp_path: Path, monkeypatch) -> None:
store = Store(str(tmp_path / "servicemap.db"))
service = _service(store, "API")
store.record_health_evidence(_component("ev-1", service.id, HealthStatus.HEALTHY))
captured: list[str] = []
original = sqlite3.Connection.execute
class Recording(sqlite3.Connection):
def execute(self, sql, parameters=(), /): # type: ignore[override]
if "health_evidence" in sql and sql.lstrip().upper().startswith(("SELECT", "WITH")):
captured.append(sql)
return original(self, sql, parameters)
real_connect = sqlite3.connect
monkeypatch.setattr(
"app.store.sqlite3.connect",
lambda *args, **kwargs: real_connect(*args, factory=Recording, **kwargs),
)
store.health_evidence_for_service(service.id)
store.health_evidence_for_services([service.id])
monkeypatch.setattr("app.store.sqlite3.connect", real_connect)
single, bulk = captured[-2], captured[-1]
for sql, params in (
(single, [service.id, service.id, 100]),
(bulk, [service.id, service.id, 100]),
):
details = _plan(store, sql, params)
assert not any(detail in {"SCAN e", "SCAN health_evidence"} for detail in details), details
def test_bulk_and_single_evidence_reads_return_the_same_rows(tmp_path: Path) -> None:
store = Store(str(tmp_path / "servicemap.db"))
api = _service(store, "API")
worker = _service(store, "Worker")
base = datetime.now(UTC) - timedelta(hours=2)
# Status flips create superseded history for the direct claim.
for index, status in enumerate([HealthStatus.HEALTHY, HealthStatus.DOWN, HealthStatus.HEALTHY]):
store.record_health_evidence(
_component(f"api-{index}", api.id, status, at=base + timedelta(minutes=index))
)
store.record_health_evidence(_component("worker-0", worker.id, HealthStatus.DOWN, at=base))
# One application claim linked to both components.
store.record_health_evidence(_application("app-0", HealthStatus.HEALTHY), service_ids=[api.id, worker.id])
bulk = store.health_evidence_for_services([api.id, worker.id])
for service in (api, worker):
single = store.health_evidence_for_service(service.id)
assert [item.id for item in bulk[service.id]] == [item.id for item in single]
assert {item.id for item in bulk[api.id]} == {"api-0", "api-1", "api-2", "app-0"}
assert {item.id for item in bulk[worker.id]} == {"worker-0", "app-0"}
limited = store.health_evidence_for_services([api.id], limit_per_service=2)
assert [item.id for item in limited[api.id]] == [
item.id for item in store.health_evidence_for_service(api.id, limit=2)
]
def test_prune_removes_only_old_superseded_history(tmp_path: Path) -> None:
store = Store(str(tmp_path / "servicemap.db"))
api = _service(store, "API")
old = datetime.now(UTC) - timedelta(days=30)
store.record_health_evidence(_component("old-down", api.id, HealthStatus.DOWN, at=old))
store.record_health_evidence(
_component("old-up", api.id, HealthStatus.HEALTHY, at=old + timedelta(minutes=1))
)
store.record_health_evidence(_component("recent-down", api.id, HealthStatus.DOWN))
store.record_health_evidence(_application("app-0", HealthStatus.HEALTHY), service_ids=[api.id])
before = store.service_health(api.id)
assert store.prune_health_evidence(7) == 2
remaining = {item.id for item in store.health_evidence_for_service(api.id)}
assert remaining == {"recent-down", "app-0"}
assert store.service_health(api.id) == before == HealthStatus.DOWN
with sqlite3.connect(store.path) as db:
orphans = db.execute(
"SELECT COUNT(*) FROM health_evidence_service_links "
"WHERE evidence_id NOT IN (SELECT id FROM health_evidence)"
).fetchone()[0]
assert orphans == 0
assert store.prune_health_evidence(0) == 0
def test_batched_discovery_context_matches_per_service_reads(tmp_path: Path) -> None:
from app.discovery_store import DiscoveryStore
from app.models import DiscoveryConfidence, DiscoveryLifecycle, RelationEvidence
store = Store(str(tmp_path / "servicemap.db"))
discovery = DiscoveryStore(store)
api = _service(store, "API")
database = _service(store, "Database")
lonely = _service(store, "Lonely")
for index, service in enumerate((api, database)):
source = discovery.save_source(
source_type="docker",
source_id=f"container-{index}",
source_name=service.name,
lifecycle=DiscoveryLifecycle.ACTIVE,
metadata={},
)
discovery.attach_source(service.id, source.id)
now = datetime.now(UTC)
discovery.save_evidence(
RelationEvidence(
id="rel-1",
source_service_id=api.id,
target_service_id=database.id,
evidence_type="compose",
confidence=DiscoveryConfidence.HIGH,
explanation="depends_on",
first_seen_at=now,
last_seen_at=now,
)
)
ids = [api.id, database.id, lonely.id]
sources = discovery.sources_for_services(ids)
relations = discovery.evidence_for_services(ids)
for service_id in ids:
assert sources[service_id] == discovery.sources_for_service(service_id)
assert relations[service_id] == discovery.evidence_for_service(service_id)
assert [item.id for item in relations[database.id]] == ["rel-1"]
assert relations[lonely.id] == [] and sources[lonely.id] == []
def _reference_effective(
graph: dict[str, list[str]], direct: dict[str, HealthStatus]
) -> dict[str, HealthStatus]:
"""The pre-v1.6.3 recursive per-path algorithm, kept as an oracle."""
def dependency(node: str, visiting: set[str]) -> HealthStatus:
if not graph[node]:
return HealthStatus.HEALTHY
states = [
effective(item, visiting) if item in graph else HealthStatus.UNKNOWN
for item in graph[node]
]
return max(states, key=lambda status: SEVERITY[status])
def effective(node: str, visiting: set[str]) -> HealthStatus:
if node in visiting:
return HealthStatus.UNKNOWN
own = direct[node]
if own == HealthStatus.MAINTENANCE:
return own
dep = dependency(node, {*visiting, node})
if own == HealthStatus.DOWN:
return own
if dep == HealthStatus.DOWN:
return HealthStatus.DEGRADED
if dep in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} and own in {
HealthStatus.HEALTHY,
HealthStatus.RECOVERING,
}:
return HealthStatus.DEGRADED
return own
return {node: effective(node, set()) for node in graph}
def test_batched_graph_matches_recursive_reference_on_random_dags() -> None:
rng = random.Random(163)
statuses = list(HealthStatus)
for _ in range(200):
size = rng.randint(1, 14)
nodes = [f"n{index}" for index in range(size)]
# Acyclic by construction: a node only depends on lower indexes.
graph = {
node: rng.sample(nodes[:index], k=rng.randint(0, min(3, index)))
for index, node in enumerate(nodes)
}
if rng.random() < 0.2:
graph[nodes[-1]].append("missing")
direct = {node: rng.choice(statuses) for node in nodes}
services = [_fake_service(node, graph[node]) for node in nodes]
effective, _ = status_maps(services, direct_statuses=direct)
assert effective == _reference_effective(graph, direct), (graph, direct)
def _fake_service(identifier: str, depends_on: list[str]):
from app.models import Service
now = datetime.now(UTC)
return Service(
id=identifier,
name=identifier,
kind=ServiceKind.HTTP,
url="https://example.test",
depends_on=depends_on,
created_at=now,
updated_at=now,
)
def test_incident_resolution_uses_a_bounded_number_of_queries(tmp_path: Path, monkeypatch) -> None:
store = Store(str(tmp_path / "servicemap.db"))
# A deep chain plus a fan-in reproduces the shape that made the old walk
# re-read every shared dependency once per path.
previous: list[str] = []
for index in range(40):
service = _service(store, f"S{index:02d}", depends_on=previous[-2:])
status = HealthStatus.DOWN if index == 0 else HealthStatus.HEALTHY
store.record_health_evidence(_component(f"ev-{index}", service.id, status))
previous.append(service.id)
connections = 0
original = store._connect
def counting_connect():
nonlocal connections
connections += 1
return original()
monkeypatch.setattr(store, "_connect", counting_connect)
overview = incident_overview(store)
assert connections <= 4, connections
assert [candidate.service_id for candidate in overview.candidates] == [previous[0]]
assert len(overview.candidates[0].affected_service_ids) == 39
health = topology_health(store)
assert health.effective[previous[0]] == HealthStatus.DOWN
assert health.effective[previous[-1]] == HealthStatus.DEGRADED
connections = 0
sync_incidents(store)
# One graph resolution plus incident journal writes; never one per node.
assert connections < 20, connections
def test_topology_request_budget_at_300_services(tmp_path: Path, monkeypatch) -> None:
store = Store(str(tmp_path / "servicemap.db"))
definitions = [
ConfigService(
name=f"Service {index}",
kind=ServiceKind.HTTP,
url=f"https://service-{index}.test",
domain="Benchmark",
application=f"Application {index // 10}",
depends_on=[f"Service {index - 1}"] if index % 10 else [],
)
for index in range(300)
]
assert store.import_config_services(definitions, "merge").created == 300
for service in store.list_services():
store.record_health_evidence(_component(f"ev-{service.id}", service.id, HealthStatus.HEALTHY))
monkeypatch.setattr(main_module, "store", store)
monkeypatch.setattr(main_module, "discovery_store", DiscoveryStore(store))
connections = 0
original = store._connect
def counting_connect():
nonlocal connections
connections += 1
return original()
monkeypatch.setattr(store, "_connect", counting_connect)
evidence_reads = 0
read_evidence = store.health_evidence_for_services
def counting_evidence(service_ids, **kwargs):
nonlocal evidence_reads
evidence_reads += 1
return read_evidence(service_ids, **kwargs)
monkeypatch.setattr(store, "health_evidence_for_services", counting_evidence)
topology_reads = 0
list_services = store.list_services
def counting_services():
nonlocal topology_reads
topology_reads += 1
return list_services()
monkeypatch.setattr(store, "list_services", counting_services)
client = TestClient(main_module.app)
budget_ms = os.environ.get("SERVICEMAP_API_P95_BUDGET_MS")
for endpoint in ("/api/v1/topology/semantic", "/api/v1/overview"):
connections = 0
evidence_reads = 0
topology_reads = 0
main_module.semantic_cache.clear()
assert client.get(endpoint).status_code == 200
assert evidence_reads == 1, (endpoint, evidence_reads)
assert topology_reads == 1, (endpoint, topology_reads)
assert connections <= 30, (endpoint, connections)
elapsed_ms = []
for _ in range(20):
if endpoint.endswith("/semantic"):
# A provider or topology change invalidates this representation in production.
main_module.semantic_cache.clear()
started = perf_counter()
response = client.get(endpoint)
elapsed_ms.append((perf_counter() - started) * 1000)
assert response.status_code == 200
p95_ms = quantiles(elapsed_ms, n=20, method="inclusive")[-1]
print(f"{endpoint} recompute p95: {p95_ms:.1f} ms")
if budget_ms is not None:
assert p95_ms <= float(budget_ms), (endpoint, p95_ms, budget_ms)