Public changes compared with the previous published export: 16 files changed. Affected areas: - backend/app: 3 files - dependencies: 5 files - deployment and CI: 2 files - documentation: 2 files - frontend/src: 1 file - project files: 1 file - tests: 2 files Changed files: - Updated: ".env.example" - Updated: "Dockerfile.unraid" - Updated: "README.md" - Updated: "backend/app/incidents.py" - Updated: "backend/app/main.py" - Updated: "backend/app/store.py" - Updated: "backend/pyproject.toml" - Updated: "backend/tests/test_docker_provider.py" - Updated: "backend/tests/test_v163_health_performance.py" - Updated: "docker-compose.broker.yml" - Updated: "docs/ARCHITECTURE.md" - Updated: "frontend/package-lock.json" - Updated: "frontend/package.json" - Updated: "frontend/src/App.tsx" - Updated: "package-lock.json" - Updated: "package.json" Export provenance/manifests refreshed; excluded from the change summary. Publish curated ServiceMap source from 8c297f756ff84aaa917ae9c48535e835f8c09ec7
399 lines
15 KiB
Python
399 lines
15 KiB
Python
"""v1.6.3 regression guards for health-evidence reads and graph resolution.
|
|
|
|
Production profiling showed 78% of server CPU in a full ``health_evidence``
|
|
scan, issued once per node of a recursive, un-memoised health walk. These
|
|
tests pin the indexed query shape, equivalence of the bulk and single reads,
|
|
the batched graph result and the bounded query count.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import random
|
|
import sqlite3
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from statistics import quantiles
|
|
from time import perf_counter
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app import main as main_module
|
|
from app.discovery_store import DiscoveryStore
|
|
|
|
from app.health import SEVERITY, status_maps, topology_health
|
|
from app.incidents import incident_overview, sync_incidents
|
|
from app.models import (
|
|
ConfigService,
|
|
EvidenceProvider,
|
|
EvidenceScope,
|
|
HealthEvidence,
|
|
HealthStatus,
|
|
ServiceCreate,
|
|
ServiceKind,
|
|
)
|
|
from app.store import Store
|
|
|
|
|
|
def _service(store: Store, name: str, depends_on: list[str] | None = None):
|
|
return store.create_service(
|
|
ServiceCreate(
|
|
name=name,
|
|
kind=ServiceKind.HTTP,
|
|
url=f"https://{name.lower()}.test",
|
|
depends_on=depends_on or [],
|
|
)
|
|
)
|
|
|
|
|
|
def _component(
|
|
evidence_id: str,
|
|
service_id: str,
|
|
status: HealthStatus,
|
|
*,
|
|
at: datetime | None = None,
|
|
) -> HealthEvidence:
|
|
now = at or datetime.now(UTC)
|
|
return HealthEvidence(
|
|
id=evidence_id,
|
|
subject_type="service",
|
|
subject_id=service_id,
|
|
service_id=service_id,
|
|
provider=EvidenceProvider.DOCKER,
|
|
provider_ref=service_id,
|
|
status=status,
|
|
scope=EvidenceScope.COMPONENT,
|
|
confidence=0.95,
|
|
observed_at=now,
|
|
valid_until=datetime.now(UTC) + timedelta(minutes=5),
|
|
received_at=now,
|
|
summary="Docker healthcheck",
|
|
)
|
|
|
|
|
|
def _application(evidence_id: str, status: HealthStatus) -> HealthEvidence:
|
|
now = datetime.now(UTC)
|
|
return HealthEvidence(
|
|
id=evidence_id,
|
|
subject_type="application",
|
|
subject_id="appops:northstar",
|
|
provider=EvidenceProvider.APPOPS,
|
|
provider_ref="northstar",
|
|
status=status,
|
|
scope=EvidenceScope.APPLICATION,
|
|
confidence=0.9,
|
|
observed_at=now,
|
|
valid_until=now + timedelta(minutes=5),
|
|
received_at=now,
|
|
summary="AppOps aggregate",
|
|
)
|
|
|
|
|
|
def _plan(store: Store, sql: str, params: list[object]) -> list[str]:
|
|
with sqlite3.connect(store.path) as db:
|
|
return [row[3] for row in db.execute(f"EXPLAIN QUERY PLAN {sql}", params)]
|
|
|
|
|
|
def test_evidence_reads_use_indexes_instead_of_full_scans(tmp_path: Path, monkeypatch) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
service = _service(store, "API")
|
|
store.record_health_evidence(_component("ev-1", service.id, HealthStatus.HEALTHY))
|
|
|
|
captured: list[str] = []
|
|
original = sqlite3.Connection.execute
|
|
|
|
class Recording(sqlite3.Connection):
|
|
def execute(self, sql, parameters=(), /): # type: ignore[override]
|
|
if "health_evidence" in sql and sql.lstrip().upper().startswith(("SELECT", "WITH")):
|
|
captured.append(sql)
|
|
return original(self, sql, parameters)
|
|
|
|
real_connect = sqlite3.connect
|
|
monkeypatch.setattr(
|
|
"app.store.sqlite3.connect",
|
|
lambda *args, **kwargs: real_connect(*args, factory=Recording, **kwargs),
|
|
)
|
|
store.health_evidence_for_service(service.id)
|
|
store.health_evidence_for_services([service.id])
|
|
monkeypatch.setattr("app.store.sqlite3.connect", real_connect)
|
|
|
|
single, bulk = captured[-2], captured[-1]
|
|
for sql, params in (
|
|
(single, [service.id, service.id, 100]),
|
|
(bulk, [service.id, service.id, 100]),
|
|
):
|
|
details = _plan(store, sql, params)
|
|
assert not any(detail in {"SCAN e", "SCAN health_evidence"} for detail in details), details
|
|
|
|
|
|
def test_bulk_and_single_evidence_reads_return_the_same_rows(tmp_path: Path) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
api = _service(store, "API")
|
|
worker = _service(store, "Worker")
|
|
base = datetime.now(UTC) - timedelta(hours=2)
|
|
# Status flips create superseded history for the direct claim.
|
|
for index, status in enumerate([HealthStatus.HEALTHY, HealthStatus.DOWN, HealthStatus.HEALTHY]):
|
|
store.record_health_evidence(
|
|
_component(f"api-{index}", api.id, status, at=base + timedelta(minutes=index))
|
|
)
|
|
store.record_health_evidence(_component("worker-0", worker.id, HealthStatus.DOWN, at=base))
|
|
# One application claim linked to both components.
|
|
store.record_health_evidence(_application("app-0", HealthStatus.HEALTHY), service_ids=[api.id, worker.id])
|
|
|
|
bulk = store.health_evidence_for_services([api.id, worker.id])
|
|
for service in (api, worker):
|
|
single = store.health_evidence_for_service(service.id)
|
|
assert [item.id for item in bulk[service.id]] == [item.id for item in single]
|
|
assert {item.id for item in bulk[api.id]} == {"api-0", "api-1", "api-2", "app-0"}
|
|
assert {item.id for item in bulk[worker.id]} == {"worker-0", "app-0"}
|
|
|
|
limited = store.health_evidence_for_services([api.id], limit_per_service=2)
|
|
assert [item.id for item in limited[api.id]] == [
|
|
item.id for item in store.health_evidence_for_service(api.id, limit=2)
|
|
]
|
|
|
|
|
|
def test_prune_removes_only_old_superseded_history(tmp_path: Path) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
api = _service(store, "API")
|
|
old = datetime.now(UTC) - timedelta(days=30)
|
|
store.record_health_evidence(_component("old-down", api.id, HealthStatus.DOWN, at=old))
|
|
store.record_health_evidence(
|
|
_component("old-up", api.id, HealthStatus.HEALTHY, at=old + timedelta(minutes=1))
|
|
)
|
|
store.record_health_evidence(_component("recent-down", api.id, HealthStatus.DOWN))
|
|
store.record_health_evidence(_application("app-0", HealthStatus.HEALTHY), service_ids=[api.id])
|
|
before = store.service_health(api.id)
|
|
|
|
assert store.prune_health_evidence(7) == 2
|
|
|
|
remaining = {item.id for item in store.health_evidence_for_service(api.id)}
|
|
assert remaining == {"recent-down", "app-0"}
|
|
assert store.service_health(api.id) == before == HealthStatus.DOWN
|
|
with sqlite3.connect(store.path) as db:
|
|
orphans = db.execute(
|
|
"SELECT COUNT(*) FROM health_evidence_service_links "
|
|
"WHERE evidence_id NOT IN (SELECT id FROM health_evidence)"
|
|
).fetchone()[0]
|
|
assert orphans == 0
|
|
assert store.prune_health_evidence(0) == 0
|
|
|
|
|
|
def test_batched_discovery_context_matches_per_service_reads(tmp_path: Path) -> None:
|
|
from app.discovery_store import DiscoveryStore
|
|
from app.models import DiscoveryConfidence, DiscoveryLifecycle, RelationEvidence
|
|
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
discovery = DiscoveryStore(store)
|
|
api = _service(store, "API")
|
|
database = _service(store, "Database")
|
|
lonely = _service(store, "Lonely")
|
|
for index, service in enumerate((api, database)):
|
|
source = discovery.save_source(
|
|
source_type="docker",
|
|
source_id=f"container-{index}",
|
|
source_name=service.name,
|
|
lifecycle=DiscoveryLifecycle.ACTIVE,
|
|
metadata={},
|
|
)
|
|
discovery.attach_source(service.id, source.id)
|
|
now = datetime.now(UTC)
|
|
discovery.save_evidence(
|
|
RelationEvidence(
|
|
id="rel-1",
|
|
source_service_id=api.id,
|
|
target_service_id=database.id,
|
|
evidence_type="compose",
|
|
confidence=DiscoveryConfidence.HIGH,
|
|
explanation="depends_on",
|
|
first_seen_at=now,
|
|
last_seen_at=now,
|
|
)
|
|
)
|
|
|
|
ids = [api.id, database.id, lonely.id]
|
|
sources = discovery.sources_for_services(ids)
|
|
relations = discovery.evidence_for_services(ids)
|
|
for service_id in ids:
|
|
assert sources[service_id] == discovery.sources_for_service(service_id)
|
|
assert relations[service_id] == discovery.evidence_for_service(service_id)
|
|
assert [item.id for item in relations[database.id]] == ["rel-1"]
|
|
assert relations[lonely.id] == [] and sources[lonely.id] == []
|
|
|
|
|
|
def _reference_effective(
|
|
graph: dict[str, list[str]], direct: dict[str, HealthStatus]
|
|
) -> dict[str, HealthStatus]:
|
|
"""The pre-v1.6.3 recursive per-path algorithm, kept as an oracle."""
|
|
|
|
def dependency(node: str, visiting: set[str]) -> HealthStatus:
|
|
if not graph[node]:
|
|
return HealthStatus.HEALTHY
|
|
states = [
|
|
effective(item, visiting) if item in graph else HealthStatus.UNKNOWN
|
|
for item in graph[node]
|
|
]
|
|
return max(states, key=lambda status: SEVERITY[status])
|
|
|
|
def effective(node: str, visiting: set[str]) -> HealthStatus:
|
|
if node in visiting:
|
|
return HealthStatus.UNKNOWN
|
|
own = direct[node]
|
|
if own == HealthStatus.MAINTENANCE:
|
|
return own
|
|
dep = dependency(node, {*visiting, node})
|
|
if own == HealthStatus.DOWN:
|
|
return own
|
|
if dep == HealthStatus.DOWN:
|
|
return HealthStatus.DEGRADED
|
|
if dep in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} and own in {
|
|
HealthStatus.HEALTHY,
|
|
HealthStatus.RECOVERING,
|
|
}:
|
|
return HealthStatus.DEGRADED
|
|
return own
|
|
|
|
return {node: effective(node, set()) for node in graph}
|
|
|
|
|
|
def test_batched_graph_matches_recursive_reference_on_random_dags() -> None:
|
|
rng = random.Random(163)
|
|
statuses = list(HealthStatus)
|
|
for _ in range(200):
|
|
size = rng.randint(1, 14)
|
|
nodes = [f"n{index}" for index in range(size)]
|
|
# Acyclic by construction: a node only depends on lower indexes.
|
|
graph = {
|
|
node: rng.sample(nodes[:index], k=rng.randint(0, min(3, index)))
|
|
for index, node in enumerate(nodes)
|
|
}
|
|
if rng.random() < 0.2:
|
|
graph[nodes[-1]].append("missing")
|
|
direct = {node: rng.choice(statuses) for node in nodes}
|
|
services = [_fake_service(node, graph[node]) for node in nodes]
|
|
effective, _ = status_maps(services, direct_statuses=direct)
|
|
assert effective == _reference_effective(graph, direct), (graph, direct)
|
|
|
|
|
|
def _fake_service(identifier: str, depends_on: list[str]):
|
|
from app.models import Service
|
|
|
|
now = datetime.now(UTC)
|
|
return Service(
|
|
id=identifier,
|
|
name=identifier,
|
|
kind=ServiceKind.HTTP,
|
|
url="https://example.test",
|
|
depends_on=depends_on,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
|
|
|
|
def test_incident_resolution_uses_a_bounded_number_of_queries(tmp_path: Path, monkeypatch) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
# A deep chain plus a fan-in reproduces the shape that made the old walk
|
|
# re-read every shared dependency once per path.
|
|
previous: list[str] = []
|
|
for index in range(40):
|
|
service = _service(store, f"S{index:02d}", depends_on=previous[-2:])
|
|
status = HealthStatus.DOWN if index == 0 else HealthStatus.HEALTHY
|
|
store.record_health_evidence(_component(f"ev-{index}", service.id, status))
|
|
previous.append(service.id)
|
|
|
|
connections = 0
|
|
original = store._connect
|
|
|
|
def counting_connect():
|
|
nonlocal connections
|
|
connections += 1
|
|
return original()
|
|
|
|
monkeypatch.setattr(store, "_connect", counting_connect)
|
|
overview = incident_overview(store)
|
|
assert connections <= 4, connections
|
|
assert [candidate.service_id for candidate in overview.candidates] == [previous[0]]
|
|
assert len(overview.candidates[0].affected_service_ids) == 39
|
|
|
|
health = topology_health(store)
|
|
assert health.effective[previous[0]] == HealthStatus.DOWN
|
|
assert health.effective[previous[-1]] == HealthStatus.DEGRADED
|
|
|
|
connections = 0
|
|
sync_incidents(store)
|
|
# One graph resolution plus incident journal writes; never one per node.
|
|
assert connections < 20, connections
|
|
|
|
|
|
def test_topology_request_budget_at_300_services(tmp_path: Path, monkeypatch) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
definitions = [
|
|
ConfigService(
|
|
name=f"Service {index}",
|
|
kind=ServiceKind.HTTP,
|
|
url=f"https://service-{index}.test",
|
|
domain="Benchmark",
|
|
application=f"Application {index // 10}",
|
|
depends_on=[f"Service {index - 1}"] if index % 10 else [],
|
|
)
|
|
for index in range(300)
|
|
]
|
|
assert store.import_config_services(definitions, "merge").created == 300
|
|
for service in store.list_services():
|
|
store.record_health_evidence(_component(f"ev-{service.id}", service.id, HealthStatus.HEALTHY))
|
|
monkeypatch.setattr(main_module, "store", store)
|
|
monkeypatch.setattr(main_module, "discovery_store", DiscoveryStore(store))
|
|
|
|
connections = 0
|
|
original = store._connect
|
|
|
|
def counting_connect():
|
|
nonlocal connections
|
|
connections += 1
|
|
return original()
|
|
|
|
monkeypatch.setattr(store, "_connect", counting_connect)
|
|
evidence_reads = 0
|
|
read_evidence = store.health_evidence_for_services
|
|
|
|
def counting_evidence(service_ids, **kwargs):
|
|
nonlocal evidence_reads
|
|
evidence_reads += 1
|
|
return read_evidence(service_ids, **kwargs)
|
|
|
|
monkeypatch.setattr(store, "health_evidence_for_services", counting_evidence)
|
|
topology_reads = 0
|
|
list_services = store.list_services
|
|
|
|
def counting_services():
|
|
nonlocal topology_reads
|
|
topology_reads += 1
|
|
return list_services()
|
|
|
|
monkeypatch.setattr(store, "list_services", counting_services)
|
|
client = TestClient(main_module.app)
|
|
budget_ms = os.environ.get("SERVICEMAP_API_P95_BUDGET_MS")
|
|
|
|
for endpoint in ("/api/v1/topology/semantic", "/api/v1/overview"):
|
|
connections = 0
|
|
evidence_reads = 0
|
|
topology_reads = 0
|
|
main_module.semantic_cache.clear()
|
|
assert client.get(endpoint).status_code == 200
|
|
assert evidence_reads == 1, (endpoint, evidence_reads)
|
|
assert topology_reads == 1, (endpoint, topology_reads)
|
|
assert connections <= 30, (endpoint, connections)
|
|
elapsed_ms = []
|
|
for _ in range(20):
|
|
if endpoint.endswith("/semantic"):
|
|
# A provider or topology change invalidates this representation in production.
|
|
main_module.semantic_cache.clear()
|
|
started = perf_counter()
|
|
response = client.get(endpoint)
|
|
elapsed_ms.append((perf_counter() - started) * 1000)
|
|
assert response.status_code == 200
|
|
p95_ms = quantiles(elapsed_ms, n=20, method="inclusive")[-1]
|
|
print(f"{endpoint} recompute p95: {p95_ms:.1f} ms")
|
|
if budget_ms is not None:
|
|
assert p95_ms <= float(budget_ms), (endpoint, p95_ms, budget_ms)
|