"""v1.6.3 regression guards for health-evidence reads and graph resolution. Production profiling showed 78% of server CPU in a full ``health_evidence`` scan, issued once per node of a recursive, un-memoised health walk. These tests pin the indexed query shape, equivalence of the bulk and single reads, the batched graph result and the bounded query count. """ from __future__ import annotations import os import random import sqlite3 from datetime import UTC, datetime, timedelta from pathlib import Path from statistics import quantiles from time import perf_counter from fastapi.testclient import TestClient from app import main as main_module from app.discovery_store import DiscoveryStore from app.health import SEVERITY, status_maps, topology_health from app.incidents import incident_overview, sync_incidents from app.models import ( ConfigService, EvidenceProvider, EvidenceScope, HealthEvidence, HealthStatus, ServiceCreate, ServiceKind, ) from app.store import Store def _service(store: Store, name: str, depends_on: list[str] | None = None): return store.create_service( ServiceCreate( name=name, kind=ServiceKind.HTTP, url=f"https://{name.lower()}.test", depends_on=depends_on or [], ) ) def _component( evidence_id: str, service_id: str, status: HealthStatus, *, at: datetime | None = None, ) -> HealthEvidence: now = at or datetime.now(UTC) return HealthEvidence( id=evidence_id, subject_type="service", subject_id=service_id, service_id=service_id, provider=EvidenceProvider.DOCKER, provider_ref=service_id, status=status, scope=EvidenceScope.COMPONENT, confidence=0.95, observed_at=now, valid_until=datetime.now(UTC) + timedelta(minutes=5), received_at=now, summary="Docker healthcheck", ) def _application(evidence_id: str, status: HealthStatus) -> HealthEvidence: now = datetime.now(UTC) return HealthEvidence( id=evidence_id, subject_type="application", subject_id="appops:northstar", provider=EvidenceProvider.APPOPS, provider_ref="northstar", status=status, scope=EvidenceScope.APPLICATION, confidence=0.9, observed_at=now, valid_until=now + timedelta(minutes=5), received_at=now, summary="AppOps aggregate", ) def _plan(store: Store, sql: str, params: list[object]) -> list[str]: with sqlite3.connect(store.path) as db: return [row[3] for row in db.execute(f"EXPLAIN QUERY PLAN {sql}", params)] def test_evidence_reads_use_indexes_instead_of_full_scans(tmp_path: Path, monkeypatch) -> None: store = Store(str(tmp_path / "servicemap.db")) service = _service(store, "API") store.record_health_evidence(_component("ev-1", service.id, HealthStatus.HEALTHY)) captured: list[str] = [] original = sqlite3.Connection.execute class Recording(sqlite3.Connection): def execute(self, sql, parameters=(), /): # type: ignore[override] if "health_evidence" in sql and sql.lstrip().upper().startswith(("SELECT", "WITH")): captured.append(sql) return original(self, sql, parameters) real_connect = sqlite3.connect monkeypatch.setattr( "app.store.sqlite3.connect", lambda *args, **kwargs: real_connect(*args, factory=Recording, **kwargs), ) store.health_evidence_for_service(service.id) store.health_evidence_for_services([service.id]) monkeypatch.setattr("app.store.sqlite3.connect", real_connect) single, bulk = captured[-2], captured[-1] for sql, params in ( (single, [service.id, service.id, 100]), (bulk, [service.id, service.id, 100]), ): details = _plan(store, sql, params) assert not any(detail in {"SCAN e", "SCAN health_evidence"} for detail in details), details def test_bulk_and_single_evidence_reads_return_the_same_rows(tmp_path: Path) -> None: store = Store(str(tmp_path / "servicemap.db")) api = _service(store, "API") worker = _service(store, "Worker") base = datetime.now(UTC) - timedelta(hours=2) # Status flips create superseded history for the direct claim. for index, status in enumerate([HealthStatus.HEALTHY, HealthStatus.DOWN, HealthStatus.HEALTHY]): store.record_health_evidence( _component(f"api-{index}", api.id, status, at=base + timedelta(minutes=index)) ) store.record_health_evidence(_component("worker-0", worker.id, HealthStatus.DOWN, at=base)) # One application claim linked to both components. store.record_health_evidence(_application("app-0", HealthStatus.HEALTHY), service_ids=[api.id, worker.id]) bulk = store.health_evidence_for_services([api.id, worker.id]) for service in (api, worker): single = store.health_evidence_for_service(service.id) assert [item.id for item in bulk[service.id]] == [item.id for item in single] assert {item.id for item in bulk[api.id]} == {"api-0", "api-1", "api-2", "app-0"} assert {item.id for item in bulk[worker.id]} == {"worker-0", "app-0"} limited = store.health_evidence_for_services([api.id], limit_per_service=2) assert [item.id for item in limited[api.id]] == [ item.id for item in store.health_evidence_for_service(api.id, limit=2) ] def test_prune_removes_only_old_superseded_history(tmp_path: Path) -> None: store = Store(str(tmp_path / "servicemap.db")) api = _service(store, "API") old = datetime.now(UTC) - timedelta(days=30) store.record_health_evidence(_component("old-down", api.id, HealthStatus.DOWN, at=old)) store.record_health_evidence( _component("old-up", api.id, HealthStatus.HEALTHY, at=old + timedelta(minutes=1)) ) store.record_health_evidence(_component("recent-down", api.id, HealthStatus.DOWN)) store.record_health_evidence(_application("app-0", HealthStatus.HEALTHY), service_ids=[api.id]) before = store.service_health(api.id) assert store.prune_health_evidence(7) == 2 remaining = {item.id for item in store.health_evidence_for_service(api.id)} assert remaining == {"recent-down", "app-0"} assert store.service_health(api.id) == before == HealthStatus.DOWN with sqlite3.connect(store.path) as db: orphans = db.execute( "SELECT COUNT(*) FROM health_evidence_service_links " "WHERE evidence_id NOT IN (SELECT id FROM health_evidence)" ).fetchone()[0] assert orphans == 0 assert store.prune_health_evidence(0) == 0 def test_batched_discovery_context_matches_per_service_reads(tmp_path: Path) -> None: from app.discovery_store import DiscoveryStore from app.models import DiscoveryConfidence, DiscoveryLifecycle, RelationEvidence store = Store(str(tmp_path / "servicemap.db")) discovery = DiscoveryStore(store) api = _service(store, "API") database = _service(store, "Database") lonely = _service(store, "Lonely") for index, service in enumerate((api, database)): source = discovery.save_source( source_type="docker", source_id=f"container-{index}", source_name=service.name, lifecycle=DiscoveryLifecycle.ACTIVE, metadata={}, ) discovery.attach_source(service.id, source.id) now = datetime.now(UTC) discovery.save_evidence( RelationEvidence( id="rel-1", source_service_id=api.id, target_service_id=database.id, evidence_type="compose", confidence=DiscoveryConfidence.HIGH, explanation="depends_on", first_seen_at=now, last_seen_at=now, ) ) ids = [api.id, database.id, lonely.id] sources = discovery.sources_for_services(ids) relations = discovery.evidence_for_services(ids) for service_id in ids: assert sources[service_id] == discovery.sources_for_service(service_id) assert relations[service_id] == discovery.evidence_for_service(service_id) assert [item.id for item in relations[database.id]] == ["rel-1"] assert relations[lonely.id] == [] and sources[lonely.id] == [] def _reference_effective( graph: dict[str, list[str]], direct: dict[str, HealthStatus] ) -> dict[str, HealthStatus]: """The pre-v1.6.3 recursive per-path algorithm, kept as an oracle.""" def dependency(node: str, visiting: set[str]) -> HealthStatus: if not graph[node]: return HealthStatus.HEALTHY states = [ effective(item, visiting) if item in graph else HealthStatus.UNKNOWN for item in graph[node] ] return max(states, key=lambda status: SEVERITY[status]) def effective(node: str, visiting: set[str]) -> HealthStatus: if node in visiting: return HealthStatus.UNKNOWN own = direct[node] if own == HealthStatus.MAINTENANCE: return own dep = dependency(node, {*visiting, node}) if own == HealthStatus.DOWN: return own if dep == HealthStatus.DOWN: return HealthStatus.DEGRADED if dep in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} and own in { HealthStatus.HEALTHY, HealthStatus.RECOVERING, }: return HealthStatus.DEGRADED return own return {node: effective(node, set()) for node in graph} def test_batched_graph_matches_recursive_reference_on_random_dags() -> None: rng = random.Random(163) statuses = list(HealthStatus) for _ in range(200): size = rng.randint(1, 14) nodes = [f"n{index}" for index in range(size)] # Acyclic by construction: a node only depends on lower indexes. graph = { node: rng.sample(nodes[:index], k=rng.randint(0, min(3, index))) for index, node in enumerate(nodes) } if rng.random() < 0.2: graph[nodes[-1]].append("missing") direct = {node: rng.choice(statuses) for node in nodes} services = [_fake_service(node, graph[node]) for node in nodes] effective, _ = status_maps(services, direct_statuses=direct) assert effective == _reference_effective(graph, direct), (graph, direct) def _fake_service(identifier: str, depends_on: list[str]): from app.models import Service now = datetime.now(UTC) return Service( id=identifier, name=identifier, kind=ServiceKind.HTTP, url="https://example.test", depends_on=depends_on, created_at=now, updated_at=now, ) def test_incident_resolution_uses_a_bounded_number_of_queries(tmp_path: Path, monkeypatch) -> None: store = Store(str(tmp_path / "servicemap.db")) # A deep chain plus a fan-in reproduces the shape that made the old walk # re-read every shared dependency once per path. previous: list[str] = [] for index in range(40): service = _service(store, f"S{index:02d}", depends_on=previous[-2:]) status = HealthStatus.DOWN if index == 0 else HealthStatus.HEALTHY store.record_health_evidence(_component(f"ev-{index}", service.id, status)) previous.append(service.id) connections = 0 original = store._connect def counting_connect(): nonlocal connections connections += 1 return original() monkeypatch.setattr(store, "_connect", counting_connect) overview = incident_overview(store) assert connections <= 4, connections assert [candidate.service_id for candidate in overview.candidates] == [previous[0]] assert len(overview.candidates[0].affected_service_ids) == 39 health = topology_health(store) assert health.effective[previous[0]] == HealthStatus.DOWN assert health.effective[previous[-1]] == HealthStatus.DEGRADED connections = 0 sync_incidents(store) # One graph resolution plus incident journal writes; never one per node. assert connections < 20, connections def test_topology_request_budget_at_300_services(tmp_path: Path, monkeypatch) -> None: store = Store(str(tmp_path / "servicemap.db")) definitions = [ ConfigService( name=f"Service {index}", kind=ServiceKind.HTTP, url=f"https://service-{index}.test", domain="Benchmark", application=f"Application {index // 10}", depends_on=[f"Service {index - 1}"] if index % 10 else [], ) for index in range(300) ] assert store.import_config_services(definitions, "merge").created == 300 for service in store.list_services(): store.record_health_evidence(_component(f"ev-{service.id}", service.id, HealthStatus.HEALTHY)) monkeypatch.setattr(main_module, "store", store) monkeypatch.setattr(main_module, "discovery_store", DiscoveryStore(store)) connections = 0 original = store._connect def counting_connect(): nonlocal connections connections += 1 return original() monkeypatch.setattr(store, "_connect", counting_connect) evidence_reads = 0 read_evidence = store.health_evidence_for_services def counting_evidence(service_ids, **kwargs): nonlocal evidence_reads evidence_reads += 1 return read_evidence(service_ids, **kwargs) monkeypatch.setattr(store, "health_evidence_for_services", counting_evidence) topology_reads = 0 list_services = store.list_services def counting_services(): nonlocal topology_reads topology_reads += 1 return list_services() monkeypatch.setattr(store, "list_services", counting_services) client = TestClient(main_module.app) budget_ms = os.environ.get("SERVICEMAP_API_P95_BUDGET_MS") for endpoint in ("/api/v1/topology/semantic", "/api/v1/overview"): connections = 0 evidence_reads = 0 topology_reads = 0 main_module.semantic_cache.clear() assert client.get(endpoint).status_code == 200 assert evidence_reads == 1, (endpoint, evidence_reads) assert topology_reads == 1, (endpoint, topology_reads) assert connections <= 30, (endpoint, connections) elapsed_ms = [] for _ in range(20): if endpoint.endswith("/semantic"): # A provider or topology change invalidates this representation in production. main_module.semantic_cache.clear() started = perf_counter() response = client.get(endpoint) elapsed_ms.append((perf_counter() - started) * 1000) assert response.status_code == 200 p95_ms = quantiles(elapsed_ms, n=20, method="inclusive")[-1] print(f"{endpoint} recompute p95: {p95_ms:.1f} ms") if budget_ms is not None: assert p95_ms <= float(budget_ms), (endpoint, p95_ms, budget_ms)