Initial public release

backend/tests/test_detection_profile_metrics_are_traceable.py

@@ -0,0 +1,78 @@
+"""Every accuracy figure shown to an operator must exist in the evidence record.
+
+The recommended profile published precision 0.6140895327792112, recall
+0.6062221049337548 and F1 0.6068607646002744. Those three numbers appear
+nowhere in this repository except the file that publishes them and the test
+that pinned them as literal strings. The only recorded evaluation of that
+model at that operating point — tile 512, overlap 64, threshold 0.15 — reported
+0.5898197518, 0.5769921004 and 0.5824578632, so the published figures were
+about two and a half points more flattering than anything that was measured,
+and a test guaranteed nobody would correct them.
+
+An operator cannot check a number that has no source. This test refuses one.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+PROFILES = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts"
+EVIDENCE = ROOT / "docs" / "CODEX_EXECUTION_LOG.md"
+
+METRIC_FIELDS = ("precision", "recall", "f1")
+# The log rounds; the source file may carry more digits of the same value.
+TOLERANCE = 1e-9
+
+
+def _published_metrics() -> list[tuple[str, str, float]]:
+    source = PROFILES.read_text(encoding="utf-8")
+    profiles = re.findall(r"id: '([^']+)',(.*?)\n  \},", source, re.S)
+    assert profiles, "no operator profiles found; the file shape changed"
+
+    published: list[tuple[str, str, float]] = []
+    for profile_id, body in profiles:
+        for field in METRIC_FIELDS:
+            match = re.search(rf"^\s*{field}: ([0-9.]+),", body, re.M)
+            assert match, f"{profile_id} publishes no {field}"
+            published.append((profile_id, field, float(match.group(1))))
+    return published
+
+
+def _recorded_values() -> list[float]:
+    text = EVIDENCE.read_text(encoding="utf-8")
+    return [float(value) for value in re.findall(r"\b0\.\d{4,}\b", text)]
+
+
+def test_every_published_accuracy_figure_appears_in_the_evidence_record() -> None:
+    recorded = _recorded_values()
+    untraceable = [
+        f"{profile_id}.{field} = {value}"
+        for profile_id, field, value in _published_metrics()
+        if not any(abs(value - candidate) <= TOLERANCE for candidate in recorded)
+    ]
+
+    assert not untraceable, (
+        "These figures are shown to operators but were never recorded in "
+        f"docs/CODEX_EXECUTION_LOG.md: {untraceable}. Publish the measurement "
+        "that was taken, or record the evaluation that produced these."
+    )
+
+
+def test_each_profile_names_the_measurement_behind_its_numbers() -> None:
+    source = PROFILES.read_text(encoding="utf-8")
+    profile_count = source.count("modelAssetId:")
+
+    assert source.count("evidenceReference:") == profile_count
+    assert source.count("backgroundGate:") == profile_count
+    assert source.count("backgroundSampleCount:") == profile_count
+
+
+def test_the_check_would_notice_an_invented_figure() -> None:
+    """Without this the test could pass because nothing ever matches."""
+
+    recorded = _recorded_values()
+
+    assert any(abs(0.5898197518 - value) <= TOLERANCE for value in recorded)
+    assert not any(abs(0.6140895327792112 - value) <= TOLERANCE for value in recorded)