Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
79 lines
3.1 KiB
Python
79 lines
3.1 KiB
Python
"""Every accuracy figure shown to an operator must exist in the evidence record.
|
|
|
|
The recommended profile published precision 0.6140895327792112, recall
|
|
0.6062221049337548 and F1 0.6068607646002744. Those three numbers appear
|
|
nowhere in this repository except the file that publishes them and the test
|
|
that pinned them as literal strings. The only recorded evaluation of that
|
|
model at that operating point — tile 512, overlap 64, threshold 0.15 — reported
|
|
0.5898197518, 0.5769921004 and 0.5824578632, so the published figures were
|
|
about two and a half points more flattering than anything that was measured,
|
|
and a test guaranteed nobody would correct them.
|
|
|
|
An operator cannot check a number that has no source. This test refuses one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
PROFILES = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts"
|
|
EVIDENCE = ROOT / "docs" / "CODEX_EXECUTION_LOG.md"
|
|
|
|
METRIC_FIELDS = ("precision", "recall", "f1")
|
|
# The log rounds; the source file may carry more digits of the same value.
|
|
TOLERANCE = 1e-9
|
|
|
|
|
|
def _published_metrics() -> list[tuple[str, str, float]]:
|
|
source = PROFILES.read_text(encoding="utf-8")
|
|
profiles = re.findall(r"id: '([^']+)',(.*?)\n \},", source, re.S)
|
|
assert profiles, "no operator profiles found; the file shape changed"
|
|
|
|
published: list[tuple[str, str, float]] = []
|
|
for profile_id, body in profiles:
|
|
for field in METRIC_FIELDS:
|
|
match = re.search(rf"^\s*{field}: ([0-9.]+),", body, re.M)
|
|
assert match, f"{profile_id} publishes no {field}"
|
|
published.append((profile_id, field, float(match.group(1))))
|
|
return published
|
|
|
|
|
|
def _recorded_values() -> list[float]:
|
|
text = EVIDENCE.read_text(encoding="utf-8")
|
|
return [float(value) for value in re.findall(r"\b0\.\d{4,}\b", text)]
|
|
|
|
|
|
def test_every_published_accuracy_figure_appears_in_the_evidence_record() -> None:
|
|
recorded = _recorded_values()
|
|
untraceable = [
|
|
f"{profile_id}.{field} = {value}"
|
|
for profile_id, field, value in _published_metrics()
|
|
if not any(abs(value - candidate) <= TOLERANCE for candidate in recorded)
|
|
]
|
|
|
|
assert not untraceable, (
|
|
"These figures are shown to operators but were never recorded in "
|
|
f"docs/CODEX_EXECUTION_LOG.md: {untraceable}. Publish the measurement "
|
|
"that was taken, or record the evaluation that produced these."
|
|
)
|
|
|
|
|
|
def test_each_profile_names_the_measurement_behind_its_numbers() -> None:
|
|
source = PROFILES.read_text(encoding="utf-8")
|
|
profile_count = source.count("modelAssetId:")
|
|
|
|
assert source.count("evidenceReference:") == profile_count
|
|
assert source.count("backgroundGate:") == profile_count
|
|
assert source.count("backgroundSampleCount:") == profile_count
|
|
|
|
|
|
def test_the_check_would_notice_an_invented_figure() -> None:
|
|
"""Without this the test could pass because nothing ever matches."""
|
|
|
|
recorded = _recorded_values()
|
|
|
|
assert any(abs(0.5898197518 - value) <= TOLERANCE for value in recorded)
|
|
assert not any(abs(0.6140895327792112 - value) <= TOLERANCE for value in recorded)
|