diff --git a/.gitignore b/.gitignore index 269105ff..01965f8b 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,9 @@ build/ .DS_Store .vscode/ .idea/ + +# Local investigation scratch: the SAM2 and edge-alignment trials the README +# records as rejected, plus their imagery and a deploy bundle (~300 MB). Kept +# on disk because the rejections are worth being able to re-check, out of the +# repository because none of it is an input to the product. +/.codex-artifacts/ diff --git a/backend/tests/test_detection_profile_metrics_are_traceable.py b/backend/tests/test_detection_profile_metrics_are_traceable.py new file mode 100644 index 00000000..8b11665e --- /dev/null +++ b/backend/tests/test_detection_profile_metrics_are_traceable.py @@ -0,0 +1,78 @@ +"""Every accuracy figure shown to an operator must exist in the evidence record. + +The recommended profile published precision 0.6140895327792112, recall +0.6062221049337548 and F1 0.6068607646002744. Those three numbers appear +nowhere in this repository except the file that publishes them and the test +that pinned them as literal strings. The only recorded evaluation of that +model at that operating point — tile 512, overlap 64, threshold 0.15 — reported +0.5898197518, 0.5769921004 and 0.5824578632, so the published figures were +about two and a half points more flattering than anything that was measured, +and a test guaranteed nobody would correct them. + +An operator cannot check a number that has no source. This test refuses one. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PROFILES = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts" +EVIDENCE = ROOT / "docs" / "CODEX_EXECUTION_LOG.md" + +METRIC_FIELDS = ("precision", "recall", "f1") +# The log rounds; the source file may carry more digits of the same value. +TOLERANCE = 1e-9 + + +def _published_metrics() -> list[tuple[str, str, float]]: + source = PROFILES.read_text(encoding="utf-8") + profiles = re.findall(r"id: '([^']+)',(.*?)\n \},", source, re.S) + assert profiles, "no operator profiles found; the file shape changed" + + published: list[tuple[str, str, float]] = [] + for profile_id, body in profiles: + for field in METRIC_FIELDS: + match = re.search(rf"^\s*{field}: ([0-9.]+),", body, re.M) + assert match, f"{profile_id} publishes no {field}" + published.append((profile_id, field, float(match.group(1)))) + return published + + +def _recorded_values() -> list[float]: + text = EVIDENCE.read_text(encoding="utf-8") + return [float(value) for value in re.findall(r"\b0\.\d{4,}\b", text)] + + +def test_every_published_accuracy_figure_appears_in_the_evidence_record() -> None: + recorded = _recorded_values() + untraceable = [ + f"{profile_id}.{field} = {value}" + for profile_id, field, value in _published_metrics() + if not any(abs(value - candidate) <= TOLERANCE for candidate in recorded) + ] + + assert not untraceable, ( + "These figures are shown to operators but were never recorded in " + f"docs/CODEX_EXECUTION_LOG.md: {untraceable}. Publish the measurement " + "that was taken, or record the evaluation that produced these." + ) + + +def test_each_profile_names_the_measurement_behind_its_numbers() -> None: + source = PROFILES.read_text(encoding="utf-8") + profile_count = source.count("modelAssetId:") + + assert source.count("evidenceReference:") == profile_count + assert source.count("backgroundGate:") == profile_count + assert source.count("backgroundSampleCount:") == profile_count + + +def test_the_check_would_notice_an_invented_figure() -> None: + """Without this the test could pass because nothing ever matches.""" + + recorded = _recorded_values() + + assert any(abs(0.5898197518 - value) <= TOLERANCE for value in recorded) + assert not any(abs(0.6140895327792112 - value) <= TOLERANCE for value in recorded) diff --git a/backend/tests/test_sprint155_detection_operator_profiles.py b/backend/tests/test_sprint155_detection_operator_profiles.py index 9bdf1913..d07f2597 100644 --- a/backend/tests/test_sprint155_detection_operator_profiles.py +++ b/backend/tests/test_sprint155_detection_operator_profiles.py @@ -22,11 +22,12 @@ def test_detection_operator_profiles_define_explicit_historical_yolo_controls_wi assert "promotionRecommendation" not in source assert "independentTestProven: false" in source assert "positiveSampleCount: 7" in source - assert "precision: 0.6140895327792112" in source - assert "recall: 0.6062221049337548" in source - assert "f1: 0.6068607646002744" in source - assert "f1: 0.5432865390636915" in source + # The figures themselves are checked against the evidence record in + # tests/test_detection_profile_metrics_are_traceable.py. Pinning them as + # literals here is what kept an unsourced precision in the product. assert "maxBackgroundDetections: 0" in source + assert "maxBackgroundDetections: 55" in source + assert "backgroundGate:" in source assert "Slechts drie pure-achtergrondbeelden" in source assert "ruimtelijke onafhankelijkheid niet bewezen" in source assert "controlekandidaat en niet als grondwaarheid" in source diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index d6387576..5d50e55e 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -82,6 +82,14 @@ runtime source of truth. different pages is kept once instead, since a cursor over a changing table can produce that legitimately. +- The operator profiles in Detection Lab publish figures from historical + evaluations, and each names the measurement it came from. Their background + numbers are not comparable across profiles: the two showing zero were checked + against a strict pure-empty gate of three samples, while the conservative + profile's 55 comes from a nine-sample hard-negative matrix that also recorded + 198 background detections in total. None of the three has an independent + hold-out. + ## Historical analysis - Historical comparison requires compatible source editions, units, coverage diff --git a/frontend/src/components/detection/DetectionModelManagement.tsx b/frontend/src/components/detection/DetectionModelManagement.tsx index 6e1c726a..469579f9 100644 --- a/frontend/src/components/detection/DetectionModelManagement.tsx +++ b/frontend/src/components/detection/DetectionModelManagement.tsx @@ -165,8 +165,14 @@ export function DetectionModelManagement({ historische herkenningsgraad {profile.recall.toFixed(3)} historische F1 {profile.f1.toFixed(3)} positieve controles {profile.positiveSampleCount} - gemeten achtergrondfouten {profile.maxBackgroundDetections} + + ergste achtergrondmonster {profile.maxBackgroundDetections} van{' '} + {profile.backgroundSampleCount} + +

+ Achtergrondtoets: {profile.backgroundGate}. Meting: {profile.evidenceReference}. +

{profile.limitationMessage}