From 5c509ca2a9aa583fe258dcfdd3272dd44d94214a Mon Sep 17 00:00:00 2001 From: Jens Date: Sun, 23 Aug 2026 00:51:53 +0200 Subject: [PATCH] publish the accuracy that was measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recommended detection profile showed operators precision 0.614, recall 0.606 and F1 0.607. Those three numbers appear nowhere in this repository except the file that publishes them and a test that pinned them as literal strings. The only recorded evaluation of that model at that operating point — tile 512, overlap 64, threshold 0.15, the exact key its promotion report recommended — reported 0.590, 0.577 and 0.582. The published figures were about two and a half points more flattering than anything that was measured, on the profile labelled "aanbevolen", and the test made sure nobody would correct them. They now carry the measured values. Worse in kind: the conservative profile reported "gemeten achtergrondfouten 0". Its nine-sample hard-negative matrix at threshold 0.35 recorded 198 background detections with 55 in the worst sample. The one number that tells an operator whether a high-precision model invents buildings on empty terrain said zero where the evidence said 55. Those zeros are not simply wrong everywhere, which is why the fix is not just a number. The other two profiles genuinely produced zero — against a strict pure-empty gate of three samples, a different and much weaker test than the nine-sample hard-negative matrix. Printing 0, 0 and 55 side by side invites a comparison the evidence does not support, so each profile now states its gate, its background sample count and the evaluation behind its figures, and the panel shows them. A test refuses any published figure that does not appear in the evidence record, with a negative control so it cannot pass by matching nothing. Pinning the numbers as literal strings is what let an unsourced precision survive; that assertion is gone. Also ignoring .codex-artifacts/ — ~300 MB of the rejected SAM2 and edge-alignment trials plus a deploy bundle. Kept on disk, out of the repository. No credentials in it; the two token scripts generate from settings. Co-Authored-By: Claude Opus 5 --- .gitignore | 6 ++ ...detection_profile_metrics_are_traceable.py | 78 +++++++++++++++++++ ...t_sprint155_detection_operator_profiles.py | 9 ++- docs/KNOWN_LIMITATIONS.md | 8 ++ .../detection/DetectionModelManagement.tsx | 8 +- .../components/detection/detectionProfiles.ts | 34 +++++--- 6 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 backend/tests/test_detection_profile_metrics_are_traceable.py diff --git a/.gitignore b/.gitignore index 269105ff..01965f8b 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,9 @@ build/ .DS_Store .vscode/ .idea/ + +# Local investigation scratch: the SAM2 and edge-alignment trials the README +# records as rejected, plus their imagery and a deploy bundle (~300 MB). Kept +# on disk because the rejections are worth being able to re-check, out of the +# repository because none of it is an input to the product. +/.codex-artifacts/ diff --git a/backend/tests/test_detection_profile_metrics_are_traceable.py b/backend/tests/test_detection_profile_metrics_are_traceable.py new file mode 100644 index 00000000..8b11665e --- /dev/null +++ b/backend/tests/test_detection_profile_metrics_are_traceable.py @@ -0,0 +1,78 @@ +"""Every accuracy figure shown to an operator must exist in the evidence record. + +The recommended profile published precision 0.6140895327792112, recall +0.6062221049337548 and F1 0.6068607646002744. Those three numbers appear +nowhere in this repository except the file that publishes them and the test +that pinned them as literal strings. The only recorded evaluation of that +model at that operating point — tile 512, overlap 64, threshold 0.15 — reported +0.5898197518, 0.5769921004 and 0.5824578632, so the published figures were +about two and a half points more flattering than anything that was measured, +and a test guaranteed nobody would correct them. + +An operator cannot check a number that has no source. This test refuses one. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PROFILES = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts" +EVIDENCE = ROOT / "docs" / "CODEX_EXECUTION_LOG.md" + +METRIC_FIELDS = ("precision", "recall", "f1") +# The log rounds; the source file may carry more digits of the same value. +TOLERANCE = 1e-9 + + +def _published_metrics() -> list[tuple[str, str, float]]: + source = PROFILES.read_text(encoding="utf-8") + profiles = re.findall(r"id: '([^']+)',(.*?)\n \},", source, re.S) + assert profiles, "no operator profiles found; the file shape changed" + + published: list[tuple[str, str, float]] = [] + for profile_id, body in profiles: + for field in METRIC_FIELDS: + match = re.search(rf"^\s*{field}: ([0-9.]+),", body, re.M) + assert match, f"{profile_id} publishes no {field}" + published.append((profile_id, field, float(match.group(1)))) + return published + + +def _recorded_values() -> list[float]: + text = EVIDENCE.read_text(encoding="utf-8") + return [float(value) for value in re.findall(r"\b0\.\d{4,}\b", text)] + + +def test_every_published_accuracy_figure_appears_in_the_evidence_record() -> None: + recorded = _recorded_values() + untraceable = [ + f"{profile_id}.{field} = {value}" + for profile_id, field, value in _published_metrics() + if not any(abs(value - candidate) <= TOLERANCE for candidate in recorded) + ] + + assert not untraceable, ( + "These figures are shown to operators but were never recorded in " + f"docs/CODEX_EXECUTION_LOG.md: {untraceable}. Publish the measurement " + "that was taken, or record the evaluation that produced these." + ) + + +def test_each_profile_names_the_measurement_behind_its_numbers() -> None: + source = PROFILES.read_text(encoding="utf-8") + profile_count = source.count("modelAssetId:") + + assert source.count("evidenceReference:") == profile_count + assert source.count("backgroundGate:") == profile_count + assert source.count("backgroundSampleCount:") == profile_count + + +def test_the_check_would_notice_an_invented_figure() -> None: + """Without this the test could pass because nothing ever matches.""" + + recorded = _recorded_values() + + assert any(abs(0.5898197518 - value) <= TOLERANCE for value in recorded) + assert not any(abs(0.6140895327792112 - value) <= TOLERANCE for value in recorded) diff --git a/backend/tests/test_sprint155_detection_operator_profiles.py b/backend/tests/test_sprint155_detection_operator_profiles.py index 9bdf1913..d07f2597 100644 --- a/backend/tests/test_sprint155_detection_operator_profiles.py +++ b/backend/tests/test_sprint155_detection_operator_profiles.py @@ -22,11 +22,12 @@ def test_detection_operator_profiles_define_explicit_historical_yolo_controls_wi assert "promotionRecommendation" not in source assert "independentTestProven: false" in source assert "positiveSampleCount: 7" in source - assert "precision: 0.6140895327792112" in source - assert "recall: 0.6062221049337548" in source - assert "f1: 0.6068607646002744" in source - assert "f1: 0.5432865390636915" in source + # The figures themselves are checked against the evidence record in + # tests/test_detection_profile_metrics_are_traceable.py. Pinning them as + # literals here is what kept an unsourced precision in the product. assert "maxBackgroundDetections: 0" in source + assert "maxBackgroundDetections: 55" in source + assert "backgroundGate:" in source assert "Slechts drie pure-achtergrondbeelden" in source assert "ruimtelijke onafhankelijkheid niet bewezen" in source assert "controlekandidaat en niet als grondwaarheid" in source diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index d6387576..5d50e55e 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -82,6 +82,14 @@ runtime source of truth. different pages is kept once instead, since a cursor over a changing table can produce that legitimately. +- The operator profiles in Detection Lab publish figures from historical + evaluations, and each names the measurement it came from. Their background + numbers are not comparable across profiles: the two showing zero were checked + against a strict pure-empty gate of three samples, while the conservative + profile's 55 comes from a nine-sample hard-negative matrix that also recorded + 198 background detections in total. None of the three has an independent + hold-out. + ## Historical analysis - Historical comparison requires compatible source editions, units, coverage diff --git a/frontend/src/components/detection/DetectionModelManagement.tsx b/frontend/src/components/detection/DetectionModelManagement.tsx index 6e1c726a..469579f9 100644 --- a/frontend/src/components/detection/DetectionModelManagement.tsx +++ b/frontend/src/components/detection/DetectionModelManagement.tsx @@ -165,8 +165,14 @@ export function DetectionModelManagement({ historische herkenningsgraad {profile.recall.toFixed(3)} historische F1 {profile.f1.toFixed(3)} positieve controles {profile.positiveSampleCount} - gemeten achtergrondfouten {profile.maxBackgroundDetections} + + ergste achtergrondmonster {profile.maxBackgroundDetections} van{' '} + {profile.backgroundSampleCount} + +

+ Achtergrondtoets: {profile.backgroundGate}. Meting: {profile.evidenceReference}. +

{profile.limitationMessage}