From 5314f16383e5ecb179729a338d569e3ade2b3576 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 8 Jul 2026 14:30:37 +0200 Subject: [PATCH] Prevent calibration evidence response overwrite --- CHANGELOG.md | 7 ++++++ ...ulti_aoi_calibration_evidence_portfolio.py | 23 ++++++++++++++++++- docs/CODEX_EXECUTION_LOG.md | 16 +++++++++++++ docs/TODO.md | 1 + .../export_detection_calibration_evidence.sh | 3 ++- 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa88e07e..bd7da0a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1378,3 +1378,10 @@ Added: - Added validation guardrails for selected project, raster dataset, reference dataset, configured non-fixture model, tile manifest and explicit local model asset. - Added regression coverage for the guided runner contract. - No backend API contracts, migrations, model downloads, provider fetching, automatic promotion or model file mutation behavior changed. + +## Sprint 142 Calibration evidence response uniqueness (2026-07-08) + +- Hardened the calibration evidence exporter so response artifacts are keyed by threshold and quality-check id. +- Prevented multi-model portfolios from overwriting runs that share the same threshold. +- Added regression coverage proving same-threshold runs are preserved in the assembled portfolio. +- No API contracts, migrations, model downloads, provider fetching or AI inference behavior changed. diff --git a/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py b/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py index af078d7d..4eac628d 100644 --- a/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py +++ b/backend/tests/test_sprint139_multi_aoi_calibration_evidence_portfolio.py @@ -57,6 +57,20 @@ def test_multi_aoi_calibration_evidence_portfolio_assembles_existing_evidence(tm "precision": 0.7, "recall": 0.3, "f1_score": 0.42, + }, + { + "threshold": 0.15, + "quality_check_id": "qc-geel-better", + "analysis_run_id": "analysis-geel-better", + "job_id": "job-geel-better", + "detection_count": 7, + "model_asset_id": "geointel-building-yolov8s-smoke-pt", + "tile_size": 640, + "tile_overlap": 64, + "quality_score": 0.55, + "precision": 0.8, + "recall": 0.42, + "f1_score": 0.55, } ], } @@ -126,6 +140,10 @@ case "$url" in role="match_candidate" quality_check_id="qc-geel" ;; + */api/v1/projects/project-geel/quality-checks/qc-geel-better/evidence/geojson) + role="match_reference" + quality_check_id="qc-geel-better" + ;; */api/v1/projects/project-mol/quality-checks/qc-mol/evidence/geojson) role="false_negative" quality_check_id="qc-mol" @@ -176,8 +194,11 @@ JSON assert portfolio["portfolio_name"] == "Kempen building model smoke" assert portfolio["model_asset_id"] == "geointel-building-yolov8s-smoke-pt" assert portfolio["sample_count"] == 2 - assert portfolio["total_evidence_feature_count"] == 2 + assert portfolio["total_evidence_feature_count"] == 3 assert {sample["sample_slug"] for sample in portfolio["samples"]} == {"geel", "mol"} + geel_sample = next(sample for sample in portfolio["samples"] if sample["sample_slug"] == "geel") + assert len(geel_sample["runs"]) == 2 + assert {run["quality_check_id"] for run in geel_sample["runs"]} == {"qc-geel", "qc-geel-better"} assert portfolio["best_sample_by_score"]["sample_slug"] == "mol" assert portfolio["best_sample_by_score"]["best_run_by_score"]["model_asset_id"] == ( "geointel-building-yolov8s-smoke-pt" diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 732eacfa..af9a1664 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5495,3 +5495,19 @@ Limitations: Next recommended pass: - Continue with V1 usability work that reduces operator confusion without expanding frozen product scope. + +## Sprint 142 Calibration evidence response uniqueness (2026-07-08) + +Changed: +- Hardened `scripts/export_detection_calibration_evidence.sh` so calibration evidence response files include both confidence threshold and quality-check id. +- Prevented same-threshold runs from different model assets or calibration jobs from overwriting each other before portfolio assembly. +- Extended the multi-AOI calibration evidence portfolio regression test with two Geel runs at the same threshold and distinct quality checks. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py -q` failed because only two evidence features were retained when three same-threshold responses were expected. +- `python -m pytest backend\tests\test_sprint139_multi_aoi_calibration_evidence_portfolio.py backend\tests\test_sprint138_calibration_evidence_bundle_smoke.py backend\tests\test_sprint137_browser_calibration_summary_evidence_script.py backend\tests\test_sprint125_detection_calibration_evidence_bundle.py -q` (`4 passed`) +- `bash -n scripts/export_detection_calibration_evidence.sh` +- `bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh` + +Open: +- Regenerate the expanded Tower calibration evidence portfolio after deploying this fix so same-threshold model comparisons are represented correctly. diff --git a/docs/TODO.md b/docs/TODO.md index 0532dc7f..169e8e2b 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -429,5 +429,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Run the first live multi-AOI calibration evidence portfolio on Tower for Geel, Mol and Turnhout. - [x] Run fresh positive-AOI matrix coverage for Balen, Herentals and Westerlo. - [x] Preserve model/tile provenance in calibration evidence bundle summaries. +- [x] Prevent same-threshold calibration evidence responses from overwriting each other in multi-model portfolios. - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout. - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles. diff --git a/scripts/export_detection_calibration_evidence.sh b/scripts/export_detection_calibration_evidence.sh index a1111045..13158986 100644 --- a/scripts/export_detection_calibration_evidence.sh +++ b/scripts/export_detection_calibration_evidence.sh @@ -157,7 +157,8 @@ echo "Output: ${CALIBRATION_EVIDENCE_DIR}" while IFS=$'\t' read -r threshold project_id quality_check_id; do threshold_label="$(printf '%s' "${threshold}" | tr '.-' 'pm')" - response_path="${CALIBRATION_EVIDENCE_DIR}/threshold_${threshold_label}_evidence_response.json" + quality_check_label="$(printf '%s' "${quality_check_id}" | tr -c 'A-Za-z0-9_.-' '_')" + response_path="${CALIBRATION_EVIDENCE_DIR}/threshold_${threshold_label}_${quality_check_label}_evidence_response.json" echo "-- Evidence threshold ${threshold}, quality_check_id ${quality_check_id} --" "${CURL_BIN}" -fsS "${BASE_URL%/}/api/v1/projects/${project_id}/quality-checks/${quality_check_id}/evidence/geojson" > "${response_path}" done < "${request_manifest}"