diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d264fd6..75cb5078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1310,3 +1310,11 @@ Added: - Detection Lab now shows linked tile manifest provenance plus preflight manifest validation, tile count and `will_run_inference` state. - Added regression coverage for the handoff contract and preserved the existing no-auto-select model guardrail. - No backend API contracts, migrations, model downloads, provider fetching or model weight mutation behavior changed. + +## Sprint 133 Detection threshold calibration UX (2026-07-08) + +- Added a Detection Lab calibration comparison panel that joins persisted detection runs with persisted QA/QC checks. +- The panel compares confidence threshold, model, detection count, precision, recall, F1, false positives and false negatives. +- Added operator guidance for best F1, best precision and lowest false-positive pressure, with a promotion guardrail to inspect evidence across AOIs before accepting a setting. +- Added regression coverage for the persisted calibration UI contract. +- No backend API contracts, migrations, model downloads, provider fetching or AI/model execution behavior changed. diff --git a/backend/tests/test_sprint133_detection_threshold_calibration_ux.py b/backend/tests/test_sprint133_detection_threshold_calibration_ux.py new file mode 100644 index 00000000..9a1057ff --- /dev/null +++ b/backend/tests/test_sprint133_detection_threshold_calibration_ux.py @@ -0,0 +1,37 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_detection_lab_exposes_persisted_threshold_calibration_comparison() -> None: + detection_lab = ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx" + app = ROOT / "frontend" / "src" / "App.tsx" + todo = ROOT / "docs" / "TODO.md" + + source = detection_lab.read_text(encoding="utf-8") + app_source = app.read_text(encoding="utf-8") + todo_source = todo.read_text(encoding="utf-8") + + assert "qualityChecks: QualityCheckRead[]" in source + assert "buildCalibrationRows(detectionRuns, qualityChecks)" in source + assert "Calibration comparison" in source + assert "Compare persisted detection runs by confidence threshold" in source + assert "Best F1 candidate" in source + assert "Best precision candidate" in source + assert "Lowest false-positive pressure" in source + assert "Threshold" in source + assert "Precision" in source + assert "Recall" in source + assert "F1" in source + assert "False positives" in source + assert "False negatives" in source + assert "Promote only after checking evidence across AOIs" in source + assert "No calibration comparison available yet" in source + assert "metricValue(check, 'f1')" in source + assert "metricValue(check, 'precision')" in source + assert "metricValue(check, 'recall')" in source + assert "metricValue(check, 'false_positives')" in source + assert "confidenceThresholdForRun(run)" in source + assert "qualityChecks={qualityChecks}" in app_source + assert "[x] Add full threshold calibration comparison UX" in todo_source diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index bf5a3cc9..1a00f7f2 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5103,6 +5103,30 @@ Limitations: Next recommended pass: - Add threshold calibration comparison UX so an operator can compare candidate thresholds before promoting a local model. +## Sprint 133 Detection threshold calibration UX (2026-07-08) + +Changed: +- Added a Detection Lab calibration comparison panel that combines existing persisted `DetectionRunRead` rows with existing persisted `QualityCheckRead`/metric rows. +- The panel shows confidence threshold, model, local model asset id, detection count, precision, recall, F1, false positives, false negatives and linked quality-check id. +- Added summary cards for best F1 candidate, best precision candidate and lowest false-positive pressure. +- Added a promotion guardrail that keeps model/threshold acceptance tied to QA evidence across AOIs instead of a single run. +- Passed project-level `qualityChecks` into Detection Lab without adding API routes, migrations or new AI execution behavior. +- Added regression coverage in `backend/tests/test_sprint133_detection_threshold_calibration_ux.py`. +- Marked the threshold calibration UX item complete in `docs/TODO.md`. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint133_detection_threshold_calibration_ux.py -q` failed while the persisted calibration comparison UI was absent. +- `python -m pytest backend\tests\test_sprint133_detection_threshold_calibration_ux.py -q` (`1 passed`) +- `python -m pytest backend\tests\test_sprint122_model_asset_activation_guardrails.py backend\tests\test_sprint123_raster_detection_handoff_operational.py backend\tests\test_sprint133_detection_threshold_calibration_ux.py -q` (`7 passed`) +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` + +Limitations: +- This is a persisted-run comparison surface only. It does not launch batch calibration sweeps from the browser and does not auto-promote model assets or thresholds. + +Next recommended pass: +- Add a guided in-app calibration runner that can queue a small explicit threshold set for one selected raster/reference pair, reusing the existing detection and QA APIs. + ## Sprint 117 Safe local YOLO model activation (2026-07-06) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index a8cc2145..66b4ac90 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -419,6 +419,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add an operator-facing local model catalog/activation workflow with SHA256, active model status and explicit threshold guidance. - [x] Block silent local model asset auto-selection in Detection Lab. - [x] Add structured raster tile manifest handoff into Detection Lab with linked preflight visibility. -- [ ] Add full threshold calibration comparison UX so detection runs can compare candidate thresholds before promotion. +- [x] Add full threshold calibration comparison UX so detection runs can compare candidate thresholds before promotion. - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout. - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 052ab980..110fba85 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -950,6 +950,7 @@ function App(): JSX.Element { detectionRunResult={detectionRunResult} detectionRunError={detectionRunError} detectionRuns={detectionRuns} + qualityChecks={qualityChecks} selectedDetectionRunId={selectedDetectionRunId} detectionItems={detectionItems} detectionClassFilter={detectionClassFilter} diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx index df78b37d..f1a6a6fd 100644 --- a/frontend/src/components/detection/DetectionLab.tsx +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -6,9 +6,26 @@ import type { DetectionRunRead, DetectionRunResponse, ModelAssetRead, + QualityCheckRead, YoloPreflightResponse, } from '../../types' +interface CalibrationRow { + analysisRunId: string + qualityCheckId: string + threshold: number | null + modelName: string + modelAssetId: string | null + detectionCount: number | null + precision: number | null + recall: number | null + f1: number | null + falsePositives: number | null + falseNegatives: number | null + score: number | null + createdAt: string | null +} + interface DetectionLabProps { detectionModels: DetectionModelCapability[] modelAssets: ModelAssetRead[] @@ -24,6 +41,7 @@ interface DetectionLabProps { detectionRunResult: DetectionRunResponse | null detectionRunError: string | null detectionRuns: DetectionRunRead[] + qualityChecks: QualityCheckRead[] selectedDetectionRunId: string detectionItems: DetectionRead[] detectionClassFilter: string @@ -71,6 +89,7 @@ export function DetectionLab({ detectionRunResult, detectionRunError, detectionRuns, + qualityChecks, selectedDetectionRunId, detectionItems, detectionClassFilter, @@ -114,6 +133,10 @@ export function DetectionLab({ const benchmarkCandidateAsset = modelAssets.find( (asset) => asset.model_asset_id === 'geointel-building-yolov8s-hardneg160r4e50-pt', ) + const calibrationRows = buildCalibrationRows(detectionRuns, qualityChecks) + const bestF1Candidate = bestCalibrationRow(calibrationRows, 'f1') + const bestPrecisionCandidate = bestCalibrationRow(calibrationRows, 'precision') + const lowestFalsePositivePressureCandidate = bestLowestCalibrationRow(calibrationRows, 'falsePositives') const detectionHasTileManifest = !detectionRequiresTileManifest || detectionTileManifestPath.trim().length > 0 const detectionRunReady = @@ -562,6 +585,69 @@ export function DetectionLab({ ) : null} +
Compare persisted detection runs by confidence threshold before promoting a model setting.
+| Threshold | +Model | +Detections | +Precision | +Recall | +F1 | +False positives | +False negatives | +Quality check | +
|---|---|---|---|---|---|---|---|---|
| {formatNullableNumber(row.threshold, 2)} | ++ {row.modelName} + {row.modelAssetId ?? 'runtime configured path'} + | +{row.detectionCount ?? 'n/a'} | +{formatNullableNumber(row.precision, 3)} | +{formatNullableNumber(row.recall, 3)} | +{formatNullableNumber(row.f1, 3)} | +{row.falsePositives ?? 'n/a'} | +{row.falseNegatives ?? 'n/a'} | +{row.qualityCheckId} | +
Run configured YOLO at multiple confidence thresholds, then compare each persisted detection run against the same reference dataset.
+