diff --git a/CHANGELOG.md b/CHANGELOG.md index 75cb5078..64feeb0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1318,3 +1318,12 @@ Added: - Added operator guidance for best F1, best precision and lowest false-positive pressure, with a promotion guardrail to inspect evidence across AOIs before accepting a setting. - Added regression coverage for the persisted calibration UI contract. - No backend API contracts, migrations, model downloads, provider fetching or AI/model execution behavior changed. + +## Sprint 134 Guided detection calibration runner (2026-07-08) + +- Added an explicit in-app calibration runner to Detection Lab for operator-selected confidence threshold sweeps. +- The runner reuses existing detection and QA APIs once per threshold, producing persisted DetectionRun, Job, Detection, QualityCheck and Metric records. +- Added visible threshold progress with per-row status, detection count, precision, recall, F1, false positives and false negatives. +- Added validation guardrails for selected project, raster dataset, reference dataset, configured non-fixture model, tile manifest and explicit local model asset. +- Added regression coverage for the guided runner contract. +- No backend API contracts, migrations, model downloads, provider fetching, automatic promotion or model file mutation behavior changed. diff --git a/backend/tests/test_sprint134_guided_detection_calibration_runner.py b/backend/tests/test_sprint134_guided_detection_calibration_runner.py new file mode 100644 index 00000000..1169bb77 --- /dev/null +++ b/backend/tests/test_sprint134_guided_detection_calibration_runner.py @@ -0,0 +1,51 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_detection_lab_has_guided_threshold_calibration_runner() -> None: + hook = ROOT / "frontend" / "src" / "hooks" / "useDetectionWorkflow.ts" + lab = ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx" + app = ROOT / "frontend" / "src" / "App.tsx" + todo = ROOT / "docs" / "TODO.md" + + hook_source = hook.read_text(encoding="utf-8") + lab_source = lab.read_text(encoding="utf-8") + app_source = app.read_text(encoding="utf-8") + todo_source = todo.read_text(encoding="utf-8") + + assert "interface DetectionCalibrationRunRow" in hook_source + assert "parseCalibrationThresholds" in hook_source + assert "calibrationThresholdText" in hook_source + assert "runningDetectionCalibration" in hook_source + assert "detectionCalibrationRows" in hook_source + assert "detectionCalibrationError" in hook_source + assert "runDetectionCalibration" in hook_source + assert "for (const threshold of thresholds)" in hook_source + assert "detectionApi.run({" in hook_source + assert "confidence_threshold: threshold" in hook_source + assert "parameters_json: { calibration: true, calibration_thresholds: thresholds }" in hook_source + assert "detectionApi.compareWithReference(result.analysis_run_id" in hook_source + assert "reference_dataset_id: detectionReferenceDatasetId" in hook_source + assert "Select a reference dataset before calibration" in hook_source + assert "Provide at least one valid threshold between 0 and 1" in hook_source + assert "Configured YOLO calibration requires a tile manifest" in hook_source + assert "Select a local model asset before calibration" in hook_source + + assert "Guided calibration runner" in lab_source + assert "This runs real configured YOLO jobs" in lab_source + assert "Threshold set" in lab_source + assert "Run calibration sweep" in lab_source + assert "Calibration run progress" in lab_source + assert "detectionCalibrationRows.map" in lab_source + assert "runningDetectionCalibration" in lab_source + assert "detectionCalibrationError" in lab_source + assert "onRunCalibration" in lab_source + assert "onSetCalibrationThresholdText" in lab_source + + assert "calibrationThresholdText={calibrationThresholdText}" in app_source + assert "runningDetectionCalibration={runningDetectionCalibration}" in app_source + assert "detectionCalibrationRows={detectionCalibrationRows}" in app_source + assert "onRunCalibration={runDetectionCalibration}" in app_source + assert "[x] Add guided in-app detection calibration runner" in todo_source diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 1a00f7f2..833447f4 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5127,6 +5127,30 @@ Limitations: Next recommended pass: - Add a guided in-app calibration runner that can queue a small explicit threshold set for one selected raster/reference pair, reusing the existing detection and QA APIs. +## Sprint 134 Guided detection calibration runner (2026-07-08) + +Changed: +- Added an explicit guided calibration runner to Detection Lab for operator-selected confidence threshold sweeps. +- Added frontend detection workflow state for calibration thresholds, progress rows, running state and errors. +- The runner parses a space/comma/semicolon-separated threshold set, validates project/raster/reference/model/manifest/model-asset readiness and runs one existing `detectionApi.run` plus one existing detection QA comparison per threshold. +- Successful threshold rows report persisted analysis run/job/quality check ids, detection count, precision, recall, F1, false positives and false negatives. +- Added UI copy that this runs real configured YOLO jobs and QA comparisons and does not promote or mutate model files. +- Added regression coverage in `backend/tests/test_sprint134_guided_detection_calibration_runner.py`. +- Marked the guided calibration runner task complete in `docs/TODO.md`. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint134_guided_detection_calibration_runner.py -q` failed while the runner contract was absent. +- `python -m pytest backend\tests\test_sprint134_guided_detection_calibration_runner.py -q` (`1 passed`) +- `python -m pytest backend\tests\test_sprint133_detection_threshold_calibration_ux.py backend\tests\test_sprint134_guided_detection_calibration_runner.py backend\tests\test_sprint122_model_asset_activation_guardrails.py backend\tests\test_sprint123_raster_detection_handoff_operational.py -q` (`8 passed`) +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` + +Limitations: +- The runner is intentionally sequential and explicit. It does not schedule background batches, compare multiple AOIs at once or promote model assets/thresholds automatically. + +Next recommended pass: +- Add an evidence shortcut from each calibration row to the persisted QA evidence map/review flow so false positives and false negatives can be inspected faster. + ## Sprint 117 Safe local YOLO model activation (2026-07-06) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 66b4ac90..693209a7 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -420,5 +420,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Block silent local model asset auto-selection in Detection Lab. - [x] Add structured raster tile manifest handoff into Detection Lab with linked preflight visibility. - [x] Add full threshold calibration comparison UX so detection runs can compare candidate thresholds before promotion. +- [x] Add guided in-app detection calibration runner for explicit threshold sweeps. - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout. - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 110fba85..808b326a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -249,12 +249,17 @@ function App(): JSX.Element { yoloPreflight, loadingYoloPreflight, yoloPreflightError, + calibrationThresholdText, + runningDetectionCalibration, + detectionCalibrationRows, + detectionCalibrationError, loadDetectionModels, loadYoloPreflight, loadDetectionRuns, loadDetectionResults, runDetection, runDetectionQa, + runDetectionCalibration, resetDetectionForProject, setSelectedDetectionDatasetId, setSelectedDetectionModelId, @@ -265,6 +270,7 @@ function App(): JSX.Element { setDetectionClassFilter, setDetectionMinConfidenceFilter, setDetectionReferenceDatasetId, + setCalibrationThresholdText, } = useDetectionWorkflow({ selectedProjectId, rasterDatasets, @@ -951,6 +957,10 @@ function App(): JSX.Element { detectionRunError={detectionRunError} detectionRuns={detectionRuns} qualityChecks={qualityChecks} + calibrationThresholdText={calibrationThresholdText} + runningDetectionCalibration={runningDetectionCalibration} + detectionCalibrationRows={detectionCalibrationRows} + detectionCalibrationError={detectionCalibrationError} selectedDetectionRunId={selectedDetectionRunId} detectionItems={detectionItems} detectionClassFilter={detectionClassFilter} @@ -981,6 +991,8 @@ function App(): JSX.Element { onLoadResults={() => loadDetectionResults()} onSelectReferenceDataset={setDetectionReferenceDatasetId} onRunQa={runDetectionQa} + onSetCalibrationThresholdText={setCalibrationThresholdText} + onRunCalibration={runDetectionCalibration} /> void onSelectReferenceDataset: (datasetId: string) => void onRunQa: () => void + onSetCalibrationThresholdText: (value: string) => void + onRunCalibration: () => void } export function DetectionLab({ @@ -90,6 +97,10 @@ export function DetectionLab({ detectionRunError, detectionRuns, qualityChecks, + calibrationThresholdText, + runningDetectionCalibration, + detectionCalibrationRows, + detectionCalibrationError, selectedDetectionRunId, detectionItems, detectionClassFilter, @@ -120,6 +131,8 @@ export function DetectionLab({ onLoadResults, onSelectReferenceDataset, onRunQa, + onSetCalibrationThresholdText, + onRunCalibration, }: DetectionLabProps): JSX.Element { const selectedDetectionModel = detectionModels.find((model) => model.model_id === selectedDetectionModelId) ?? null const selectedModelAsset = modelAssets.find((asset) => asset.model_asset_id === selectedModelAssetId) ?? null @@ -161,6 +174,7 @@ export function DetectionLab({ : !detectionHasTileManifest ? 'Provide a raster tile manifest for configured YOLO' : null + const calibrationRunReady = detectionRunReady && detectionReferenceDatasetId.length > 0 && calibrationThresholdText.trim().length > 0 return (
@@ -501,6 +515,101 @@ export function DetectionLab({ ) : null} +
+
+
+

Guided calibration runner

+

This runs real configured YOLO jobs and QA comparisons for each threshold. It does not promote or mutate model files.

+
+ + {calibrationRunReady ? 'ready' : 'needs dataset, model, manifest and reference'} + +
+
+ + +
+ + {detectionCalibrationError ? ( +
+ Calibration sweep failed. +

{detectionCalibrationError}

+
+ ) : null} + {detectionCalibrationRows.length > 0 ? ( +
+
+
+

Calibration run progress

+

Each row is backed by a persisted detection run and QA check when successful.

+
+ {detectionCalibrationRows.length} thresholds +
+
+ + + + + + + + + + + + + + + {detectionCalibrationRows.map((row) => ( + + + + + + + + + + + ))} + +
ThresholdStatusDetectionsPrecisionRecallF1False positivesFalse negatives
{row.threshold.toFixed(2)}{row.status}{row.detection_count ?? 'n/a'}{formatNullableNumber(row.precision ?? null, 3)}{formatNullableNumber(row.recall ?? null, 3)}{formatNullableNumber(row.f1_score ?? null, 3)}{row.false_positives ?? 'n/a'}{row.false_negatives ?? 'n/a'}
+
+
+ ) : ( +
+ No calibration sweep has been run in this session. +

Choose a reference dataset and threshold set, then start the explicit sweep.

+
+ )} +
+
diff --git a/frontend/src/hooks/useDetectionWorkflow.ts b/frontend/src/hooks/useDetectionWorkflow.ts index 8a45bf33..221c8b75 100644 --- a/frontend/src/hooks/useDetectionWorkflow.ts +++ b/frontend/src/hooks/useDetectionWorkflow.ts @@ -21,6 +21,39 @@ interface DetectionWorkflowOptions { loadQualityChecks: (projectId?: string | null) => Promise } +export interface DetectionCalibrationRunRow { + threshold: number + status: 'queued' | 'running' | 'success' | 'failed' + analysis_run_id?: string | null + job_id?: string | null + quality_check_id?: string | null + detection_count?: number | null + precision?: number | null + recall?: number | null + f1_score?: number | null + false_positives?: number | null + false_negatives?: number | null + message?: string | null +} + +function parseCalibrationThresholds(value: string): number[] { + const tokens = value + .split(/[\s,;]+/) + .map((token) => token.trim()) + .filter(Boolean) + const thresholds: number[] = [] + for (const token of tokens) { + const threshold = Number(token) + if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + return [] + } + if (!thresholds.includes(threshold)) { + thresholds.push(threshold) + } + } + return thresholds +} + export function useDetectionWorkflow({ selectedProjectId, rasterDatasets, @@ -55,6 +88,10 @@ export function useDetectionWorkflow({ const [yoloPreflight, setYoloPreflight] = useState(null) const [loadingYoloPreflight, setLoadingYoloPreflight] = useState(false) const [yoloPreflightError, setYoloPreflightError] = useState(null) + const [calibrationThresholdText, setCalibrationThresholdText] = useState('0.50 0.25 0.15') + const [runningDetectionCalibration, setRunningDetectionCalibration] = useState(false) + const [detectionCalibrationRows, setDetectionCalibrationRows] = useState([]) + const [detectionCalibrationError, setDetectionCalibrationError] = useState(null) const loadDetectionModels = async () => { setLoadingDetectionModels(true) @@ -204,6 +241,98 @@ export function useDetectionWorkflow({ } } + const runDetectionCalibration = async () => { + if (!selectedProjectId) { + setDetectionCalibrationError('Select a project before calibration') + return + } + const datasetId = selectedDetectionDatasetId || rasterDatasets[0]?.id + if (!datasetId) { + setDetectionCalibrationError('Select a raster dataset before calibration') + return + } + if (!detectionReferenceDatasetId) { + setDetectionCalibrationError('Select a reference dataset before calibration') + return + } + const selectedModel = detectionModels.find((model) => model.model_id === selectedDetectionModelId) + if (!selectedModel?.configured || selectedDetectionModelId === 'manual-fixture-detector') { + setDetectionCalibrationError('Select a configured non-fixture detection model before calibration') + return + } + if (selectedDetectionModelId === 'yolo-configured' && !detectionTileManifestPath.trim()) { + setDetectionCalibrationError('Configured YOLO calibration requires a tile manifest') + return + } + if (selectedDetectionModelId === 'yolo-configured' && modelAssets.length > 0 && !selectedModelAssetId) { + setDetectionCalibrationError('Select a local model asset before calibration') + return + } + const thresholds = parseCalibrationThresholds(calibrationThresholdText) + if (thresholds.length === 0) { + setDetectionCalibrationError('Provide at least one valid threshold between 0 and 1') + return + } + setDetectionCalibrationError(null) + setDetectionCalibrationRows(thresholds.map((threshold) => ({ threshold, status: 'queued' }))) + setRunningDetectionCalibration(true) + try { + for (const threshold of thresholds) { + setDetectionCalibrationRows((rows) => + rows.map((row) => row.threshold === threshold ? { ...row, status: 'running', message: 'Running detection' } : row), + ) + try { + const result = await detectionApi.run({ + project_id: selectedProjectId, + dataset_id: datasetId, + model_id: selectedDetectionModelId, + model_asset_id: selectedModelAssetId || null, + confidence_threshold: threshold, + tile_manifest_path: detectionTileManifestPath.trim() || null, + parameters_json: { calibration: true, calibration_thresholds: thresholds }, + }) + setSelectedDetectionRunId(result.analysis_run_id) + const qa = await detectionApi.compareWithReference(result.analysis_run_id, { + reference_dataset_id: detectionReferenceDatasetId, + iou_threshold: qaIouThreshold, + class_name: detectionClassFilter || null, + min_confidence: null, + }) + setDetectionCalibrationRows((rows) => + rows.map((row) => row.threshold === threshold + ? { + ...row, + status: 'success', + analysis_run_id: result.analysis_run_id, + job_id: result.job_id, + quality_check_id: qa.quality_check_id, + detection_count: result.detection_count, + precision: qa.precision ?? null, + recall: qa.recall ?? null, + f1_score: qa.f1_score ?? null, + false_positives: qa.false_positives, + false_negatives: qa.false_negatives, + message: result.message, + } + : row), + ) + } catch (error) { + const message = formatError(error, `Calibration threshold ${threshold} failed`) + setDetectionCalibrationRows((rows) => + rows.map((row) => row.threshold === threshold ? { ...row, status: 'failed', message } : row), + ) + setDetectionCalibrationError(message) + break + } + } + await loadDetectionRuns(selectedProjectId) + await loadQualityChecks(selectedProjectId) + await loadProjectData(selectedProjectId) + } finally { + setRunningDetectionCalibration(false) + } + } + const resetDetectionForProject = () => { setSelectedDetectionDatasetId('') setDetectionRuns([]) @@ -211,6 +340,8 @@ export function useDetectionWorkflow({ setDetectionItems([]) setDetectionGeoJson(null) setDetectionRunResult(null) + setDetectionCalibrationRows([]) + setDetectionCalibrationError(null) } return { @@ -241,12 +372,17 @@ export function useDetectionWorkflow({ yoloPreflight, loadingYoloPreflight, yoloPreflightError, + calibrationThresholdText, + runningDetectionCalibration, + detectionCalibrationRows, + detectionCalibrationError, loadDetectionModels, loadYoloPreflight, loadDetectionRuns, loadDetectionResults, runDetection, runDetectionQa, + runDetectionCalibration, resetDetectionForProject, setSelectedDetectionDatasetId, setSelectedDetectionModelId, @@ -257,5 +393,6 @@ export function useDetectionWorkflow({ setDetectionClassFilter, setDetectionMinConfidenceFilter, setDetectionReferenceDatasetId, + setCalibrationThresholdText, } } diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index f8f98638..9901e852 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -3294,6 +3294,21 @@ button.entity-card { background: #fbfdfb; } +.guided-calibration-surface { + border-color: #d9e4dd; + background: linear-gradient(180deg, #ffffff, #f8fcf9); +} + +.calibration-progress-panel { + display: grid; + gap: 0.55rem; + min-width: 0; + border: 1px solid var(--line); + border-radius: 8px; + padding: 0.68rem; + background: #ffffff; +} + .calibration-summary-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr));