calibrate a confidence threshold from one inference pass

Threshold calibration ran the model over every tile once per threshold — three
GPU passes to compare 0.50, 0.25 and 0.15 on a hundred-tile raster. The answer
is already in a single run at the lowest value: detections above a higher cut
are a subset of it, and duplicate suppression walks candidates in descending
confidence, so a lower-confidence box can never displace a higher-confidence
one. The kept set above any cut is identical whichever threshold the run used,
which is what makes one pass sufficient rather than merely cheaper.

QA now takes calibration_thresholds and reads each operating point off the same
precision/recall walk it already performs, marking the F1-optimal cut. The lab
runs inference once and fills its table from the sweep.

The contract test asserted the per-threshold loop by name, pinning the waste it
was meant to describe. It now states what calibration owes an operator: a row
per requested threshold, from one run.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 19:37:19 +02:00
co-authored by Claude Opus 5
parent 8a26007281
commit 2cd2c49389
11 changed files with 303 additions and 52 deletions
@@ -648,8 +648,11 @@ export function DetectionLab({
</thead>
<tbody>
{detectionCalibrationRows.map((row) => (
<tr key={row.threshold}>
<td>{row.threshold.toFixed(2)}</td>
<tr key={row.threshold} className={row.best_f1 ? 'calibration-row-best' : undefined}>
<td>
{row.threshold.toFixed(2)}
{row.best_f1 ? <span className="status-badge"> beste F1</span> : null}
</td>
<td>{row.status}</td>
<td>{row.detection_count ?? 'n.v.t.'}</td>
<td>{formatNullableNumber(row.precision ?? null, 3)}</td>
+61 -48
View File
@@ -50,6 +50,8 @@ export interface DetectionCalibrationRunRow {
f1_score?: number | null
false_positives?: number | null
false_negatives?: number | null
/** The F1-optimal cut among the requested thresholds. */
best_f1?: boolean
message?: string | null
}
@@ -481,57 +483,68 @@ export function useDetectionWorkflow({
setDetectionCalibrationRows(thresholds.map((threshold) => ({ threshold, status: 'queued' })))
setRunningDetectionCalibration(true)
try {
for (const threshold of thresholds) {
setDetectionCalibrationRows((rows) =>
rows.map((row) => row.threshold === threshold ? { ...row, status: 'running', message: 'Running detection' } : row),
)
try {
const result = await detectionApi.run({
project_id: selectedProjectId,
dataset_id: datasetId,
model_id: selectedDetectionModelId,
model_asset_id: selectedModelAssetId || null,
confidence_threshold: threshold,
tile_manifest_path: detectionTileManifestPath.trim() || null,
parameters_json: { calibration: true, calibration_thresholds: thresholds },
})
setSelectedDetectionRunId(result.analysis_run_id)
const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, {
reference_dataset_id: detectionReferenceDatasetId,
iou_threshold: qaIouThreshold,
class_name: detectionClassFilter || null,
min_confidence: null,
})
setDetectionCalibrationRows((rows) =>
rows.map((row) => row.threshold === threshold
? {
...row,
status: 'success',
analysis_run_id: result.analysis_run_id,
job_id: result.job_id,
quality_check_id: qa.quality_check_id,
detection_count: result.detection_count,
precision: qa.precision ?? null,
recall: qa.recall ?? null,
f1_score: qa.f1_score ?? null,
false_positives: qa.false_positives,
false_negatives: qa.false_negatives,
message: result.message,
}
: row),
)
} catch (error) {
const message = formatError(error, `Calibration threshold ${threshold} failed`)
setDetectionCalibrationRows((rows) =>
rows.map((row) => row.threshold === threshold ? { ...row, status: 'failed', message } : row),
)
setDetectionCalibrationError(message)
break
}
}
// One inference pass answers every threshold. Detections above a higher
// cut are a subset of a lower-cut run, and suppression walks candidates
// in descending confidence, so the kept set above a cut does not depend
// on the threshold the run used. Running the model per threshold spent N
// GPU passes to reproduce identical numbers.
const lowestThreshold = Math.min(...thresholds)
setDetectionCalibrationRows((rows) =>
rows.map((row) => ({ ...row, status: 'running', message: 'Eén inferentie voor alle drempels' })),
)
const result = await detectionApi.run({
project_id: selectedProjectId,
dataset_id: datasetId,
model_id: selectedDetectionModelId,
model_asset_id: selectedModelAssetId || null,
confidence_threshold: lowestThreshold,
tile_manifest_path: detectionTileManifestPath.trim() || null,
parameters_json: { calibration: true, calibration_thresholds: thresholds },
})
setSelectedDetectionRunId(result.analysis_run_id)
const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, {
reference_dataset_id: detectionReferenceDatasetId,
iou_threshold: qaIouThreshold,
class_name: detectionClassFilter || null,
min_confidence: null,
calibration_thresholds: thresholds,
})
const sweep = new Map((qa.calibration_sweep ?? []).map((point) => [point.min_confidence, point]))
setDetectionCalibrationRows((rows) =>
rows.map((row) => {
const point = sweep.get(row.threshold)
if (!point) {
return { ...row, status: 'failed', message: 'Geen meetpunt voor deze drempel' }
}
return {
...row,
status: 'success',
analysis_run_id: result.analysis_run_id,
job_id: result.job_id,
quality_check_id: qa.quality_check_id,
detection_count: point.candidate_count,
precision: point.precision,
recall: point.recall,
f1_score: point.f1_score,
false_positives: point.false_positives,
false_negatives: point.false_negatives,
best_f1: point.best_f1_in_sweep,
message: point.best_f1_in_sweep ? 'Beste F1 in deze reeks' : null,
}
}),
)
await loadDetectionRuns(selectedProjectId)
await loadQualityChecks(selectedProjectId)
await loadProjectData(selectedProjectId)
} catch (error) {
const message = formatError(error, 'Calibration failed')
setDetectionCalibrationRows((rows) =>
rows.map((row) => (row.status === 'success' ? row : { ...row, status: 'failed', message })),
)
setDetectionCalibrationError(message)
} finally {
setRunningDetectionCalibration(false)
}
+20
View File
@@ -1431,6 +1431,25 @@ export interface DetectionQaRequest {
iou_threshold: number
class_name?: string | null
min_confidence?: number | null
/**
* Confidence cuts to report next to the run's own operating point. They are
* read off one matching pass, so a sweep costs no extra inference.
*/
calibration_thresholds?: number[]
}
/** One confidence cut, derived from a single run rather than a run of its own. */
export interface DetectionCalibrationPoint {
min_confidence: number
confidence_threshold: number | null
candidate_count: number
true_positives: number
false_positives: number
false_negatives: number
precision: number | null
recall: number | null
f1_score: number | null
best_f1_in_sweep: boolean
}
export interface DetectionQaResult {
@@ -1486,6 +1505,7 @@ export interface DetectionQaResult {
envelope_precision_recall_curve?: PrecisionRecallCurve
}
precision_recall_curve?: PrecisionRecallCurve
calibration_sweep?: DetectionCalibrationPoint[]
}
/**