calibrate a confidence threshold from one inference pass
Threshold calibration ran the model over every tile once per threshold — three GPU passes to compare 0.50, 0.25 and 0.15 on a hundred-tile raster. The answer is already in a single run at the lowest value: detections above a higher cut are a subset of it, and duplicate suppression walks candidates in descending confidence, so a lower-confidence box can never displace a higher-confidence one. The kept set above any cut is identical whichever threshold the run used, which is what makes one pass sufficient rather than merely cheaper. QA now takes calibration_thresholds and reads each operating point off the same precision/recall walk it already performs, marking the F1-optimal cut. The lab runs inference once and fills its table from the sweep. The contract test asserted the per-threshold loop by name, pinning the waste it was meant to describe. It now states what calibration owes an operator: a row per requested threshold, from one run. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -648,8 +648,11 @@ export function DetectionLab({
|
||||
</thead>
|
||||
<tbody>
|
||||
{detectionCalibrationRows.map((row) => (
|
||||
<tr key={row.threshold}>
|
||||
<td>{row.threshold.toFixed(2)}</td>
|
||||
<tr key={row.threshold} className={row.best_f1 ? 'calibration-row-best' : undefined}>
|
||||
<td>
|
||||
{row.threshold.toFixed(2)}
|
||||
{row.best_f1 ? <span className="status-badge"> beste F1</span> : null}
|
||||
</td>
|
||||
<td>{row.status}</td>
|
||||
<td>{row.detection_count ?? 'n.v.t.'}</td>
|
||||
<td>{formatNullableNumber(row.precision ?? null, 3)}</td>
|
||||
|
||||
@@ -50,6 +50,8 @@ export interface DetectionCalibrationRunRow {
|
||||
f1_score?: number | null
|
||||
false_positives?: number | null
|
||||
false_negatives?: number | null
|
||||
/** The F1-optimal cut among the requested thresholds. */
|
||||
best_f1?: boolean
|
||||
message?: string | null
|
||||
}
|
||||
|
||||
@@ -481,57 +483,68 @@ export function useDetectionWorkflow({
|
||||
setDetectionCalibrationRows(thresholds.map((threshold) => ({ threshold, status: 'queued' })))
|
||||
setRunningDetectionCalibration(true)
|
||||
try {
|
||||
for (const threshold of thresholds) {
|
||||
setDetectionCalibrationRows((rows) =>
|
||||
rows.map((row) => row.threshold === threshold ? { ...row, status: 'running', message: 'Running detection' } : row),
|
||||
)
|
||||
try {
|
||||
const result = await detectionApi.run({
|
||||
project_id: selectedProjectId,
|
||||
dataset_id: datasetId,
|
||||
model_id: selectedDetectionModelId,
|
||||
model_asset_id: selectedModelAssetId || null,
|
||||
confidence_threshold: threshold,
|
||||
tile_manifest_path: detectionTileManifestPath.trim() || null,
|
||||
parameters_json: { calibration: true, calibration_thresholds: thresholds },
|
||||
})
|
||||
setSelectedDetectionRunId(result.analysis_run_id)
|
||||
const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, {
|
||||
reference_dataset_id: detectionReferenceDatasetId,
|
||||
iou_threshold: qaIouThreshold,
|
||||
class_name: detectionClassFilter || null,
|
||||
min_confidence: null,
|
||||
})
|
||||
setDetectionCalibrationRows((rows) =>
|
||||
rows.map((row) => row.threshold === threshold
|
||||
? {
|
||||
...row,
|
||||
status: 'success',
|
||||
analysis_run_id: result.analysis_run_id,
|
||||
job_id: result.job_id,
|
||||
quality_check_id: qa.quality_check_id,
|
||||
detection_count: result.detection_count,
|
||||
precision: qa.precision ?? null,
|
||||
recall: qa.recall ?? null,
|
||||
f1_score: qa.f1_score ?? null,
|
||||
false_positives: qa.false_positives,
|
||||
false_negatives: qa.false_negatives,
|
||||
message: result.message,
|
||||
}
|
||||
: row),
|
||||
)
|
||||
} catch (error) {
|
||||
const message = formatError(error, `Calibration threshold ${threshold} failed`)
|
||||
setDetectionCalibrationRows((rows) =>
|
||||
rows.map((row) => row.threshold === threshold ? { ...row, status: 'failed', message } : row),
|
||||
)
|
||||
setDetectionCalibrationError(message)
|
||||
break
|
||||
}
|
||||
}
|
||||
// One inference pass answers every threshold. Detections above a higher
|
||||
// cut are a subset of a lower-cut run, and suppression walks candidates
|
||||
// in descending confidence, so the kept set above a cut does not depend
|
||||
// on the threshold the run used. Running the model per threshold spent N
|
||||
// GPU passes to reproduce identical numbers.
|
||||
const lowestThreshold = Math.min(...thresholds)
|
||||
setDetectionCalibrationRows((rows) =>
|
||||
rows.map((row) => ({ ...row, status: 'running', message: 'Eén inferentie voor alle drempels' })),
|
||||
)
|
||||
const result = await detectionApi.run({
|
||||
project_id: selectedProjectId,
|
||||
dataset_id: datasetId,
|
||||
model_id: selectedDetectionModelId,
|
||||
model_asset_id: selectedModelAssetId || null,
|
||||
confidence_threshold: lowestThreshold,
|
||||
tile_manifest_path: detectionTileManifestPath.trim() || null,
|
||||
parameters_json: { calibration: true, calibration_thresholds: thresholds },
|
||||
})
|
||||
setSelectedDetectionRunId(result.analysis_run_id)
|
||||
|
||||
const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, {
|
||||
reference_dataset_id: detectionReferenceDatasetId,
|
||||
iou_threshold: qaIouThreshold,
|
||||
class_name: detectionClassFilter || null,
|
||||
min_confidence: null,
|
||||
calibration_thresholds: thresholds,
|
||||
})
|
||||
|
||||
const sweep = new Map((qa.calibration_sweep ?? []).map((point) => [point.min_confidence, point]))
|
||||
setDetectionCalibrationRows((rows) =>
|
||||
rows.map((row) => {
|
||||
const point = sweep.get(row.threshold)
|
||||
if (!point) {
|
||||
return { ...row, status: 'failed', message: 'Geen meetpunt voor deze drempel' }
|
||||
}
|
||||
return {
|
||||
...row,
|
||||
status: 'success',
|
||||
analysis_run_id: result.analysis_run_id,
|
||||
job_id: result.job_id,
|
||||
quality_check_id: qa.quality_check_id,
|
||||
detection_count: point.candidate_count,
|
||||
precision: point.precision,
|
||||
recall: point.recall,
|
||||
f1_score: point.f1_score,
|
||||
false_positives: point.false_positives,
|
||||
false_negatives: point.false_negatives,
|
||||
best_f1: point.best_f1_in_sweep,
|
||||
message: point.best_f1_in_sweep ? 'Beste F1 in deze reeks' : null,
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
await loadDetectionRuns(selectedProjectId)
|
||||
await loadQualityChecks(selectedProjectId)
|
||||
await loadProjectData(selectedProjectId)
|
||||
} catch (error) {
|
||||
const message = formatError(error, 'Calibration failed')
|
||||
setDetectionCalibrationRows((rows) =>
|
||||
rows.map((row) => (row.status === 'success' ? row : { ...row, status: 'failed', message })),
|
||||
)
|
||||
setDetectionCalibrationError(message)
|
||||
} finally {
|
||||
setRunningDetectionCalibration(false)
|
||||
}
|
||||
|
||||
@@ -1431,6 +1431,25 @@ export interface DetectionQaRequest {
|
||||
iou_threshold: number
|
||||
class_name?: string | null
|
||||
min_confidence?: number | null
|
||||
/**
|
||||
* Confidence cuts to report next to the run's own operating point. They are
|
||||
* read off one matching pass, so a sweep costs no extra inference.
|
||||
*/
|
||||
calibration_thresholds?: number[]
|
||||
}
|
||||
|
||||
/** One confidence cut, derived from a single run rather than a run of its own. */
|
||||
export interface DetectionCalibrationPoint {
|
||||
min_confidence: number
|
||||
confidence_threshold: number | null
|
||||
candidate_count: number
|
||||
true_positives: number
|
||||
false_positives: number
|
||||
false_negatives: number
|
||||
precision: number | null
|
||||
recall: number | null
|
||||
f1_score: number | null
|
||||
best_f1_in_sweep: boolean
|
||||
}
|
||||
|
||||
export interface DetectionQaResult {
|
||||
@@ -1486,6 +1505,7 @@ export interface DetectionQaResult {
|
||||
envelope_precision_recall_curve?: PrecisionRecallCurve
|
||||
}
|
||||
precision_recall_curve?: PrecisionRecallCurve
|
||||
calibration_sweep?: DetectionCalibrationPoint[]
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user