Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

calibrate a confidence threshold from one inference pass

Browse Source 
Threshold calibration ran the model over every tile once per threshold — three
GPU passes to compare 0.50, 0.25 and 0.15 on a hundred-tile raster. The answer
is already in a single run at the lowest value: detections above a higher cut
are a subset of it, and duplicate suppression walks candidates in descending
confidence, so a lower-confidence box can never displace a higher-confidence
one. The kept set above any cut is identical whichever threshold the run used,
which is what makes one pass sufficient rather than merely cheaper.

QA now takes calibration_thresholds and reads each operating point off the same
precision/recall walk it already performs, marking the F1-optimal cut. The lab
runs inference once and fills its table from the sweep.

The contract test asserted the per-threshold loop by name, pinning the waste it
was meant to describe. It now states what calibration owes an operator: a row
per requested threshold, from one run.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 19:37:19 +02:00
co-authored by Claude Opus 5
parent 8a26007281
commit 2cd2c49389
11 changed files with 303 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/hooks/useDetectionWorkflow.ts
+61 -48
View File
@@ -50,6 +50,8 @@ export interface DetectionCalibrationRunRow {
f1_score?: number | null
false_positives?: number | null
false_negatives?: number | null
/** The F1-optimal cut among the requested thresholds. */
best_f1?: boolean
message?: string | null
}
@@ -481,57 +483,68 @@ export function useDetectionWorkflow({
setDetectionCalibrationRows(thresholds.map((threshold) => ({ threshold, status: 'queued' })))
setRunningDetectionCalibration(true)
try {
for (const threshold of thresholds) {
setDetectionCalibrationRows((rows) =>
rows.map((row) => row.threshold === threshold ? { ...row, status: 'running', message: 'Running detection' } : row),
)
try {
const result = await detectionApi.run({
project_id: selectedProjectId,
dataset_id: datasetId,
model_id: selectedDetectionModelId,
model_asset_id: selectedModelAssetId || null,
confidence_threshold: threshold,
tile_manifest_path: detectionTileManifestPath.trim() || null,
parameters_json: { calibration: true, calibration_thresholds: thresholds },
})
setSelectedDetectionRunId(result.analysis_run_id)
const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, {
reference_dataset_id: detectionReferenceDatasetId,
iou_threshold: qaIouThreshold,
class_name: detectionClassFilter || null,
min_confidence: null,
})
setDetectionCalibrationRows((rows) =>
rows.map((row) => row.threshold === threshold
? {
...row,
status: 'success',
analysis_run_id: result.analysis_run_id,
job_id: result.job_id,
quality_check_id: qa.quality_check_id,
detection_count: result.detection_count,
precision: qa.precision ?? null,
recall: qa.recall ?? null,
f1_score: qa.f1_score ?? null,
false_positives: qa.false_positives,
false_negatives: qa.false_negatives,
message: result.message,
}
: row),
)
} catch (error) {
const message = formatError(error, `Calibration threshold ${threshold} failed`)
setDetectionCalibrationRows((rows) =>
rows.map((row) => row.threshold === threshold ? { ...row, status: 'failed', message } : row),
)
setDetectionCalibrationError(message)
break
}
}
// One inference pass answers every threshold. Detections above a higher
// cut are a subset of a lower-cut run, and suppression walks candidates
// in descending confidence, so the kept set above a cut does not depend
// on the threshold the run used. Running the model per threshold spent N
// GPU passes to reproduce identical numbers.
const lowestThreshold = Math.min(...thresholds)
setDetectionCalibrationRows((rows) =>
rows.map((row) => ({ ...row, status: 'running', message: 'Eén inferentie voor alle drempels' })),
)
const result = await detectionApi.run({
project_id: selectedProjectId,
dataset_id: datasetId,
model_id: selectedDetectionModelId,
model_asset_id: selectedModelAssetId || null,
confidence_threshold: lowestThreshold,
tile_manifest_path: detectionTileManifestPath.trim() || null,
parameters_json: { calibration: true, calibration_thresholds: thresholds },
})
setSelectedDetectionRunId(result.analysis_run_id)
const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, {
reference_dataset_id: detectionReferenceDatasetId,
iou_threshold: qaIouThreshold,
class_name: detectionClassFilter || null,
min_confidence: null,
calibration_thresholds: thresholds,
})
const sweep = new Map((qa.calibration_sweep ?? []).map((point) => [point.min_confidence, point]))
setDetectionCalibrationRows((rows) =>
rows.map((row) => {
const point = sweep.get(row.threshold)
if (!point) {
return { ...row, status: 'failed', message: 'Geen meetpunt voor deze drempel' }
}
return {
...row,
status: 'success',
analysis_run_id: result.analysis_run_id,
job_id: result.job_id,
quality_check_id: qa.quality_check_id,
detection_count: point.candidate_count,
precision: point.precision,
recall: point.recall,
f1_score: point.f1_score,
false_positives: point.false_positives,
false_negatives: point.false_negatives,
best_f1: point.best_f1_in_sweep,
message: point.best_f1_in_sweep ? 'Beste F1 in deze reeks' : null,
}
}),
)
await loadDetectionRuns(selectedProjectId)
await loadQualityChecks(selectedProjectId)
await loadProjectData(selectedProjectId)
} catch (error) {
const message = formatError(error, 'Calibration failed')
setDetectionCalibrationRows((rows) =>
rows.map((row) => (row.status === 'success' ? row : { ...row, status: 'failed', message })),
)
setDetectionCalibrationError(message)
} finally {
setRunningDetectionCalibration(false)
}