calibrate a confidence threshold from one inference pass

Threshold calibration ran the model over every tile once per threshold — three
GPU passes to compare 0.50, 0.25 and 0.15 on a hundred-tile raster. The answer
is already in a single run at the lowest value: detections above a higher cut
are a subset of it, and duplicate suppression walks candidates in descending
confidence, so a lower-confidence box can never displace a higher-confidence
one. The kept set above any cut is identical whichever threshold the run used,
which is what makes one pass sufficient rather than merely cheaper.

QA now takes calibration_thresholds and reads each operating point off the same
precision/recall walk it already performs, marking the F1-optimal cut. The lab
runs inference once and fills its table from the sweep.

The contract test asserted the per-threshold loop by name, pinning the waste it
was meant to describe. It now states what calibration owes an operator: a row
per requested threshold, from one run.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 19:37:19 +02:00
co-authored by Claude Opus 5
parent 8a26007281
commit 2cd2c49389
11 changed files with 303 additions and 52 deletions
@@ -118,6 +118,66 @@ class DetectionMetricsService:
area += (recalls[index] - recalls[index - 1]) * precisions[index]
return area
@staticmethod
def operating_point(curve: dict[str, Any], *, min_confidence: float) -> dict[str, Any]:
"""The metrics that hold when detections below ``min_confidence`` are dropped.
Read off the curve rather than recomputed: the curve already walked the
ranking once, and every threshold is a prefix of that walk. Running the
model again per threshold would spend N GPU passes to reproduce numbers
that are already here.
"""
points = curve.get("points") or []
reference_count = int(curve.get("reference_count") or 0)
admitted = [point for point in points if point["confidence_threshold"] >= min_confidence]
# Points are cumulative down the ranking, so the last admitted one is
# the complete tally at this cut.
tally = admitted[-1] if admitted else None
true_positives = int(tally["true_positives"]) if tally else 0
false_positives = int(tally["false_positives"]) if tally else 0
false_negatives = max(0, reference_count - true_positives)
candidate_count = true_positives + false_positives
precision = true_positives / candidate_count if candidate_count else None
recall = true_positives / reference_count if reference_count else None
f1_score = None
if precision is not None and recall is not None:
f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
return {
"min_confidence": min_confidence,
"confidence_threshold": tally["confidence_threshold"] if tally else None,
"candidate_count": candidate_count,
"true_positives": true_positives,
"false_positives": false_positives,
"false_negatives": false_negatives,
"precision": precision,
"recall": recall,
"f1_score": f1_score,
}
@staticmethod
def calibration_sweep(curve: dict[str, Any], *, thresholds: list[float]) -> list[dict[str, Any]]:
"""Every requested operating point, strictest first, from one curve.
Recall cannot fall as the cut loosens — that monotonicity is exactly
why a single run answers the whole sweep.
"""
ordered = sorted({float(value) for value in thresholds}, reverse=True)
rows = [
DetectionMetricsService.operating_point(curve, min_confidence=value) for value in ordered
]
best_f1 = max((row["f1_score"] or 0.0) for row in rows) if rows else 0.0
marked = False
for row in rows:
is_best = not marked and (row["f1_score"] or 0.0) == best_f1
row["best_f1_in_sweep"] = is_best
marked = marked or is_best
return rows
@staticmethod
def precision_recall_curve(
candidates: list[tuple[dict[str, Any], BaseGeometry]],