diff --git a/backend/app/api/routes/detection.py b/backend/app/api/routes/detection.py index cb5506b4..3384f613 100644 --- a/backend/app/api/routes/detection.py +++ b/backend/app/api/routes/detection.py @@ -249,5 +249,6 @@ def compare_detection_run_with_reference( iou_threshold=payload.iou_threshold, class_name=payload.class_name, min_confidence=payload.min_confidence, + calibration_thresholds=payload.calibration_thresholds, ) ) diff --git a/backend/app/schemas/detection.py b/backend/app/schemas/detection.py index ddae2bb1..d706a084 100644 --- a/backend/app/schemas/detection.py +++ b/backend/app/schemas/detection.py @@ -73,6 +73,9 @@ class DetectionQaRequest(BaseModel): iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) class_name: str | None = None min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) + # Confidence cuts to report alongside the run's own operating point. They + # are read off the one matching pass, so a sweep costs no extra inference. + calibration_thresholds: list[float] = Field(default_factory=list, max_length=32) class DetectionRunResponse(BaseModel): diff --git a/backend/app/schemas/qa.py b/backend/app/schemas/qa.py index cafd4bfb..2d23056b 100644 --- a/backend/app/schemas/qa.py +++ b/backend/app/schemas/qa.py @@ -121,6 +121,8 @@ class AnalysisQaResponse(BaseModel): coverage: dict[str, Any] | None = None temporal_compatibility: dict[str, Any] | None = None box_to_footprint_diagnostics: dict[str, Any] | None = None + precision_recall_curve: dict[str, Any] | None = None + calibration_sweep: list[dict[str, Any]] = Field(default_factory=list) match_evidence: list[dict[str, Any]] = Field(default_factory=list) false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list) false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list) diff --git a/backend/app/services/detection_metrics_service.py b/backend/app/services/detection_metrics_service.py index b69fe003..562d255a 100644 --- a/backend/app/services/detection_metrics_service.py +++ b/backend/app/services/detection_metrics_service.py @@ -118,6 +118,66 @@ class DetectionMetricsService: area += (recalls[index] - recalls[index - 1]) * precisions[index] return area + @staticmethod + def operating_point(curve: dict[str, Any], *, min_confidence: float) -> dict[str, Any]: + """The metrics that hold when detections below ``min_confidence`` are dropped. + + Read off the curve rather than recomputed: the curve already walked the + ranking once, and every threshold is a prefix of that walk. Running the + model again per threshold would spend N GPU passes to reproduce numbers + that are already here. + """ + + points = curve.get("points") or [] + reference_count = int(curve.get("reference_count") or 0) + admitted = [point for point in points if point["confidence_threshold"] >= min_confidence] + # Points are cumulative down the ranking, so the last admitted one is + # the complete tally at this cut. + tally = admitted[-1] if admitted else None + + true_positives = int(tally["true_positives"]) if tally else 0 + false_positives = int(tally["false_positives"]) if tally else 0 + false_negatives = max(0, reference_count - true_positives) + candidate_count = true_positives + false_positives + + precision = true_positives / candidate_count if candidate_count else None + recall = true_positives / reference_count if reference_count else None + f1_score = None + if precision is not None and recall is not None: + f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0 + + return { + "min_confidence": min_confidence, + "confidence_threshold": tally["confidence_threshold"] if tally else None, + "candidate_count": candidate_count, + "true_positives": true_positives, + "false_positives": false_positives, + "false_negatives": false_negatives, + "precision": precision, + "recall": recall, + "f1_score": f1_score, + } + + @staticmethod + def calibration_sweep(curve: dict[str, Any], *, thresholds: list[float]) -> list[dict[str, Any]]: + """Every requested operating point, strictest first, from one curve. + + Recall cannot fall as the cut loosens — that monotonicity is exactly + why a single run answers the whole sweep. + """ + + ordered = sorted({float(value) for value in thresholds}, reverse=True) + rows = [ + DetectionMetricsService.operating_point(curve, min_confidence=value) for value in ordered + ] + best_f1 = max((row["f1_score"] or 0.0) for row in rows) if rows else 0.0 + marked = False + for row in rows: + is_best = not marked and (row["f1_score"] or 0.0) == best_f1 + row["best_f1_in_sweep"] = is_best + marked = marked or is_best + return rows + @staticmethod def precision_recall_curve( candidates: list[tuple[dict[str, Any], BaseGeometry]], diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 81d98b27..e63fb82c 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -377,6 +377,7 @@ class DetectionService: iou_threshold: float = 0.5, class_name: str | None = None, min_confidence: float | None = None, + calibration_thresholds: list[float] | None = None, ) -> dict[str, Any]: run = db.get(AnalysisRun, analysis_run_id) if not run or run.analysis_type != "detection": @@ -563,6 +564,15 @@ class DetectionService: reference_geometries, iou_threshold=iou_threshold, ) + # Every requested confidence cut, answered from that one matching pass. + # Re-running inference per threshold spends N GPU passes to reproduce + # numbers already present here: suppression walks candidates in + # descending confidence, so the kept set above a cut does not depend on + # the threshold the run itself used. + calibration_sweep = DetectionMetricsService.calibration_sweep( + precision_recall_curve, + thresholds=list(calibration_thresholds or []), + ) mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None @@ -598,6 +608,7 @@ class DetectionService: "temporal_compatibility": temporal_compatibility, "box_to_footprint_diagnostics": box_to_footprint_diagnostics, "precision_recall_curve": precision_recall_curve, + "calibration_sweep": calibration_sweep, "match_evidence": evidence.match_evidence, "false_positive_evidence": evidence.false_positive_evidence, "false_negative_evidence": evidence.false_negative_evidence, @@ -647,6 +658,7 @@ class DetectionService: "temporal_compatibility": temporal_compatibility, "box_to_footprint_diagnostics": box_to_footprint_diagnostics, "precision_recall_curve": precision_recall_curve, + "calibration_sweep": calibration_sweep, "match_evidence": evidence.match_evidence, "false_positive_evidence": evidence.false_positive_evidence, "false_negative_evidence": evidence.false_negative_evidence, diff --git a/backend/tests/test_detection_calibration_from_one_run.py b/backend/tests/test_detection_calibration_from_one_run.py new file mode 100644 index 00000000..5681dae7 --- /dev/null +++ b/backend/tests/test_detection_calibration_from_one_run.py @@ -0,0 +1,119 @@ +"""Calibrating a confidence threshold does not need one inference run per value. + +The workbench ran the model over every tile once per threshold — three GPU +passes to compare 0.50, 0.25 and 0.15. The answer is already in a single run at +the lowest value: detections above a higher cut are a subset of it, and +suppression walks candidates in descending confidence, so a lower-confidence +box can never displace a higher-confidence one. The kept set above any cut is +therefore identical whichever threshold the run used. + +One matching pass produces every operating point exactly, so the sweep is free +rather than N times the cost. +""" + +from __future__ import annotations + +import pytest +from shapely.geometry import box + +from app.services.detection_metrics_service import DetectionMetricsService + + +def _candidate(name: str, geometry, confidence: float): + return ({"id": name, "confidence": confidence}, geometry) + + +def _reference(name: str, geometry): + return ({"id": name}, geometry) + + +REFERENCES = [ + _reference("r1", box(0, 0, 1, 1)), + _reference("r2", box(5, 5, 6, 6)), + _reference("r3", box(10, 10, 11, 11)), +] +CANDIDATES = [ + _candidate("hit-high", box(0, 0, 1, 1), 0.90), + _candidate("hit-mid", box(5, 5, 6, 6), 0.40), + _candidate("junk-low", box(30, 30, 31, 31), 0.20), +] + + +def _curve(): + return DetectionMetricsService.precision_recall_curve(CANDIDATES, REFERENCES, iou_threshold=0.5) + + +class TestOperatingPoints: + def test_a_strict_cut_keeps_only_the_confident_detection(self) -> None: + point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.5) + + assert point["true_positives"] == 1 + assert point["false_positives"] == 0 + assert point["false_negatives"] == 2 + assert point["precision"] == pytest.approx(1.0) + assert point["recall"] == pytest.approx(1 / 3) + + def test_a_looser_cut_finds_more_and_stays_exact(self) -> None: + point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.3) + + assert point["true_positives"] == 2 + assert point["false_positives"] == 0 + assert point["recall"] == pytest.approx(2 / 3) + + def test_the_loosest_cut_admits_the_false_positive(self) -> None: + point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.1) + + assert point["true_positives"] == 2 + assert point["false_positives"] == 1 + assert point["precision"] == pytest.approx(2 / 3) + + def test_a_cut_above_every_detection_finds_nothing_but_still_reports(self) -> None: + point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.99) + + assert point["true_positives"] == 0 + assert point["false_negatives"] == 3 + assert point["recall"] == pytest.approx(0.0) + assert point["precision"] is None + + def test_the_requested_threshold_is_echoed_back(self) -> None: + point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.42) + + assert point["min_confidence"] == pytest.approx(0.42) + # The nearest actual operating point sits at the detection's own + # confidence, which is what the numbers describe. + assert point["confidence_threshold"] == pytest.approx(0.9) + + +class TestSweep: + def test_a_sweep_returns_one_row_per_requested_threshold(self) -> None: + rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1]) + + assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1] + + def test_the_sweep_is_ordered_from_strict_to_loose(self) -> None: + rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.1, 0.5, 0.3]) + + assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1] + + def test_a_repeated_threshold_is_asked_once(self) -> None: + rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.3, 0.3]) + + assert len(rows) == 1 + + def test_recall_never_falls_as_the_cut_loosens(self) -> None: + """The monotonicity that makes one run sufficient.""" + + rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.9, 0.5, 0.3, 0.1]) + recalls = [row["recall"] for row in rows] + + assert recalls == sorted(recalls) + + def test_an_empty_sweep_is_not_an_error(self) -> None: + assert DetectionMetricsService.calibration_sweep(_curve(), thresholds=[]) == [] + + def test_the_sweep_marks_the_f1_optimal_row(self) -> None: + rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1]) + best = [row for row in rows if row["best_f1_in_sweep"]] + + assert len(best) == 1 + assert best[0]["f1_score"] == max(row["f1_score"] for row in rows) diff --git a/backend/tests/test_sprint134_guided_detection_calibration_runner.py b/backend/tests/test_sprint134_guided_detection_calibration_runner.py index 9ae2d43a..190804e2 100644 --- a/backend/tests/test_sprint134_guided_detection_calibration_runner.py +++ b/backend/tests/test_sprint134_guided_detection_calibration_runner.py @@ -22,9 +22,15 @@ def test_detection_lab_has_guided_threshold_calibration_runner() -> None: assert "detectionCalibrationRows" in hook_source assert "detectionCalibrationError" in hook_source assert "runDetectionCalibration" in hook_source - assert "for (const threshold of thresholds)" in hook_source assert "detectionApi.run({" in hook_source - assert "confidence_threshold: threshold" in hook_source + # Every requested threshold is reported, but from one inference pass at the + # lowest cut: detections above a higher cut are a subset of it, and + # suppression walks candidates in descending confidence, so the kept set + # above a cut does not depend on the threshold the run used. Asserting the + # old per-threshold loop pinned N GPU passes that produced identical numbers. + assert "confidence_threshold: lowestThreshold" in hook_source + assert "calibration_thresholds: thresholds" in hook_source + assert "calibration_sweep" in hook_source assert "parameters_json: { calibration: true, calibration_thresholds: thresholds }" in hook_source assert "detectionApi.compareWithReference(result.analysis_run_id" in hook_source assert "reference_dataset_id: detectionReferenceDatasetId" in hook_source diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index b46a9d3b..c6757ee2 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1913,6 +1913,18 @@ transaction timestamp, so ordering by `created_at` left the assignment — and therefore the score and the false-positive evidence shown to a reviewer — undefined between identical runs. +`calibration_thresholds` asks for named confidence cuts alongside the run's own +operating point. They are read off the same matching pass, so a sweep costs no +extra inference at all. Each row reports the tally, precision, recall and F1 at +that cut, and `best_f1_in_sweep` marks the F1-optimal one. + +This replaces re-running the model once per threshold. Detections above a +higher cut are a subset of a lower-cut run, and duplicate suppression walks +candidates in descending confidence, so a lower-confidence box can never +displace a higher-confidence one: the kept set above any cut is identical +whichever threshold the run itself used. Three thresholds therefore cost one +GPU pass rather than three, and produce the same numbers. + The response also returns `precision_recall_curve`: precision, recall and F1 at every confidence value present in the run, plus `average_precision`, `best_f1` and `best_f1_threshold`. A single F1 describes one operating point and cannot diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx index 8ddd6106..589fda90 100644 --- a/frontend/src/components/detection/DetectionLab.tsx +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -648,8 +648,11 @@ export function DetectionLab({ {detectionCalibrationRows.map((row) => ( - - {row.threshold.toFixed(2)} + + + {row.threshold.toFixed(2)} + {row.best_f1 ? beste F1 : null} + {row.status} {row.detection_count ?? 'n.v.t.'} {formatNullableNumber(row.precision ?? null, 3)} diff --git a/frontend/src/hooks/useDetectionWorkflow.ts b/frontend/src/hooks/useDetectionWorkflow.ts index 334a4350..0680af87 100644 --- a/frontend/src/hooks/useDetectionWorkflow.ts +++ b/frontend/src/hooks/useDetectionWorkflow.ts @@ -50,6 +50,8 @@ export interface DetectionCalibrationRunRow { f1_score?: number | null false_positives?: number | null false_negatives?: number | null + /** The F1-optimal cut among the requested thresholds. */ + best_f1?: boolean message?: string | null } @@ -481,57 +483,68 @@ export function useDetectionWorkflow({ setDetectionCalibrationRows(thresholds.map((threshold) => ({ threshold, status: 'queued' }))) setRunningDetectionCalibration(true) try { - for (const threshold of thresholds) { - setDetectionCalibrationRows((rows) => - rows.map((row) => row.threshold === threshold ? { ...row, status: 'running', message: 'Running detection' } : row), - ) - try { - const result = await detectionApi.run({ - project_id: selectedProjectId, - dataset_id: datasetId, - model_id: selectedDetectionModelId, - model_asset_id: selectedModelAssetId || null, - confidence_threshold: threshold, - tile_manifest_path: detectionTileManifestPath.trim() || null, - parameters_json: { calibration: true, calibration_thresholds: thresholds }, - }) - setSelectedDetectionRunId(result.analysis_run_id) - const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, { - reference_dataset_id: detectionReferenceDatasetId, - iou_threshold: qaIouThreshold, - class_name: detectionClassFilter || null, - min_confidence: null, - }) - setDetectionCalibrationRows((rows) => - rows.map((row) => row.threshold === threshold - ? { - ...row, - status: 'success', - analysis_run_id: result.analysis_run_id, - job_id: result.job_id, - quality_check_id: qa.quality_check_id, - detection_count: result.detection_count, - precision: qa.precision ?? null, - recall: qa.recall ?? null, - f1_score: qa.f1_score ?? null, - false_positives: qa.false_positives, - false_negatives: qa.false_negatives, - message: result.message, - } - : row), - ) - } catch (error) { - const message = formatError(error, `Calibration threshold ${threshold} failed`) - setDetectionCalibrationRows((rows) => - rows.map((row) => row.threshold === threshold ? { ...row, status: 'failed', message } : row), - ) - setDetectionCalibrationError(message) - break - } - } + // One inference pass answers every threshold. Detections above a higher + // cut are a subset of a lower-cut run, and suppression walks candidates + // in descending confidence, so the kept set above a cut does not depend + // on the threshold the run used. Running the model per threshold spent N + // GPU passes to reproduce identical numbers. + const lowestThreshold = Math.min(...thresholds) + setDetectionCalibrationRows((rows) => + rows.map((row) => ({ ...row, status: 'running', message: 'Eén inferentie voor alle drempels' })), + ) + const result = await detectionApi.run({ + project_id: selectedProjectId, + dataset_id: datasetId, + model_id: selectedDetectionModelId, + model_asset_id: selectedModelAssetId || null, + confidence_threshold: lowestThreshold, + tile_manifest_path: detectionTileManifestPath.trim() || null, + parameters_json: { calibration: true, calibration_thresholds: thresholds }, + }) + setSelectedDetectionRunId(result.analysis_run_id) + + const qa = await detectionApi.compareWithReference(result.analysis_run_id, selectedProjectId, { + reference_dataset_id: detectionReferenceDatasetId, + iou_threshold: qaIouThreshold, + class_name: detectionClassFilter || null, + min_confidence: null, + calibration_thresholds: thresholds, + }) + + const sweep = new Map((qa.calibration_sweep ?? []).map((point) => [point.min_confidence, point])) + setDetectionCalibrationRows((rows) => + rows.map((row) => { + const point = sweep.get(row.threshold) + if (!point) { + return { ...row, status: 'failed', message: 'Geen meetpunt voor deze drempel' } + } + return { + ...row, + status: 'success', + analysis_run_id: result.analysis_run_id, + job_id: result.job_id, + quality_check_id: qa.quality_check_id, + detection_count: point.candidate_count, + precision: point.precision, + recall: point.recall, + f1_score: point.f1_score, + false_positives: point.false_positives, + false_negatives: point.false_negatives, + best_f1: point.best_f1_in_sweep, + message: point.best_f1_in_sweep ? 'Beste F1 in deze reeks' : null, + } + }), + ) + await loadDetectionRuns(selectedProjectId) await loadQualityChecks(selectedProjectId) await loadProjectData(selectedProjectId) + } catch (error) { + const message = formatError(error, 'Calibration failed') + setDetectionCalibrationRows((rows) => + rows.map((row) => (row.status === 'success' ? row : { ...row, status: 'failed', message })), + ) + setDetectionCalibrationError(message) } finally { setRunningDetectionCalibration(false) } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index ac0301e4..b2bbc31d 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1431,6 +1431,25 @@ export interface DetectionQaRequest { iou_threshold: number class_name?: string | null min_confidence?: number | null + /** + * Confidence cuts to report next to the run's own operating point. They are + * read off one matching pass, so a sweep costs no extra inference. + */ + calibration_thresholds?: number[] +} + +/** One confidence cut, derived from a single run rather than a run of its own. */ +export interface DetectionCalibrationPoint { + min_confidence: number + confidence_threshold: number | null + candidate_count: number + true_positives: number + false_positives: number + false_negatives: number + precision: number | null + recall: number | null + f1_score: number | null + best_f1_in_sweep: boolean } export interface DetectionQaResult { @@ -1486,6 +1505,7 @@ export interface DetectionQaResult { envelope_precision_recall_curve?: PrecisionRecallCurve } precision_recall_curve?: PrecisionRecallCurve + calibration_sweep?: DetectionCalibrationPoint[] } /**