From 1a1a9af6e71842fa1c50c92dc6edd44322bfdab1 Mon Sep 17 00:00:00 2001 From: Jens Date: Sat, 22 Aug 2026 20:47:30 +0200 Subject: [PATCH] give segmentation QA the same threshold-independent view as detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detection QA reports a precision/recall curve, average precision and a calibration sweep; segmentation QA reported a single operating point. Both rank their outputs by confidence, so the same view applies, and the asymmetry meant the two panels answered different questions about comparable runs — an inconsistency introduced when detection gained the curve. Co-Authored-By: Claude Opus 5 --- backend/app/api/routes/segmentation.py | 1 + backend/app/schemas/segmentation.py | 2 ++ backend/app/services/segmentation_service.py | 21 +++++++++++++++++++ .../test_sprint9_segmentation_foundation.py | 5 +++++ docs/API_CONTRACTS.md | 7 ++++++- frontend/src/types.ts | 2 ++ 6 files changed, 37 insertions(+), 1 deletion(-) diff --git a/backend/app/api/routes/segmentation.py b/backend/app/api/routes/segmentation.py index 7894de3c..9846b5d8 100644 --- a/backend/app/api/routes/segmentation.py +++ b/backend/app/api/routes/segmentation.py @@ -223,5 +223,6 @@ def compare_segmentation_run_with_reference( iou_threshold=payload.iou_threshold, class_name=payload.class_name, min_confidence=payload.min_confidence, + calibration_thresholds=payload.calibration_thresholds, ) ) diff --git a/backend/app/schemas/segmentation.py b/backend/app/schemas/segmentation.py index 5098c251..f82415ac 100644 --- a/backend/app/schemas/segmentation.py +++ b/backend/app/schemas/segmentation.py @@ -32,6 +32,8 @@ class SegmentationQaRequest(BaseModel): iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) class_name: str | None = None min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) + # Read off the one matching pass, exactly as for detection. + calibration_thresholds: list[float] = Field(default_factory=list, max_length=32) class SegmentationRunResponse(BaseModel): diff --git a/backend/app/services/segmentation_service.py b/backend/app/services/segmentation_service.py index b94af6c3..999a7c4d 100644 --- a/backend/app/services/segmentation_service.py +++ b/backend/app/services/segmentation_service.py @@ -21,6 +21,7 @@ from app.schemas.segmentation import ( SegmentationRunResponse, ) from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon +from app.services.detection_metrics_service import DetectionMetricsService from app.services.detection_qa_service import DetectionQaService from app.services.detection_service import DetectionService from app.services.dataset_consumption_gate_service import DatasetConsumptionGate @@ -328,6 +329,7 @@ class SegmentationService: iou_threshold: float = 0.5, class_name: str | None = None, min_confidence: float | None = None, + calibration_thresholds: list[float] | None = None, ) -> dict[str, Any]: run = db.get(AnalysisRun, analysis_run_id) if not run or run.analysis_type != "segmentation": @@ -474,6 +476,18 @@ class SegmentationService: reference_geometries, iou_threshold, ) + # Segmentation confidences rank the same way detections do, so the same + # threshold-independent view applies. Detection had it and segmentation + # did not, which made the two QA panels answer different questions. + precision_recall_curve = DetectionMetricsService.precision_recall_curve( + candidate_geometries, + reference_geometries, + iou_threshold=iou_threshold, + ) + calibration_sweep = DetectionMetricsService.calibration_sweep( + precision_recall_curve, + thresholds=list(calibration_thresholds or []), + ) mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None @@ -505,6 +519,8 @@ class SegmentationService: "warnings": coverage_warnings + evidence.warnings, "unsupported_geometry": evidence.unsupported, "coverage": coverage_summary, + "precision_recall_curve": precision_recall_curve, + "calibration_sweep": calibration_sweep, "match_evidence": evidence.match_evidence, "false_positive_evidence": evidence.false_positive_evidence, "false_negative_evidence": evidence.false_negative_evidence, @@ -516,6 +532,9 @@ class SegmentationService: "mean_iou": mean_iou, "false_positive_count": evidence.false_positives, "false_negative_count": evidence.false_negatives, + "average_precision": precision_recall_curve["average_precision"], + "best_f1": precision_recall_curve["best_f1"], + "best_f1_threshold": precision_recall_curve["best_f1_threshold"], }, ) return { @@ -537,6 +556,8 @@ class SegmentationService: "iou_threshold": iou_threshold, "warnings": coverage_warnings + evidence.warnings, "coverage": coverage_summary, + "precision_recall_curve": precision_recall_curve, + "calibration_sweep": calibration_sweep, "match_evidence": evidence.match_evidence, "false_positive_evidence": evidence.false_positive_evidence, "false_negative_evidence": evidence.false_negative_evidence, diff --git a/backend/tests/test_sprint9_segmentation_foundation.py b/backend/tests/test_sprint9_segmentation_foundation.py index b35dd933..ce2cfeca 100644 --- a/backend/tests/test_sprint9_segmentation_foundation.py +++ b/backend/tests/test_sprint9_segmentation_foundation.py @@ -362,6 +362,11 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None: "mean_iou", "false_positive_count", "false_negative_count", + # Threshold-independent, as detection QA already reported. The two + # panels previously answered different questions about the same run. + "average_precision", + "best_f1", + "best_f1_threshold", ] diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 7e5b5acd..c235932e 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1981,7 +1981,12 @@ compare two models whose calibration differs; the curve can. `average_precision` existing ones. Segmentation QA (`POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference`) -applies the same tile-coverage clipping and returns the same `coverage` block. +applies the same tile-coverage clipping and returns the same `coverage` block, +and now also the same `precision_recall_curve`, `calibration_thresholds` / +`calibration_sweep` and average-precision metrics. Segmentation confidences rank +the same way detection confidences do, so the same threshold-independent view +applies; without it the two QA panels answered different questions about +comparable runs. Without it, every reference feature outside the inferred tiles counted as a false negative and recall was understated by an arbitrary amount. diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 535c9ca9..aba100bd 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1620,6 +1620,8 @@ export interface SegmentationQaRequest { iou_threshold: number class_name?: string | null min_confidence?: number | null + /** Read off the one matching pass, exactly as for detection. */ + calibration_thresholds?: number[] } export type SegmentationQaResult = DetectionQaResult