give segmentation QA the same threshold-independent view as detection
Detection QA reports a precision/recall curve, average precision and a calibration sweep; segmentation QA reported a single operating point. Both rank their outputs by confidence, so the same view applies, and the asymmetry meant the two panels answered different questions about comparable runs — an inconsistency introduced when detection gained the curve. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -223,5 +223,6 @@ def compare_segmentation_run_with_reference(
|
||||
iou_threshold=payload.iou_threshold,
|
||||
class_name=payload.class_name,
|
||||
min_confidence=payload.min_confidence,
|
||||
calibration_thresholds=payload.calibration_thresholds,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -32,6 +32,8 @@ class SegmentationQaRequest(BaseModel):
|
||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
class_name: str | None = None
|
||||
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
# Read off the one matching pass, exactly as for detection.
|
||||
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class SegmentationRunResponse(BaseModel):
|
||||
|
||||
@@ -21,6 +21,7 @@ from app.schemas.segmentation import (
|
||||
SegmentationRunResponse,
|
||||
)
|
||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||
from app.services.detection_metrics_service import DetectionMetricsService
|
||||
from app.services.detection_qa_service import DetectionQaService
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
@@ -328,6 +329,7 @@ class SegmentationService:
|
||||
iou_threshold: float = 0.5,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
calibration_thresholds: list[float] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "segmentation":
|
||||
@@ -474,6 +476,18 @@ class SegmentationService:
|
||||
reference_geometries,
|
||||
iou_threshold,
|
||||
)
|
||||
# Segmentation confidences rank the same way detections do, so the same
|
||||
# threshold-independent view applies. Detection had it and segmentation
|
||||
# did not, which made the two QA panels answer different questions.
|
||||
precision_recall_curve = DetectionMetricsService.precision_recall_curve(
|
||||
candidate_geometries,
|
||||
reference_geometries,
|
||||
iou_threshold=iou_threshold,
|
||||
)
|
||||
calibration_sweep = DetectionMetricsService.calibration_sweep(
|
||||
precision_recall_curve,
|
||||
thresholds=list(calibration_thresholds or []),
|
||||
)
|
||||
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
|
||||
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
|
||||
recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
|
||||
@@ -505,6 +519,8 @@ class SegmentationService:
|
||||
"warnings": coverage_warnings + evidence.warnings,
|
||||
"unsupported_geometry": evidence.unsupported,
|
||||
"coverage": coverage_summary,
|
||||
"precision_recall_curve": precision_recall_curve,
|
||||
"calibration_sweep": calibration_sweep,
|
||||
"match_evidence": evidence.match_evidence,
|
||||
"false_positive_evidence": evidence.false_positive_evidence,
|
||||
"false_negative_evidence": evidence.false_negative_evidence,
|
||||
@@ -516,6 +532,9 @@ class SegmentationService:
|
||||
"mean_iou": mean_iou,
|
||||
"false_positive_count": evidence.false_positives,
|
||||
"false_negative_count": evidence.false_negatives,
|
||||
"average_precision": precision_recall_curve["average_precision"],
|
||||
"best_f1": precision_recall_curve["best_f1"],
|
||||
"best_f1_threshold": precision_recall_curve["best_f1_threshold"],
|
||||
},
|
||||
)
|
||||
return {
|
||||
@@ -537,6 +556,8 @@ class SegmentationService:
|
||||
"iou_threshold": iou_threshold,
|
||||
"warnings": coverage_warnings + evidence.warnings,
|
||||
"coverage": coverage_summary,
|
||||
"precision_recall_curve": precision_recall_curve,
|
||||
"calibration_sweep": calibration_sweep,
|
||||
"match_evidence": evidence.match_evidence,
|
||||
"false_positive_evidence": evidence.false_positive_evidence,
|
||||
"false_negative_evidence": evidence.false_negative_evidence,
|
||||
|
||||
Reference in New Issue
Block a user