give segmentation QA the same threshold-independent view as detection
Detection QA reports a precision/recall curve, average precision and a calibration sweep; segmentation QA reported a single operating point. Both rank their outputs by confidence, so the same view applies, and the asymmetry meant the two panels answered different questions about comparable runs — an inconsistency introduced when detection gained the curve. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -223,5 +223,6 @@ def compare_segmentation_run_with_reference(
|
||||
iou_threshold=payload.iou_threshold,
|
||||
class_name=payload.class_name,
|
||||
min_confidence=payload.min_confidence,
|
||||
calibration_thresholds=payload.calibration_thresholds,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -32,6 +32,8 @@ class SegmentationQaRequest(BaseModel):
|
||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
class_name: str | None = None
|
||||
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
# Read off the one matching pass, exactly as for detection.
|
||||
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class SegmentationRunResponse(BaseModel):
|
||||
|
||||
@@ -21,6 +21,7 @@ from app.schemas.segmentation import (
|
||||
SegmentationRunResponse,
|
||||
)
|
||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||
from app.services.detection_metrics_service import DetectionMetricsService
|
||||
from app.services.detection_qa_service import DetectionQaService
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
@@ -328,6 +329,7 @@ class SegmentationService:
|
||||
iou_threshold: float = 0.5,
|
||||
class_name: str | None = None,
|
||||
min_confidence: float | None = None,
|
||||
calibration_thresholds: list[float] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "segmentation":
|
||||
@@ -474,6 +476,18 @@ class SegmentationService:
|
||||
reference_geometries,
|
||||
iou_threshold,
|
||||
)
|
||||
# Segmentation confidences rank the same way detections do, so the same
|
||||
# threshold-independent view applies. Detection had it and segmentation
|
||||
# did not, which made the two QA panels answer different questions.
|
||||
precision_recall_curve = DetectionMetricsService.precision_recall_curve(
|
||||
candidate_geometries,
|
||||
reference_geometries,
|
||||
iou_threshold=iou_threshold,
|
||||
)
|
||||
calibration_sweep = DetectionMetricsService.calibration_sweep(
|
||||
precision_recall_curve,
|
||||
thresholds=list(calibration_thresholds or []),
|
||||
)
|
||||
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
|
||||
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
|
||||
recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
|
||||
@@ -505,6 +519,8 @@ class SegmentationService:
|
||||
"warnings": coverage_warnings + evidence.warnings,
|
||||
"unsupported_geometry": evidence.unsupported,
|
||||
"coverage": coverage_summary,
|
||||
"precision_recall_curve": precision_recall_curve,
|
||||
"calibration_sweep": calibration_sweep,
|
||||
"match_evidence": evidence.match_evidence,
|
||||
"false_positive_evidence": evidence.false_positive_evidence,
|
||||
"false_negative_evidence": evidence.false_negative_evidence,
|
||||
@@ -516,6 +532,9 @@ class SegmentationService:
|
||||
"mean_iou": mean_iou,
|
||||
"false_positive_count": evidence.false_positives,
|
||||
"false_negative_count": evidence.false_negatives,
|
||||
"average_precision": precision_recall_curve["average_precision"],
|
||||
"best_f1": precision_recall_curve["best_f1"],
|
||||
"best_f1_threshold": precision_recall_curve["best_f1_threshold"],
|
||||
},
|
||||
)
|
||||
return {
|
||||
@@ -537,6 +556,8 @@ class SegmentationService:
|
||||
"iou_threshold": iou_threshold,
|
||||
"warnings": coverage_warnings + evidence.warnings,
|
||||
"coverage": coverage_summary,
|
||||
"precision_recall_curve": precision_recall_curve,
|
||||
"calibration_sweep": calibration_sweep,
|
||||
"match_evidence": evidence.match_evidence,
|
||||
"false_positive_evidence": evidence.false_positive_evidence,
|
||||
"false_negative_evidence": evidence.false_negative_evidence,
|
||||
|
||||
@@ -362,6 +362,11 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
|
||||
"mean_iou",
|
||||
"false_positive_count",
|
||||
"false_negative_count",
|
||||
# Threshold-independent, as detection QA already reported. The two
|
||||
# panels previously answered different questions about the same run.
|
||||
"average_precision",
|
||||
"best_f1",
|
||||
"best_f1_threshold",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -1981,7 +1981,12 @@ compare two models whose calibration differs; the curve can. `average_precision`
|
||||
existing ones.
|
||||
|
||||
Segmentation QA (`POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference`)
|
||||
applies the same tile-coverage clipping and returns the same `coverage` block.
|
||||
applies the same tile-coverage clipping and returns the same `coverage` block,
|
||||
and now also the same `precision_recall_curve`, `calibration_thresholds` /
|
||||
`calibration_sweep` and average-precision metrics. Segmentation confidences rank
|
||||
the same way detection confidences do, so the same threshold-independent view
|
||||
applies; without it the two QA panels answered different questions about
|
||||
comparable runs.
|
||||
Without it, every reference feature outside the inferred tiles counted as a
|
||||
false negative and recall was understated by an arbitrary amount.
|
||||
|
||||
|
||||
@@ -1620,6 +1620,8 @@ export interface SegmentationQaRequest {
|
||||
iou_threshold: number
|
||||
class_name?: string | null
|
||||
min_confidence?: number | null
|
||||
/** Read off the one matching pass, exactly as for detection. */
|
||||
calibration_thresholds?: number[]
|
||||
}
|
||||
|
||||
export type SegmentationQaResult = DetectionQaResult
|
||||
|
||||
Reference in New Issue
Block a user