give segmentation QA the same threshold-independent view as detection
Detection QA reports a precision/recall curve, average precision and a calibration sweep; segmentation QA reported a single operating point. Both rank their outputs by confidence, so the same view applies, and the asymmetry meant the two panels answered different questions about comparable runs — an inconsistency introduced when detection gained the curve. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -223,5 +223,6 @@ def compare_segmentation_run_with_reference(
|
|||||||
iou_threshold=payload.iou_threshold,
|
iou_threshold=payload.iou_threshold,
|
||||||
class_name=payload.class_name,
|
class_name=payload.class_name,
|
||||||
min_confidence=payload.min_confidence,
|
min_confidence=payload.min_confidence,
|
||||||
|
calibration_thresholds=payload.calibration_thresholds,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ class SegmentationQaRequest(BaseModel):
|
|||||||
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||||
class_name: str | None = None
|
class_name: str | None = None
|
||||||
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||||
|
# Read off the one matching pass, exactly as for detection.
|
||||||
|
calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
|
||||||
|
|
||||||
|
|
||||||
class SegmentationRunResponse(BaseModel):
|
class SegmentationRunResponse(BaseModel):
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from app.schemas.segmentation import (
|
|||||||
SegmentationRunResponse,
|
SegmentationRunResponse,
|
||||||
)
|
)
|
||||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||||
|
from app.services.detection_metrics_service import DetectionMetricsService
|
||||||
from app.services.detection_qa_service import DetectionQaService
|
from app.services.detection_qa_service import DetectionQaService
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
@@ -328,6 +329,7 @@ class SegmentationService:
|
|||||||
iou_threshold: float = 0.5,
|
iou_threshold: float = 0.5,
|
||||||
class_name: str | None = None,
|
class_name: str | None = None,
|
||||||
min_confidence: float | None = None,
|
min_confidence: float | None = None,
|
||||||
|
calibration_thresholds: list[float] | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
if not run or run.analysis_type != "segmentation":
|
if not run or run.analysis_type != "segmentation":
|
||||||
@@ -474,6 +476,18 @@ class SegmentationService:
|
|||||||
reference_geometries,
|
reference_geometries,
|
||||||
iou_threshold,
|
iou_threshold,
|
||||||
)
|
)
|
||||||
|
# Segmentation confidences rank the same way detections do, so the same
|
||||||
|
# threshold-independent view applies. Detection had it and segmentation
|
||||||
|
# did not, which made the two QA panels answer different questions.
|
||||||
|
precision_recall_curve = DetectionMetricsService.precision_recall_curve(
|
||||||
|
candidate_geometries,
|
||||||
|
reference_geometries,
|
||||||
|
iou_threshold=iou_threshold,
|
||||||
|
)
|
||||||
|
calibration_sweep = DetectionMetricsService.calibration_sweep(
|
||||||
|
precision_recall_curve,
|
||||||
|
thresholds=list(calibration_thresholds or []),
|
||||||
|
)
|
||||||
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
|
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
|
||||||
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
|
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
|
||||||
recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
|
recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
|
||||||
@@ -505,6 +519,8 @@ class SegmentationService:
|
|||||||
"warnings": coverage_warnings + evidence.warnings,
|
"warnings": coverage_warnings + evidence.warnings,
|
||||||
"unsupported_geometry": evidence.unsupported,
|
"unsupported_geometry": evidence.unsupported,
|
||||||
"coverage": coverage_summary,
|
"coverage": coverage_summary,
|
||||||
|
"precision_recall_curve": precision_recall_curve,
|
||||||
|
"calibration_sweep": calibration_sweep,
|
||||||
"match_evidence": evidence.match_evidence,
|
"match_evidence": evidence.match_evidence,
|
||||||
"false_positive_evidence": evidence.false_positive_evidence,
|
"false_positive_evidence": evidence.false_positive_evidence,
|
||||||
"false_negative_evidence": evidence.false_negative_evidence,
|
"false_negative_evidence": evidence.false_negative_evidence,
|
||||||
@@ -516,6 +532,9 @@ class SegmentationService:
|
|||||||
"mean_iou": mean_iou,
|
"mean_iou": mean_iou,
|
||||||
"false_positive_count": evidence.false_positives,
|
"false_positive_count": evidence.false_positives,
|
||||||
"false_negative_count": evidence.false_negatives,
|
"false_negative_count": evidence.false_negatives,
|
||||||
|
"average_precision": precision_recall_curve["average_precision"],
|
||||||
|
"best_f1": precision_recall_curve["best_f1"],
|
||||||
|
"best_f1_threshold": precision_recall_curve["best_f1_threshold"],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
@@ -537,6 +556,8 @@ class SegmentationService:
|
|||||||
"iou_threshold": iou_threshold,
|
"iou_threshold": iou_threshold,
|
||||||
"warnings": coverage_warnings + evidence.warnings,
|
"warnings": coverage_warnings + evidence.warnings,
|
||||||
"coverage": coverage_summary,
|
"coverage": coverage_summary,
|
||||||
|
"precision_recall_curve": precision_recall_curve,
|
||||||
|
"calibration_sweep": calibration_sweep,
|
||||||
"match_evidence": evidence.match_evidence,
|
"match_evidence": evidence.match_evidence,
|
||||||
"false_positive_evidence": evidence.false_positive_evidence,
|
"false_positive_evidence": evidence.false_positive_evidence,
|
||||||
"false_negative_evidence": evidence.false_negative_evidence,
|
"false_negative_evidence": evidence.false_negative_evidence,
|
||||||
|
|||||||
@@ -362,6 +362,11 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
|
|||||||
"mean_iou",
|
"mean_iou",
|
||||||
"false_positive_count",
|
"false_positive_count",
|
||||||
"false_negative_count",
|
"false_negative_count",
|
||||||
|
# Threshold-independent, as detection QA already reported. The two
|
||||||
|
# panels previously answered different questions about the same run.
|
||||||
|
"average_precision",
|
||||||
|
"best_f1",
|
||||||
|
"best_f1_threshold",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1981,7 +1981,12 @@ compare two models whose calibration differs; the curve can. `average_precision`
|
|||||||
existing ones.
|
existing ones.
|
||||||
|
|
||||||
Segmentation QA (`POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference`)
|
Segmentation QA (`POST /api/v1/segmentation/runs/{analysis_run_id}/qa/reference`)
|
||||||
applies the same tile-coverage clipping and returns the same `coverage` block.
|
applies the same tile-coverage clipping and returns the same `coverage` block,
|
||||||
|
and now also the same `precision_recall_curve`, `calibration_thresholds` /
|
||||||
|
`calibration_sweep` and average-precision metrics. Segmentation confidences rank
|
||||||
|
the same way detection confidences do, so the same threshold-independent view
|
||||||
|
applies; without it the two QA panels answered different questions about
|
||||||
|
comparable runs.
|
||||||
Without it, every reference feature outside the inferred tiles counted as a
|
Without it, every reference feature outside the inferred tiles counted as a
|
||||||
false negative and recall was understated by an arbitrary amount.
|
false negative and recall was understated by an arbitrary amount.
|
||||||
|
|
||||||
|
|||||||
@@ -1620,6 +1620,8 @@ export interface SegmentationQaRequest {
|
|||||||
iou_threshold: number
|
iou_threshold: number
|
||||||
class_name?: string | null
|
class_name?: string | null
|
||||||
min_confidence?: number | null
|
min_confidence?: number | null
|
||||||
|
/** Read off the one matching pass, exactly as for detection. */
|
||||||
|
calibration_thresholds?: number[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SegmentationQaResult = DetectionQaResult
|
export type SegmentationQaResult = DetectionQaResult
|
||||||
|
|||||||
Reference in New Issue
Block a user