JensandClaude Opus 5
2b968b74cf
The greedy IoU matcher gave a reference to whichever candidate was offered first. Row order decided that, and every detection in a run shares one transaction timestamp, so ordering by created_at left the assignment undefined: the same QA run over the same data produced different mean IoU, and the geometry shown to a reviewer as a false positive could be the better of two detections. Candidates are now ranked by confidence with feature identity as tiebreaker, which is also the COCO/PASCAL rule. A single precision/recall/F1 triple describes one operating point, so two models cannot be compared from it: a conservatively calibrated model looks worse at a low confidence cut and better at a high one without detecting anything differently. DetectionMetricsService adds the full curve, average precision and the threshold where F1 actually peaks. Also: - report the population the metrics were computed over, so matches + false_positives equals candidate_feature_count even under an area filter; raw dataset totals move to the _raw fields; - state whether candidates are axis-aligned boxes or footprint polygons. A box can never reach IoU 1 against a rotated building, so the strict score has a ceiling that has nothing to do with detection quality. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
121 lines
3.8 KiB
Python
121 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from typing import Any
|
|
from uuid import UUID
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from app.schemas.common import GeoJsonFeatureCollection
|
|
|
|
|
|
class QaProviderComparisonRequest(BaseModel):
|
|
candidate_dataset_id: UUID
|
|
reference_dataset_id: UUID
|
|
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
|
area_id: UUID | None = None
|
|
|
|
|
|
class QaProviderComparisonResult(BaseModel):
|
|
status: str
|
|
warnings: list[str] = Field(default_factory=list)
|
|
# Counts of the population that was actually matched, so that
|
|
# ``matches + false_positives == candidate_feature_count`` holds even when
|
|
# an area filter or an unparseable geometry removed features. The ``_raw``
|
|
# fields keep the untouched dataset totals visible next to them.
|
|
candidate_feature_count: int
|
|
reference_feature_count: int
|
|
candidate_feature_count_raw: int | None = None
|
|
reference_feature_count_raw: int | None = None
|
|
matches: int
|
|
false_positives: int
|
|
false_negatives: int
|
|
precision: float | None
|
|
recall: float | None
|
|
f1_score: float | None
|
|
mean_iou: float | None
|
|
iou_threshold: float
|
|
unsupported_geometry: bool = False
|
|
unsupported_geometries: list[str] = Field(default_factory=list)
|
|
match_evidence: list[dict] = Field(default_factory=list)
|
|
false_positive_evidence: list[dict] = Field(default_factory=list)
|
|
false_negative_evidence: list[dict] = Field(default_factory=list)
|
|
generated_at: datetime
|
|
|
|
|
|
class MetricRead(BaseModel):
|
|
id: UUID
|
|
quality_check_id: UUID | None = None
|
|
analysis_run_id: UUID | None = None
|
|
metric_key: str
|
|
metric_value: float | None = None
|
|
metric_unit: str | None = None
|
|
label: str | None = None
|
|
metadata_json: dict | None = None
|
|
created_at: datetime | None = None
|
|
|
|
model_config = {"from_attributes": True}
|
|
|
|
|
|
class QualityCheckRead(BaseModel):
|
|
id: UUID
|
|
project_id: UUID
|
|
job_id: UUID | None = None
|
|
analysis_run_id: UUID | None = None
|
|
candidate_dataset_id: UUID | None = None
|
|
reference_dataset_id: UUID
|
|
check_type: str
|
|
status: str
|
|
score: float | None = None
|
|
parameters_json: dict | None = None
|
|
findings_json: dict | None = None
|
|
created_at: datetime | None = None
|
|
completed_at: datetime | None = None
|
|
metrics: list[MetricRead] = Field(default_factory=list)
|
|
|
|
model_config = {"from_attributes": True}
|
|
|
|
|
|
class QualityCheckList(BaseModel):
|
|
items: list[QualityCheckRead]
|
|
total: int
|
|
limit: int
|
|
offset: int
|
|
|
|
|
|
class QualityEvidenceResponse(BaseModel):
|
|
quality_check_id: UUID
|
|
project_id: UUID
|
|
candidate_dataset_id: UUID | None = None
|
|
reference_dataset_id: UUID
|
|
analysis_run_id: UUID | None = None
|
|
feature_count: int
|
|
warnings: list[str] = Field(default_factory=list)
|
|
geojson: GeoJsonFeatureCollection
|
|
|
|
|
|
class AnalysisQaResponse(BaseModel):
|
|
status: str
|
|
quality_check_id: UUID
|
|
analysis_run_id: UUID
|
|
reference_dataset_id: UUID
|
|
candidate_feature_count: int
|
|
reference_feature_count: int
|
|
candidate_feature_count_raw: int | None = None
|
|
reference_feature_count_raw: int | None = None
|
|
matches: int
|
|
false_positives: int
|
|
false_negatives: int
|
|
precision: float | None = None
|
|
recall: float | None = None
|
|
f1_score: float | None = None
|
|
mean_iou: float | None = None
|
|
iou_threshold: float
|
|
warnings: list[str] = Field(default_factory=list)
|
|
coverage: dict[str, Any] | None = None
|
|
temporal_compatibility: dict[str, Any] | None = None
|
|
box_to_footprint_diagnostics: dict[str, Any] | None = None
|
|
match_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
|
false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
|
false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)
|