JensandClaude Opus 5
2cd2c49389
Threshold calibration ran the model over every tile once per threshold — three GPU passes to compare 0.50, 0.25 and 0.15 on a hundred-tile raster. The answer is already in a single run at the lowest value: detections above a higher cut are a subset of it, and duplicate suppression walks candidates in descending confidence, so a lower-confidence box can never displace a higher-confidence one. The kept set above any cut is identical whichever threshold the run used, which is what makes one pass sufficient rather than merely cheaper. QA now takes calibration_thresholds and reads each operating point off the same precision/recall walk it already performs, marking the F1-optimal cut. The lab runs inference once and fills its table from the sweep. The contract test asserted the per-threshold loop by name, pinning the waste it was meant to describe. It now states what calibration owes an operator: a row per requested threshold, from one run. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
129 lines
4.2 KiB
Python
129 lines
4.2 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from typing import Any
|
|
from uuid import UUID
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from app.schemas.common import GeoJsonFeatureCollection
|
|
|
|
|
|
class QaProviderComparisonRequest(BaseModel):
|
|
candidate_dataset_id: UUID
|
|
reference_dataset_id: UUID
|
|
iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
|
area_id: UUID | None = None
|
|
|
|
|
|
class QaProviderComparisonResult(BaseModel):
|
|
status: str
|
|
warnings: list[str] = Field(default_factory=list)
|
|
# Counts of the population that was actually matched, so that
|
|
# ``matches + false_positives == candidate_feature_count`` holds even when
|
|
# an area filter or an unparseable geometry removed features. The ``_raw``
|
|
# fields keep the untouched dataset totals visible next to them.
|
|
candidate_feature_count: int
|
|
reference_feature_count: int
|
|
candidate_feature_count_raw: int | None = None
|
|
reference_feature_count_raw: int | None = None
|
|
matches: int
|
|
false_positives: int
|
|
false_negatives: int
|
|
precision: float | None
|
|
recall: float | None
|
|
f1_score: float | None
|
|
mean_iou: float | None
|
|
iou_threshold: float
|
|
unsupported_geometry: bool = False
|
|
unsupported_geometries: list[str] = Field(default_factory=list)
|
|
match_evidence: list[dict] = Field(default_factory=list)
|
|
false_positive_evidence: list[dict] = Field(default_factory=list)
|
|
false_negative_evidence: list[dict] = Field(default_factory=list)
|
|
generated_at: datetime
|
|
|
|
|
|
class MetricRead(BaseModel):
|
|
id: UUID
|
|
quality_check_id: UUID | None = None
|
|
analysis_run_id: UUID | None = None
|
|
metric_key: str
|
|
metric_value: float | None = None
|
|
metric_unit: str | None = None
|
|
label: str | None = None
|
|
metadata_json: dict | None = None
|
|
created_at: datetime | None = None
|
|
|
|
model_config = {"from_attributes": True}
|
|
|
|
|
|
class QualityCheckRead(BaseModel):
|
|
id: UUID
|
|
project_id: UUID
|
|
job_id: UUID | None = None
|
|
analysis_run_id: UUID | None = None
|
|
candidate_dataset_id: UUID | None = None
|
|
reference_dataset_id: UUID
|
|
check_type: str
|
|
status: str
|
|
score: float | None = None
|
|
parameters_json: dict | None = None
|
|
findings_json: dict | None = None
|
|
created_at: datetime | None = None
|
|
completed_at: datetime | None = None
|
|
metrics: list[MetricRead] = Field(default_factory=list)
|
|
|
|
model_config = {"from_attributes": True}
|
|
|
|
|
|
class QualityCheckList(BaseModel):
|
|
items: list[QualityCheckRead]
|
|
total: int
|
|
limit: int
|
|
offset: int
|
|
|
|
|
|
class QualityEvidenceResponse(BaseModel):
|
|
quality_check_id: UUID
|
|
project_id: UUID
|
|
candidate_dataset_id: UUID | None = None
|
|
reference_dataset_id: UUID
|
|
analysis_run_id: UUID | None = None
|
|
# The overlay is capped so a regional check stays reviewable; the counts in
|
|
# the quality check itself are always complete.
|
|
feature_count: int
|
|
total_feature_count: int | None = None
|
|
role_counts: dict[str, int] = Field(default_factory=dict)
|
|
truncated: bool = False
|
|
limit: int | None = None
|
|
warnings: list[str] = Field(default_factory=list)
|
|
geojson: GeoJsonFeatureCollection
|
|
|
|
|
|
class AnalysisQaResponse(BaseModel):
|
|
status: str
|
|
quality_check_id: UUID
|
|
analysis_run_id: UUID
|
|
reference_dataset_id: UUID
|
|
candidate_feature_count: int
|
|
reference_feature_count: int
|
|
candidate_feature_count_raw: int | None = None
|
|
reference_feature_count_raw: int | None = None
|
|
matches: int
|
|
false_positives: int
|
|
false_negatives: int
|
|
precision: float | None = None
|
|
recall: float | None = None
|
|
f1_score: float | None = None
|
|
mean_iou: float | None = None
|
|
iou_threshold: float
|
|
warnings: list[str] = Field(default_factory=list)
|
|
coverage: dict[str, Any] | None = None
|
|
temporal_compatibility: dict[str, Any] | None = None
|
|
box_to_footprint_diagnostics: dict[str, Any] | None = None
|
|
precision_recall_curve: dict[str, Any] | None = None
|
|
calibration_sweep: list[dict[str, Any]] = Field(default_factory=list)
|
|
match_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
|
false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
|
|
false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)
|