JensandClaude Opus 5
f6eced1b94
The workbench ranks model variants by a stored F1, each measured at that variant's own confidence threshold. A conservatively calibrated detector then looks worse than a liberal one without detecting anything differently: the number says as much about the threshold as about the model. POST /detection/runs/compare ranks on average precision instead, which describes the whole ranking a model produced, and keeps each run's own-threshold F1 visible next to it so the difference between the two readings is auditable. Comparability comes before the ranking. Runs over different source rasters, scored against different references, without a proven inference footprint, or covering a different evaluated population are not alternatives to one another, and no metric makes them so. The report names which of those applies and still returns the numbers — they are simply not a ranking. Each run is scored through the same QA path the workbench uses, so a comparison and the persisted quality checks cannot drift apart. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
358 lines
9.9 KiB
Python
358 lines
9.9 KiB
Python
from __future__ import annotations
|
|
|
|
from .common import (
|
|
ApiErrorEnvelope,
|
|
ApiErrorItem,
|
|
Envelope,
|
|
GeoJsonFeature,
|
|
GeoJsonFeatureCollection,
|
|
ItemList,
|
|
PaginationEnvelope,
|
|
)
|
|
from .coverage import (
|
|
CoverageBBox,
|
|
CoverageCatalogResponse,
|
|
CoverageResolutionItem,
|
|
CoverageResolveRequest,
|
|
CoverageResolveResponse,
|
|
CoverageSourceContract,
|
|
)
|
|
from .project import ProjectCreate, ProjectDeleteResult, ProjectList, ProjectRead, ProjectUpdate
|
|
from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
|
|
from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
|
|
from .dataset import DatasetCreateResponse, DatasetList
|
|
from .source_freshness import (
|
|
SourceFreshnessItem,
|
|
SourceFreshnessReport,
|
|
SourceFreshnessSummary,
|
|
SourceIntegritySummary,
|
|
)
|
|
from .source_catalog import (
|
|
SourceCatalogProbeItem,
|
|
SourceCatalogProbeReport,
|
|
SourceCatalogProbeSummary,
|
|
)
|
|
from .source_registry import (
|
|
DatasetProvenanceRead,
|
|
DatasetLineageEdgeRead,
|
|
DatasetQuarantineRead,
|
|
SourceRegistryDetailRead,
|
|
SourceRegistryRead,
|
|
SourceSnapshotRead,
|
|
)
|
|
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
|
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
|
from .official_vector import (
|
|
OfficialVectorAcquireRequest,
|
|
OfficialVectorAcquisitionResult,
|
|
OfficialVectorProductRead,
|
|
)
|
|
from .detection import (
|
|
DetectionListResponse,
|
|
DetectionModelCapability,
|
|
DetectionModelsResponse,
|
|
DetectionQaRequest,
|
|
DetectionRead,
|
|
DetectionRunListResponse,
|
|
DetectionRunRead,
|
|
DetectionRunRequest,
|
|
DetectionComparisonRequest,
|
|
DetectionComparisonResponse,
|
|
DetectionRunResponse,
|
|
ModelAssetListResponse,
|
|
ModelAssetRead,
|
|
YoloPreflightResponse,
|
|
)
|
|
from .detection_review import DetectionReviewList, DetectionReviewRead, DetectionReviewSummary, DetectionReviewUpsert
|
|
from .segmentation import (
|
|
SegmentationListResponse,
|
|
SegmentationModelCapability,
|
|
SegmentationModelsResponse,
|
|
SegmentationQaRequest,
|
|
SegmentationRead,
|
|
SegmentationRunListResponse,
|
|
SegmentationRunRead,
|
|
SegmentationRunRequest,
|
|
SegmentationRunResponse,
|
|
)
|
|
from .health import HealthResponse, SystemCapabilities
|
|
from .job import JobCreate, JobList, JobRead, JobStatus
|
|
from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult, OrthophotoProductRead
|
|
from .dhmv import (
|
|
DhmvAcquireRequest,
|
|
DhmvAcquisitionResult,
|
|
DhmvProductRead,
|
|
TerrainMetric,
|
|
TerrainPartitionSelectionRequest,
|
|
TerrainSelectionRequest,
|
|
TerrainSelectionResponse,
|
|
TerrainSelectionSummary,
|
|
)
|
|
from .spw_terrain import (
|
|
SpwTerrainAcquireRequest,
|
|
SpwTerrainAcquisitionResult,
|
|
SpwTerrainProductRead,
|
|
)
|
|
from .flood_hazard import (
|
|
FloodHazardAcquireRequest,
|
|
FloodHazardAcquisitionResult,
|
|
FloodHazardMetric,
|
|
FloodHazardPartitionSelectionRequest,
|
|
FloodHazardProductRead,
|
|
FloodHazardSelectionRequest,
|
|
FloodHazardSelectionResponse,
|
|
FloodHazardSelectionSummary,
|
|
)
|
|
from .bathymetry import (
|
|
BathymetryPartitionFinalizeRequest,
|
|
BathymetryPartitionFinalizationResult,
|
|
BathymetryProfileAcquireRequest,
|
|
BathymetryProfileAcquisitionResult,
|
|
BathymetryRasterMetric,
|
|
BathymetryRasterSelectionRequest,
|
|
BathymetryRasterSelectionResponse,
|
|
BathymetryRasterSelectionSummary,
|
|
BathymetrySourceProbeRead,
|
|
BathymetrySourceRead,
|
|
MdkBathymetryAcquireRequest,
|
|
MdkBathymetryAcquisitionResult,
|
|
)
|
|
from .thematic_raster import (
|
|
ThematicRasterAcquireRequest,
|
|
ThematicRasterAcquisitionResult,
|
|
ThematicRasterMetric,
|
|
ThematicRasterProductRead,
|
|
ThematicRasterSelectionRequest,
|
|
ThematicRasterSelectionResponse,
|
|
ThematicRasterSelectionSummary,
|
|
)
|
|
from .external import (
|
|
ExternalFetchRequest,
|
|
ExternalFetchResponse,
|
|
ProviderCapabilitiesResponse,
|
|
ProviderCapabilityResponse,
|
|
ProviderImportRequest,
|
|
ProviderImportResponse,
|
|
ProviderLayersResponse,
|
|
ProviderStatusResponse,
|
|
)
|
|
from .export import (
|
|
ExportContentResponse,
|
|
ExportCreateResponse,
|
|
ExportListResponse,
|
|
ExportRead,
|
|
GeoJsonExportRequest,
|
|
MetadataExportRequest,
|
|
ReportExportRequest,
|
|
)
|
|
from .qa import (
|
|
AnalysisQaResponse,
|
|
QaProviderComparisonRequest,
|
|
QaProviderComparisonResult,
|
|
QualityEvidenceResponse,
|
|
)
|
|
from .operations import (
|
|
RasterClipRequest,
|
|
RasterIndexBaseRequest,
|
|
RasterMetadataResponse,
|
|
RasterNdviRequest,
|
|
RasterNdwiRequest,
|
|
RasterNdbiRequest,
|
|
RasterOperationResult,
|
|
RasterPreviewResponse,
|
|
RasterReprojectRequest,
|
|
RasterReprojectResponse,
|
|
RasterStatsResponse,
|
|
RasterTileManifest,
|
|
RasterTileManifestTile,
|
|
RasterTileRequest,
|
|
RasterTileResponse,
|
|
VectorBBoxResponse,
|
|
VectorBufferRequest,
|
|
VectorClipRequest,
|
|
VectorIntersectRequest,
|
|
VectorOperationRequest,
|
|
VectorOperationResult,
|
|
VectorSelectionBBox,
|
|
VectorSelectionDeriveRequest,
|
|
VectorSelectionRequest,
|
|
VectorSelectionResponse,
|
|
VectorSelectionMetric,
|
|
VectorSelectionSummary,
|
|
VectorStatsRequest,
|
|
VectorStatsResponse,
|
|
)
|
|
|
|
__all__ = [
|
|
"Envelope",
|
|
"ItemList",
|
|
"GeoJsonFeature",
|
|
"GeoJsonFeatureCollection",
|
|
"ApiErrorEnvelope",
|
|
"ApiErrorItem",
|
|
"PaginationEnvelope",
|
|
"CoverageBBox",
|
|
"CoverageCatalogResponse",
|
|
"CoverageResolutionItem",
|
|
"CoverageResolveRequest",
|
|
"CoverageResolveResponse",
|
|
"CoverageSourceContract",
|
|
"ProjectCreate",
|
|
"ProjectRead",
|
|
"ProjectUpdate",
|
|
"ProjectList",
|
|
"ProjectDeleteResult",
|
|
"AreaCreate",
|
|
"AreaRead",
|
|
"AreaUpdate",
|
|
"AreaList",
|
|
"ChangeDetectionRequest",
|
|
"ChangeDetectionSummary",
|
|
"DatasetCreateResponse",
|
|
"DatasetList",
|
|
"SourceFreshnessItem",
|
|
"SourceFreshnessReport",
|
|
"SourceFreshnessSummary",
|
|
"SourceIntegritySummary",
|
|
"SourceCatalogProbeItem",
|
|
"SourceCatalogProbeReport",
|
|
"SourceCatalogProbeSummary",
|
|
"SourceRegistryRead",
|
|
"SourceRegistryDetailRead",
|
|
"SourceSnapshotRead",
|
|
"DatasetLineageEdgeRead",
|
|
"DatasetQuarantineRead",
|
|
"DatasetProvenanceRead",
|
|
"GrbRefreshLayerPlan",
|
|
"GrbRefreshPlan",
|
|
"GrbRefreshPlanSummary",
|
|
"GrbAcquireRequest",
|
|
"GrbAcquisitionResult",
|
|
"GrbProductRead",
|
|
"OfficialVectorAcquireRequest",
|
|
"OfficialVectorAcquisitionResult",
|
|
"OfficialVectorProductRead",
|
|
"DetectionListResponse",
|
|
"DetectionModelCapability",
|
|
"DetectionModelsResponse",
|
|
"DetectionQaRequest",
|
|
"DetectionRead",
|
|
"DetectionRunListResponse",
|
|
"DetectionRunRead",
|
|
"DetectionRunRequest",
|
|
"DetectionComparisonRequest",
|
|
"DetectionComparisonResponse",
|
|
"DetectionRunResponse",
|
|
"ModelAssetListResponse",
|
|
"ModelAssetRead",
|
|
"YoloPreflightResponse",
|
|
"DetectionReviewList",
|
|
"DetectionReviewRead",
|
|
"DetectionReviewSummary",
|
|
"DetectionReviewUpsert",
|
|
"SegmentationListResponse",
|
|
"SegmentationModelCapability",
|
|
"SegmentationModelsResponse",
|
|
"SegmentationQaRequest",
|
|
"SegmentationRead",
|
|
"SegmentationRunListResponse",
|
|
"SegmentationRunRead",
|
|
"SegmentationRunRequest",
|
|
"SegmentationRunResponse",
|
|
"HealthResponse",
|
|
"SystemCapabilities",
|
|
"JobCreate",
|
|
"JobList",
|
|
"JobRead",
|
|
"JobStatus",
|
|
"OrthophotoAcquireRequest",
|
|
"OrthophotoAcquisitionResult",
|
|
"OrthophotoProductRead",
|
|
"DhmvAcquireRequest",
|
|
"DhmvAcquisitionResult",
|
|
"DhmvProductRead",
|
|
"SpwTerrainAcquireRequest",
|
|
"SpwTerrainAcquisitionResult",
|
|
"SpwTerrainProductRead",
|
|
"TerrainMetric",
|
|
"TerrainPartitionSelectionRequest",
|
|
"TerrainSelectionRequest",
|
|
"TerrainSelectionResponse",
|
|
"TerrainSelectionSummary",
|
|
"FloodHazardAcquireRequest",
|
|
"FloodHazardAcquisitionResult",
|
|
"FloodHazardMetric",
|
|
"FloodHazardPartitionSelectionRequest",
|
|
"FloodHazardProductRead",
|
|
"FloodHazardSelectionRequest",
|
|
"FloodHazardSelectionResponse",
|
|
"FloodHazardSelectionSummary",
|
|
"BathymetryProfileAcquireRequest",
|
|
"BathymetryProfileAcquisitionResult",
|
|
"BathymetryRasterMetric",
|
|
"BathymetryRasterSelectionRequest",
|
|
"BathymetryRasterSelectionResponse",
|
|
"BathymetryRasterSelectionSummary",
|
|
"BathymetryPartitionFinalizeRequest",
|
|
"BathymetryPartitionFinalizationResult",
|
|
"BathymetrySourceProbeRead",
|
|
"BathymetrySourceRead",
|
|
"MdkBathymetryAcquireRequest",
|
|
"MdkBathymetryAcquisitionResult",
|
|
"ThematicRasterAcquireRequest",
|
|
"ThematicRasterAcquisitionResult",
|
|
"ThematicRasterMetric",
|
|
"ThematicRasterProductRead",
|
|
"ThematicRasterSelectionRequest",
|
|
"ThematicRasterSelectionResponse",
|
|
"ThematicRasterSelectionSummary",
|
|
"VectorBBoxResponse",
|
|
"VectorClipRequest",
|
|
"VectorBufferRequest",
|
|
"VectorIntersectRequest",
|
|
"VectorOperationRequest",
|
|
"VectorOperationResult",
|
|
"VectorSelectionBBox",
|
|
"VectorSelectionDeriveRequest",
|
|
"VectorSelectionRequest",
|
|
"VectorSelectionResponse",
|
|
"VectorSelectionMetric",
|
|
"VectorSelectionSummary",
|
|
"RasterClipRequest",
|
|
"RasterStatsResponse",
|
|
"RasterReprojectRequest",
|
|
"RasterReprojectResponse",
|
|
"RasterTileRequest",
|
|
"RasterMetadataResponse",
|
|
"RasterOperationResult",
|
|
"RasterPreviewResponse",
|
|
"RasterTileManifestTile",
|
|
"RasterTileManifest",
|
|
"RasterTileResponse",
|
|
"RasterIndexBaseRequest",
|
|
"RasterNdviRequest",
|
|
"RasterNdwiRequest",
|
|
"RasterNdbiRequest",
|
|
"VectorStatsRequest",
|
|
"VectorStatsResponse",
|
|
"ExternalFetchRequest",
|
|
"ExternalFetchResponse",
|
|
"ProviderCapabilitiesResponse",
|
|
"ProviderCapabilityResponse",
|
|
"ProviderImportRequest",
|
|
"ProviderImportResponse",
|
|
"ProviderLayersResponse",
|
|
"ProviderStatusResponse",
|
|
"GeoJsonExportRequest",
|
|
"MetadataExportRequest",
|
|
"ReportExportRequest",
|
|
"ExportRead",
|
|
"ExportCreateResponse",
|
|
"ExportListResponse",
|
|
"ExportContentResponse",
|
|
"QaProviderComparisonRequest",
|
|
"QaProviderComparisonResult",
|
|
"AnalysisQaResponse",
|
|
"QualityEvidenceResponse",
|
|
]
|