Add detection threshold calibration comparison

2026-07-08 12:33:07 +02:00
parent 46204edbf4
commit 0e182ad69d
7 changed files with 333 additions and 1 deletions
@@ -950,6 +950,7 @@ function App(): JSX.Element {
                detectionRunResult={detectionRunResult}
                detectionRunError={detectionRunError}
                detectionRuns={detectionRuns}
+                qualityChecks={qualityChecks}
                selectedDetectionRunId={selectedDetectionRunId}
                detectionItems={detectionItems}
                detectionClassFilter={detectionClassFilter}
@@ -6,9 +6,26 @@ import type {
  DetectionRunRead,
  DetectionRunResponse,
  ModelAssetRead,
+  QualityCheckRead,
  YoloPreflightResponse,
 } from '../../types'

+interface CalibrationRow {
+  analysisRunId: string
+  qualityCheckId: string
+  threshold: number | null
+  modelName: string
+  modelAssetId: string | null
+  detectionCount: number | null
+  precision: number | null
+  recall: number | null
+  f1: number | null
+  falsePositives: number | null
+  falseNegatives: number | null
+  score: number | null
+  createdAt: string | null
+}
+
 interface DetectionLabProps {
  detectionModels: DetectionModelCapability[]
  modelAssets: ModelAssetRead[]
@@ -24,6 +41,7 @@ interface DetectionLabProps {
  detectionRunResult: DetectionRunResponse | null
  detectionRunError: string | null
  detectionRuns: DetectionRunRead[]
+  qualityChecks: QualityCheckRead[]
  selectedDetectionRunId: string
  detectionItems: DetectionRead[]
  detectionClassFilter: string
@@ -71,6 +89,7 @@ export function DetectionLab({
  detectionRunResult,
  detectionRunError,
  detectionRuns,
+  qualityChecks,
  selectedDetectionRunId,
  detectionItems,
  detectionClassFilter,
@@ -114,6 +133,10 @@ export function DetectionLab({
  const benchmarkCandidateAsset = modelAssets.find(
    (asset) => asset.model_asset_id === 'geointel-building-yolov8s-hardneg160r4e50-pt',
  )
+  const calibrationRows = buildCalibrationRows(detectionRuns, qualityChecks)
+  const bestF1Candidate = bestCalibrationRow(calibrationRows, 'f1')
+  const bestPrecisionCandidate = bestCalibrationRow(calibrationRows, 'precision')
+  const lowestFalsePositivePressureCandidate = bestLowestCalibrationRow(calibrationRows, 'falsePositives')
  const detectionHasTileManifest =
    !detectionRequiresTileManifest || detectionTileManifestPath.trim().length > 0
  const detectionRunReady =
@@ -562,6 +585,69 @@ export function DetectionLab({
        ) : null}
      </div>

+      <div className="ai-lab-results-surface calibration-comparison-surface" aria-label="Detection calibration comparison">
+        <div className="panel-title-row">
+          <div>
+            <h3>Calibration comparison</h3>
+            <p className="muted">Compare persisted detection runs by confidence threshold before promoting a model setting.</p>
+          </div>
+          <span className="count-pill">{calibrationRows.length} rows</span>
+        </div>
+        {calibrationRows.length > 0 ? (
+          <>
+            <div className="calibration-summary-grid" aria-label="Calibration comparison winners">
+              <CalibrationSummaryCard title="Best F1 candidate" row={bestF1Candidate} metric="f1" />
+              <CalibrationSummaryCard title="Best precision candidate" row={bestPrecisionCandidate} metric="precision" />
+              <CalibrationSummaryCard title="Lowest false-positive pressure" row={lowestFalsePositivePressureCandidate} metric="falsePositives" />
+            </div>
+            <div className="table-scroll">
+              <table>
+                <thead>
+                  <tr>
+                    <th>Threshold</th>
+                    <th>Model</th>
+                    <th>Detections</th>
+                    <th>Precision</th>
+                    <th>Recall</th>
+                    <th>F1</th>
+                    <th>False positives</th>
+                    <th>False negatives</th>
+                    <th>Quality check</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {calibrationRows.map((row) => (
+                    <tr key={`${row.analysisRunId}-${row.qualityCheckId}`}>
+                      <td>{formatNullableNumber(row.threshold, 2)}</td>
+                      <td>
+                        <strong>{row.modelName}</strong>
+                        <span className="table-subtle">{row.modelAssetId ?? 'runtime configured path'}</span>
+                      </td>
+                      <td>{row.detectionCount ?? 'n/a'}</td>
+                      <td>{formatNullableNumber(row.precision, 3)}</td>
+                      <td>{formatNullableNumber(row.recall, 3)}</td>
+                      <td>{formatNullableNumber(row.f1, 3)}</td>
+                      <td>{row.falsePositives ?? 'n/a'}</td>
+                      <td>{row.falseNegatives ?? 'n/a'}</td>
+                      <td>{row.qualityCheckId}</td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            </div>
+            <div className="lab-action-guardrail">
+              <span>Promotion guardrail</span>
+              <strong>Promote only after checking evidence across AOIs, false positives and false negatives.</strong>
+            </div>
+          </>
+        ) : (
+          <div className="result-state result-state-empty">
+            <strong>No calibration comparison available yet.</strong>
+            <p>Run configured YOLO at multiple confidence thresholds, then compare each persisted detection run against the same reference dataset.</p>
+          </div>
+        )}
+      </div>
+
      <div className="ai-lab-qa-surface" aria-label="Detection QA controls and results">
        <h3>Detection QA</h3>
        <label>
@@ -601,6 +687,33 @@ export function DetectionLab({
  )
 }

+function CalibrationSummaryCard({
+  title,
+  row,
+  metric,
+}: {
+  title: string
+  row: CalibrationRow | null
+  metric: 'f1' | 'precision' | 'falsePositives'
+}): JSX.Element {
+  return (
+    <div className={row ? 'calibration-summary-card calibration-summary-card-ready' : 'calibration-summary-card'}>
+      <span>{title}</span>
+      {row ? (
+        <>
+          <strong>{metric === 'falsePositives' ? row.falsePositives ?? 'n/a' : formatNullableNumber(row[metric], 3)}</strong>
+          <p>Threshold {formatNullableNumber(row.threshold, 2)} · {row.modelName}</p>
+        </>
+      ) : (
+        <>
+          <strong>n/a</strong>
+          <p>Persisted QA metrics are required.</p>
+        </>
+      )}
+    </div>
+  )
+}
+
 function formatModelAssetSize(sizeBytes: number): string {
  if (sizeBytes >= 1024 * 1024) {
    return `${(sizeBytes / (1024 * 1024)).toFixed(1)} MB`
@@ -610,3 +723,100 @@ function formatModelAssetSize(sizeBytes: number): string {
  }
  return `${sizeBytes} B`
 }
+
+function buildCalibrationRows(detectionRuns: DetectionRunRead[], qualityChecks: QualityCheckRead[]): CalibrationRow[] {
+  const runById = new Map(detectionRuns.map((run) => [run.id, run]))
+  return qualityChecks
+    .flatMap((check) => {
+      const run = check.analysis_run_id ? runById.get(check.analysis_run_id) : null
+      if (!run || run.analysis_type !== 'detection') {
+        return []
+      }
+      const threshold = confidenceThresholdForRun(run)
+      return [{
+        analysisRunId: run.id,
+        qualityCheckId: check.id,
+        threshold,
+        modelName: run.model_name ?? 'configured detection',
+        modelAssetId: stringFromRecord(run.parameters_json, 'model_asset_id'),
+        detectionCount: numberFromRecord(run.result_json, 'detection_count'),
+        precision: metricValue(check, 'precision'),
+        recall: metricValue(check, 'recall'),
+        f1: metricValue(check, 'f1'),
+        falsePositives: metricValue(check, 'false_positives'),
+        falseNegatives: metricValue(check, 'false_negatives'),
+        score: check.score ?? null,
+        createdAt: check.completed_at ?? check.created_at ?? run.finished_at ?? run.created_at ?? null,
+      }]
+    })
+    .sort((left, right) => {
+      const createdDiff = Date.parse(right.createdAt ?? '') - Date.parse(left.createdAt ?? '')
+      if (Number.isFinite(createdDiff) && createdDiff !== 0) {
+        return createdDiff
+      }
+      return (right.threshold ?? -1) - (left.threshold ?? -1)
+    })
+}
+
+function confidenceThresholdForRun(run: DetectionRunRead): number | null {
+  return (
+    numberFromRecord(run.parameters_json, 'confidence_threshold') ??
+    numberFromRecord(run.result_json, 'confidence_threshold') ??
+    null
+  )
+}
+
+function metricValue(check: QualityCheckRead, key: string): number | null {
+  const aliases = key === 'f1' ? ['f1', 'f1_score'] : [key]
+  for (const alias of aliases) {
+    const metric = check.metrics.find((item) => item.metric_key === alias)
+    if (typeof metric?.metric_value === 'number' && Number.isFinite(metric.metric_value)) {
+      return metric.metric_value
+    }
+    const findingValue = numberFromRecord(check.findings_json, alias)
+    if (findingValue !== null) {
+      return findingValue
+    }
+  }
+  return key === 'f1' && typeof check.score === 'number' ? check.score : null
+}
+
+function bestCalibrationRow(rows: CalibrationRow[], metric: 'f1' | 'precision'): CalibrationRow | null {
+  return rows.reduce<CalibrationRow | null>((best, row) => {
+    const value = row[metric]
+    if (value === null) {
+      return best
+    }
+    if (!best || value > (best[metric] ?? Number.NEGATIVE_INFINITY)) {
+      return row
+    }
+    return best
+  }, null)
+}
+
+function bestLowestCalibrationRow(rows: CalibrationRow[], metric: 'falsePositives'): CalibrationRow | null {
+  return rows.reduce<CalibrationRow | null>((best, row) => {
+    const value = row[metric]
+    if (value === null) {
+      return best
+    }
+    if (!best || value < (best[metric] ?? Number.POSITIVE_INFINITY)) {
+      return row
+    }
+    return best
+  }, null)
+}
+
+function numberFromRecord(record: Record<string, unknown> | null | undefined, key: string): number | null {
+  const value = record?.[key]
+  return typeof value === 'number' && Number.isFinite(value) ? value : null
+}
+
+function stringFromRecord(record: Record<string, unknown> | null | undefined, key: string): string | null {
+  const value = record?.[key]
+  return typeof value === 'string' && value.trim().length > 0 ? value : null
+}
+
+function formatNullableNumber(value: number | null, digits: number): string {
+  return typeof value === 'number' && Number.isFinite(value) ? value.toFixed(digits) : 'n/a'
+}
@@ -3289,6 +3289,58 @@ button.entity-card {
  background: #ffffff;
 }

+.calibration-comparison-surface {
+  border-color: #d6dfda;
+  background: #fbfdfb;
+}
+
+.calibration-summary-grid {
+  display: grid;
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  gap: 0.62rem;
+  min-width: 0;
+}
+
+.calibration-summary-card {
+  display: grid;
+  gap: 0.18rem;
+  min-width: 0;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  padding: 0.64rem;
+  background: #ffffff;
+}
+
+.calibration-summary-card-ready {
+  border-color: #b8d8c5;
+  background: #f7fcf8;
+}
+
+.calibration-summary-card span {
+  color: var(--muted);
+  font-size: 0.76rem;
+  font-weight: 800;
+  text-transform: uppercase;
+}
+
+.calibration-summary-card strong {
+  color: var(--ink);
+  font-size: 1.25rem;
+}
+
+.calibration-summary-card p,
+.table-subtle {
+  margin: 0;
+  color: var(--muted);
+  font-size: 0.78rem;
+  line-height: 1.35;
+}
+
+.table-subtle {
+  display: block;
+  overflow-wrap: anywhere;
+}
+
 .ai-lab-section-header {
  display: flex;
  min-width: 0;