Add guided detection calibration runner

2026-07-08 13:11:02 +02:00
parent 0e182ad69d
commit c0bebd609b
8 changed files with 358 additions and 0 deletions
@@ -249,12 +249,17 @@ function App(): JSX.Element {
    yoloPreflight,
    loadingYoloPreflight,
    yoloPreflightError,
+    calibrationThresholdText,
+    runningDetectionCalibration,
+    detectionCalibrationRows,
+    detectionCalibrationError,
    loadDetectionModels,
    loadYoloPreflight,
    loadDetectionRuns,
    loadDetectionResults,
    runDetection,
    runDetectionQa,
+    runDetectionCalibration,
    resetDetectionForProject,
    setSelectedDetectionDatasetId,
    setSelectedDetectionModelId,
@@ -265,6 +270,7 @@ function App(): JSX.Element {
    setDetectionClassFilter,
    setDetectionMinConfidenceFilter,
    setDetectionReferenceDatasetId,
+    setCalibrationThresholdText,
  } = useDetectionWorkflow({
    selectedProjectId,
    rasterDatasets,
@@ -951,6 +957,10 @@ function App(): JSX.Element {
                detectionRunError={detectionRunError}
                detectionRuns={detectionRuns}
                qualityChecks={qualityChecks}
+                calibrationThresholdText={calibrationThresholdText}
+                runningDetectionCalibration={runningDetectionCalibration}
+                detectionCalibrationRows={detectionCalibrationRows}
+                detectionCalibrationError={detectionCalibrationError}
                selectedDetectionRunId={selectedDetectionRunId}
                detectionItems={detectionItems}
                detectionClassFilter={detectionClassFilter}
@@ -981,6 +991,8 @@ function App(): JSX.Element {
                onLoadResults={() => loadDetectionResults()}
                onSelectReferenceDataset={setDetectionReferenceDatasetId}
                onRunQa={runDetectionQa}
+                onSetCalibrationThresholdText={setCalibrationThresholdText}
+                onRunCalibration={runDetectionCalibration}
              />

              <SegmentationLab
@@ -9,6 +9,7 @@ import type {
  QualityCheckRead,
  YoloPreflightResponse,
 } from '../../types'
+import type { DetectionCalibrationRunRow } from '../../hooks/useDetectionWorkflow'

 interface CalibrationRow {
  analysisRunId: string
@@ -42,6 +43,10 @@ interface DetectionLabProps {
  detectionRunError: string | null
  detectionRuns: DetectionRunRead[]
  qualityChecks: QualityCheckRead[]
+  calibrationThresholdText: string
+  runningDetectionCalibration: boolean
+  detectionCalibrationRows: DetectionCalibrationRunRow[]
+  detectionCalibrationError: string | null
  selectedDetectionRunId: string
  detectionItems: DetectionRead[]
  detectionClassFilter: string
@@ -72,6 +77,8 @@ interface DetectionLabProps {
  onLoadResults: () => void
  onSelectReferenceDataset: (datasetId: string) => void
  onRunQa: () => void
+  onSetCalibrationThresholdText: (value: string) => void
+  onRunCalibration: () => void
 }

 export function DetectionLab({
@@ -90,6 +97,10 @@ export function DetectionLab({
  detectionRunError,
  detectionRuns,
  qualityChecks,
+  calibrationThresholdText,
+  runningDetectionCalibration,
+  detectionCalibrationRows,
+  detectionCalibrationError,
  selectedDetectionRunId,
  detectionItems,
  detectionClassFilter,
@@ -120,6 +131,8 @@ export function DetectionLab({
  onLoadResults,
  onSelectReferenceDataset,
  onRunQa,
+  onSetCalibrationThresholdText,
+  onRunCalibration,
 }: DetectionLabProps): JSX.Element {
  const selectedDetectionModel = detectionModels.find((model) => model.model_id === selectedDetectionModelId) ?? null
  const selectedModelAsset = modelAssets.find((asset) => asset.model_asset_id === selectedModelAssetId) ?? null
@@ -161,6 +174,7 @@ export function DetectionLab({
            : !detectionHasTileManifest
              ? 'Provide a raster tile manifest for configured YOLO'
              : null
+  const calibrationRunReady = detectionRunReady && detectionReferenceDatasetId.length > 0 && calibrationThresholdText.trim().length > 0

  return (
    <section className="workspace-panel ai-lab-shell detection-lab-shell">
@@ -501,6 +515,101 @@ export function DetectionLab({
        ) : null}
      </div>

+      <div className="ai-lab-run-surface guided-calibration-surface" aria-label="Guided calibration runner">
+        <div className="ai-lab-section-header">
+          <div>
+            <h3>Guided calibration runner</h3>
+            <p>This runs real configured YOLO jobs and QA comparisons for each threshold. It does not promote or mutate model files.</p>
+          </div>
+          <span className={calibrationRunReady ? 'status-badge status-badge-ready' : 'status-badge'}>
+            {calibrationRunReady ? 'ready' : 'needs dataset, model, manifest and reference'}
+          </span>
+        </div>
+        <div className="lab-form-grid">
+          <label>
+            Threshold set
+            <input
+              type="text"
+              value={calibrationThresholdText}
+              onChange={(event) => onSetCalibrationThresholdText(event.target.value)}
+              placeholder="0.50 0.25 0.15"
+            />
+            <span className="field-guidance">Use spaces, commas or semicolons. Values must be between 0 and 1.</span>
+          </label>
+          <label>
+            Reference dataset
+            <select value={detectionReferenceDatasetId} onChange={(event) => onSelectReferenceDataset(event.target.value)}>
+              <option value="">Select reference dataset</option>
+              {referenceDatasets.map((dataset) => (
+                <option key={dataset.id} value={dataset.id}>
+                  {dataset.name}
+                </option>
+              ))}
+            </select>
+          </label>
+        </div>
+        <button
+          className="primary-action"
+          type="button"
+          onClick={onRunCalibration}
+          disabled={runningDetectionCalibration || !calibrationRunReady}
+        >
+          Run calibration sweep
+        </button>
+        {detectionCalibrationError ? (
+          <div className="result-state result-state-error">
+            <strong>Calibration sweep failed.</strong>
+            <p>{detectionCalibrationError}</p>
+          </div>
+        ) : null}
+        {detectionCalibrationRows.length > 0 ? (
+          <div className="calibration-progress-panel" aria-label="Calibration run progress">
+            <div className="panel-title-row">
+              <div>
+                <h3>Calibration run progress</h3>
+                <p className="muted">Each row is backed by a persisted detection run and QA check when successful.</p>
+              </div>
+              <span className="count-pill">{detectionCalibrationRows.length} thresholds</span>
+            </div>
+            <div className="table-scroll">
+              <table>
+                <thead>
+                  <tr>
+                    <th>Threshold</th>
+                    <th>Status</th>
+                    <th>Detections</th>
+                    <th>Precision</th>
+                    <th>Recall</th>
+                    <th>F1</th>
+                    <th>False positives</th>
+                    <th>False negatives</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {detectionCalibrationRows.map((row) => (
+                    <tr key={row.threshold}>
+                      <td>{row.threshold.toFixed(2)}</td>
+                      <td>{row.status}</td>
+                      <td>{row.detection_count ?? 'n/a'}</td>
+                      <td>{formatNullableNumber(row.precision ?? null, 3)}</td>
+                      <td>{formatNullableNumber(row.recall ?? null, 3)}</td>
+                      <td>{formatNullableNumber(row.f1_score ?? null, 3)}</td>
+                      <td>{row.false_positives ?? 'n/a'}</td>
+                      <td>{row.false_negatives ?? 'n/a'}</td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            </div>
+          </div>
+        ) : (
+          <div className="result-state result-state-empty">
+            <strong>No calibration sweep has been run in this session.</strong>
+            <p>Choose a reference dataset and threshold set, then start the explicit sweep.</p>
+          </div>
+        )}
+      </div>
+
      <div className="ai-lab-results-surface" aria-label="Detection results">
        <div className="panel-title-row">
          <div>
@@ -21,6 +21,39 @@ interface DetectionWorkflowOptions {
  loadQualityChecks: (projectId?: string | null) => Promise<QualityCheckRead[] | void>
 }

+export interface DetectionCalibrationRunRow {
+  threshold: number
+  status: 'queued' | 'running' | 'success' | 'failed'
+  analysis_run_id?: string | null
+  job_id?: string | null
+  quality_check_id?: string | null
+  detection_count?: number | null
+  precision?: number | null
+  recall?: number | null
+  f1_score?: number | null
+  false_positives?: number | null
+  false_negatives?: number | null
+  message?: string | null
+}
+
+function parseCalibrationThresholds(value: string): number[] {
+  const tokens = value
+    .split(/[\s,;]+/)
+    .map((token) => token.trim())
+    .filter(Boolean)
+  const thresholds: number[] = []
+  for (const token of tokens) {
+    const threshold = Number(token)
+    if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
+      return []
+    }
+    if (!thresholds.includes(threshold)) {
+      thresholds.push(threshold)
+    }
+  }
+  return thresholds
+}
+
 export function useDetectionWorkflow({
  selectedProjectId,
  rasterDatasets,
@@ -55,6 +88,10 @@ export function useDetectionWorkflow({
  const [yoloPreflight, setYoloPreflight] = useState<YoloPreflightResponse | null>(null)
  const [loadingYoloPreflight, setLoadingYoloPreflight] = useState(false)
  const [yoloPreflightError, setYoloPreflightError] = useState<string | null>(null)
+  const [calibrationThresholdText, setCalibrationThresholdText] = useState('0.50 0.25 0.15')
+  const [runningDetectionCalibration, setRunningDetectionCalibration] = useState(false)
+  const [detectionCalibrationRows, setDetectionCalibrationRows] = useState<DetectionCalibrationRunRow[]>([])
+  const [detectionCalibrationError, setDetectionCalibrationError] = useState<string | null>(null)

  const loadDetectionModels = async () => {
    setLoadingDetectionModels(true)
@@ -204,6 +241,98 @@ export function useDetectionWorkflow({
    }
  }

+  const runDetectionCalibration = async () => {
+    if (!selectedProjectId) {
+      setDetectionCalibrationError('Select a project before calibration')
+      return
+    }
+    const datasetId = selectedDetectionDatasetId || rasterDatasets[0]?.id
+    if (!datasetId) {
+      setDetectionCalibrationError('Select a raster dataset before calibration')
+      return
+    }
+    if (!detectionReferenceDatasetId) {
+      setDetectionCalibrationError('Select a reference dataset before calibration')
+      return
+    }
+    const selectedModel = detectionModels.find((model) => model.model_id === selectedDetectionModelId)
+    if (!selectedModel?.configured || selectedDetectionModelId === 'manual-fixture-detector') {
+      setDetectionCalibrationError('Select a configured non-fixture detection model before calibration')
+      return
+    }
+    if (selectedDetectionModelId === 'yolo-configured' && !detectionTileManifestPath.trim()) {
+      setDetectionCalibrationError('Configured YOLO calibration requires a tile manifest')
+      return
+    }
+    if (selectedDetectionModelId === 'yolo-configured' && modelAssets.length > 0 && !selectedModelAssetId) {
+      setDetectionCalibrationError('Select a local model asset before calibration')
+      return
+    }
+    const thresholds = parseCalibrationThresholds(calibrationThresholdText)
+    if (thresholds.length === 0) {
+      setDetectionCalibrationError('Provide at least one valid threshold between 0 and 1')
+      return
+    }
+    setDetectionCalibrationError(null)
+    setDetectionCalibrationRows(thresholds.map((threshold) => ({ threshold, status: 'queued' })))
+    setRunningDetectionCalibration(true)
+    try {
+      for (const threshold of thresholds) {
+        setDetectionCalibrationRows((rows) =>
+          rows.map((row) => row.threshold === threshold ? { ...row, status: 'running', message: 'Running detection' } : row),
+        )
+        try {
+          const result = await detectionApi.run({
+            project_id: selectedProjectId,
+            dataset_id: datasetId,
+            model_id: selectedDetectionModelId,
+            model_asset_id: selectedModelAssetId || null,
+            confidence_threshold: threshold,
+            tile_manifest_path: detectionTileManifestPath.trim() || null,
+            parameters_json: { calibration: true, calibration_thresholds: thresholds },
+          })
+          setSelectedDetectionRunId(result.analysis_run_id)
+          const qa = await detectionApi.compareWithReference(result.analysis_run_id, {
+            reference_dataset_id: detectionReferenceDatasetId,
+            iou_threshold: qaIouThreshold,
+            class_name: detectionClassFilter || null,
+            min_confidence: null,
+          })
+          setDetectionCalibrationRows((rows) =>
+            rows.map((row) => row.threshold === threshold
+              ? {
+                  ...row,
+                  status: 'success',
+                  analysis_run_id: result.analysis_run_id,
+                  job_id: result.job_id,
+                  quality_check_id: qa.quality_check_id,
+                  detection_count: result.detection_count,
+                  precision: qa.precision ?? null,
+                  recall: qa.recall ?? null,
+                  f1_score: qa.f1_score ?? null,
+                  false_positives: qa.false_positives,
+                  false_negatives: qa.false_negatives,
+                  message: result.message,
+                }
+              : row),
+          )
+        } catch (error) {
+          const message = formatError(error, `Calibration threshold ${threshold} failed`)
+          setDetectionCalibrationRows((rows) =>
+            rows.map((row) => row.threshold === threshold ? { ...row, status: 'failed', message } : row),
+          )
+          setDetectionCalibrationError(message)
+          break
+        }
+      }
+      await loadDetectionRuns(selectedProjectId)
+      await loadQualityChecks(selectedProjectId)
+      await loadProjectData(selectedProjectId)
+    } finally {
+      setRunningDetectionCalibration(false)
+    }
+  }
+
  const resetDetectionForProject = () => {
    setSelectedDetectionDatasetId('')
    setDetectionRuns([])
@@ -211,6 +340,8 @@ export function useDetectionWorkflow({
    setDetectionItems([])
    setDetectionGeoJson(null)
    setDetectionRunResult(null)
+    setDetectionCalibrationRows([])
+    setDetectionCalibrationError(null)
  }

  return {
@@ -241,12 +372,17 @@ export function useDetectionWorkflow({
    yoloPreflight,
    loadingYoloPreflight,
    yoloPreflightError,
+    calibrationThresholdText,
+    runningDetectionCalibration,
+    detectionCalibrationRows,
+    detectionCalibrationError,
    loadDetectionModels,
    loadYoloPreflight,
    loadDetectionRuns,
    loadDetectionResults,
    runDetection,
    runDetectionQa,
+    runDetectionCalibration,
    resetDetectionForProject,
    setSelectedDetectionDatasetId,
    setSelectedDetectionModelId,
@@ -257,5 +393,6 @@ export function useDetectionWorkflow({
    setDetectionClassFilter,
    setDetectionMinConfidenceFilter,
    setDetectionReferenceDatasetId,
+    setCalibrationThresholdText,
  }
 }
@@ -3294,6 +3294,21 @@ button.entity-card {
  background: #fbfdfb;
 }

+.guided-calibration-surface {
+  border-color: #d9e4dd;
+  background: linear-gradient(180deg, #ffffff, #f8fcf9);
+}
+
+.calibration-progress-panel {
+  display: grid;
+  gap: 0.55rem;
+  min-width: 0;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  padding: 0.68rem;
+  background: #ffffff;
+}
+
 .calibration-summary-grid {
  display: grid;
  grid-template-columns: repeat(3, minmax(0, 1fr));