From 90048ffb4ac019ca01f6127527e41023b35e7465 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 20:19:10 +0200 Subject: [PATCH] Add detection operator profiles --- CHANGELOG.md | 7 ++ ...nt122_model_asset_activation_guardrails.py | 7 +- ...t_sprint155_detection_operator_profiles.py | 46 +++++++++++++ docs/AI_PIPELINES.md | 8 +++ docs/CODEX_EXECUTION_LOG.md | 29 ++++++++ docs/TODO.md | 2 +- frontend/README.md | 1 + frontend/src/App.tsx | 2 + .../src/components/detection/DetectionLab.tsx | 66 ++++++++++++++++--- .../components/detection/detectionProfiles.ts | 47 +++++++++++++ frontend/src/hooks/useDetectionWorkflow.ts | 12 ++++ frontend/src/styles/app.css | 61 +++++++++++++++++ 12 files changed, 275 insertions(+), 13 deletions(-) create mode 100644 backend/tests/test_sprint155_detection_operator_profiles.py create mode 100644 frontend/src/components/detection/detectionProfiles.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ff4a44d3..7d60d890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 155 Detection operator profiles (2026-07-09) + +- Added explicit Detection Lab operator profiles for the inactive `geointel-building-yolov8s-aoi1024bg512r3e50-pt` local model asset. +- Added a balanced review profile at confidence threshold `0.15` and a conservative review profile at `0.35`, with persisted gate metrics shown in the UI. +- Kept both profiles clearly marked as candidate-only and not default-approved because the promotion recommendation remains `none` and background false-positive pressure still blocks automatic activation. +- No model download behavior, API contract, migration, provider fetching, fake detection output or active runtime default changed. + ## Sprint 154 Background-aware AOI1024 YOLOv8s candidate gate (2026-07-09) - Exported and audited background-aware AOI1024 training dataset `/app/storage/operator-data/yolo-building-aoi1024-bgaware512r3`; the audit passed with 162 tiles, 117 positive tiles, 45 negative tiles, 21,530 labels and no warnings. diff --git a/backend/tests/test_sprint122_model_asset_activation_guardrails.py b/backend/tests/test_sprint122_model_asset_activation_guardrails.py index 96b9b0d0..1a08283d 100644 --- a/backend/tests/test_sprint122_model_asset_activation_guardrails.py +++ b/backend/tests/test_sprint122_model_asset_activation_guardrails.py @@ -20,10 +20,9 @@ def test_detection_lab_explains_explicit_model_asset_and_threshold_selection() - assert "Explicit model asset" in lab assert "No model file is selected automatically" in lab - assert "Current benchmark candidate" in lab - assert "geointel-building-yolov8s-hardneg160r4e50-pt" in lab - assert "Recommended starting threshold" in lab - assert "0.25" in lab + assert "Operator profiles" in lab + assert "DETECTION_OPERATOR_PROFILES" in lab + assert "Candidate only - not default-approved" in lab assert "will_download_models" in lab diff --git a/backend/tests/test_sprint155_detection_operator_profiles.py b/backend/tests/test_sprint155_detection_operator_profiles.py new file mode 100644 index 00000000..36c5c7b6 --- /dev/null +++ b/backend/tests/test_sprint155_detection_operator_profiles.py @@ -0,0 +1,46 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_detection_operator_profiles_define_explicit_non_default_yolo_candidates() -> None: + profiles = ROOT / "frontend" / "src" / "components" / "detection" / "detectionProfiles.ts" + source = profiles.read_text(encoding="utf-8") + + assert "DETECTION_OPERATOR_PROFILES" in source + assert "geointel-building-yolov8s-aoi1024bg512r3e50-pt" in source + assert "balanced-review" in source + assert "conservative-review" in source + assert "confidenceThreshold: 0.15" in source + assert "confidenceThreshold: 0.35" in source + assert "defaultApproved: false" in source + assert "promotionRecommendation: 'none'" in source + assert "false-positive pressure" in source + + +def test_detection_lab_surfaces_profiles_as_deliberate_operator_actions() -> None: + lab = (ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx").read_text( + encoding="utf-8" + ) + + assert "DETECTION_OPERATOR_PROFILES" in lab + assert "Operator profiles" in lab + assert "profile.displayName" in lab + assert "profile.confidenceThreshold" in lab + assert "Candidate only - not default-approved" in lab + assert "Apply profile" in lab + assert "onApplyOperatorProfile(profile)" in lab + assert "Recommended starting threshold: 0.25" not in lab + + +def test_detection_workflow_applies_profiles_without_auto_selecting_assets() -> None: + hook = (ROOT / "frontend" / "src" / "hooks" / "useDetectionWorkflow.ts").read_text(encoding="utf-8") + app = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8") + + assert "applyDetectionOperatorProfile" in hook + assert "setSelectedDetectionModelId('yolo-configured')" in hook + assert "setSelectedModelAssetId(profile.modelAssetId)" in hook + assert "setDetectionConfidenceThreshold(profile.confidenceThreshold)" in hook + assert "setSelectedModelAssetId(assetResponse.items[0]" not in hook + assert "onApplyOperatorProfile={applyDetectionOperatorProfile}" in app diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index eb084a93..35f1d321 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -260,6 +260,14 @@ as false-positive pressure. It does not run QA/QC or invent reference metrics for empty/sparse background AOIs. The first expanded local model improved dense AOI F1, but Kasterlee-bos false positives block default promotion. +The current inactive AOI1024 background-aware local model asset, +`geointel-building-yolov8s-aoi1024bg512r3e50-pt`, is exposed in Detection Lab +only through deliberate operator profiles. `balanced-review` applies threshold +`0.15` for the strongest positive-AOI F1 observed so far; `conservative-review` +applies threshold `0.35` for higher precision review. Both profiles remain +candidate-only, not default-approved, because the promotion recommendation is +still `none` and background false-positive pressure has not passed the gate. + To compare the same model/tile/threshold grid across all prepared operator samples, use: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index b7ffdeb6..715a21b3 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -6168,3 +6168,32 @@ Open: - Build a V1 operator detection profile layer in the UI/docs: expose `balanced` (`threshold=0.15`) and `conservative review` (`threshold=0.35`) as explicit choices for local model assets, with clear warning that the model is not a default-approved detector. - Clean the background corpus classification: separate pure-empty AOIs from sparse-building contextual AOIs, then retrain or recalibrate against that cleaner gate. + +# Sprint 155 - Detection operator profiles + +## What changed + +- Added `frontend/src/components/detection/detectionProfiles.ts` with explicit operator profiles for the inactive `geointel-building-yolov8s-aoi1024bg512r3e50-pt` local model asset. +- Exposed two deliberate Detection Lab actions: + - `balanced-review`: confidence threshold `0.15`, positive-AOI F1 `0.5074022485589402`, precision `0.636639`, recall `0.424258`, max background detections `103`. + - `conservative-review`: confidence threshold `0.35`, positive-AOI F1 `0.32086574003576274`, precision `0.840006`, recall `0.202135`, max background detections `55`. +- Applying a profile selects `yolo-configured`, the local model asset id and the profile threshold. It does not auto-select assets on model catalog load and does not promote the candidate as a default detector. +- Detection Lab now marks both profiles as `Candidate only - not default-approved` because the promotion recommendation remains `none`. +- Updated frontend, AI pipeline and TODO documentation. + +## What was tested + +- Added regression coverage in `backend/tests/test_sprint155_detection_operator_profiles.py`. +- Ran `python -m pytest tests/test_sprint155_detection_operator_profiles.py tests/test_sprint122_model_asset_activation_guardrails.py -q`. +- Ran `python -m pytest` in `backend`: 435 passed. +- Ran `python -m compileall backend/app`. +- Ran `cd frontend && npm run typecheck`. +- Ran `cd frontend && npm run build`. +- Ran `bash scripts/run_readiness_check.sh`. +- Ran `cd backend && python -m alembic heads` and `cd backend && python -m alembic upgrade head --sql`. +- Ran `bash -n scripts/live_migration_smoke.sh`. + +## Known limitations + +- The profiles are review/demo aids only. The background corpus still needs to be split into pure-empty negatives and sparse-building contextual AOIs before retraining or recalibrating for a default detector decision. +- No backend API contract, migration, provider fetching, fake detection output, model download behavior or active runtime default changed. diff --git a/docs/TODO.md b/docs/TODO.md index 6043371d..d6aa2524 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -120,7 +120,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Export and audit AOI1024 clean-label variants; select `yolo-building-aoi1024-visible050-minpx8` as the first audit-passing 512px training candidate. - [x] Train and gate `geointel-building-yolov8s-aoi1024clean512e50-pt` through seven positive AOIs and nine hard-negative/background AOIs. - [x] Train and gate background-aware `geointel-building-yolov8s-aoi1024bg512r3e50-pt`; it is the strongest positive-AOI candidate so far but remains inactive because full background-candidate false-positive pressure still blocks default promotion. -- [ ] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass. +- [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass. - [ ] Split the background corpus into pure-empty negatives and sparse-building contextual AOIs, then retrain or recalibrate against the cleaner gate. - [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads. diff --git a/frontend/README.md b/frontend/README.md index bc0c4400..604dab64 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -122,6 +122,7 @@ AI Lab run controls explicitly explain when no raster dataset is available, inst - Detection Lab now exposes the `yolo-configured` capability reported by the backend. - When `yolo-configured` is selected, users can provide an existing raster tile manifest path. - Detection Lab lists local model assets from `GET /api/v1/detection/model-assets` so operators can choose an existing mounted model file instead of editing only one hidden `YOLO_MODEL_PATH` slot. +- Detection Lab exposes explicit operator profiles for the current inactive local AOI1024 building detector: balanced review at threshold `0.15` and conservative review at threshold `0.35`. Applying a profile deliberately selects the local model asset and threshold; it does not approve or promote a default model. - Detection Lab includes a read-only YOLO runtime preflight panel with backend status, dependency visibility, local model configuration, `torch`/`ultralytics` versions, CUDA state and `YOLO_CONFIG_DIR`. - The UI still does not download models or create fake detections; backend status and error codes remain the source of truth. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index cf5fbfde..325d63a5 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -260,6 +260,7 @@ function App(): JSX.Element { runDetection, runDetectionQa, runDetectionCalibration, + applyDetectionOperatorProfile, resetDetectionForProject, setSelectedDetectionDatasetId, setSelectedDetectionModelId, @@ -994,6 +995,7 @@ function App(): JSX.Element { onSetCalibrationThresholdText={setCalibrationThresholdText} onRunCalibration={runDetectionCalibration} onOpenCalibrationEvidence={openQualityEvidenceOnMap} + onApplyOperatorProfile={applyDetectionOperatorProfile} /> void onRunCalibration: () => void onOpenCalibrationEvidence: (qualityCheckId: string) => void + onApplyOperatorProfile: (profile: DetectionOperatorProfile) => void } export function DetectionLab({ @@ -135,6 +137,7 @@ export function DetectionLab({ onSetCalibrationThresholdText, onRunCalibration, onOpenCalibrationEvidence, + onApplyOperatorProfile, }: DetectionLabProps): JSX.Element { const selectedDetectionModel = detectionModels.find((model) => model.model_id === selectedDetectionModelId) ?? null const selectedModelAsset = modelAssets.find((asset) => asset.model_asset_id === selectedModelAssetId) ?? null @@ -145,9 +148,6 @@ export function DetectionLab({ const detectionModelUiRunnable = detectionModelReady && selectedDetectionModelId !== 'manual-fixture-detector' const detectionHasExplicitModelAsset = selectedDetectionModelId !== 'yolo-configured' || modelAssets.length === 0 || selectedModelAssetId.length > 0 - const benchmarkCandidateAsset = modelAssets.find( - (asset) => asset.model_asset_id === 'geointel-building-yolov8s-hardneg160r4e50-pt', - ) const calibrationRows = buildCalibrationRows(detectionRuns, qualityChecks) const bestF1Candidate = bestCalibrationRow(calibrationRows, 'f1') const bestPrecisionCandidate = bestCalibrationRow(calibrationRows, 'precision') @@ -251,12 +251,62 @@ export function DetectionLab({ {selectedModelAsset ? 'asset selected' : 'no explicit asset'} - {benchmarkCandidateAsset ? ( +
+ Operator profiles +

+ Candidate profiles apply a local model asset and confidence threshold only after an explicit click. + Candidate only - not default-approved while the promotion recommendation remains none. +

+
+
+ {DETECTION_OPERATOR_PROFILES.map((profile) => { + const profileAsset = modelAssets.find((asset) => asset.model_asset_id === profile.modelAssetId) + const profileSelected = + selectedModelAssetId === profile.modelAssetId && + Math.abs(detectionConfidenceThreshold - profile.confidenceThreshold) < 0.0001 + return ( +
+
+ {profile.displayName} + + {profile.defaultApproved ? 'default-approved' : 'Candidate only - not default-approved'} + +
+

{profile.description}

+
+ threshold {profile.confidenceThreshold.toFixed(2)} + precision {profile.precision.toFixed(3)} + recall {profile.recall.toFixed(3)} + F1 {profile.f1.toFixed(3)} + max background FP {profile.maxBackgroundDetections} +
+
+ asset: {profile.modelAssetId} + promotionRecommendation: {profile.promotionRecommendation} + available: {profileAsset ? 'yes' : 'not mounted'} +
+

{profile.limitationMessage}

+ +
+ ) + })} +
+ {selectedModelAsset ? (
- Current benchmark candidate + Selected model asset status

- {benchmarkCandidateAsset.display_name} is available for deliberate evaluation. Recommended starting threshold: 0.25. - Keep it operator-selected until hard-negative false positives are reduced. + {selectedModelAsset.display_name} is operator-selected. Keep local candidates inactive until persisted + promotion evidence explicitly recommends default activation.

) : null} @@ -469,7 +519,7 @@ export function DetectionLab({ /> {selectedDetectionModelId === 'yolo-configured' ? ( - Recommended starting threshold: 0.25 for the current local YOLOv8s benchmark candidate. + Use an operator profile for the current local YOLOv8s candidate, or enter a threshold manually for calibration. ) : null} diff --git a/frontend/src/components/detection/detectionProfiles.ts b/frontend/src/components/detection/detectionProfiles.ts new file mode 100644 index 00000000..dea1cb69 --- /dev/null +++ b/frontend/src/components/detection/detectionProfiles.ts @@ -0,0 +1,47 @@ +export interface DetectionOperatorProfile { + id: string + displayName: string + modelAssetId: string + confidenceThreshold: number + defaultApproved: boolean + promotionRecommendation: 'none' | 'promote_candidate' + precision: number + recall: number + f1: number + maxBackgroundDetections: number + description: string + limitationMessage: string +} + +export const DETECTION_OPERATOR_PROFILES: DetectionOperatorProfile[] = [ + { + id: 'balanced-review', + displayName: 'Balanced review', + modelAssetId: 'geointel-building-yolov8s-aoi1024bg512r3e50-pt', + confidenceThreshold: 0.15, + defaultApproved: false, + promotionRecommendation: 'none', + precision: 0.636639, + recall: 0.424258, + f1: 0.5074022485589402, + maxBackgroundDetections: 103, + description: 'Best positive-AOI F1 profile for deliberate operator review of the inactive AOI1024 model asset.', + limitationMessage: + 'Candidate only because false-positive pressure still blocks default promotion on the background/hard-negative gate.', + }, + { + id: 'conservative-review', + displayName: 'Conservative review', + modelAssetId: 'geointel-building-yolov8s-aoi1024bg512r3e50-pt', + confidenceThreshold: 0.35, + defaultApproved: false, + promotionRecommendation: 'none', + precision: 0.840006, + recall: 0.202135, + f1: 0.32086574003576274, + maxBackgroundDetections: 55, + description: 'Higher-precision profile for demos or review sessions where fewer false positives matter more than recall.', + limitationMessage: + 'Candidate only because false-positive pressure remains visible; use it deliberately and inspect persisted QA evidence.', + }, +] diff --git a/frontend/src/hooks/useDetectionWorkflow.ts b/frontend/src/hooks/useDetectionWorkflow.ts index 221c8b75..3972122c 100644 --- a/frontend/src/hooks/useDetectionWorkflow.ts +++ b/frontend/src/hooks/useDetectionWorkflow.ts @@ -21,6 +21,11 @@ interface DetectionWorkflowOptions { loadQualityChecks: (projectId?: string | null) => Promise } +interface DetectionOperatorProfileSelection { + modelAssetId: string + confidenceThreshold: number +} + export interface DetectionCalibrationRunRow { threshold: number status: 'queued' | 'running' | 'success' | 'failed' @@ -333,6 +338,12 @@ export function useDetectionWorkflow({ } } + const applyDetectionOperatorProfile = (profile: DetectionOperatorProfileSelection) => { + setSelectedDetectionModelId('yolo-configured') + setSelectedModelAssetId(profile.modelAssetId) + setDetectionConfidenceThreshold(profile.confidenceThreshold) + } + const resetDetectionForProject = () => { setSelectedDetectionDatasetId('') setDetectionRuns([]) @@ -383,6 +394,7 @@ export function useDetectionWorkflow({ runDetection, runDetectionQa, runDetectionCalibration, + applyDetectionOperatorProfile, resetDetectionForProject, setSelectedDetectionDatasetId, setSelectedDetectionModelId, diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index e6c566a5..60f7b118 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -3473,6 +3473,67 @@ button.entity-card { line-height: 1.35; } +.operator-profile-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr)); + gap: 0.62rem; +} + +.operator-profile-card { + display: grid; + gap: 0.48rem; + min-width: 0; + border: 1px solid #d8e3de; + border-radius: 8px; + padding: 0.72rem; + background: #ffffff; +} + +.operator-profile-card-selected { + border-color: var(--accent); + background: #f7fffc; + box-shadow: 0 0 0 3px rgba(15, 118, 110, 0.1); +} + +.operator-profile-card-header { + display: flex; + min-width: 0; + align-items: flex-start; + justify-content: space-between; + gap: 0.6rem; +} + +.operator-profile-card-header strong { + min-width: 0; + color: var(--text); + font-size: 0.94rem; + line-height: 1.25; +} + +.operator-profile-card p { + margin: 0; + color: var(--muted); + font-size: 0.8rem; + line-height: 1.35; +} + +.operator-profile-metrics { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(6.4rem, 1fr)); + gap: 0.36rem; +} + +.operator-profile-metrics span { + border: 1px solid #e0e9e4; + border-radius: 6px; + padding: 0.34rem 0.42rem; + background: #f9fbfa; + color: var(--text); + font-size: 0.76rem; + font-weight: 700; + overflow-wrap: anywhere; +} + .field-guidance { display: block; margin-top: 0.24rem;