From 0ed71c6f1ec294a83a7aee6577f5c5ed9d06b331 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 8 Jul 2026 04:03:50 +0200 Subject: [PATCH] Harden detection model asset selection --- CHANGELOG.md | 9 ++++ ...nt122_model_asset_activation_guardrails.py | 37 ++++++++++++++ docs/CODEX_EXECUTION_LOG.md | 30 +++++++++++ docs/TODO.md | 5 +- .../src/components/detection/DetectionLab.tsx | 50 +++++++++++++++++-- frontend/src/hooks/useDetectionWorkflow.ts | 3 +- frontend/src/styles/app.css | 28 +++++++++++ 7 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 backend/tests/test_sprint122_model_asset_activation_guardrails.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 83e82711..60314cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1292,3 +1292,12 @@ Added: - Observed dense QA F1 scores up to `0.6380` and safest current threshold behavior around `0.25`. - Kept the model inactive by default because the `kasterlee_bos` hard-negative sample still produced 10 detections at threshold `0.25`. - No repository code, API contracts, migrations, product behavior, provider fetching or AI dependency strategy changed in this benchmark pass. + +## Sprint 122 Detection model asset activation guardrails (2026-07-08) + +- Hardened Detection Lab so local model assets are no longer auto-selected when the backend reports available model files. +- Required an explicit local model asset choice before configured YOLO can be submitted when local assets exist. +- Added local model asset details in the run surface: active runtime env status, SHA-256 preview, file size, path and `will_download_models`. +- Surfaced the current YOLOv8s hard-negative benchmark candidate and recommended starting threshold `0.25` as operator guidance. +- Added regression coverage for the no-auto-select behavior and UI guardrail copy. +- No backend API contracts, migrations, model downloads, provider fetching or model weight mutation behavior changed. diff --git a/backend/tests/test_sprint122_model_asset_activation_guardrails.py b/backend/tests/test_sprint122_model_asset_activation_guardrails.py new file mode 100644 index 00000000..96b9b0d0 --- /dev/null +++ b/backend/tests/test_sprint122_model_asset_activation_guardrails.py @@ -0,0 +1,37 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_detection_workflow_does_not_auto_select_runtime_model_assets() -> None: + hook = (ROOT / "frontend" / "src" / "hooks" / "useDetectionWorkflow.ts").read_text(encoding="utf-8") + + assert "const activeAsset = assetResponse.items.find" not in hook + assert "setSelectedModelAssetId(activeAsset?.model_asset_id ?? '')" not in hook + assert "setSelectedModelAssetId(assetResponse.items[0]" not in hook + assert "setSelectedModelAssetId('')" in hook + + +def test_detection_lab_explains_explicit_model_asset_and_threshold_selection() -> None: + lab = (ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx").read_text( + encoding="utf-8" + ) + + assert "Explicit model asset" in lab + assert "No model file is selected automatically" in lab + assert "Current benchmark candidate" in lab + assert "geointel-building-yolov8s-hardneg160r4e50-pt" in lab + assert "Recommended starting threshold" in lab + assert "0.25" in lab + assert "will_download_models" in lab + + +def test_detection_run_readiness_requires_explicit_asset_when_local_assets_exist() -> None: + lab = (ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx").read_text( + encoding="utf-8" + ) + + assert "detectionHasExplicitModelAsset" in lab + assert "Select a local model asset deliberately" in lab + assert "local model asset" in lab diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 0187215c..f17328f9 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5000,6 +5000,36 @@ Limitations: Next recommended pass: - Continue with V1 usability work that reduces operator confusion without expanding frozen product scope. +## Sprint 122 Detection model asset activation guardrails (2026-07-08) + +Changed: +- Hardened Detection Lab local model handling so reported runtime model assets are read-only choices and are not auto-selected by the frontend hook. +- Configured YOLO run readiness now blocks submission when local model assets exist but no explicit `model_asset_id` has been selected. +- Added an explicit model asset section with active runtime env status, `will_download_models`, SHA-256 preview, file size and mounted model path. +- Surfaced the current benchmark candidate `geointel-building-yolov8s-hardneg160r4e50-pt` with recommended starting threshold `0.25`. +- Added compact UI guidance styling for the benchmark/threshold warning. +- Added regression coverage in `backend/tests/test_sprint122_model_asset_activation_guardrails.py`. +- Updated `CHANGELOG.md` and `docs/TODO.md`. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint122_model_asset_activation_guardrails.py -q` failed on the previous auto-selection behavior and missing guardrail copy. +- `python -m pytest backend\tests\test_sprint122_model_asset_activation_guardrails.py -q` (`3 passed`) +- `python -m pytest backend\tests\test_sprint118_yolo_preflight_ui.py backend\tests\test_sprint103_ai_lab_run_readiness.py backend\tests\test_sprint104_ai_lab_action_guardrails.py backend\tests\test_model_asset_catalog.py backend\tests\test_sprint122_model_asset_activation_guardrails.py -q` (`16 passed`) +- `python -m compileall backend/app` +- `python -m pytest backend\tests -q` (`408 passed`) +- `cd frontend && npm run typecheck` +- `cd frontend && npm run build` + +Open: +- Full threshold calibration comparison UI is still pending; this pass adds safe single-threshold guidance and explicit asset choice only. + +Limitations: +- Frontend guardrail only; no backend API contracts, migrations, provider fetching, model downloads or model weight mutation behavior changed. +- The active runtime env model can still be configured by operators through existing deployment/env tooling, but the Detection Lab no longer silently chooses a local asset from the catalog for a run. + +Next recommended pass: +- Add threshold calibration comparison UX over existing persisted runs so candidate models can be promoted with visible precision/recall/F1 and hard-negative counts. + ## Operator YOLOv8s hard-negative model benchmark (2026-07-08) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index a6eff2e3..2bbc8a47 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -416,7 +416,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add operator-only tile-level YOLO dataset export with overlapping windows and deterministic negative tile retention. - [x] Run tile-level training on Tower and accept/reject the resulting local model through the persisted QA/QC matrix. - [x] Train/evaluate a YOLOv8s hard-negative local building-detector candidate on Tower and keep it inactive because hard-negative false positives remain. -- [ ] Add an operator-facing local model catalog/activation workflow with SHA256, active model status and explicit threshold guidance. -- [ ] Add threshold calibration UX so detection runs do not silently rely on an unsafe default confidence. +- [x] Add an operator-facing local model catalog/activation workflow with SHA256, active model status and explicit threshold guidance. +- [x] Block silent local model asset auto-selection in Detection Lab. +- [ ] Add full threshold calibration comparison UX so detection runs can compare candidate thresholds before promotion. - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout. - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles. diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx index 06a2cb2c..1e546f91 100644 --- a/frontend/src/components/detection/DetectionLab.tsx +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -109,10 +109,20 @@ export function DetectionLab({ const detectionHasModel = selectedDetectionModel !== null const detectionModelReady = Boolean(selectedDetectionModel?.configured) const detectionModelUiRunnable = detectionModelReady && selectedDetectionModelId !== 'manual-fixture-detector' + const detectionHasExplicitModelAsset = + selectedDetectionModelId !== 'yolo-configured' || modelAssets.length === 0 || selectedModelAssetId.length > 0 + const benchmarkCandidateAsset = modelAssets.find( + (asset) => asset.model_asset_id === 'geointel-building-yolov8s-hardneg160r4e50-pt', + ) const detectionHasTileManifest = !detectionRequiresTileManifest || detectionTileManifestPath.trim().length > 0 const detectionRunReady = - Boolean(selectedProjectId) && detectionHasDataset && detectionHasModel && detectionModelUiRunnable && detectionHasTileManifest + Boolean(selectedProjectId) && + detectionHasDataset && + detectionHasModel && + detectionModelUiRunnable && + detectionHasExplicitModelAsset && + detectionHasTileManifest const detectionRunBlockedReason = !selectedProjectId ? 'Select or create a project first' : !detectionHasDataset @@ -123,6 +133,8 @@ export function DetectionLab({ ? 'Fixture model is explicit test/demo-only' : !detectionModelReady ? selectedDetectionModel?.limitation_message ?? 'Selected model is not configured' + : !detectionHasExplicitModelAsset + ? 'Select a local model asset deliberately' : !detectionHasTileManifest ? 'Provide a raster tile manifest for configured YOLO' : null @@ -193,17 +205,26 @@ export function DetectionLab({
-

Local model assets

-

Select an existing model file mounted into the backend runtime. GeoIntel does not download model weights.

+

Explicit model asset

+

Local model assets are read-only. No model file is selected automatically. Choose an existing local model asset before submitting configured YOLO.

- {selectedModelAsset ? 'asset selected' : 'using configured path'} + {selectedModelAsset ? 'asset selected' : 'no explicit asset'}
+ {benchmarkCandidateAsset ? ( +
+ Current benchmark candidate +

+ {benchmarkCandidateAsset.display_name} is available for deliberate evaluation. Recommended starting threshold: 0.25. + Keep it operator-selected until hard-negative false positives are reduced. +

+
+ ) : null}