Harden detection model asset selection
This commit is contained in:
@@ -1292,3 +1292,12 @@ Added:
|
||||
- Observed dense QA F1 scores up to `0.6380` and safest current threshold behavior around `0.25`.
|
||||
- Kept the model inactive by default because the `kasterlee_bos` hard-negative sample still produced 10 detections at threshold `0.25`.
|
||||
- No repository code, API contracts, migrations, product behavior, provider fetching or AI dependency strategy changed in this benchmark pass.
|
||||
|
||||
## Sprint 122 Detection model asset activation guardrails (2026-07-08)
|
||||
|
||||
- Hardened Detection Lab so local model assets are no longer auto-selected when the backend reports available model files.
|
||||
- Required an explicit local model asset choice before configured YOLO can be submitted when local assets exist.
|
||||
- Added local model asset details in the run surface: active runtime env status, SHA-256 preview, file size, path and `will_download_models`.
|
||||
- Surfaced the current YOLOv8s hard-negative benchmark candidate and recommended starting threshold `0.25` as operator guidance.
|
||||
- Added regression coverage for the no-auto-select behavior and UI guardrail copy.
|
||||
- No backend API contracts, migrations, model downloads, provider fetching or model weight mutation behavior changed.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_detection_workflow_does_not_auto_select_runtime_model_assets() -> None:
|
||||
hook = (ROOT / "frontend" / "src" / "hooks" / "useDetectionWorkflow.ts").read_text(encoding="utf-8")
|
||||
|
||||
assert "const activeAsset = assetResponse.items.find" not in hook
|
||||
assert "setSelectedModelAssetId(activeAsset?.model_asset_id ?? '')" not in hook
|
||||
assert "setSelectedModelAssetId(assetResponse.items[0]" not in hook
|
||||
assert "setSelectedModelAssetId('')" in hook
|
||||
|
||||
|
||||
def test_detection_lab_explains_explicit_model_asset_and_threshold_selection() -> None:
|
||||
lab = (ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "Explicit model asset" in lab
|
||||
assert "No model file is selected automatically" in lab
|
||||
assert "Current benchmark candidate" in lab
|
||||
assert "geointel-building-yolov8s-hardneg160r4e50-pt" in lab
|
||||
assert "Recommended starting threshold" in lab
|
||||
assert "0.25" in lab
|
||||
assert "will_download_models" in lab
|
||||
|
||||
|
||||
def test_detection_run_readiness_requires_explicit_asset_when_local_assets_exist() -> None:
|
||||
lab = (ROOT / "frontend" / "src" / "components" / "detection" / "DetectionLab.tsx").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert "detectionHasExplicitModelAsset" in lab
|
||||
assert "Select a local model asset deliberately" in lab
|
||||
assert "local model asset" in lab
|
||||
@@ -5000,6 +5000,36 @@ Limitations:
|
||||
Next recommended pass:
|
||||
- Continue with V1 usability work that reduces operator confusion without expanding frozen product scope.
|
||||
|
||||
## Sprint 122 Detection model asset activation guardrails (2026-07-08)
|
||||
|
||||
Changed:
|
||||
- Hardened Detection Lab local model handling so reported runtime model assets are read-only choices and are not auto-selected by the frontend hook.
|
||||
- Configured YOLO run readiness now blocks submission when local model assets exist but no explicit `model_asset_id` has been selected.
|
||||
- Added an explicit model asset section with active runtime env status, `will_download_models`, SHA-256 preview, file size and mounted model path.
|
||||
- Surfaced the current benchmark candidate `geointel-building-yolov8s-hardneg160r4e50-pt` with recommended starting threshold `0.25`.
|
||||
- Added compact UI guidance styling for the benchmark/threshold warning.
|
||||
- Added regression coverage in `backend/tests/test_sprint122_model_asset_activation_guardrails.py`.
|
||||
- Updated `CHANGELOG.md` and `docs/TODO.md`.
|
||||
|
||||
Tested:
|
||||
- Red step: `python -m pytest backend\tests\test_sprint122_model_asset_activation_guardrails.py -q` failed on the previous auto-selection behavior and missing guardrail copy.
|
||||
- `python -m pytest backend\tests\test_sprint122_model_asset_activation_guardrails.py -q` (`3 passed`)
|
||||
- `python -m pytest backend\tests\test_sprint118_yolo_preflight_ui.py backend\tests\test_sprint103_ai_lab_run_readiness.py backend\tests\test_sprint104_ai_lab_action_guardrails.py backend\tests\test_model_asset_catalog.py backend\tests\test_sprint122_model_asset_activation_guardrails.py -q` (`16 passed`)
|
||||
- `python -m compileall backend/app`
|
||||
- `python -m pytest backend\tests -q` (`408 passed`)
|
||||
- `cd frontend && npm run typecheck`
|
||||
- `cd frontend && npm run build`
|
||||
|
||||
Open:
|
||||
- Full threshold calibration comparison UI is still pending; this pass adds safe single-threshold guidance and explicit asset choice only.
|
||||
|
||||
Limitations:
|
||||
- Frontend guardrail only; no backend API contracts, migrations, provider fetching, model downloads or model weight mutation behavior changed.
|
||||
- The active runtime env model can still be configured by operators through existing deployment/env tooling, but the Detection Lab no longer silently chooses a local asset from the catalog for a run.
|
||||
|
||||
Next recommended pass:
|
||||
- Add threshold calibration comparison UX over existing persisted runs so candidate models can be promoted with visible precision/recall/F1 and hard-negative counts.
|
||||
|
||||
## Operator YOLOv8s hard-negative model benchmark (2026-07-08)
|
||||
|
||||
Changed:
|
||||
|
||||
+3
-2
@@ -416,7 +416,8 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Add operator-only tile-level YOLO dataset export with overlapping windows and deterministic negative tile retention.
|
||||
- [x] Run tile-level training on Tower and accept/reject the resulting local model through the persisted QA/QC matrix.
|
||||
- [x] Train/evaluate a YOLOv8s hard-negative local building-detector candidate on Tower and keep it inactive because hard-negative false positives remain.
|
||||
- [ ] Add an operator-facing local model catalog/activation workflow with SHA256, active model status and explicit threshold guidance.
|
||||
- [ ] Add threshold calibration UX so detection runs do not silently rely on an unsafe default confidence.
|
||||
- [x] Add an operator-facing local model catalog/activation workflow with SHA256, active model status and explicit threshold guidance.
|
||||
- [x] Block silent local model asset auto-selection in Detection Lab.
|
||||
- [ ] Add full threshold calibration comparison UX so detection runs can compare candidate thresholds before promotion.
|
||||
- [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout.
|
||||
- [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles.
|
||||
|
||||
@@ -109,10 +109,20 @@ export function DetectionLab({
|
||||
const detectionHasModel = selectedDetectionModel !== null
|
||||
const detectionModelReady = Boolean(selectedDetectionModel?.configured)
|
||||
const detectionModelUiRunnable = detectionModelReady && selectedDetectionModelId !== 'manual-fixture-detector'
|
||||
const detectionHasExplicitModelAsset =
|
||||
selectedDetectionModelId !== 'yolo-configured' || modelAssets.length === 0 || selectedModelAssetId.length > 0
|
||||
const benchmarkCandidateAsset = modelAssets.find(
|
||||
(asset) => asset.model_asset_id === 'geointel-building-yolov8s-hardneg160r4e50-pt',
|
||||
)
|
||||
const detectionHasTileManifest =
|
||||
!detectionRequiresTileManifest || detectionTileManifestPath.trim().length > 0
|
||||
const detectionRunReady =
|
||||
Boolean(selectedProjectId) && detectionHasDataset && detectionHasModel && detectionModelUiRunnable && detectionHasTileManifest
|
||||
Boolean(selectedProjectId) &&
|
||||
detectionHasDataset &&
|
||||
detectionHasModel &&
|
||||
detectionModelUiRunnable &&
|
||||
detectionHasExplicitModelAsset &&
|
||||
detectionHasTileManifest
|
||||
const detectionRunBlockedReason = !selectedProjectId
|
||||
? 'Select or create a project first'
|
||||
: !detectionHasDataset
|
||||
@@ -123,6 +133,8 @@ export function DetectionLab({
|
||||
? 'Fixture model is explicit test/demo-only'
|
||||
: !detectionModelReady
|
||||
? selectedDetectionModel?.limitation_message ?? 'Selected model is not configured'
|
||||
: !detectionHasExplicitModelAsset
|
||||
? 'Select a local model asset deliberately'
|
||||
: !detectionHasTileManifest
|
||||
? 'Provide a raster tile manifest for configured YOLO'
|
||||
: null
|
||||
@@ -193,17 +205,26 @@ export function DetectionLab({
|
||||
<div className="ai-lab-model-surface" aria-label="Local model asset selection">
|
||||
<div className="ai-lab-section-header">
|
||||
<div>
|
||||
<h3>Local model assets</h3>
|
||||
<p>Select an existing model file mounted into the backend runtime. GeoIntel does not download model weights.</p>
|
||||
<h3>Explicit model asset</h3>
|
||||
<p>Local model assets are read-only. No model file is selected automatically. Choose an existing local model asset before submitting configured YOLO.</p>
|
||||
</div>
|
||||
<span className={selectedModelAsset ? 'status-badge status-badge-ready' : 'status-badge'}>
|
||||
{selectedModelAsset ? 'asset selected' : 'using configured path'}
|
||||
{selectedModelAsset ? 'asset selected' : 'no explicit asset'}
|
||||
</span>
|
||||
</div>
|
||||
{benchmarkCandidateAsset ? (
|
||||
<div className="model-asset-guidance">
|
||||
<strong>Current benchmark candidate</strong>
|
||||
<p>
|
||||
{benchmarkCandidateAsset.display_name} is available for deliberate evaluation. Recommended starting threshold: 0.25.
|
||||
Keep it operator-selected until hard-negative false positives are reduced.
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
<label>
|
||||
Local model file
|
||||
<select value={selectedModelAssetId} onChange={(event) => onSelectModelAsset(event.target.value)}>
|
||||
<option value="">Use configured YOLO_MODEL_PATH</option>
|
||||
<option value="">Select explicit local model asset</option>
|
||||
{modelAssets.map((asset) => (
|
||||
<option key={asset.model_asset_id} value={asset.model_asset_id}>
|
||||
{asset.display_name} {asset.active ? '(active)' : ''}
|
||||
@@ -221,6 +242,8 @@ export function DetectionLab({
|
||||
<div className="result-summary-card">
|
||||
<p>File: {selectedModelAsset.filename}</p>
|
||||
<p>Status: {selectedModelAsset.status}</p>
|
||||
<p>Active runtime env model: {selectedModelAsset.active ? 'yes' : 'no'}</p>
|
||||
<p>will_download_models: {selectedModelAsset.will_download_models ? 'yes' : 'no'}</p>
|
||||
<p>Size: {formatModelAssetSize(selectedModelAsset.size_bytes)}</p>
|
||||
<p>SHA-256: {selectedModelAsset.sha256.slice(0, 12)}</p>
|
||||
<p>Path: {selectedModelAsset.model_path}</p>
|
||||
@@ -340,6 +363,18 @@ export function DetectionLab({
|
||||
: 'Not required for this model'}
|
||||
</strong>
|
||||
</div>
|
||||
<div className={detectionHasExplicitModelAsset ? 'lab-readiness-item lab-readiness-item-ready' : 'lab-readiness-item'}>
|
||||
<span>Local model asset</span>
|
||||
<strong>
|
||||
{selectedDetectionModelId === 'yolo-configured'
|
||||
? selectedModelAsset
|
||||
? selectedModelAsset.display_name
|
||||
: modelAssets.length > 0
|
||||
? 'Select a local model asset deliberately'
|
||||
: 'No local assets reported; backend configured path only'
|
||||
: 'Not required for this model'}
|
||||
</strong>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className={detectionRunReady ? 'lab-action-guardrail lab-action-guardrail-ready' : 'lab-action-guardrail'}>
|
||||
@@ -384,6 +419,11 @@ export function DetectionLab({
|
||||
value={detectionConfidenceThreshold}
|
||||
onChange={(event) => onSetConfidenceThreshold(Number(event.target.value))}
|
||||
/>
|
||||
{selectedDetectionModelId === 'yolo-configured' ? (
|
||||
<span className="field-guidance">
|
||||
Recommended starting threshold: 0.25 for the current local YOLOv8s benchmark candidate.
|
||||
</span>
|
||||
) : null}
|
||||
</label>
|
||||
</div>
|
||||
{selectedDetectionModelId === 'yolo-configured' ? (
|
||||
|
||||
@@ -72,9 +72,8 @@ export function useDetectionWorkflow({
|
||||
try {
|
||||
const assetResponse = await detectionApi.listModelAssets()
|
||||
setModelAssets(assetResponse.items)
|
||||
const activeAsset = assetResponse.items.find((asset) => asset.active) ?? assetResponse.items[0] ?? null
|
||||
if (!assetResponse.items.some((asset) => asset.model_asset_id === selectedModelAssetId)) {
|
||||
setSelectedModelAssetId(activeAsset?.model_asset_id ?? '')
|
||||
setSelectedModelAssetId('')
|
||||
}
|
||||
} catch (error) {
|
||||
setModelAssets([])
|
||||
|
||||
@@ -3369,6 +3369,34 @@ button.entity-card {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.model-asset-guidance {
|
||||
display: grid;
|
||||
gap: 0.22rem;
|
||||
border: 1px solid #d8e3de;
|
||||
border-left: 4px solid #315a92;
|
||||
border-radius: 8px;
|
||||
padding: 0.58rem 0.68rem;
|
||||
background: #f8fbff;
|
||||
}
|
||||
|
||||
.model-asset-guidance strong {
|
||||
color: var(--text);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.model-asset-guidance p,
|
||||
.field-guidance {
|
||||
margin: 0;
|
||||
color: var(--muted);
|
||||
font-size: 0.78rem;
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.field-guidance {
|
||||
display: block;
|
||||
margin-top: 0.24rem;
|
||||
}
|
||||
|
||||
.lab-action-guardrail {
|
||||
display: flex;
|
||||
min-width: 0;
|
||||
|
||||
Reference in New Issue
Block a user