From 1c16313ae657d90e4ab25269fc945ed308df1c3f Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 02:21:42 +0200 Subject: [PATCH] Promote expanded building detector profile --- CHANGELOG.md | 9 +++++++ ...t_sprint155_detection_operator_profiles.py | 10 ++++--- docs/AI_PIPELINES.md | 24 +++++++++-------- docs/CODEX_EXECUTION_LOG.md | 22 +++++++++++++++ docs/TODO.md | 7 ++--- .../src/components/detection/DetectionLab.tsx | 3 ++- .../components/detection/detectionProfiles.ts | 27 ++++++++++--------- 7 files changed, 71 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bfe197f..72e32ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 173 Expanded building model promotion (2026-07-13) + +- Trained and fully gated the inactive `geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt` candidate from the 20-source expanded real-data corpus. +- The recommended `512` tile / `64` overlap / `0.15` confidence profile reached mean precision `0.6471`, recall `0.4700` and F1 `0.5433` across seven positive AOIs; all three pure-empty background AOIs remained at zero detections. +- Compared fixed-threshold persisted evidence against the previous active model and reduced the false-negative rate in every validated positive AOI. +- Activated the exact promoted local model through the guarded dry-run-first helper; no model was downloaded and no fake inference or QA result was introduced. +- Updated Detection Lab operator profiles so the promoted expanded model is the recommended balanced review choice while the previous high-precision model remains available as a legacy conservative profile. +- Persistent small-building misses remain the primary model-quality limitation and still require operator QA/QC. + ## Sprint 172 CPU AI image build hardening (2026-07-12) - Reordered the Unraid all-in-one Docker build so backend source changes reuse the Python/GIS/AI dependency layer. diff --git a/backend/tests/test_sprint155_detection_operator_profiles.py b/backend/tests/test_sprint155_detection_operator_profiles.py index ff3b4f7b..1ce30630 100644 --- a/backend/tests/test_sprint155_detection_operator_profiles.py +++ b/backend/tests/test_sprint155_detection_operator_profiles.py @@ -9,17 +9,19 @@ def test_detection_operator_profiles_define_explicit_yolo_candidates_and_promote source = profiles.read_text(encoding="utf-8") assert "DETECTION_OPERATOR_PROFILES" in source + assert "geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt" in source assert "geointel-building-yolov8s-aoi1024bg512r3e50-pt" in source - assert "balanced-review" in source + assert "expanded-balanced-review" in source assert "conservative-review" in source assert "confidenceThreshold: 0.15" in source assert "confidenceThreshold: 0.35" in source - assert "defaultApproved: false" in source assert "defaultApproved: true" in source - assert "promotionRecommendation: 'none'" in source assert "promotionRecommendation: 'promote_candidate'" in source + assert "positiveSampleCount: 7" in source + assert "f1: 0.5432865390636915" in source + assert "maxBackgroundDetections: 0" in source assert "pure-empty gate passed" in source - assert "false-positive pressure" in source + assert "persistent small-building misses" in source def test_detection_lab_surfaces_profiles_as_deliberate_operator_actions() -> None: diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index cfca347d..b13304b3 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -335,15 +335,17 @@ hard-negative gates, then run `sparse_building_context` as a separate review matrix. The first expanded local model improved dense AOI F1, but Kasterlee-bos false positives block default promotion. -The AOI1024 background-aware local model asset, -`geointel-building-yolov8s-aoi1024bg512r3e50-pt`, is exposed in Detection Lab -only through deliberate operator profiles. `balanced-review` applies threshold -`0.15` for the strongest positive-AOI F1 observed so far, but remains -candidate-only because pure-empty false-positive pressure failed at that -threshold. `conservative-review` applies threshold `0.35` and is marked as the -promoted candidate after the split-background report passed the strict -pure-empty gate. Sparse-context detections remain review-only evidence, not a -default-promotion blocker. +The expanded-AOI local model asset, +`geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt`, is the current +recommended Detection Lab operator profile. Use tile size `512`, overlap `64` +and confidence threshold `0.15`. Persisted QA/QC across seven positive AOIs +measured mean precision `0.6471`, recall `0.4700` and F1 `0.5433`; the strict +three-sample pure-empty background gate produced zero detections. The previous +`geointel-building-yolov8s-aoi1024bg512r3e50-pt` model remains available as a +legacy conservative `0.35` review profile. Sparse-context detections remain +review-only evidence, not a default-promotion blocker. Persistent misses are +concentrated in small buildings, so every production-like run still requires +persisted QA/QC against suitable reference data. To update a Tower/Unraid `.env` from a promoted report, use the guarded activation helper. It validates the exact report candidate key, verifies that @@ -353,8 +355,8 @@ when `--apply` is supplied: ```bash python scripts/activate_promoted_yolo_candidate.py \ - --promotion-report artifacts/detection-model-promotion/split-aware/aoi1024bg512r3e50-high-threshold-split-20260710T222934Z/detection_model_promotion_report.json \ - --candidate-key 'geointel-building-yolov8s-aoi1024bg512r3e50-pt|512|64|0.35' \ + --promotion-report artifacts/detection-model-promotion/split-aware/aoi1024expandedminpx4vis035e50-split/detection_model_promotion_report.json \ + --candidate-key 'geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt|512|64|0.15' \ --models-dir /mnt/user/appdata/geointel/models \ --env-file /mnt/user/appdata/geointel/.env \ --json diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index cafe2307..82328b8a 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7039,3 +7039,25 @@ Open: - The expanded live export exposed that Arendonk-heide remained configured as a holdout while all of its low-variance tiles were correctly filtered out. - Added `retained_validation_sample_slugs` and `empty_validation_sample_slugs` to tile dataset summaries so configured and actual validation coverage cannot be confused. - Added a focused regression test and kept filtering behavior unchanged; no blank tile was reintroduced. + +# Sprint 173 - Expanded building model promotion + +## Runtime evidence + +- Completed inactive 50-epoch CPU training for `geointel-building-yolov8s-aoi1024expandedminpx4vis035e50.pt` from the expanded 20-source real-data corpus. +- Trained-model SHA256: `a8a79cf5b0bdc19a0245acc322cf77232c335e222bd5f3c00a17d5f29402c196`. +- Training summary recorded 45,892 labels across 171 retained tiles; final training metrics were precision `0.55558`, recall `0.35206`, mAP50 `0.27440` and mAP50-95 `0.10107`. +- The persisted seven-AOI positive matrix recommended tile size `512`, overlap `64` and threshold `0.15`: mean precision `0.6470590036`, recall `0.4699913837`, F1 `0.5432865391`, and minimum per-AOI F1 `0.4897494305`. +- The strict pure-empty gate covered Postel, Lommel and Arendonk across all tested thresholds and produced zero detections. Sparse-building contextual AOIs remained review-only evidence because real GRB buildings are present there. +- Fixed-threshold persisted false-negative comparison used identical reference feature populations and reduced the false-negative rate in all seven positive AOIs versus the previous active `0.35` profile. +- The guarded activation helper first returned `ready_to_apply`, resolved the exact local model asset and reported no downloads or inference. The reviewed `--apply` pass updated only `GEOINTEL_INSTALL_AI`, `YOLO_ENABLED`, `YOLO_MODELS_DIR` and `YOLO_MODEL_PATH` in the Tower environment. + +## UI and operator behavior + +- Detection Lab now recommends the promoted expanded-AOI model at threshold `0.15` and surfaces its seven-AOI coverage plus QA metrics. +- The previous AOI1024 background-aware model remains mounted as an explicit legacy high-precision `0.35` review profile. +- Profile selection remains deliberate: it selects a mounted local asset and run threshold only; it does not mutate runtime environment, download a model or start inference automatically. + +## Remaining limitation + +- Persistent small-building misses remain the weakest quality bucket. Continue with targeted evidence review and data coverage before considering another training run; do not infer production accuracy from aggregate F1 alone. diff --git a/docs/TODO.md b/docs/TODO.md index 0a6b0952..b4bdec61 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -121,7 +121,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Export and audit AOI1024 clean-label variants; select `yolo-building-aoi1024-visible050-minpx8` as the first audit-passing 512px training candidate. - [x] Train and gate `geointel-building-yolov8s-aoi1024clean512e50-pt` through seven positive AOIs and nine hard-negative/background AOIs. - [x] Train and gate background-aware `geointel-building-yolov8s-aoi1024bg512r3e50-pt`; it is the strongest positive-AOI candidate so far but remains inactive because full background-candidate false-positive pressure still blocks default promotion. -- [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` remains candidate-only, while conservative high-precision review around threshold `0.35` is marked promoted after the pure-empty split-background gate passed. +- [x] Add explicit operator detection profiles for local model assets and promote the expanded-AOI balanced review profile at threshold `0.15` after positive and pure-empty split-background gates passed. - [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance. - [x] Add a split background-corpus matrix runner and report builder that runs pure-empty and sparse-context matrices separately. - [x] Teach the model promotion report to consume split background summaries so only `pure_empty_negative` blocks default promotion and `sparse_building_context` stays review-only. @@ -135,8 +135,9 @@ This file now starts with the current implementation status. Older preparation/b - [x] Regenerate the AOI1024 cleanpx YOLO dataset with low-variance negative filtering and rerun visual contact-sheet QA before training. - [x] Train one inactive candidate from the filtered AOI1024 cleanpx YOLO dataset and gate it through the positive-AOI plus split-background promotion workflow; reject it because mean positive F1 remains below gate. - [x] Add deterministic dataset/base/trained-model SHA256 provenance to future operator training summaries. -- [ ] Review per-AOI false-negative evidence for the weakest AOIs and improve positive sample/label geometry coverage before another training candidate. -- [ ] Apply promoted V1 default building detector only after explicit operator review of the emitted `.env` updates, followed by rebuild/restart and browser/runtime smoke. +- [x] Review per-AOI false-negative evidence, expand positive sample/label coverage and verify the resulting candidate improves false-negative rate in every validated AOI. +- [ ] Complete rebuild/restart and browser/runtime smoke for the guarded promoted V1 building detector activation. +- [ ] Expand focused small-building training evidence only after reviewing persistent false negatives from the promoted model; do not start another blind training run. ## Sprint 8 status diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx index d9ed152f..7c47704a 100644 --- a/frontend/src/components/detection/DetectionLab.tsx +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -281,6 +281,7 @@ export function DetectionLab({ precision {profile.precision.toFixed(3)} recall {profile.recall.toFixed(3)} F1 {profile.f1.toFixed(3)} + positive AOIs {profile.positiveSampleCount} max background FP {profile.maxBackgroundDetections}
@@ -519,7 +520,7 @@ export function DetectionLab({ /> {selectedDetectionModelId === 'yolo-configured' ? ( - Use an operator profile for the current local YOLOv8s candidate, or enter a threshold manually for calibration. + Use a validated operator profile, or enter a threshold manually for calibration. ) : null} diff --git a/frontend/src/components/detection/detectionProfiles.ts b/frontend/src/components/detection/detectionProfiles.ts index 29026995..93fd7ad6 100644 --- a/frontend/src/components/detection/detectionProfiles.ts +++ b/frontend/src/components/detection/detectionProfiles.ts @@ -8,6 +8,7 @@ export interface DetectionOperatorProfile { precision: number recall: number f1: number + positiveSampleCount: number maxBackgroundDetections: number description: string limitationMessage: string @@ -15,19 +16,20 @@ export interface DetectionOperatorProfile { export const DETECTION_OPERATOR_PROFILES: DetectionOperatorProfile[] = [ { - id: 'balanced-review', - displayName: 'Balanced review', - modelAssetId: 'geointel-building-yolov8s-aoi1024bg512r3e50-pt', + id: 'expanded-balanced-review', + displayName: 'Recommended balanced review', + modelAssetId: 'geointel-building-yolov8s-aoi1024expandedminpx4vis035e50-pt', confidenceThreshold: 0.15, - defaultApproved: false, - promotionRecommendation: 'none', - precision: 0.636639, - recall: 0.424258, - f1: 0.5074022485589402, - maxBackgroundDetections: 46, - description: 'Best positive-AOI F1 profile for deliberate operator review of the inactive AOI1024 model asset.', + defaultApproved: true, + promotionRecommendation: 'promote_candidate', + precision: 0.6470590036169351, + recall: 0.4699913836847832, + f1: 0.5432865390636915, + positiveSampleCount: 7, + maxBackgroundDetections: 0, + description: 'Recommended expanded-AOI profile for balanced building review across the validated Kempen samples.', limitationMessage: - 'Candidate only because pure-empty false-positive pressure still blocks default promotion on the background gate.', + 'Default-approved after the pure-empty gate passed; persistent small-building misses still require operator QA.', }, { id: 'conservative-review', @@ -39,8 +41,9 @@ export const DETECTION_OPERATOR_PROFILES: DetectionOperatorProfile[] = [ precision: 0.840006, recall: 0.202135, f1: 0.32086574003576274, + positiveSampleCount: 7, maxBackgroundDetections: 0, - description: 'Promoted high-precision profile for demos or review sessions where fewer false positives matter more than recall.', + description: 'Legacy high-precision profile for review sessions where fewer false positives matter more than recall.', limitationMessage: 'Default-approved after the split-background pure-empty gate passed; sparse-context detections remain review-only evidence.', },