From 006819ca51c4a3e1ae96fc6a7ad82848c6d9e379 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 14:52:06 +0200 Subject: [PATCH] Support multi-sample promotion evidence --- CHANGELOG.md | 3 + ...int143_detection_model_promotion_report.py | 97 +++++++++++++++++++ docs/CODEX_EXECUTION_LOG.md | 13 ++- docs/TODO.md | 5 +- .../build_detection_model_promotion_report.py | 41 +++++--- 5 files changed, 142 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30878aa4..b0257a58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,13 @@ - Fixed the operator YOLO training wrapper so the all-in-one runtime defaults to `/opt/geointel/venv/bin/python` when present, while still falling back to `python3` for local shells. - Raised the Nginx upload limit to `250m` in both the compose frontend proxy and Unraid all-in-one proxy after a live 1024px GeoTIFF QA upload hit `413 Request Entity Too Large`. - Raised Nginx proxy read/send timeouts to `600s` after a long low-threshold persisted YOLO/QA run hit `504 Gateway Timeout`. +- Hardened `build_detection_model_promotion_report.py` so it correctly accepts both calibration evidence portfolios and multi-sample quality summaries as positive evidence inputs. - Prepared a larger Tower operator sample manifest at `/app/storage/operator-data/operator-samples-1024` using explicit `1024x1024` rasters and doubled AOI half-size. - Exported and audited `/app/storage/operator-data/yolo-building-aoi1024-visible025`: 144 tiles, 117 positive tiles, 27 negative tiles, 15,079 labels and `min_label_visible_ratio=0.25`; audit remains `needs_attention` because median label area is still below gate. - Trained inactive local model asset `geointel-building-yolov8s-aoi1024visible025e50-pt` from the AOI1024 dataset. Ultralytics validation ended at approximately precision `0.275`, recall `0.331`, mAP50 `0.188` and mAP50-95 `0.0716`. - Redeployed Tower and verified the previously failing Geel low-threshold persisted YOLO/QA path now completes instead of returning `504`; the run produced F1 `0.09136212624584718`, so the model remains rejected for default use. +- Ran the full four-sample AOI1024 positive matrix and nine-sample hard-negative matrix. Best positive result was Westerlo threshold `0.15` with F1 `0.28703703703703703`; background pressure still reached 59 detections at threshold `0.25`, 107 at `0.15` and 226 at `0.05`. +- Generated the AOI1024 promotion report after the parser fix; recommended candidate remains `none`. - The candidate remains inactive and must pass persisted detection QA/QC plus background/hard-negative promotion gates before default activation. - No API contract, migration, product feature, provider fetching, fake detection data or active model default changed. diff --git a/backend/tests/test_sprint143_detection_model_promotion_report.py b/backend/tests/test_sprint143_detection_model_promotion_report.py index ade6d76e..f1441dac 100644 --- a/backend/tests/test_sprint143_detection_model_promotion_report.py +++ b/backend/tests/test_sprint143_detection_model_promotion_report.py @@ -252,3 +252,100 @@ def test_detection_model_promotion_report_uses_portfolio_and_tile_defaults( report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8")) assert report["recommended_candidate"]["candidate_key"] == "candidate-from-portfolio|640|64|0.25" + + +def test_detection_model_promotion_report_accepts_multi_sample_quality_summary( + tmp_path: Path, +) -> None: + script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py" + + positive_path = tmp_path / "multi_sample_quality_summary.json" + positive_path.write_text( + json.dumps( + { + "items": [ + { + "sample_slug": "geel", + "model_asset_id": "candidate-multi", + "tile_size": 512, + "tile_overlap": 64, + "threshold": 0.15, + "precision": 0.2, + "recall": 0.1, + "f1": 0.1333333333, + }, + { + "sample_slug": "retie", + "model_asset_id": "candidate-multi", + "tile_size": 512, + "tile_overlap": 64, + "threshold": 0.15, + "precision": 0.3, + "recall": 0.2, + "f1_score": 0.24, + }, + ] + } + ), + encoding="utf-8", + ) + + background_path = tmp_path / "hard_negative_matrix_summary.json" + background_path.write_text( + json.dumps( + { + "items": [ + { + "sample_slug": "postel_bos", + "model_asset_id": "candidate-multi", + "tile_size": 512, + "tile_overlap": 64, + "threshold": 0.15, + "detection_count": 0, + }, + { + "sample_slug": "lommel_heide", + "model_asset_id": "candidate-multi", + "tile_size": 512, + "tile_overlap": 64, + "threshold": 0.15, + "detection_count": 0, + }, + ] + } + ), + encoding="utf-8", + ) + + output_dir = tmp_path / "promotion-report" + subprocess.run( + [ + "python", + str(script_path), + "--positive-portfolio", + str(positive_path), + "--hard-negative-summary", + str(background_path), + "--output-dir", + str(output_dir), + "--min-positive-samples", + "2", + "--min-background-samples", + "2", + "--min-mean-f1", + "0.1", + "--max-background-detections-per-sample", + "0", + ], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + + report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8")) + decision = report["candidate_decisions"][0] + assert decision["candidate_key"] == "candidate-multi|512|64|0.15" + assert decision["positive_sample_count"] == 2 + assert decision["mean_f1"] > 0.18 + assert decision["promotion_status"] == "promote_candidate" diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index b1850722..57f6691a 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5965,6 +5965,16 @@ Open: - the previous Geel 1024px upload `413` no longer occurs - the previous Geel low-threshold persisted YOLO/QA `504` no longer occurs - Geel `threshold=0.05` completed with 2,612 detections, 5,091 raw candidates, 2,479 suppressed duplicates, precision `0.06316998468606431`, recall `0.165` and F1 `0.09136212624584718` +- Full AOI1024 positive persisted QA matrix: + - Geel best F1: `0.09136212624584718` at threshold `0.05` + - Turnhout best F1: `0.058721074894252295` at threshold `0.05` + - Retie best F1: `0.15621436716077539` at threshold `0.15` + - Westerlo best F1: `0.28703703703703703` at threshold `0.15` +- Full AOI1024 hard-negative/background matrix: + - pure empty Postel/Lommel/Arendonk samples stayed at or near zero detections + - mixed background candidates still produced false-positive pressure: max detections were `59` at threshold `0.25`, `107` at `0.15` and `226` at `0.05` +- Fixed `scripts/build_detection_model_promotion_report.py` after discovering it accepted `multi_sample_quality_summary.json` but counted positive samples as `0`. The report now supports both `samples[].runs[]` portfolios and `items[]` multi-sample summaries. +- AOI1024 promotion report path: `artifacts/detection-model-promotion/aoi1024visible025e50-full/detection_model_promotion_report.md`; recommendation remains `none`. ## Known limitations @@ -5972,7 +5982,8 @@ Open: - Several dense 1024 GRB reference exports reached the current 1000-feature source cap. Treat those samples as useful but potentially reference-capped until the provider query path supports paging or smaller dense AOIs are chosen. - The trained model is intentionally inactive. It needs persisted detection QA/QC matrix evidence and background/hard-negative evidence before default promotion. - The first completed persisted Geel QA run confirms the candidate is not promotion-ready: recall improves at low threshold, but false-positive pressure is too high. +- The complete positive/background evidence confirms the candidate is not promotion-ready. The bottleneck is label/source quality and sample design, not runtime configuration. ## Next recommended pass -- Run the remaining AOI1024 persisted QA samples and background/hard-negative checks, then decide whether label/source paging or additional AOI quality work comes before another training run. +- Fix dense GRB reference completeness first: add provider-side paging or split dense AOIs so reference exports do not cap at 1000 features, then regenerate AOI1024 labels before another training run. diff --git a/docs/TODO.md b/docs/TODO.md index 60c8ed1d..2e3f5f42 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -468,6 +468,9 @@ This file now starts with the current implementation status. Older preparation/b - [x] Fix the all-in-one/compose Nginx upload limit after live 1024px GeoTIFF uploads hit `413 Request Entity Too Large`. - [x] Fix the all-in-one/compose Nginx proxy timeout after low-threshold persisted YOLO/QA runs hit `504 Gateway Timeout`. - [x] Rerun the previously failing Geel low-threshold persisted QA/QC path for `geointel-building-yolov8s-aoi1024visible025e50-pt` after redeploying upload/timeout fixes. -- [ ] Finish remaining AOI1024 positive-sample matrix coverage for `geointel-building-yolov8s-aoi1024visible025e50-pt`. +- [x] Finish remaining AOI1024 positive-sample matrix coverage for `geointel-building-yolov8s-aoi1024visible025e50-pt`. +- [x] Run AOI1024 background/hard-negative matrix for `geointel-building-yolov8s-aoi1024visible025e50-pt`. +- [x] Fix promotion-report parsing for `multi_sample_quality_summary.json` inputs. +- [x] Generate AOI1024 promotion report and keep recommended candidate as `none`. - [ ] Add GRB paging or smaller dense AOI sampling before trusting 1000-feature-capped dense reference exports as full ground truth. - [ ] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation. diff --git a/scripts/build_detection_model_promotion_report.py b/scripts/build_detection_model_promotion_report.py index 6781b7ba..2dd36534 100644 --- a/scripts/build_detection_model_promotion_report.py +++ b/scripts/build_detection_model_promotion_report.py @@ -109,24 +109,35 @@ def collect_positive_runs( ) -> dict[CandidateKey, list[dict[str, Any]]]: best_by_candidate_sample: dict[tuple[CandidateKey, str], dict[str, Any]] = {} portfolio_model_asset_id = portfolio.get("model_asset_id") + + positive_runs: list[tuple[str, dict[str, Any]]] = [] for sample in portfolio.get("samples") or []: sample_slug = str(sample.get("sample_slug") or "unknown") for run in sample.get("runs") or []: - key = as_candidate_key( - run, - fallback_model_asset_id=str(portfolio_model_asset_id) if portfolio_model_asset_id else None, - fallback_tile_size=default_tile_size, - fallback_tile_overlap=default_tile_overlap, - ) - f1 = numeric(run, "f1_score") - if key is None or f1 is None: - continue - enriched = dict(run) - enriched["sample_slug"] = sample_slug - existing = best_by_candidate_sample.get((key, sample_slug)) - existing_f1 = numeric(existing or {}, "f1_score") - if existing is None or existing_f1 is None or f1 > existing_f1: - best_by_candidate_sample[(key, sample_slug)] = enriched + positive_runs.append((sample_slug, run)) + for item in portfolio.get("items") or []: + sample_slug = str(item.get("sample_slug") or "unknown") + positive_runs.append((sample_slug, item)) + + for sample_slug, run in positive_runs: + key = as_candidate_key( + run, + fallback_model_asset_id=str(portfolio_model_asset_id) if portfolio_model_asset_id else None, + fallback_tile_size=default_tile_size, + fallback_tile_overlap=default_tile_overlap, + ) + f1 = numeric(run, "f1_score") + if f1 is None: + f1 = numeric(run, "f1") + if key is None or f1 is None: + continue + enriched = dict(run) + enriched["f1_score"] = f1 + enriched["sample_slug"] = sample_slug + existing = best_by_candidate_sample.get((key, sample_slug)) + existing_f1 = numeric(existing or {}, "f1_score") + if existing is None or existing_f1 is None or f1 > existing_f1: + best_by_candidate_sample[(key, sample_slug)] = enriched grouped: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list) for (key, _sample_slug), run in best_by_candidate_sample.items():