Support multi-sample promotion evidence

CHANGELOG.md

@@ -12,10 +12,13 @@
 - Fixed the operator YOLO training wrapper so the all-in-one runtime defaults to `/opt/geointel/venv/bin/python` when present, while still falling back to `python3` for local shells.
 - Raised the Nginx upload limit to `250m` in both the compose frontend proxy and Unraid all-in-one proxy after a live 1024px GeoTIFF QA upload hit `413 Request Entity Too Large`.
 - Raised Nginx proxy read/send timeouts to `600s` after a long low-threshold persisted YOLO/QA run hit `504 Gateway Timeout`.
+- Hardened `build_detection_model_promotion_report.py` so it correctly accepts both calibration evidence portfolios and multi-sample quality summaries as positive evidence inputs.
 - Prepared a larger Tower operator sample manifest at `/app/storage/operator-data/operator-samples-1024` using explicit `1024x1024` rasters and doubled AOI half-size.
 - Exported and audited `/app/storage/operator-data/yolo-building-aoi1024-visible025`: 144 tiles, 117 positive tiles, 27 negative tiles, 15,079 labels and `min_label_visible_ratio=0.25`; audit remains `needs_attention` because median label area is still below gate.
 - Trained inactive local model asset `geointel-building-yolov8s-aoi1024visible025e50-pt` from the AOI1024 dataset. Ultralytics validation ended at approximately precision `0.275`, recall `0.331`, mAP50 `0.188` and mAP50-95 `0.0716`.
 - Redeployed Tower and verified the previously failing Geel low-threshold persisted YOLO/QA path now completes instead of returning `504`; the run produced F1 `0.09136212624584718`, so the model remains rejected for default use.
+- Ran the full four-sample AOI1024 positive matrix and nine-sample hard-negative matrix. Best positive result was Westerlo threshold `0.15` with F1 `0.28703703703703703`; background pressure still reached 59 detections at threshold `0.25`, 107 at `0.15` and 226 at `0.05`.
+- Generated the AOI1024 promotion report after the parser fix; recommended candidate remains `none`.
 - The candidate remains inactive and must pass persisted detection QA/QC plus background/hard-negative promotion gates before default activation.
 - No API contract, migration, product feature, provider fetching, fake detection data or active model default changed.
 

backend/tests/test_sprint143_detection_model_promotion_report.py

@@ -252,3 +252,100 @@ def test_detection_model_promotion_report_uses_portfolio_and_tile_defaults(
 
     report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8"))
     assert report["recommended_candidate"]["candidate_key"] == "candidate-from-portfolio|640|64|0.25"
+
+
+def test_detection_model_promotion_report_accepts_multi_sample_quality_summary(
+    tmp_path: Path,
+) -> None:
+    script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py"
+
+    positive_path = tmp_path / "multi_sample_quality_summary.json"
+    positive_path.write_text(
+        json.dumps(
+            {
+                "items": [
+                    {
+                        "sample_slug": "geel",
+                        "model_asset_id": "candidate-multi",
+                        "tile_size": 512,
+                        "tile_overlap": 64,
+                        "threshold": 0.15,
+                        "precision": 0.2,
+                        "recall": 0.1,
+                        "f1": 0.1333333333,
+                    },
+                    {
+                        "sample_slug": "retie",
+                        "model_asset_id": "candidate-multi",
+                        "tile_size": 512,
+                        "tile_overlap": 64,
+                        "threshold": 0.15,
+                        "precision": 0.3,
+                        "recall": 0.2,
+                        "f1_score": 0.24,
+                    },
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    background_path = tmp_path / "hard_negative_matrix_summary.json"
+    background_path.write_text(
+        json.dumps(
+            {
+                "items": [
+                    {
+                        "sample_slug": "postel_bos",
+                        "model_asset_id": "candidate-multi",
+                        "tile_size": 512,
+                        "tile_overlap": 64,
+                        "threshold": 0.15,
+                        "detection_count": 0,
+                    },
+                    {
+                        "sample_slug": "lommel_heide",
+                        "model_asset_id": "candidate-multi",
+                        "tile_size": 512,
+                        "tile_overlap": 64,
+                        "threshold": 0.15,
+                        "detection_count": 0,
+                    },
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    output_dir = tmp_path / "promotion-report"
+    subprocess.run(
+        [
+            "python",
+            str(script_path),
+            "--positive-portfolio",
+            str(positive_path),
+            "--hard-negative-summary",
+            str(background_path),
+            "--output-dir",
+            str(output_dir),
+            "--min-positive-samples",
+            "2",
+            "--min-background-samples",
+            "2",
+            "--min-mean-f1",
+            "0.1",
+            "--max-background-detections-per-sample",
+            "0",
+        ],
+        cwd=ROOT,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+
+    report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8"))
+    decision = report["candidate_decisions"][0]
+    assert decision["candidate_key"] == "candidate-multi|512|64|0.15"
+    assert decision["positive_sample_count"] == 2
+    assert decision["mean_f1"] > 0.18
+    assert decision["promotion_status"] == "promote_candidate"

docs/CODEX_EXECUTION_LOG.md

@@ -5965,6 +5965,16 @@ Open:
   - the previous Geel 1024px upload `413` no longer occurs
   - the previous Geel low-threshold persisted YOLO/QA `504` no longer occurs
   - Geel `threshold=0.05` completed with 2,612 detections, 5,091 raw candidates, 2,479 suppressed duplicates, precision `0.06316998468606431`, recall `0.165` and F1 `0.09136212624584718`
+- Full AOI1024 positive persisted QA matrix:
+  - Geel best F1: `0.09136212624584718` at threshold `0.05`
+  - Turnhout best F1: `0.058721074894252295` at threshold `0.05`
+  - Retie best F1: `0.15621436716077539` at threshold `0.15`
+  - Westerlo best F1: `0.28703703703703703` at threshold `0.15`
+- Full AOI1024 hard-negative/background matrix:
+  - pure empty Postel/Lommel/Arendonk samples stayed at or near zero detections
+  - mixed background candidates still produced false-positive pressure: max detections were `59` at threshold `0.25`, `107` at `0.15` and `226` at `0.05`
+- Fixed `scripts/build_detection_model_promotion_report.py` after discovering it accepted `multi_sample_quality_summary.json` but counted positive samples as `0`. The report now supports both `samples[].runs[]` portfolios and `items[]` multi-sample summaries.
+- AOI1024 promotion report path: `artifacts/detection-model-promotion/aoi1024visible025e50-full/detection_model_promotion_report.md`; recommendation remains `none`.
 
 ## Known limitations
 
@@ -5972,7 +5982,8 @@ Open:
 - Several dense 1024 GRB reference exports reached the current 1000-feature source cap. Treat those samples as useful but potentially reference-capped until the provider query path supports paging or smaller dense AOIs are chosen.
 - The trained model is intentionally inactive. It needs persisted detection QA/QC matrix evidence and background/hard-negative evidence before default promotion.
 - The first completed persisted Geel QA run confirms the candidate is not promotion-ready: recall improves at low threshold, but false-positive pressure is too high.
+- The complete positive/background evidence confirms the candidate is not promotion-ready. The bottleneck is label/source quality and sample design, not runtime configuration.
 
 ## Next recommended pass
 
-- Run the remaining AOI1024 persisted QA samples and background/hard-negative checks, then decide whether label/source paging or additional AOI quality work comes before another training run.
+- Fix dense GRB reference completeness first: add provider-side paging or split dense AOIs so reference exports do not cap at 1000 features, then regenerate AOI1024 labels before another training run.

docs/TODO.md

@@ -468,6 +468,9 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Fix the all-in-one/compose Nginx upload limit after live 1024px GeoTIFF uploads hit `413 Request Entity Too Large`.
 - [x] Fix the all-in-one/compose Nginx proxy timeout after low-threshold persisted YOLO/QA runs hit `504 Gateway Timeout`.
 - [x] Rerun the previously failing Geel low-threshold persisted QA/QC path for `geointel-building-yolov8s-aoi1024visible025e50-pt` after redeploying upload/timeout fixes.
-- [ ] Finish remaining AOI1024 positive-sample matrix coverage for `geointel-building-yolov8s-aoi1024visible025e50-pt`.
+- [x] Finish remaining AOI1024 positive-sample matrix coverage for `geointel-building-yolov8s-aoi1024visible025e50-pt`.
+- [x] Run AOI1024 background/hard-negative matrix for `geointel-building-yolov8s-aoi1024visible025e50-pt`.
+- [x] Fix promotion-report parsing for `multi_sample_quality_summary.json` inputs.
+- [x] Generate AOI1024 promotion report and keep recommended candidate as `none`.
 - [ ] Add GRB paging or smaller dense AOI sampling before trusting 1000-feature-capped dense reference exports as full ground truth.
 - [ ] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation.

scripts/build_detection_model_promotion_report.py

@@ -109,24 +109,35 @@ def collect_positive_runs(
 ) -> dict[CandidateKey, list[dict[str, Any]]]:
     best_by_candidate_sample: dict[tuple[CandidateKey, str], dict[str, Any]] = {}
     portfolio_model_asset_id = portfolio.get("model_asset_id")
+
+    positive_runs: list[tuple[str, dict[str, Any]]] = []
     for sample in portfolio.get("samples") or []:
         sample_slug = str(sample.get("sample_slug") or "unknown")
         for run in sample.get("runs") or []:
-            key = as_candidate_key(
-                run,
-                fallback_model_asset_id=str(portfolio_model_asset_id) if portfolio_model_asset_id else None,
-                fallback_tile_size=default_tile_size,
-                fallback_tile_overlap=default_tile_overlap,
-            )
-            f1 = numeric(run, "f1_score")
-            if key is None or f1 is None:
-                continue
-            enriched = dict(run)
-            enriched["sample_slug"] = sample_slug
-            existing = best_by_candidate_sample.get((key, sample_slug))
-            existing_f1 = numeric(existing or {}, "f1_score")
-            if existing is None or existing_f1 is None or f1 > existing_f1:
-                best_by_candidate_sample[(key, sample_slug)] = enriched
+            positive_runs.append((sample_slug, run))
+    for item in portfolio.get("items") or []:
+        sample_slug = str(item.get("sample_slug") or "unknown")
+        positive_runs.append((sample_slug, item))
+
+    for sample_slug, run in positive_runs:
+        key = as_candidate_key(
+            run,
+            fallback_model_asset_id=str(portfolio_model_asset_id) if portfolio_model_asset_id else None,
+            fallback_tile_size=default_tile_size,
+            fallback_tile_overlap=default_tile_overlap,
+        )
+        f1 = numeric(run, "f1_score")
+        if f1 is None:
+            f1 = numeric(run, "f1")
+        if key is None or f1 is None:
+            continue
+        enriched = dict(run)
+        enriched["f1_score"] = f1
+        enriched["sample_slug"] = sample_slug
+        existing = best_by_candidate_sample.get((key, sample_slug))
+        existing_f1 = numeric(existing or {}, "f1_score")
+        if existing is None or existing_f1 is None or f1 > existing_f1:
+            best_by_candidate_sample[(key, sample_slug)] = enriched
 
     grouped: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list)
     for (key, _sample_slug), run in best_by_candidate_sample.items():