Add detection model promotion report

CHANGELOG.md

@@ -7,6 +7,14 @@
 
 # Changelog
 
+## Sprint 143 Detection model promotion decision report (2026-07-08)
+
+- Added `scripts/build_detection_model_promotion_report.py` for operator-only model promotion review.
+- The report combines positive-AOI calibration evidence portfolios with hard-negative/background matrix summaries.
+- Candidate decisions are grouped by model asset, tile size, overlap and threshold, then gated by positive sample count, background sample count, mean F1 and maximum background detections per sample.
+- Added regression coverage for promoting a clean candidate and rejecting a candidate with background false-positive pressure.
+- No backend API, migration, frontend runtime, model weight, model download, inference or provider-fetching behavior changed.
+
 ## Sprint 141 Expanded positive-AOI matrix and portfolio metadata hardening (2026-07-08)
 
 - Ran a fresh Tower quality matrix for Balen, Herentals and Westerlo using `geointel-building-yolov8n-expanded160e50-pt` and `geointel-building-yolov8n-hardneg160r8e40-pt`.

backend/tests/test_sprint143_detection_model_promotion_report.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1].parent
+
+
+def test_detection_model_promotion_report_combines_positive_and_background_gates(
+    tmp_path: Path,
+) -> None:
+    script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py"
+    assert script_path.exists()
+
+    positive_path = tmp_path / "positive_portfolio.json"
+    positive_path.write_text(
+        json.dumps(
+            {
+                "portfolio_name": "Positive AOI portfolio",
+                "samples": [
+                    {
+                        "sample_slug": "geel",
+                        "runs": [
+                            {
+                                "model_asset_id": "candidate-clean",
+                                "tile_size": 640,
+                                "tile_overlap": 64,
+                                "threshold": 0.25,
+                                "quality_score": 0.42,
+                                "precision": 0.7,
+                                "recall": 0.3,
+                                "f1_score": 0.42,
+                            },
+                            {
+                                "model_asset_id": "candidate-leaky",
+                                "tile_size": 640,
+                                "tile_overlap": 64,
+                                "threshold": 0.05,
+                                "quality_score": 0.55,
+                                "precision": 0.6,
+                                "recall": 0.52,
+                                "f1_score": 0.55,
+                            },
+                        ],
+                    },
+                    {
+                        "sample_slug": "mol",
+                        "runs": [
+                            {
+                                "model_asset_id": "candidate-clean",
+                                "tile_size": 640,
+                                "tile_overlap": 64,
+                                "threshold": 0.25,
+                                "quality_score": 0.38,
+                                "precision": 0.64,
+                                "recall": 0.27,
+                                "f1_score": 0.38,
+                            },
+                            {
+                                "model_asset_id": "candidate-leaky",
+                                "tile_size": 640,
+                                "tile_overlap": 64,
+                                "threshold": 0.05,
+                                "quality_score": 0.5,
+                                "precision": 0.55,
+                                "recall": 0.46,
+                                "f1_score": 0.5,
+                            },
+                        ],
+                    },
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    background_path = tmp_path / "hard_negative_matrix_summary.json"
+    background_path.write_text(
+        json.dumps(
+            {
+                "items": [
+                    {
+                        "sample_slug": "postel_bos",
+                        "model_asset_id": "candidate-clean",
+                        "tile_size": 640,
+                        "tile_overlap": 64,
+                        "threshold": 0.25,
+                        "detection_count": 0,
+                    },
+                    {
+                        "sample_slug": "lommel_heide",
+                        "model_asset_id": "candidate-clean",
+                        "tile_size": 640,
+                        "tile_overlap": 64,
+                        "threshold": 0.25,
+                        "detection_count": 0,
+                    },
+                    {
+                        "sample_slug": "postel_bos",
+                        "model_asset_id": "candidate-leaky",
+                        "tile_size": 640,
+                        "tile_overlap": 64,
+                        "threshold": 0.05,
+                        "detection_count": 3,
+                    },
+                    {
+                        "sample_slug": "lommel_heide",
+                        "model_asset_id": "candidate-leaky",
+                        "tile_size": 640,
+                        "tile_overlap": 64,
+                        "threshold": 0.05,
+                        "detection_count": 1,
+                    },
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    output_dir = tmp_path / "promotion-report"
+    result = subprocess.run(
+        [
+            "python",
+            str(script_path),
+            "--positive-portfolio",
+            str(positive_path),
+            "--hard-negative-summary",
+            str(background_path),
+            "--output-dir",
+            str(output_dir),
+            "--min-positive-samples",
+            "2",
+            "--min-background-samples",
+            "2",
+            "--min-mean-f1",
+            "0.35",
+            "--max-background-detections-per-sample",
+            "0",
+        ],
+        cwd=ROOT,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+
+    assert "Detection model promotion report passed" in result.stdout
+    report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8"))
+    decisions = {
+        item["candidate_key"]: item["promotion_status"]
+        for item in report["candidate_decisions"]
+    }
+    assert decisions["candidate-clean|640|64|0.25"] == "promote_candidate"
+    assert decisions["candidate-leaky|640|64|0.05"] == "reject"
+
+    leaky = next(
+        item
+        for item in report["candidate_decisions"]
+        if item["candidate_key"] == "candidate-leaky|640|64|0.05"
+    )
+    assert "background_false_positive_pressure" in leaky["rejection_reasons"]
+    assert leaky["max_background_detections"] == 3
+    assert report["recommended_candidate"]["candidate_key"] == "candidate-clean|640|64|0.25"
+
+    markdown = (output_dir / "detection_model_promotion_report.md").read_text(encoding="utf-8")
+    assert "candidate-clean" in markdown
+    assert "candidate-leaky" in markdown
+    assert "background_false_positive_pressure" in markdown

docs/CODEX_EXECUTION_LOG.md

@@ -1,3 +1,25 @@
+## Sprint 143 Detection model promotion decision report (2026-07-08)
+
+Changed:
+- Added `scripts/build_detection_model_promotion_report.py` as operator-only evidence tooling.
+- The script combines a positive-AOI `calibration_evidence_portfolio.json` with one or more `hard_negative_matrix_summary.json` files.
+- Candidate rows are grouped by `model_asset_id`, `tile_size`, `tile_overlap` and `threshold`.
+- Promotion gates are explicit:
+  - minimum positive sample count
+  - minimum background sample count
+  - minimum mean positive F1
+  - maximum background detections per sample
+- Added readiness `py_compile` coverage for the new script.
+- Documented the Tower command in `scripts/README.md`.
+
+Tested:
+- Red step: `python -m pytest backend\tests\test_sprint143_detection_model_promotion_report.py -q` failed because the report script did not exist.
+- `python -m pytest backend\tests\test_sprint143_detection_model_promotion_report.py -q` (`1 passed`)
+
+Open:
+- Run the report against the regenerated 7-AOI positive portfolio and live hard-negative summaries on Tower.
+- Use the report as a promotion gate only; it must not mutate the active YOLO model configuration.
+
 ## Sprint 141 Expanded positive-AOI matrix and portfolio metadata hardening (2026-07-08)
 
 Changed:

docs/TODO.md

@@ -430,5 +430,6 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Run fresh positive-AOI matrix coverage for Balen, Herentals and Westerlo.
 - [x] Preserve model/tile provenance in calibration evidence bundle summaries.
 - [x] Prevent same-threshold calibration evidence responses from overwriting each other in multi-model portfolios.
+- [x] Add a model promotion decision report that combines positive-AOI score with hard-negative false-positive pressure.
 - [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout.
 - [ ] Add negative/background AOIs so the next tile dataset is not all positive tiles.

scripts/README.md

@@ -551,6 +551,24 @@ Tower-local model evaluation status:
 - Do not silently activate this model as a default. Apply it only as an explicit
   operator choice until the model catalog/threshold workflow is hardened.
 
+Build a model promotion decision report from an existing positive-AOI evidence
+portfolio and one or more hard-negative/background summaries:
+
+```bash
+python scripts/build_detection_model_promotion_report.py \
+  --positive-portfolio /mnt/user/appdata/geointel/artifacts/detection-calibration-portfolio/positive-aoi-expanded-20260708/output/calibration_evidence_portfolio.json \
+  --hard-negative-summary /mnt/user/appdata/geointel/artifacts/detection-hard-negatives/hardneg160r8e40-live/hard_negative_matrix_summary.json \
+  --output-dir /mnt/user/appdata/geointel/artifacts/detection-model-promotion/expanded-positive-vs-hard-negative-20260708
+```
+
+The report writes `detection_model_promotion_report.json` and
+`detection_model_promotion_report.md`. It groups candidates by
+`model_asset_id`, tile size, tile overlap and confidence threshold, then applies
+explicit gates for positive-AOI sample count, background sample count, mean F1
+and maximum background detections per sample. It is evidence/report tooling
+only: it does not run inference, mutate application data, download models or
+change the active YOLO configuration.
+
 Clean old offline demo export artifacts without touching uploaded source data:
 
 ```bash

scripts/build_detection_model_promotion_report.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""Build an operator-only detection model promotion report.
+
+The report combines persisted positive-AOI QA portfolio evidence with
+hard-negative/background detection-count summaries. It does not run inference,
+mutate application state, download models or change active model configuration.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from statistics import mean
+from typing import Any
+
+
+@dataclass(frozen=True)
+class CandidateKey:
+    model_asset_id: str
+    tile_size: int
+    tile_overlap: int
+    threshold: float
+
+    @property
+    def label(self) -> str:
+        threshold = ("%f" % self.threshold).rstrip("0").rstrip(".")
+        return f"{self.model_asset_id}|{self.tile_size}|{self.tile_overlap}|{threshold}"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Build a detection model promotion report from positive and hard-negative evidence."
+    )
+    parser.add_argument("--positive-portfolio", required=True, help="Path to calibration_evidence_portfolio.json")
+    parser.add_argument(
+        "--hard-negative-summary",
+        action="append",
+        required=True,
+        help="Path to hard_negative_matrix_summary.json. May be supplied multiple times.",
+    )
+    parser.add_argument("--output-dir", required=True, help="Directory for JSON and Markdown report output.")
+    parser.add_argument("--min-positive-samples", type=int, default=3)
+    parser.add_argument("--min-background-samples", type=int, default=3)
+    parser.add_argument("--min-mean-f1", type=float, default=0.25)
+    parser.add_argument("--max-background-detections-per-sample", type=int, default=0)
+    return parser.parse_args()
+
+
+def read_json(path: Path) -> dict[str, Any]:
+    if not path.is_file():
+        raise SystemExit(f"JSON input is not readable: {path}")
+    return json.loads(path.read_text(encoding="utf-8-sig"))
+
+
+def as_candidate_key(item: dict[str, Any]) -> CandidateKey | None:
+    model_asset_id = item.get("model_asset_id") or item.get("model_request")
+    if not model_asset_id:
+        return None
+    try:
+        return CandidateKey(
+            model_asset_id=str(model_asset_id),
+            tile_size=int(item.get("tile_size")),
+            tile_overlap=int(item.get("tile_overlap")),
+            threshold=float(item.get("threshold")),
+        )
+    except (TypeError, ValueError):
+        return None
+
+
+def numeric(item: dict[str, Any], key: str) -> float | None:
+    value = item.get(key)
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def collect_positive_runs(portfolio: dict[str, Any]) -> dict[CandidateKey, list[dict[str, Any]]]:
+    best_by_candidate_sample: dict[tuple[CandidateKey, str], dict[str, Any]] = {}
+    for sample in portfolio.get("samples") or []:
+        sample_slug = str(sample.get("sample_slug") or "unknown")
+        for run in sample.get("runs") or []:
+            key = as_candidate_key(run)
+            f1 = numeric(run, "f1_score")
+            if key is None or f1 is None:
+                continue
+            enriched = dict(run)
+            enriched["sample_slug"] = sample_slug
+            existing = best_by_candidate_sample.get((key, sample_slug))
+            existing_f1 = numeric(existing or {}, "f1_score")
+            if existing is None or existing_f1 is None or f1 > existing_f1:
+                best_by_candidate_sample[(key, sample_slug)] = enriched
+
+    grouped: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list)
+    for (key, _sample_slug), run in best_by_candidate_sample.items():
+        grouped[key].append(run)
+    return grouped
+
+
+def collect_background_runs(summary_paths: list[Path]) -> dict[CandidateKey, list[dict[str, Any]]]:
+    best_by_candidate_sample: dict[tuple[CandidateKey, str], dict[str, Any]] = {}
+    for path in summary_paths:
+        payload = read_json(path)
+        for item in payload.get("items") or []:
+            key = as_candidate_key(item)
+            sample_slug = str(item.get("sample_slug") or "unknown")
+            if key is None:
+                continue
+            enriched = dict(item)
+            enriched["source_summary_path"] = str(path)
+            detection_count = int(enriched.get("detection_count") or 0)
+            existing = best_by_candidate_sample.get((key, sample_slug))
+            existing_count = int((existing or {}).get("detection_count") or 0)
+            if existing is None or detection_count > existing_count:
+                best_by_candidate_sample[(key, sample_slug)] = enriched
+
+    grouped: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list)
+    for (key, _sample_slug), item in best_by_candidate_sample.items():
+        grouped[key].append(item)
+    return grouped
+
+
+def average_metric(runs: list[dict[str, Any]], metric: str) -> float | None:
+    values = [numeric(run, metric) for run in runs]
+    values = [value for value in values if value is not None]
+    return mean(values) if values else None
+
+
+def build_decisions(args: argparse.Namespace) -> dict[str, Any]:
+    portfolio_path = Path(args.positive_portfolio)
+    positive_portfolio = read_json(portfolio_path)
+    background_paths = [Path(path) for path in args.hard_negative_summary]
+    positive = collect_positive_runs(positive_portfolio)
+    background = collect_background_runs(background_paths)
+
+    decisions = []
+    for key in sorted(set(positive) | set(background), key=lambda item: item.label):
+        positive_runs = positive.get(key, [])
+        background_runs = background.get(key, [])
+        f1_values = [numeric(run, "f1_score") for run in positive_runs]
+        f1_values = [value for value in f1_values if value is not None]
+        background_counts = [int(run.get("detection_count") or 0) for run in background_runs]
+
+        mean_f1 = mean(f1_values) if f1_values else None
+        min_f1 = min(f1_values) if f1_values else None
+        total_background = sum(background_counts)
+        max_background = max(background_counts, default=None)
+
+        rejection_reasons: list[str] = []
+        if len(positive_runs) < args.min_positive_samples:
+            rejection_reasons.append("insufficient_positive_samples")
+        if len(background_runs) < args.min_background_samples:
+            rejection_reasons.append("insufficient_background_samples")
+        if mean_f1 is None or mean_f1 < args.min_mean_f1:
+            rejection_reasons.append("positive_mean_f1_below_gate")
+        if max_background is None or max_background > args.max_background_detections_per_sample:
+            rejection_reasons.append("background_false_positive_pressure")
+
+        decisions.append(
+            {
+                "candidate_key": key.label,
+                "model_asset_id": key.model_asset_id,
+                "tile_size": key.tile_size,
+                "tile_overlap": key.tile_overlap,
+                "threshold": key.threshold,
+                "promotion_status": "reject" if rejection_reasons else "promote_candidate",
+                "rejection_reasons": rejection_reasons,
+                "positive_sample_count": len(positive_runs),
+                "background_sample_count": len(background_runs),
+                "mean_f1": mean_f1,
+                "min_f1": min_f1,
+                "mean_precision": average_metric(positive_runs, "precision"),
+                "mean_recall": average_metric(positive_runs, "recall"),
+                "total_background_detections": total_background,
+                "max_background_detections": max_background,
+                "positive_samples": sorted(str(run.get("sample_slug")) for run in positive_runs),
+                "background_samples": sorted(
+                    {
+                        str(run.get("sample_slug")): int(run.get("detection_count") or 0)
+                        for run in background_runs
+                    }.items()
+                ),
+            }
+        )
+
+    promoted = [item for item in decisions if item["promotion_status"] == "promote_candidate"]
+    recommended = max(
+        promoted,
+        key=lambda item: (
+            item["mean_f1"] if item["mean_f1"] is not None else float("-inf"),
+            -(item["total_background_detections"] or 0),
+        ),
+        default=None,
+    )
+
+    return {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "positive_portfolio_path": str(portfolio_path),
+        "hard_negative_summary_paths": [str(path) for path in background_paths],
+        "gates": {
+            "min_positive_samples": args.min_positive_samples,
+            "min_background_samples": args.min_background_samples,
+            "min_mean_f1": args.min_mean_f1,
+            "max_background_detections_per_sample": args.max_background_detections_per_sample,
+        },
+        "candidate_count": len(decisions),
+        "recommended_candidate": recommended,
+        "candidate_decisions": decisions,
+    }
+
+
+def write_markdown(report: dict[str, Any], path: Path) -> None:
+    lines = [
+        "# Detection Model Promotion Report",
+        "",
+        f"- Generated: {report['generated_at']}",
+        f"- Positive portfolio: `{report['positive_portfolio_path']}`",
+        f"- Hard-negative summaries: {len(report['hard_negative_summary_paths'])}",
+        f"- Candidates: {report['candidate_count']}",
+        "",
+        "## Gates",
+        "",
+    ]
+    for key, value in report["gates"].items():
+        lines.append(f"- {key}: `{value}`")
+    lines.extend(["", "## Recommendation", ""])
+    recommended = report.get("recommended_candidate")
+    if recommended:
+        lines.append(f"- Promote candidate for operator review: `{recommended['candidate_key']}`")
+    else:
+        lines.append("- No candidate passed all positive and hard-negative gates.")
+    lines.extend(["", "## Candidate Decisions", ""])
+    for item in report["candidate_decisions"]:
+        reasons = ", ".join(item["rejection_reasons"]) or "none"
+        lines.extend(
+            [
+                f"### `{item['candidate_key']}`",
+                "",
+                f"- Status: `{item['promotion_status']}`",
+                f"- Rejection reasons: `{reasons}`",
+                f"- Positive samples: `{item['positive_sample_count']}`",
+                f"- Background samples: `{item['background_sample_count']}`",
+                f"- Mean F1: `{item['mean_f1']}`",
+                f"- Mean precision: `{item['mean_precision']}`",
+                f"- Mean recall: `{item['mean_recall']}`",
+                f"- Max background detections: `{item['max_background_detections']}`",
+                f"- Total background detections: `{item['total_background_detections']}`",
+                "",
+            ]
+        )
+    path.write_text("\n".join(lines), encoding="utf-8")
+
+
+def main() -> None:
+    args = parse_args()
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    report = build_decisions(args)
+    report_path = output_dir / "detection_model_promotion_report.json"
+    markdown_path = output_dir / "detection_model_promotion_report.md"
+    report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
+    write_markdown(report, markdown_path)
+
+    print("Detection model promotion report passed")
+    print(f"Report JSON: {report_path}")
+    print(f"Report Markdown: {markdown_path}")
+    recommended = report.get("recommended_candidate")
+    print(f"Recommended candidate: {recommended['candidate_key'] if recommended else 'none'}")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_readiness_check.sh

@@ -44,6 +44,7 @@ ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
 ${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
 ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
 ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
+${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
 ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m compileall backend/app