feat: add coverage-aware Mol benchmark

CHANGELOG.md

@@ -7,6 +7,14 @@
 
 # Changelog
 
+## Sprint 185 Coverage-aware Mol operational benchmark (2026-07-14)
+
+- Extended real detection quality-matrix evidence with persisted inference coverage, raw/evaluated/excluded/clipped populations and diagnostic-only box-to-footprint mismatch counts.
+- Preserved municipality, operational-zone, split and source-reference metadata in combined multi-sample summaries.
+- Added a fail-closed Mol benchmark report that groups exact model/tile/overlap/threshold candidates and gates four independent positive holdouts plus a pure-empty background control.
+- Kept canonical footprint-IoU metrics authoritative and made model-quality rejection a reported evidence outcome rather than a hidden runner failure or automatic model mutation.
+- Added readiness, all-in-one image and focused pass/reject regression coverage without changing APIs, migrations, inference behavior or dependencies.
+
 ## Sprint 184 Detection QA coverage and matching diagnostics (2026-07-14)
 
 - Clipped configured-YOLO candidate and reference QA populations to the union of the exact persisted inference tile footprints before canonical IoU matching.

backend/README.md

@@ -494,7 +494,10 @@ Mol additionally has operational holdouts for Achterbos, Gompel, Donk and
 Postel, with Mol center as the historical baseline and Postel-bos as a separate
 background control. Prepare and execute that pack with the documented
 `prepare_operator_real_data_samples.py` and
-`run_mol_operational_validation.sh` commands in `scripts/README.md`.
+`run_mol_operational_validation.sh` commands in `scripts/README.md`. The runner
+produces a coverage-aware operational decision report: canonical footprint-IoU
+metrics remain authoritative, reference-envelope matches remain diagnostic,
+and no report can activate or mutate a model asset.
 
 For municipality-wide navigation, run
 `/app/scripts/provision_mol_municipality_workspace.py` inside the all-in-one

backend/tests/test_sprint185_mol_coverage_benchmark.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import subprocess
+import sys
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def positive_item(slug: str, *, f1: float = 0.6, coverage: bool = True) -> dict:
+    matches = 60
+    false_positives = 30
+    false_negatives = 50
+    return {
+        "sample_slug": slug,
+        "sample_display_name": f"Mol {slug}",
+        "municipality": "Mol",
+        "operational_zone": slug,
+        "recommended_split": "val",
+        "model_asset_id": "mol-model",
+        "tile_size": 512,
+        "tile_overlap": 64,
+        "threshold": 0.15,
+        "project_id": f"project-{slug}",
+        "area_id": f"area-{slug}",
+        "analysis_run_id": f"run-{slug}",
+        "quality_check_id": f"quality-{slug}",
+        "detection_count": 90,
+        "precision": 0.6666666667,
+        "recall": 0.5454545455,
+        "f1_score": f1,
+        "mean_iou": 0.65,
+        "matches": matches,
+        "false_positives": false_positives,
+        "false_negatives": false_negatives,
+        "coverage_applied": coverage,
+        "coverage_mode": "persisted_tile_manifest_union",
+        "coverage_tile_count": 9,
+        "candidate_raw_count": 90,
+        "candidate_evaluated_count": 90,
+        "candidate_excluded_outside_count": 0,
+        "candidate_clipped_boundary_count": 3,
+        "reference_raw_count": 112,
+        "reference_evaluated_count": 110,
+        "reference_excluded_outside_count": 2,
+        "reference_clipped_boundary_count": 4,
+        "reference_coverage_ratio": 110 / 112,
+        "diagnostic_only": True,
+        "strict_matches": matches,
+        "envelope_matches": 72,
+        "possible_box_to_footprint_mismatch_count": 12,
+        "envelope_precision": 0.8,
+        "envelope_recall": 0.65,
+        "envelope_f1_score": 0.717,
+    }
+
+
+def write_inputs(tmp_path: Path, *, rejected: bool = False) -> tuple[Path, Path, Path]:
+    slugs = ["mol_achterbos", "mol_gompel", "mol_donk", "mol_postel"]
+    items = [
+        positive_item(slug, f1=0.05 if rejected and slug == "mol_postel" else 0.6, coverage=not rejected)
+        for slug in slugs
+    ]
+    positive_path = tmp_path / "positive.json"
+    positive_path.write_text(json.dumps({"items": items}), encoding="utf-8")
+    background_path = tmp_path / "background.json"
+    background_path.write_text(
+        json.dumps(
+            {
+                "items": [
+                    {
+                        "sample_slug": "postel_bos",
+                        "background_category": "pure_empty_negative",
+                        "model_asset_id": "mol-model",
+                        "tile_size": 512,
+                        "tile_overlap": 64,
+                        "threshold": 0.15,
+                        "project_id": "project-background",
+                        "area_id": "area-background",
+                        "analysis_run_id": "run-background",
+                        "tile_count": 9,
+                        "detection_count": 2 if rejected else 0,
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+    manifest_path = tmp_path / "manifest.json"
+    manifest_path.write_text(
+        json.dumps(
+            {
+                "samples": [
+                    {
+                        "sample_slug": slug,
+                        "display_name": f"Mol {slug}",
+                        "municipality": "Mol",
+                        "operational_zone": slug,
+                        "recommended_split": "val",
+                    }
+                    for slug in slugs
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+    return positive_path, background_path, manifest_path
+
+
+def run_report(tmp_path: Path, *, rejected: bool = False) -> dict:
+    positive, background, manifest = write_inputs(tmp_path, rejected=rejected)
+    output_dir = tmp_path / "report"
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(ROOT / "scripts" / "build_mol_operational_benchmark_report.py"),
+            "--positive-summary",
+            str(positive),
+            "--background-summary",
+            str(background),
+            "--manifest-path",
+            str(manifest),
+            "--output-dir",
+            str(output_dir),
+            "--base-url",
+            "http://example.test",
+        ],
+        cwd=ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    assert "Mol operational benchmark report passed" in result.stdout
+    assert (output_dir / "mol_operational_benchmark_report.md").is_file()
+    return json.loads((output_dir / "mol_operational_benchmark_report.json").read_text(encoding="utf-8"))
+
+
+def test_mol_benchmark_accepts_coverage_safe_multi_zone_evidence(tmp_path: Path) -> None:
+    report = run_report(tmp_path)
+
+    assert report["status"] == "accepted"
+    assert report["recommendation"] == "retain_or_promote_candidate"
+    decision = report["recommended_candidate"]
+    assert decision["decision"] == "operationally_accepted"
+    assert decision["positive_sample_count"] == 4
+    assert decision["background_sample_count"] == 1
+    assert decision["total_references_raw"] == 448
+    assert decision["total_references_evaluated"] == 440
+    assert decision["total_box_to_footprint_mismatch_count"] == 48
+    assert decision["total_background_detections"] == 0
+    assert decision["failed_gates"] == []
+
+
+def test_mol_benchmark_rejects_missing_coverage_zone_collapse_and_background_pressure(tmp_path: Path) -> None:
+    report = run_report(tmp_path, rejected=True)
+
+    assert report["status"] == "review_required"
+    assert report["recommended_candidate"] is None
+    decision = report["candidate_decisions"][0]
+    assert decision["decision"] == "review_required"
+    assert "coverage_provenance" in decision["failed_gates"]
+    assert "minimum_zone_f1" in decision["failed_gates"]
+    assert "background_false_positive_pressure" in decision["failed_gates"]
+
+
+def test_mol_benchmark_is_wired_into_existing_operator_pipeline() -> None:
+    matrix = (ROOT / "scripts" / "run_detection_quality_matrix.sh").read_text(encoding="utf-8")
+    multi = (ROOT / "scripts" / "run_multi_sample_detection_quality_matrix.sh").read_text(encoding="utf-8")
+    runner = (ROOT / "scripts" / "run_mol_operational_validation.sh").read_text(encoding="utf-8")
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+    dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
+
+    assert 'findings.get("coverage")' in matrix
+    assert 'findings.get("box_to_footprint_diagnostics")' in matrix
+    assert '"reference_coverage_ratio"' in matrix
+    assert '"possible_box_to_footprint_mismatch_count"' in matrix
+    assert 'enriched["operational_zone"]' in multi
+    assert "build_mol_operational_benchmark_report.py" in runner
+    assert "MOL_MIN_REFERENCE_COVERAGE" in runner
+    assert "py_compile scripts/build_mol_operational_benchmark_report.py" in readiness
+    assert "COPY scripts/build_mol_operational_benchmark_report.py" in dockerfile
+    assert "fixture_mode" not in runner
+    assert "manual-fixture-detector" not in runner

deploy/unraid/Dockerfile.all-in-one

@@ -93,6 +93,7 @@ COPY scripts/run_operator_hard_negative_detection_matrix.sh /app/scripts/run_ope
 COPY scripts/run_background_corpus_split_matrix.sh /app/scripts/run_background_corpus_split_matrix.sh
 COPY scripts/build_background_corpus_split_report.py /app/scripts/build_background_corpus_split_report.py
 COPY scripts/build_detection_model_promotion_report.py /app/scripts/build_detection_model_promotion_report.py
+COPY scripts/build_mol_operational_benchmark_report.py /app/scripts/build_mol_operational_benchmark_report.py
 COPY scripts/run_split_background_promotion_workflow.sh /app/scripts/run_split_background_promotion_workflow.sh
 COPY scripts/activate_promoted_yolo_candidate.py /app/scripts/activate_promoted_yolo_candidate.py
 COPY deploy/unraid/nginx-all-in-one.conf /etc/nginx/conf.d/default.conf

docs/CODEX_EXECUTION_LOG.md

@@ -7659,3 +7659,42 @@ Open:
   from envelope diagnostics. The next safe model step is an evidence-led
   footprint-label/matching review, followed by a fresh bounded Mol multi-zone
   benchmark before any activation or retraining decision.
+
+# Sprint 185 - Coverage-aware Mol operational benchmark
+
+## Implementation
+
+- Extended the existing real-data quality matrix summaries with the exact
+  persisted tile-coverage population, raw/evaluated/excluded/clipped candidate
+  and reference counts, CRS/tile provenance and diagnostic-only
+  box-to-footprint match gap.
+- Preserved manifest-backed Mol municipality, operational-zone, validation
+  split, WGS84 bounds and source reference counts in the multi-sample summary.
+- Added `build_mol_operational_benchmark_report.py`. It groups evidence by the
+  exact model asset, tile size, overlap and confidence threshold and refuses to
+  blend different candidate configurations.
+- Added explicit gates for four independent positive Mol holdouts, one
+  pure-empty background control, complete coverage provenance, minimum 95%
+  reference coverage, mean F1 `0.25`, minimum per-zone F1 `0.10` and zero
+  background detections. Envelope matches remain diagnostic and cannot satisfy
+  the canonical F1 gates.
+- Wired the report into the persistent Mol operator runner, readiness gate and
+  all-in-one image. The report does not activate, replace, download or train a
+  model and does not change APIs, migrations or inference behavior.
+
+## Local validation
+
+- Focused Sprint 126/127/178/184/185 regression set passed: `14` tests.
+- Full readiness passed with `512` backend tests, one Alembic head
+  `202606120900`, 81 audited API operations, frontend typecheck and Vite 7.3.6
+  production build.
+- `npm audit --audit-level=high` reported zero vulnerabilities.
+- Shell syntax checks passed for the single-, multi-sample and Mol operational
+  runners. Fixture tests prove both an accepted four-zone result and explicit
+  rejection for missing coverage, one-zone collapse and background pressure.
+
+## Next pass
+
+- Deploy the operator tooling to Tower and run the current active local model
+  over Achterbos, Gompel, Donk, Postel and the Postel-bos pure-empty control.
+  Record the coverage-aware operational decision before changing model state.

docs/TODO.md

@@ -27,6 +27,8 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Persist combined Mol operator evidence under the Unraid storage mount so reports survive all-in-one container replacement.
 - [x] Visually review Mol Postel and Donk false-positive/false-negative evidence, classify the dominant error modes and only then decide whether another model-training pass is justified.
 - [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training.
+- [x] Add a coverage-aware Mol multi-zone benchmark report with explicit positive-zone, per-zone collapse, reference-coverage and pure-empty background gates.
+- [ ] Execute the refreshed coverage-aware Mol operational benchmark against the active local model and record the resulting retain/review decision.
 - [x] Backend FastAPI foundation, health endpoint and service structure.
 - [x] React/TypeScript frontend foundation and MapLibre workbench.
 - [x] Map layer visibility, opacity and feature property inspection.

scripts/README.md

@@ -261,9 +261,20 @@ confidence `0.15` and QA IoU `0.25`. Every positive run persists Project, Area,
 Dataset, Job, AnalysisRun, Detection, QualityCheck, Metric and Export records.
 The background run persists its project, AOI, raster, job, analysis and
 detections but intentionally does not invent QA metrics for an empty or sparse
-reference context. The combined JSON/Markdown summary reports
+reference context. Each matrix row now preserves the exact persisted inference
-`evidence_ready`; this records completed evidence and is not an automatic model
+coverage counts and the diagnostic-only reference-envelope comparison beside
-promotion decision.
+the canonical footprint-IoU metrics.
+
+The runner also writes `mol_operational_benchmark_report.json` and `.md`. The
+default operational gates require four positive holdouts, one background
+control, coverage provenance for every positive run, at least 95% reference
+coverage in every zone, mean F1 at least `0.25`, per-zone F1 at least `0.10`
+and zero detections in each pure-empty control. Override the numeric gates only
+through the documented `MOL_MIN_MEAN_F1`, `MOL_MIN_ZONE_F1`,
+`MOL_MIN_REFERENCE_COVERAGE` and `MOL_MAX_BACKGROUND_DETECTIONS` variables.
+An `accepted` report records bounded operational evidence; it does not mutate
+the active model. A `review_required` report is still a successful benchmark
+execution but explicitly blocks a promotion recommendation.
 
 For model-training candidates, prepare a larger operator-only sample manifest so
 tile overlap can create meaningful context instead of one tile per source
@@ -344,7 +355,10 @@ logs plus `quality_matrix_summary.json` under
 set. The summary ranks `best_by_score`, `best_by_recall` and
 `best_by_precision` so the next model decision is based on persisted
 `QualityCheck`/`Metric` evidence rather than visual guesses. It does not create
-provider data, use fixtures or download model weights.
+provider data, use fixtures or download model weights. Coverage-aware rows also
+record raw/evaluated/excluded/clipped candidate and reference counts, tile
+coverage provenance and the separately labelled box-to-footprint diagnostic
+gap.
 
 Run the same matrix across every prepared operator sample:
 

scripts/build_mol_operational_benchmark_report.py

@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+import argparse
+from collections import defaultdict
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+from statistics import fmean
+from typing import Any
+
+
+CandidateKey = tuple[str, int, int, float]
+
+
+def load_json(path: Path) -> dict[str, Any]:
+    if not path.is_file():
+        raise SystemExit(f"Input is not readable: {path}")
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8-sig"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise SystemExit(f"Input is not valid JSON: {path}: {exc}") from exc
+    if not isinstance(payload, dict):
+        raise SystemExit(f"Input root must be an object: {path}")
+    return payload
+
+
+def candidate_key(item: dict[str, Any]) -> CandidateKey:
+    model_asset_id = str(item.get("model_asset_id") or "").strip()
+    if not model_asset_id:
+        raise SystemExit("Benchmark item is missing model_asset_id")
+    try:
+        tile_size = int(item["tile_size"])
+        tile_overlap = int(item["tile_overlap"])
+        threshold = round(float(item["threshold"]), 8)
+    except (KeyError, TypeError, ValueError) as exc:
+        raise SystemExit(f"Benchmark item has an invalid model configuration: {item}") from exc
+    return model_asset_id, tile_size, tile_overlap, threshold
+
+
+def candidate_label(key: CandidateKey) -> str:
+    return f"{key[0]}|{key[1]}|{key[2]}|{key[3]:.8g}"
+
+
+def number(item: dict[str, Any], key: str) -> float:
+    value = item.get(key)
+    if value is None:
+        raise SystemExit(f"Positive benchmark item is missing {key}: {item.get('sample_slug')}")
+    try:
+        return float(value)
+    except (TypeError, ValueError) as exc:
+        raise SystemExit(f"Positive benchmark item has invalid {key}: {item.get('sample_slug')}") from exc
+
+
+def integer(item: dict[str, Any], key: str) -> int:
+    value = item.get(key)
+    if value is None:
+        raise SystemExit(f"Benchmark item is missing {key}: {item.get('sample_slug')}")
+    try:
+        return int(value)
+    except (TypeError, ValueError) as exc:
+        raise SystemExit(f"Benchmark item has invalid {key}: {item.get('sample_slug')}") from exc
+
+
+def coverage_ratio(item: dict[str, Any]) -> float:
+    explicit = item.get("reference_coverage_ratio")
+    if explicit is not None:
+        return float(explicit)
+    raw_count = integer(item, "reference_raw_count")
+    evaluated_count = integer(item, "reference_evaluated_count")
+    if raw_count < 1:
+        raise SystemExit(f"Positive benchmark item has no raw references: {item.get('sample_slug')}")
+    return evaluated_count / raw_count
+
+
+def gate(name: str, passed: bool, observed: Any, required: str) -> dict[str, Any]:
+    return {"name": name, "passed": bool(passed), "observed": observed, "required": required}
+
+
+def zone_row(item: dict[str, Any], manifest_samples: dict[str, dict[str, Any]]) -> dict[str, Any]:
+    slug = str(item.get("sample_slug") or "").strip().lower()
+    if not slug:
+        raise SystemExit("Positive benchmark item is missing sample_slug")
+    manifest = manifest_samples.get(slug, {})
+    matches = integer(item, "matches")
+    false_positives = integer(item, "false_positives")
+    false_negatives = integer(item, "false_negatives")
+    strict_matches = item.get("strict_matches")
+    if strict_matches is not None and int(strict_matches) != matches:
+        raise SystemExit(f"Diagnostic strict match count drifted from canonical matches for {slug}")
+    return {
+        "sample_slug": slug,
+        "display_name": item.get("sample_display_name") or manifest.get("display_name") or slug,
+        "municipality": item.get("municipality") or manifest.get("municipality"),
+        "operational_zone": item.get("operational_zone") or manifest.get("operational_zone"),
+        "recommended_split": item.get("recommended_split") or manifest.get("recommended_split"),
+        "project_id": item.get("project_id"),
+        "area_id": item.get("area_id"),
+        "analysis_run_id": item.get("analysis_run_id"),
+        "quality_check_id": item.get("quality_check_id"),
+        "detection_count": integer(item, "detection_count"),
+        "precision": number(item, "precision"),
+        "recall": number(item, "recall"),
+        "f1_score": number(item, "f1_score"),
+        "mean_iou": number(item, "mean_iou"),
+        "matches": matches,
+        "false_positives": false_positives,
+        "false_negatives": false_negatives,
+        "coverage_applied": item.get("coverage_applied") is True,
+        "coverage_mode": item.get("coverage_mode"),
+        "coverage_tile_count": integer(item, "coverage_tile_count"),
+        "reference_raw_count": integer(item, "reference_raw_count"),
+        "reference_evaluated_count": integer(item, "reference_evaluated_count"),
+        "reference_excluded_outside_count": integer(item, "reference_excluded_outside_count"),
+        "reference_clipped_boundary_count": integer(item, "reference_clipped_boundary_count"),
+        "reference_coverage_ratio": coverage_ratio(item),
+        "candidate_raw_count": integer(item, "candidate_raw_count"),
+        "candidate_evaluated_count": integer(item, "candidate_evaluated_count"),
+        "candidate_excluded_outside_count": integer(item, "candidate_excluded_outside_count"),
+        "candidate_clipped_boundary_count": integer(item, "candidate_clipped_boundary_count"),
+        "diagnostic_only": item.get("diagnostic_only") is True,
+        "strict_matches": int(strict_matches) if strict_matches is not None else None,
+        "envelope_matches": integer(item, "envelope_matches"),
+        "possible_box_to_footprint_mismatch_count": integer(item, "possible_box_to_footprint_mismatch_count"),
+        "envelope_precision": number(item, "envelope_precision"),
+        "envelope_recall": number(item, "envelope_recall"),
+        "envelope_f1_score": number(item, "envelope_f1_score"),
+    }
+
+
+def evaluate_candidate(
+    key: CandidateKey,
+    positive_items: list[dict[str, Any]],
+    background_items: list[dict[str, Any]],
+    manifest_samples: dict[str, dict[str, Any]],
+    args: argparse.Namespace,
+) -> dict[str, Any]:
+    zones = [zone_row(item, manifest_samples) for item in positive_items]
+    zone_slugs = [row["sample_slug"] for row in zones]
+    if len(zone_slugs) != len(set(zone_slugs)):
+        raise SystemExit(f"Candidate has duplicate positive sample rows: {candidate_label(key)}")
+    background_slugs = [str(item.get("sample_slug") or "").strip().lower() for item in background_items]
+    if any(not slug for slug in background_slugs) or len(background_slugs) != len(set(background_slugs)):
+        raise SystemExit(f"Candidate has invalid or duplicate background rows: {candidate_label(key)}")
+
+    f1_values = [row["f1_score"] for row in zones]
+    precision_values = [row["precision"] for row in zones]
+    recall_values = [row["recall"] for row in zones]
+    coverage_ratios = [row["reference_coverage_ratio"] for row in zones]
+    background_detections = [integer(item, "detection_count") for item in background_items]
+    total_matches = sum(row["matches"] for row in zones)
+    total_fp = sum(row["false_positives"] for row in zones)
+    total_fn = sum(row["false_negatives"] for row in zones)
+    micro_precision = total_matches / (total_matches + total_fp) if total_matches + total_fp else None
+    micro_recall = total_matches / (total_matches + total_fn) if total_matches + total_fn else None
+    micro_f1 = None
+    if micro_precision is not None and micro_recall is not None:
+        micro_f1 = 2 * micro_precision * micro_recall / (micro_precision + micro_recall) if micro_precision + micro_recall else 0.0
+
+    gates = [
+        gate("positive_sample_count", len(zones) >= args.min_positive_samples, len(zones), f">={args.min_positive_samples}"),
+        gate("background_sample_count", len(background_items) >= args.min_background_samples, len(background_items), f">={args.min_background_samples}"),
+        gate("coverage_provenance", all(row["coverage_applied"] for row in zones), sum(row["coverage_applied"] for row in zones), "all positive runs"),
+        gate("coverage_ratio", bool(coverage_ratios) and min(coverage_ratios) >= args.min_reference_coverage_ratio, min(coverage_ratios) if coverage_ratios else None, f">={args.min_reference_coverage_ratio}"),
+        gate("diagnostic_separation", all(row["diagnostic_only"] for row in zones), sum(row["diagnostic_only"] for row in zones), "all positive runs"),
+        gate("mean_f1", bool(f1_values) and fmean(f1_values) >= args.min_mean_f1, fmean(f1_values) if f1_values else None, f">={args.min_mean_f1}"),
+        gate("minimum_zone_f1", bool(f1_values) and min(f1_values) >= args.min_zone_f1, min(f1_values) if f1_values else None, f">={args.min_zone_f1}"),
+        gate("background_false_positive_pressure", bool(background_detections) and max(background_detections) <= args.max_background_detections, max(background_detections) if background_detections else None, f"<={args.max_background_detections} per sample"),
+    ]
+    failed_gates = [item["name"] for item in gates if not item["passed"]]
+    return {
+        "candidate_key": candidate_label(key),
+        "model_asset_id": key[0],
+        "tile_size": key[1],
+        "tile_overlap": key[2],
+        "threshold": key[3],
+        "decision": "operationally_accepted" if not failed_gates else "review_required",
+        "failed_gates": failed_gates,
+        "gates": gates,
+        "positive_sample_count": len(zones),
+        "background_sample_count": len(background_items),
+        "mean_precision": fmean(precision_values) if precision_values else None,
+        "mean_recall": fmean(recall_values) if recall_values else None,
+        "mean_f1": fmean(f1_values) if f1_values else None,
+        "minimum_zone_f1": min(f1_values) if f1_values else None,
+        "micro_precision": micro_precision,
+        "micro_recall": micro_recall,
+        "micro_f1": micro_f1,
+        "total_matches": total_matches,
+        "total_false_positives": total_fp,
+        "total_false_negatives": total_fn,
+        "total_background_detections": sum(background_detections),
+        "max_background_detections": max(background_detections) if background_detections else None,
+        "minimum_reference_coverage_ratio": min(coverage_ratios) if coverage_ratios else None,
+        "total_references_raw": sum(row["reference_raw_count"] for row in zones),
+        "total_references_evaluated": sum(row["reference_evaluated_count"] for row in zones),
+        "total_references_excluded_outside": sum(row["reference_excluded_outside_count"] for row in zones),
+        "total_references_clipped_boundary": sum(row["reference_clipped_boundary_count"] for row in zones),
+        "total_candidates_raw": sum(row["candidate_raw_count"] for row in zones),
+        "total_candidates_evaluated": sum(row["candidate_evaluated_count"] for row in zones),
+        "total_box_to_footprint_mismatch_count": sum(row["possible_box_to_footprint_mismatch_count"] for row in zones),
+        "zones": zones,
+        "background_controls": [
+            {
+                "sample_slug": str(item.get("sample_slug") or "").strip().lower(),
+                "background_category": item.get("background_category"),
+                "project_id": item.get("project_id"),
+                "area_id": item.get("area_id"),
+                "analysis_run_id": item.get("analysis_run_id"),
+                "tile_count": item.get("tile_count"),
+                "detection_count": integer(item, "detection_count"),
+            }
+            for item in background_items
+        ],
+    }
+
+
+def render_markdown(report: dict[str, Any]) -> str:
+    lines = [
+        "# Mol operational detection benchmark",
+        "",
+        f"- Generated: `{report['generated_at']}`",
+        f"- Status: `{report['status']}`",
+        f"- Recommendation: `{report['recommendation']}`",
+        f"- Candidate configurations: `{report['candidate_count']}`",
+        "",
+        "Canonical metrics remain footprint-IoU based. Envelope results are diagnostic only.",
+        "",
+    ]
+    for decision in report["candidate_decisions"]:
+        lines.extend(
+            [
+                f"## {decision['candidate_key']}",
+                "",
+                f"- Decision: `{decision['decision']}`",
+                f"- Mean/minimum F1: `{decision['mean_f1']:.4f}` / `{decision['minimum_zone_f1']:.4f}`",
+                f"- Micro precision/recall/F1: `{decision['micro_precision']:.4f}` / `{decision['micro_recall']:.4f}` / `{decision['micro_f1']:.4f}`",
+                f"- Canonical matches / FP / FN: `{decision['total_matches']}` / `{decision['total_false_positives']}` / `{decision['total_false_negatives']}`",
+                f"- Reference coverage: `{decision['total_references_evaluated']}` / `{decision['total_references_raw']}` evaluated; `{decision['total_references_excluded_outside']}` outside; `{decision['total_references_clipped_boundary']}` boundary-clipped",
+                f"- Diagnostic box-to-footprint gap: `{decision['total_box_to_footprint_mismatch_count']}`",
+                f"- Background detections: `{decision['total_background_detections']}`",
+                f"- Failed gates: `{', '.join(decision['failed_gates']) or 'none'}`",
+                "",
+                "| Zone | Precision | Recall | F1 | Coverage | Strict | Envelope | Gap |",
+                "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+            ]
+        )
+        for zone in decision["zones"]:
+            lines.append(
+                f"| {zone['display_name']} | {zone['precision']:.4f} | {zone['recall']:.4f} | {zone['f1_score']:.4f} | "
+                f"{zone['reference_evaluated_count']}/{zone['reference_raw_count']} | {zone['matches']} | {zone['envelope_matches']} | "
+                f"{zone['possible_box_to_footprint_mismatch_count']} |"
+            )
+        lines.append("")
+    return "\n".join(lines) + "\n"
+
+
+def build_report(args: argparse.Namespace) -> dict[str, Any]:
+    positive = load_json(args.positive_summary)
+    background = load_json(args.background_summary)
+    manifest = load_json(args.manifest_path)
+    positive_items = positive.get("items") or []
+    background_items = background.get("items") or []
+    if not isinstance(positive_items, list) or not positive_items:
+        raise SystemExit("Positive summary contains no benchmark items")
+    if not isinstance(background_items, list) or not background_items:
+        raise SystemExit("Background summary contains no benchmark items")
+    manifest_rows = manifest.get("samples") or []
+    manifest_samples = {
+        str(item.get("sample_slug") or "").strip().lower(): item
+        for item in manifest_rows
+        if isinstance(item, dict) and item.get("sample_slug")
+    }
+
+    grouped_positive: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list)
+    grouped_background: dict[CandidateKey, list[dict[str, Any]]] = defaultdict(list)
+    for item in positive_items:
+        if not isinstance(item, dict):
+            raise SystemExit("Positive summary contains a non-object item")
+        grouped_positive[candidate_key(item)].append(item)
+    for item in background_items:
+        if not isinstance(item, dict):
+            raise SystemExit("Background summary contains a non-object item")
+        grouped_background[candidate_key(item)].append(item)
+
+    decisions = [
+        evaluate_candidate(key, items, grouped_background.get(key, []), manifest_samples, args)
+        for key, items in sorted(grouped_positive.items(), key=lambda entry: candidate_label(entry[0]))
+    ]
+    accepted = [item for item in decisions if item["decision"] == "operationally_accepted"]
+    recommended = max(
+        accepted,
+        key=lambda item: (item["mean_f1"], item["minimum_zone_f1"], item["mean_precision"]),
+        default=None,
+    )
+    report = {
+        "schema_version": 1,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "status": "accepted" if recommended else "review_required",
+        "recommendation": "retain_or_promote_candidate" if recommended else "do_not_promote_retraining_review_required",
+        "base_url": args.base_url,
+        "positive_summary_path": str(args.positive_summary),
+        "background_summary_path": str(args.background_summary),
+        "operator_sample_manifest_path": str(args.manifest_path),
+        "gate_configuration": {
+            "min_positive_samples": args.min_positive_samples,
+            "min_background_samples": args.min_background_samples,
+            "min_mean_f1": args.min_mean_f1,
+            "min_zone_f1": args.min_zone_f1,
+            "min_reference_coverage_ratio": args.min_reference_coverage_ratio,
+            "max_background_detections": args.max_background_detections,
+        },
+        "candidate_count": len(decisions),
+        "recommended_candidate": recommended,
+        "candidate_decisions": decisions,
+    }
+    return report
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Build a coverage-aware Mol operational detection benchmark report.")
+    parser.add_argument("--positive-summary", type=Path, required=True)
+    parser.add_argument("--background-summary", type=Path, required=True)
+    parser.add_argument("--manifest-path", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--base-url", default="")
+    parser.add_argument("--min-positive-samples", type=int, default=4)
+    parser.add_argument("--min-background-samples", type=int, default=1)
+    parser.add_argument("--min-mean-f1", type=float, default=0.25)
+    parser.add_argument("--min-zone-f1", type=float, default=0.10)
+    parser.add_argument("--min-reference-coverage-ratio", type=float, default=0.95)
+    parser.add_argument("--max-background-detections", type=int, default=0)
+    args = parser.parse_args()
+    if args.min_positive_samples < 1 or args.min_background_samples < 1:
+        parser.error("sample gates must be at least 1")
+    if not 0 <= args.min_mean_f1 <= 1 or not 0 <= args.min_zone_f1 <= 1:
+        parser.error("F1 gates must be between 0 and 1")
+    if not 0 < args.min_reference_coverage_ratio <= 1:
+        parser.error("coverage ratio gate must be greater than 0 and at most 1")
+    if args.max_background_detections < 0:
+        parser.error("background detection gate cannot be negative")
+    return args
+
+
+def main() -> int:
+    args = parse_args()
+    report = build_report(args)
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    json_path = args.output_dir / "mol_operational_benchmark_report.json"
+    markdown_path = args.output_dir / "mol_operational_benchmark_report.md"
+    json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
+    markdown_path.write_text(render_markdown(report), encoding="utf-8")
+    print("Mol operational benchmark report passed")
+    print(f"Status: {report['status']}")
+    print(f"Recommendation: {report['recommendation']}")
+    print(f"JSON: {json_path}")
+    print(f"Markdown: {markdown_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_detection_quality_matrix.sh

@@ -286,6 +286,13 @@ metrics = {
     if metric.get("metric_key")
 }
 findings = quality_check.get("findings_json") or {}
+coverage = findings.get("coverage") or {}
+diagnostics = findings.get("box_to_footprint_diagnostics") or {}
+reference_raw_count = coverage.get("reference_raw_count")
+reference_evaluated_count = coverage.get("reference_evaluated_count")
+reference_coverage_ratio = None
+if isinstance(reference_raw_count, int) and reference_raw_count > 0 and isinstance(reference_evaluated_count, int):
+    reference_coverage_ratio = reference_evaluated_count / reference_raw_count
 summary = {
     "model_request": model_request,
     "model_asset_id": selected_model_asset_id,
@@ -312,6 +319,27 @@ summary = {
     "matches": findings.get("matches"),
     "false_positives": findings.get("false_positives"),
     "false_negatives": findings.get("false_negatives"),
+    "coverage_applied": coverage.get("applied", False),
+    "coverage_mode": coverage.get("mode"),
+    "coverage_tile_count": coverage.get("tile_count", 0),
+    "coverage_source_crs_values": coverage.get("source_crs_values") or [],
+    "candidate_raw_count": coverage.get("candidate_raw_count"),
+    "candidate_evaluated_count": coverage.get("candidate_evaluated_count"),
+    "candidate_excluded_outside_count": coverage.get("candidate_excluded_outside_count"),
+    "candidate_clipped_boundary_count": coverage.get("candidate_clipped_boundary_count"),
+    "reference_raw_count": reference_raw_count,
+    "reference_evaluated_count": reference_evaluated_count,
+    "reference_excluded_outside_count": coverage.get("reference_excluded_outside_count"),
+    "reference_clipped_boundary_count": coverage.get("reference_clipped_boundary_count"),
+    "reference_coverage_ratio": reference_coverage_ratio,
+    "diagnostic_only": diagnostics.get("diagnostic_only"),
+    "diagnostic_method": diagnostics.get("diagnostic_method"),
+    "strict_matches": diagnostics.get("strict_matches"),
+    "envelope_matches": diagnostics.get("envelope_matches"),
+    "possible_box_to_footprint_mismatch_count": diagnostics.get("possible_box_to_footprint_mismatch_count"),
+    "envelope_precision": diagnostics.get("envelope_precision"),
+    "envelope_recall": diagnostics.get("envelope_recall"),
+    "envelope_f1_score": diagnostics.get("envelope_f1_score"),
     "export_id": export_id,
     "run_log": run_log,
 }
@@ -319,7 +347,8 @@ with open(output_path, "w", encoding="utf-8") as handle:
     json.dump(summary, handle, indent=2, sort_keys=True)
 print(
     "model={model} tile={tile} overlap={overlap} threshold={threshold} detections={detections} raw={raw} suppressed={suppressed} "
-    "score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn}".format(
+    "score={score} precision={precision} recall={recall} f1={f1} matches={matches} fp={fp} fn={fn} "
+    "coverage={coverage} diagnostic_gap={diagnostic_gap}".format(
         model=summary["model_asset_id"],
         tile=summary["tile_size"],
         overlap=summary["tile_overlap"],
@@ -334,6 +363,8 @@ print(
         matches=summary["matches"],
         fp=summary["false_positives"],
         fn=summary["false_negatives"],
+        coverage=summary["reference_coverage_ratio"],
+        diagnostic_gap=summary["possible_box_to_footprint_mismatch_count"],
     )
 )
 PY
@@ -381,10 +412,10 @@ with open(summary_path, "w", encoding="utf-8") as handle:
 
 print("")
 print("Detection quality matrix summary")
-print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
+print("model\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn\tcoverage\tdiagnostic_gap")
 for item in items:
     print(
-        "{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
+        "{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}\t{reference_coverage_ratio}\t{possible_box_to_footprint_mismatch_count}".format(
             **item
         )
     )

scripts/run_mol_operational_validation.sh

@@ -17,6 +17,10 @@ Optional environment:
   QUALITY_TILE_OVERLAPS        Default: 64.
   QUALITY_THRESHOLDS           Default: 0.15.
   REAL_IOU_THRESHOLD           Default: 0.25.
+  MOL_MIN_MEAN_F1              Operational gate, default: 0.25.
+  MOL_MIN_ZONE_F1              Per-zone collapse gate, default: 0.10.
+  MOL_MIN_REFERENCE_COVERAGE   Minimum evaluated/raw reference ratio, default: 0.95.
+  MOL_MAX_BACKGROUND_DETECTIONS Maximum detections per pure-empty control, default: 0.
 
 The runner never downloads weights, fetches product providers or uses fixture
 outputs. Prepare the documented real orthophoto/GRB files explicitly first.
@@ -41,6 +45,10 @@ QUALITY_TILE_SIZES="${QUALITY_TILE_SIZES:-512}"
 QUALITY_TILE_OVERLAPS="${QUALITY_TILE_OVERLAPS:-64}"
 QUALITY_THRESHOLDS="${QUALITY_THRESHOLDS:-0.15}"
 REAL_IOU_THRESHOLD="${REAL_IOU_THRESHOLD:-0.25}"
+MOL_MIN_MEAN_F1="${MOL_MIN_MEAN_F1:-0.25}"
+MOL_MIN_ZONE_F1="${MOL_MIN_ZONE_F1:-0.10}"
+MOL_MIN_REFERENCE_COVERAGE="${MOL_MIN_REFERENCE_COVERAGE:-0.95}"
+MOL_MAX_BACKGROUND_DETECTIONS="${MOL_MAX_BACKGROUND_DETECTIONS:-0}"
 
 if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
   usage
@@ -213,3 +221,16 @@ lines = [
 (output_dir / "mol_operational_validation_summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
 print(json.dumps({"status": summary["status"], "summary_path": str(summary_path)}, indent=2))
 PY
+
+"${PYTHON_BIN}" scripts/build_mol_operational_benchmark_report.py \
+  --positive-summary "${positive_output_dir}/multi_sample_quality_summary.json" \
+  --background-summary "${background_output_dir}/hard_negative_matrix_summary.json" \
+  --manifest-path "${OPERATOR_SAMPLE_MANIFEST_PATH}" \
+  --output-dir "${MOL_VALIDATION_OUTPUT_DIR}" \
+  --base-url "${BASE_URL}" \
+  --min-positive-samples 4 \
+  --min-background-samples 1 \
+  --min-mean-f1 "${MOL_MIN_MEAN_F1}" \
+  --min-zone-f1 "${MOL_MIN_ZONE_F1}" \
+  --min-reference-coverage-ratio "${MOL_MIN_REFERENCE_COVERAGE}" \
+  --max-background-detections "${MOL_MAX_BACKGROUND_DETECTIONS}"

scripts/run_multi_sample_detection_quality_matrix.sh

@@ -184,6 +184,12 @@ for summary_path in sorted(glob.glob(str(output_dir / "*" / "quality_matrix_summ
     for item in items:
         enriched = dict(item)
         enriched["sample_slug"] = sample_slug
+        enriched["sample_display_name"] = sample_metadata.get("display_name")
+        enriched["municipality"] = sample_metadata.get("municipality")
+        enriched["operational_zone"] = sample_metadata.get("operational_zone")
+        enriched["recommended_split"] = sample_metadata.get("recommended_split")
+        enriched["manifest_reference_feature_count"] = sample_metadata.get("reference_feature_count")
+        enriched["wgs84_bbox"] = sample_metadata.get("wgs84_bbox")
         flat_items.append(enriched)
     sample_summaries.append(
         {
@@ -232,10 +238,10 @@ summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding=
 
 print("")
 print("Multi-sample detection quality summary")
-print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
+print("sample\tzone\tmodel\ttile\toverlap\tthreshold\tdetections\traw\tsuppressed\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn\tcoverage\tdiagnostic_gap")
 for item in flat_items:
     print(
-        "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
+        "{sample_slug}\t{operational_zone}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{raw_detection_count}\t{suppressed_detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}\t{reference_coverage_ratio}\t{possible_box_to_footprint_mismatch_count}".format(
             **item
         )
     )

scripts/run_readiness_check.sh

@@ -47,6 +47,7 @@ ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
 ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
 ${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
 ${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
+${PYTHON_BIN} -m py_compile scripts/build_mol_operational_benchmark_report.py
 ${PYTHON_BIN} -m py_compile scripts/build_background_corpus_split_report.py
 ${PYTHON_BIN} -m py_compile scripts/build_fixed_threshold_evidence_portfolio_inputs.py
 ${PYTHON_BIN} -m py_compile scripts/audit_detection_false_negative_evidence.py