Add background split matrix runner

2026-07-10 02:15:15 +02:00
parent 64dac0d9b7
commit 251aa7b044
9 changed files with 461 additions and 1 deletions
@@ -473,6 +473,26 @@ as the best current experimental dense-AOI candidate, not as a V1 default.
 Run a dedicated hard-negative matrix against documented background candidates
 before changing model defaults:

+```bash
+OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos dessel_heide ravels_bos meerhout_bos geel_bel arendonk_heide herenthout_bos" \
+QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
+QUALITY_TILE_SIZES="512" \
+QUALITY_TILE_OVERLAPS="64" \
+QUALITY_THRESHOLDS="0.35 0.15" \
+BACKGROUND_SPLIT_OUTPUT_DIR=artifacts/detection-hard-negatives/aoi1024bg512r3e50-split \
+bash scripts/run_background_corpus_split_matrix.sh http://192.168.10.150:1202
+```
+
+The split runner executes the strict `pure_empty_negative` matrix and the
+review-only `sparse_building_context` matrix as separate runs, then writes
+`background_corpus_split_summary.json` and
+`background_corpus_split_summary.md`. Use the pure-empty block for the
+default-promotion false-positive gate; use sparse-context results as review
+evidence only.
+
+The lower-level hard-negative matrix can still be run directly:
+
 ```bash
 OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
 OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
@@ -0,0 +1,174 @@
+"""Build a split background-corpus detection summary.
+
+This operator helper combines two hard-negative matrix summaries:
+
+- pure-empty negatives: strict false-positive gate for default promotion.
+- sparse-building context: review-only evidence, not a precision/recall proxy.
+
+It does not run inference, fetch providers, mutate models or promote defaults.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+PURE_EMPTY_CATEGORY = "pure_empty_negative"
+SPARSE_CONTEXT_CATEGORY = "sparse_building_context"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Combine split background-corpus hard-negative summaries.")
+    parser.add_argument("--pure-empty-summary", type=Path, required=True)
+    parser.add_argument("--sparse-context-summary", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    return parser.parse_args()
+
+
+def load_summary(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        raise SystemExit(f"Summary file is not readable: {path}")
+    payload = json.loads(path.read_text(encoding="utf-8-sig"))
+    items = payload.get("items") or []
+    if not isinstance(items, list) or not items:
+        raise SystemExit(f"Summary has no items: {path}")
+    return payload
+
+
+def validate_category(summary: dict[str, Any], expected_category: str, label: str) -> None:
+    categories = {
+        str(item.get("background_category") or "")
+        for item in summary.get("items") or []
+    }
+    if categories != {expected_category}:
+        raise SystemExit(
+            f"{label} summary must contain only {expected_category} items; found {sorted(categories)}"
+        )
+
+
+def max_number(items: list[dict[str, Any]], key: str) -> float:
+    values = [float(item.get(key) or 0) for item in items]
+    return max(values, default=0.0)
+
+
+def total_int(items: list[dict[str, Any]], key: str) -> int:
+    return sum(int(item.get(key) or 0) for item in items)
+
+
+def category_block(summary: dict[str, Any], category: str, *, review_only: bool) -> dict[str, Any]:
+    items = list(summary.get("items") or [])
+    sample_slugs = sorted({str(item.get("sample_slug") or "") for item in items if item.get("sample_slug")})
+    max_detection_count = int(max_number(items, "detection_count"))
+    block: dict[str, Any] = {
+        "category": category,
+        "review_only": review_only,
+        "sample_count": len(sample_slugs),
+        "run_count": len(items),
+        "sample_slugs": sample_slugs,
+        "total_detection_count": total_int(items, "detection_count"),
+        "max_detection_count": max_detection_count,
+        "max_false_positive_pressure": max_number(items, "false_positive_pressure"),
+        "best_by_lowest_pressure": summary.get("best_by_lowest_pressure"),
+        "background_category_counts": summary.get("background_category_counts") or {},
+    }
+    if not review_only:
+        block["passes_zero_detection_gate"] = max_detection_count == 0
+    return block
+
+
+def build_markdown(report: dict[str, Any]) -> str:
+    strict = report["strict_default_gate"]
+    context = report["context_review"]
+    lines = [
+        "# Background corpus split summary",
+        "",
+        f"- Generated: `{report['generated_at']}`",
+        f"- Recommended next step: `{report['recommended_next_step']}`",
+        "",
+        "## Strict default gate",
+        "",
+        f"- Category: `{strict['category']}`",
+        f"- Samples: `{strict['sample_count']}`",
+        f"- Runs: `{strict['run_count']}`",
+        f"- Max detections: `{strict['max_detection_count']}`",
+        f"- Total detections: `{strict['total_detection_count']}`",
+        f"- Max false-positive pressure: `{strict['max_false_positive_pressure']}`",
+        f"- Passes zero-detection gate: `{strict['passes_zero_detection_gate']}`",
+        "",
+        "## Sparse-context review",
+        "",
+        f"- Category: `{context['category']}`",
+        f"- Samples: `{context['sample_count']}`",
+        f"- Runs: `{context['run_count']}`",
+        f"- Max detections: `{context['max_detection_count']}`",
+        f"- Total detections: `{context['total_detection_count']}`",
+        f"- Max false-positive pressure: `{context['max_false_positive_pressure']}`",
+        "- Interpretation: review-only evidence, not a default-promotion precision/recall gate.",
+        "",
+        "## Source summaries",
+        "",
+        f"- Pure-empty summary: `{report['source_summaries']['pure_empty_negative']}`",
+        f"- Sparse-context summary: `{report['source_summaries']['sparse_building_context']}`",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def build_split_report(
+    *,
+    pure_empty_summary_path: Path,
+    sparse_context_summary_path: Path,
+    output_dir: Path,
+) -> dict[str, Any]:
+    pure_summary = load_summary(pure_empty_summary_path)
+    sparse_summary = load_summary(sparse_context_summary_path)
+    validate_category(pure_summary, PURE_EMPTY_CATEGORY, "pure-empty")
+    validate_category(sparse_summary, SPARSE_CONTEXT_CATEGORY, "sparse-context")
+
+    strict_block = category_block(pure_summary, PURE_EMPTY_CATEGORY, review_only=False)
+    context_block = category_block(sparse_summary, SPARSE_CONTEXT_CATEGORY, review_only=True)
+    recommended_next_step = (
+        "retrain_or_recalibrate_after_review"
+        if not strict_block["passes_zero_detection_gate"] or context_block["max_detection_count"] > 0
+        else "eligible_for_positive_aoi_gate_review"
+    )
+    report = {
+        "schema_version": 1,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "source_summaries": {
+            PURE_EMPTY_CATEGORY: str(pure_empty_summary_path),
+            SPARSE_CONTEXT_CATEGORY: str(sparse_context_summary_path),
+        },
+        "strict_default_gate": strict_block,
+        "context_review": context_block,
+        "recommended_next_step": recommended_next_step,
+    }
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "background_corpus_split_summary.json").write_text(
+        json.dumps(report, indent=2, sort_keys=True),
+        encoding="utf-8",
+    )
+    (output_dir / "background_corpus_split_summary.md").write_text(build_markdown(report), encoding="utf-8")
+    return report
+
+
+def main() -> int:
+    args = parse_args()
+    report = build_split_report(
+        pure_empty_summary_path=args.pure_empty_summary,
+        sparse_context_summary_path=args.sparse_context_summary,
+        output_dir=args.output_dir,
+    )
+    print(args.output_dir / "background_corpus_split_summary.json")
+    print(f"strict_default_gate_passed={report['strict_default_gate']['passes_zero_detection_gate']}")
+    print(f"recommended_next_step={report['recommended_next_step']}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+Usage:
+  OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+  QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
+  QUALITY_THRESHOLDS="0.35 0.15" \
+  bash scripts/run_background_corpus_split_matrix.sh [base_url]
+
+Optional environment:
+  BACKGROUND_SPLIT_OUTPUT_DIR        Output directory, default: artifacts/detection-hard-negatives/background-split/<timestamp>.
+  OPERATOR_SAMPLE_MANIFEST_PATH      Manifest from prepare_operator_real_data_samples.py.
+  OPERATOR_BACKGROUND_SAMPLE_SLUGS   Optional comma/space separated slug filter applied to both categories.
+  QUALITY_MODEL_ASSET_IDS            Forwarded to run_operator_hard_negative_detection_matrix.sh.
+  QUALITY_TILE_SIZES                 Forwarded to run_operator_hard_negative_detection_matrix.sh.
+  QUALITY_TILE_OVERLAPS              Forwarded to run_operator_hard_negative_detection_matrix.sh.
+  QUALITY_THRESHOLDS                 Forwarded to run_operator_hard_negative_detection_matrix.sh.
+
+Runs two live hard-negative matrices from the same manifest:
+  1. pure_empty_negative: strict default-promotion false-positive gate.
+  2. sparse_building_context: review-only contextual evidence.
+
+The script does not upload reference vectors, run QA/QC, use fixture detections,
+fetch providers, download model weights or promote model defaults.
+EOF
+}
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
+BACKGROUND_SPLIT_OUTPUT_DIR="${BACKGROUND_SPLIT_OUTPUT_DIR:-artifacts/detection-hard-negatives/background-split/$(date -u +%Y%m%dT%H%M%SZ)}"
+
+if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
+  usage
+  exit 0
+fi
+
+if [ -n "${PYTHON_BIN:-}" ]; then
+  PYTHON_BIN="${PYTHON_BIN}"
+else
+  PYTHON_BIN=""
+  for candidate in python3 python.exe python; do
+    if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
+      PYTHON_BIN="${candidate}"
+      break
+    fi
+  done
+fi
+
+if [ -z "${PYTHON_BIN}" ]; then
+  echo "A Python interpreter is required for background split reporting" >&2
+  exit 1
+fi
+
+mkdir -p "${BACKGROUND_SPLIT_OUTPUT_DIR}"
+
+pure_output="${BACKGROUND_SPLIT_OUTPUT_DIR}/pure_empty_negative"
+sparse_output="${BACKGROUND_SPLIT_OUTPUT_DIR}/sparse_building_context"
+
+echo "== GeoIntel background corpus split matrix =="
+echo "Base URL: ${BASE_URL}"
+echo "Output: ${BACKGROUND_SPLIT_OUTPUT_DIR}"
+echo "-- Running strict pure-empty default gate --"
+OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
+HARD_NEGATIVE_OUTPUT_DIR="${pure_output}" \
+bash scripts/run_operator_hard_negative_detection_matrix.sh "${BASE_URL}"
+
+echo "-- Running sparse-context review matrix --"
+OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context" \
+HARD_NEGATIVE_OUTPUT_DIR="${sparse_output}" \
+bash scripts/run_operator_hard_negative_detection_matrix.sh "${BASE_URL}"
+
+"${PYTHON_BIN}" scripts/build_background_corpus_split_report.py \
+  --pure-empty-summary "${pure_output}/hard_negative_matrix_summary.json" \
+  --sparse-context-summary "${sparse_output}/hard_negative_matrix_summary.json" \
+  --output-dir "${BACKGROUND_SPLIT_OUTPUT_DIR}"
+
+echo "Background corpus split summary: ${BACKGROUND_SPLIT_OUTPUT_DIR}/background_corpus_split_summary.json"
@@ -46,6 +46,7 @@ ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
 ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
 ${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
 ${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
+${PYTHON_BIN} -m py_compile scripts/build_background_corpus_split_report.py
 ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m compileall backend/app
@@ -67,6 +68,7 @@ bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh
 bash -n scripts/run_detection_quality_matrix.sh
 bash -n scripts/run_multi_sample_detection_quality_matrix.sh
 bash -n scripts/run_operator_hard_negative_detection_matrix.sh
+bash -n scripts/run_background_corpus_split_matrix.sh
 bash -n scripts/train_operator_yolo_detector.sh
 bash -n scripts/verify_workbench_default_state.sh
 bash -n scripts/verify_workbench_interactions.sh