Add background split matrix runner
This commit is contained in:
@@ -7,6 +7,13 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 157 Background split matrix runner (2026-07-10)
|
||||
|
||||
- Added `scripts/run_background_corpus_split_matrix.sh` to run pure-empty and sparse-context hard-negative matrices separately from one operator command.
|
||||
- Added `scripts/build_background_corpus_split_report.py` to combine both hard-negative summaries into `background_corpus_split_summary.json` and `.md`.
|
||||
- Added readiness coverage and tests for the split runner/report contract.
|
||||
- No model default, backend API, database migration, provider fetching, fake detection output or model download behavior changed.
|
||||
|
||||
## Sprint 156 Background corpus classification (2026-07-10)
|
||||
|
||||
- Added explicit operator background categories to prepared sample manifests: `pure_empty_negative` when GRB returns zero reference buildings and `sparse_building_context` when contextual GRB buildings are present.
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_split_report_builder():
|
||||
script_path = ROOT / "scripts" / "build_background_corpus_split_report.py"
|
||||
assert script_path.exists()
|
||||
spec = importlib.util.spec_from_file_location("background_split_report_builder", script_path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def write_summary(path: Path, *, category: str, detections: list[int]) -> None:
|
||||
items = [
|
||||
{
|
||||
"sample_slug": f"{category}_{index}",
|
||||
"background_category": category,
|
||||
"model_asset_id": "candidate-model",
|
||||
"tile_size": 512,
|
||||
"tile_overlap": 64,
|
||||
"threshold": 0.35,
|
||||
"tile_count": 4,
|
||||
"detection_count": detection_count,
|
||||
"false_positive_pressure": detection_count / 4,
|
||||
}
|
||||
for index, detection_count in enumerate(detections, start=1)
|
||||
]
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"generated_at": "2026-07-10T00:00:00+00:00",
|
||||
"sample_count": len(items),
|
||||
"run_count": len(items),
|
||||
"background_category_counts": {category: len(items)},
|
||||
"best_by_lowest_pressure": min(items, key=lambda item: item["false_positive_pressure"]),
|
||||
"items": items,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_split_report_builder_creates_strict_gate_and_context_review(tmp_path: Path) -> None:
|
||||
module = load_split_report_builder()
|
||||
pure_summary = tmp_path / "pure_empty.json"
|
||||
sparse_summary = tmp_path / "sparse_context.json"
|
||||
output_dir = tmp_path / "split-report"
|
||||
write_summary(pure_summary, category="pure_empty_negative", detections=[0, 2])
|
||||
write_summary(sparse_summary, category="sparse_building_context", detections=[1, 5])
|
||||
|
||||
report = module.build_split_report(
|
||||
pure_empty_summary_path=pure_summary,
|
||||
sparse_context_summary_path=sparse_summary,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
assert report["strict_default_gate"]["category"] == "pure_empty_negative"
|
||||
assert report["strict_default_gate"]["passes_zero_detection_gate"] is False
|
||||
assert report["strict_default_gate"]["max_detection_count"] == 2
|
||||
assert report["context_review"]["category"] == "sparse_building_context"
|
||||
assert report["context_review"]["review_only"] is True
|
||||
assert report["context_review"]["max_detection_count"] == 5
|
||||
assert report["recommended_next_step"] == "retrain_or_recalibrate_after_review"
|
||||
assert (output_dir / "background_corpus_split_summary.json").exists()
|
||||
markdown = (output_dir / "background_corpus_split_summary.md").read_text(encoding="utf-8")
|
||||
assert "Strict default gate" in markdown
|
||||
assert "Sparse-context review" in markdown
|
||||
|
||||
|
||||
def test_split_report_builder_rejects_wrong_summary_category(tmp_path: Path) -> None:
|
||||
module = load_split_report_builder()
|
||||
wrong_summary = tmp_path / "wrong.json"
|
||||
sparse_summary = tmp_path / "sparse.json"
|
||||
write_summary(wrong_summary, category="sparse_building_context", detections=[0])
|
||||
write_summary(sparse_summary, category="sparse_building_context", detections=[0])
|
||||
|
||||
try:
|
||||
module.build_split_report(
|
||||
pure_empty_summary_path=wrong_summary,
|
||||
sparse_context_summary_path=sparse_summary,
|
||||
output_dir=tmp_path / "out",
|
||||
)
|
||||
except SystemExit as exc:
|
||||
assert "pure_empty_negative" in str(exc)
|
||||
else: # pragma: no cover - defensive assertion for the contract.
|
||||
raise AssertionError("wrong category summary should fail")
|
||||
|
||||
|
||||
def test_split_matrix_runner_invokes_both_background_categories() -> None:
|
||||
runner = ROOT / "scripts" / "run_background_corpus_split_matrix.sh"
|
||||
assert runner.exists()
|
||||
source = runner.read_text(encoding="utf-8")
|
||||
|
||||
assert "run_operator_hard_negative_detection_matrix.sh" in source
|
||||
assert "OPERATOR_BACKGROUND_CATEGORIES=\"pure_empty_negative\"" in source
|
||||
assert "OPERATOR_BACKGROUND_CATEGORIES=\"sparse_building_context\"" in source
|
||||
assert "build_background_corpus_split_report.py" in source
|
||||
assert "background_corpus_split_summary.json" in source
|
||||
assert "/qa/reference" not in source
|
||||
assert "fixture_mode" not in source
|
||||
|
||||
|
||||
def test_readiness_covers_split_matrix_runner() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "py_compile scripts/build_background_corpus_split_report.py" in readiness
|
||||
assert "bash -n scripts/run_background_corpus_split_matrix.sh" in readiness
|
||||
@@ -245,6 +245,23 @@ detections or treat AI detections as ground truth without QA/QC. The same
|
||||
candidate should also pass the background false-positive matrix before it is
|
||||
considered as a default:
|
||||
|
||||
```bash
|
||||
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||
OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \
|
||||
QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
|
||||
QUALITY_TILE_SIZES="512" \
|
||||
QUALITY_TILE_OVERLAPS="64" \
|
||||
QUALITY_THRESHOLDS="0.35 0.15" \
|
||||
BACKGROUND_SPLIT_OUTPUT_DIR=artifacts/detection-hard-negatives/aoi1024bg512r3e50-split \
|
||||
bash scripts/run_background_corpus_split_matrix.sh http://192.168.10.150:1202
|
||||
```
|
||||
|
||||
The split runner writes `background_corpus_split_summary.json` and Markdown
|
||||
handoff output with a strict `pure_empty_negative` gate and a separate
|
||||
review-only `sparse_building_context` block.
|
||||
|
||||
The underlying single-category matrix remains available:
|
||||
|
||||
```bash
|
||||
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
||||
|
||||
@@ -6235,3 +6235,43 @@ Open:
|
||||
- `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` for the strict default-promotion false-positive gate.
|
||||
- `OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` for contextual review evidence.
|
||||
- Retrain or recalibrate the inactive AOI1024 local model candidate only after those two matrices are available.
|
||||
|
||||
# Sprint 157 - Background split matrix runner
|
||||
|
||||
## What changed
|
||||
|
||||
- Added `scripts/run_background_corpus_split_matrix.sh` as the operator wrapper for the next Tower run.
|
||||
- The wrapper runs `scripts/run_operator_hard_negative_detection_matrix.sh` twice:
|
||||
- `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` for the strict default-promotion false-positive gate.
|
||||
- `OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` for review-only contextual evidence.
|
||||
- Added `scripts/build_background_corpus_split_report.py` to combine both summaries into:
|
||||
- `background_corpus_split_summary.json`
|
||||
- `background_corpus_split_summary.md`
|
||||
- The combined report records `strict_default_gate`, `context_review`, `passes_zero_detection_gate`, max detection counts and the recommended next step.
|
||||
- Added readiness coverage for the new Python and Bash scripts.
|
||||
- Updated operator pipeline docs, TODO and changelog.
|
||||
|
||||
## What was tested
|
||||
|
||||
- Added regression coverage in `backend/tests/test_sprint157_background_split_matrix_runner.py`.
|
||||
- Ran `python -m pytest tests/test_sprint157_background_split_matrix_runner.py -q`.
|
||||
- Ran `python -m pytest tests/test_sprint157_background_split_matrix_runner.py tests/test_sprint156_background_corpus_classification.py tests/test_sprint132_operator_hard_negative_matrix.py -q`: 9 passed.
|
||||
- Ran `python -m py_compile scripts/build_background_corpus_split_report.py`.
|
||||
- Ran `bash -n scripts/run_background_corpus_split_matrix.sh`.
|
||||
- Ran `python -m compileall backend/app`.
|
||||
- Ran `python -m pytest` in `backend`: 443 passed.
|
||||
- Ran `cd frontend && npm run typecheck`.
|
||||
- Ran `cd frontend && npm run build`.
|
||||
- Ran `bash scripts/run_readiness_check.sh`.
|
||||
- Ran `cd backend && python -m alembic heads` and `cd backend && python -m alembic upgrade head --sql`.
|
||||
- Ran `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/run_background_corpus_split_matrix.sh`.
|
||||
|
||||
## Known limitations
|
||||
|
||||
- This pass adds orchestration/report tooling only. It does not run live inference on Tower, retrain YOLO, rerun the split matrices or change any model default.
|
||||
- No backend API contract, database migration, provider fetching, fake detection output or model download behavior changed.
|
||||
|
||||
## Next recommended pass
|
||||
|
||||
- Rebuild/redeploy the runtime, regenerate the operator manifest if needed, then run `scripts/run_background_corpus_split_matrix.sh` against `http://192.168.10.150:1202`.
|
||||
- Use the emitted split report to decide whether to retrain, recalibrate thresholds or keep the AOI1024 candidate operator-only.
|
||||
|
||||
+2
-1
@@ -122,7 +122,8 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Train and gate background-aware `geointel-building-yolov8s-aoi1024bg512r3e50-pt`; it is the strongest positive-AOI candidate so far but remains inactive because full background-candidate false-positive pressure still blocks default promotion.
|
||||
- [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass.
|
||||
- [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance.
|
||||
- [ ] Retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix.
|
||||
- [x] Add a split background-corpus matrix runner and report builder that runs pure-empty and sparse-context matrices separately.
|
||||
- [ ] Rerun split background matrices on Tower after rebuild, then retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix.
|
||||
- [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads.
|
||||
|
||||
## Sprint 8 status
|
||||
|
||||
@@ -473,6 +473,26 @@ as the best current experimental dense-AOI candidate, not as a V1 default.
|
||||
Run a dedicated hard-negative matrix against documented background candidates
|
||||
before changing model defaults:
|
||||
|
||||
```bash
|
||||
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||
OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos dessel_heide ravels_bos meerhout_bos geel_bel arendonk_heide herenthout_bos" \
|
||||
QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
|
||||
QUALITY_TILE_SIZES="512" \
|
||||
QUALITY_TILE_OVERLAPS="64" \
|
||||
QUALITY_THRESHOLDS="0.35 0.15" \
|
||||
BACKGROUND_SPLIT_OUTPUT_DIR=artifacts/detection-hard-negatives/aoi1024bg512r3e50-split \
|
||||
bash scripts/run_background_corpus_split_matrix.sh http://192.168.10.150:1202
|
||||
```
|
||||
|
||||
The split runner executes the strict `pure_empty_negative` matrix and the
|
||||
review-only `sparse_building_context` matrix as separate runs, then writes
|
||||
`background_corpus_split_summary.json` and
|
||||
`background_corpus_split_summary.md`. Use the pure-empty block for the
|
||||
default-promotion false-positive gate; use sparse-context results as review
|
||||
evidence only.
|
||||
|
||||
The lower-level hard-negative matrix can still be run directly:
|
||||
|
||||
```bash
|
||||
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Build a split background-corpus detection summary.
|
||||
|
||||
This operator helper combines two hard-negative matrix summaries:
|
||||
|
||||
- pure-empty negatives: strict false-positive gate for default promotion.
|
||||
- sparse-building context: review-only evidence, not a precision/recall proxy.
|
||||
|
||||
It does not run inference, fetch providers, mutate models or promote defaults.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PURE_EMPTY_CATEGORY = "pure_empty_negative"
|
||||
SPARSE_CONTEXT_CATEGORY = "sparse_building_context"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Combine split background-corpus hard-negative summaries.")
|
||||
parser.add_argument("--pure-empty-summary", type=Path, required=True)
|
||||
parser.add_argument("--sparse-context-summary", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_summary(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise SystemExit(f"Summary file is not readable: {path}")
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
items = payload.get("items") or []
|
||||
if not isinstance(items, list) or not items:
|
||||
raise SystemExit(f"Summary has no items: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def validate_category(summary: dict[str, Any], expected_category: str, label: str) -> None:
|
||||
categories = {
|
||||
str(item.get("background_category") or "")
|
||||
for item in summary.get("items") or []
|
||||
}
|
||||
if categories != {expected_category}:
|
||||
raise SystemExit(
|
||||
f"{label} summary must contain only {expected_category} items; found {sorted(categories)}"
|
||||
)
|
||||
|
||||
|
||||
def max_number(items: list[dict[str, Any]], key: str) -> float:
|
||||
values = [float(item.get(key) or 0) for item in items]
|
||||
return max(values, default=0.0)
|
||||
|
||||
|
||||
def total_int(items: list[dict[str, Any]], key: str) -> int:
|
||||
return sum(int(item.get(key) or 0) for item in items)
|
||||
|
||||
|
||||
def category_block(summary: dict[str, Any], category: str, *, review_only: bool) -> dict[str, Any]:
|
||||
items = list(summary.get("items") or [])
|
||||
sample_slugs = sorted({str(item.get("sample_slug") or "") for item in items if item.get("sample_slug")})
|
||||
max_detection_count = int(max_number(items, "detection_count"))
|
||||
block: dict[str, Any] = {
|
||||
"category": category,
|
||||
"review_only": review_only,
|
||||
"sample_count": len(sample_slugs),
|
||||
"run_count": len(items),
|
||||
"sample_slugs": sample_slugs,
|
||||
"total_detection_count": total_int(items, "detection_count"),
|
||||
"max_detection_count": max_detection_count,
|
||||
"max_false_positive_pressure": max_number(items, "false_positive_pressure"),
|
||||
"best_by_lowest_pressure": summary.get("best_by_lowest_pressure"),
|
||||
"background_category_counts": summary.get("background_category_counts") or {},
|
||||
}
|
||||
if not review_only:
|
||||
block["passes_zero_detection_gate"] = max_detection_count == 0
|
||||
return block
|
||||
|
||||
|
||||
def build_markdown(report: dict[str, Any]) -> str:
|
||||
strict = report["strict_default_gate"]
|
||||
context = report["context_review"]
|
||||
lines = [
|
||||
"# Background corpus split summary",
|
||||
"",
|
||||
f"- Generated: `{report['generated_at']}`",
|
||||
f"- Recommended next step: `{report['recommended_next_step']}`",
|
||||
"",
|
||||
"## Strict default gate",
|
||||
"",
|
||||
f"- Category: `{strict['category']}`",
|
||||
f"- Samples: `{strict['sample_count']}`",
|
||||
f"- Runs: `{strict['run_count']}`",
|
||||
f"- Max detections: `{strict['max_detection_count']}`",
|
||||
f"- Total detections: `{strict['total_detection_count']}`",
|
||||
f"- Max false-positive pressure: `{strict['max_false_positive_pressure']}`",
|
||||
f"- Passes zero-detection gate: `{strict['passes_zero_detection_gate']}`",
|
||||
"",
|
||||
"## Sparse-context review",
|
||||
"",
|
||||
f"- Category: `{context['category']}`",
|
||||
f"- Samples: `{context['sample_count']}`",
|
||||
f"- Runs: `{context['run_count']}`",
|
||||
f"- Max detections: `{context['max_detection_count']}`",
|
||||
f"- Total detections: `{context['total_detection_count']}`",
|
||||
f"- Max false-positive pressure: `{context['max_false_positive_pressure']}`",
|
||||
"- Interpretation: review-only evidence, not a default-promotion precision/recall gate.",
|
||||
"",
|
||||
"## Source summaries",
|
||||
"",
|
||||
f"- Pure-empty summary: `{report['source_summaries']['pure_empty_negative']}`",
|
||||
f"- Sparse-context summary: `{report['source_summaries']['sparse_building_context']}`",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_split_report(
|
||||
*,
|
||||
pure_empty_summary_path: Path,
|
||||
sparse_context_summary_path: Path,
|
||||
output_dir: Path,
|
||||
) -> dict[str, Any]:
|
||||
pure_summary = load_summary(pure_empty_summary_path)
|
||||
sparse_summary = load_summary(sparse_context_summary_path)
|
||||
validate_category(pure_summary, PURE_EMPTY_CATEGORY, "pure-empty")
|
||||
validate_category(sparse_summary, SPARSE_CONTEXT_CATEGORY, "sparse-context")
|
||||
|
||||
strict_block = category_block(pure_summary, PURE_EMPTY_CATEGORY, review_only=False)
|
||||
context_block = category_block(sparse_summary, SPARSE_CONTEXT_CATEGORY, review_only=True)
|
||||
recommended_next_step = (
|
||||
"retrain_or_recalibrate_after_review"
|
||||
if not strict_block["passes_zero_detection_gate"] or context_block["max_detection_count"] > 0
|
||||
else "eligible_for_positive_aoi_gate_review"
|
||||
)
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"source_summaries": {
|
||||
PURE_EMPTY_CATEGORY: str(pure_empty_summary_path),
|
||||
SPARSE_CONTEXT_CATEGORY: str(sparse_context_summary_path),
|
||||
},
|
||||
"strict_default_gate": strict_block,
|
||||
"context_review": context_block,
|
||||
"recommended_next_step": recommended_next_step,
|
||||
}
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "background_corpus_split_summary.json").write_text(
|
||||
json.dumps(report, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(output_dir / "background_corpus_split_summary.md").write_text(build_markdown(report), encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
report = build_split_report(
|
||||
pure_empty_summary_path=args.pure_empty_summary,
|
||||
sparse_context_summary_path=args.sparse_context_summary,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
print(args.output_dir / "background_corpus_split_summary.json")
|
||||
print(f"strict_default_gate_passed={report['strict_default_gate']['passes_zero_detection_gate']}")
|
||||
print(f"recommended_next_step={report['recommended_next_step']}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
Usage:
|
||||
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||
QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
|
||||
QUALITY_THRESHOLDS="0.35 0.15" \
|
||||
bash scripts/run_background_corpus_split_matrix.sh [base_url]
|
||||
|
||||
Optional environment:
|
||||
BACKGROUND_SPLIT_OUTPUT_DIR Output directory, default: artifacts/detection-hard-negatives/background-split/<timestamp>.
|
||||
OPERATOR_SAMPLE_MANIFEST_PATH Manifest from prepare_operator_real_data_samples.py.
|
||||
OPERATOR_BACKGROUND_SAMPLE_SLUGS Optional comma/space separated slug filter applied to both categories.
|
||||
QUALITY_MODEL_ASSET_IDS Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||
QUALITY_TILE_SIZES Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||
QUALITY_TILE_OVERLAPS Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||
QUALITY_THRESHOLDS Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||
|
||||
Runs two live hard-negative matrices from the same manifest:
|
||||
1. pure_empty_negative: strict default-promotion false-positive gate.
|
||||
2. sparse_building_context: review-only contextual evidence.
|
||||
|
||||
The script does not upload reference vectors, run QA/QC, use fixture detections,
|
||||
fetch providers, download model weights or promote model defaults.
|
||||
EOF
|
||||
}
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
|
||||
BACKGROUND_SPLIT_OUTPUT_DIR="${BACKGROUND_SPLIT_OUTPUT_DIR:-artifacts/detection-hard-negatives/background-split/$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
|
||||
if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -n "${PYTHON_BIN:-}" ]; then
|
||||
PYTHON_BIN="${PYTHON_BIN}"
|
||||
else
|
||||
PYTHON_BIN=""
|
||||
for candidate in python3 python.exe python; do
|
||||
if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
|
||||
PYTHON_BIN="${candidate}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "${PYTHON_BIN}" ]; then
|
||||
echo "A Python interpreter is required for background split reporting" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${BACKGROUND_SPLIT_OUTPUT_DIR}"
|
||||
|
||||
pure_output="${BACKGROUND_SPLIT_OUTPUT_DIR}/pure_empty_negative"
|
||||
sparse_output="${BACKGROUND_SPLIT_OUTPUT_DIR}/sparse_building_context"
|
||||
|
||||
echo "== GeoIntel background corpus split matrix =="
|
||||
echo "Base URL: ${BASE_URL}"
|
||||
echo "Output: ${BACKGROUND_SPLIT_OUTPUT_DIR}"
|
||||
echo "-- Running strict pure-empty default gate --"
|
||||
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
||||
HARD_NEGATIVE_OUTPUT_DIR="${pure_output}" \
|
||||
bash scripts/run_operator_hard_negative_detection_matrix.sh "${BASE_URL}"
|
||||
|
||||
echo "-- Running sparse-context review matrix --"
|
||||
OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context" \
|
||||
HARD_NEGATIVE_OUTPUT_DIR="${sparse_output}" \
|
||||
bash scripts/run_operator_hard_negative_detection_matrix.sh "${BASE_URL}"
|
||||
|
||||
"${PYTHON_BIN}" scripts/build_background_corpus_split_report.py \
|
||||
--pure-empty-summary "${pure_output}/hard_negative_matrix_summary.json" \
|
||||
--sparse-context-summary "${sparse_output}/hard_negative_matrix_summary.json" \
|
||||
--output-dir "${BACKGROUND_SPLIT_OUTPUT_DIR}"
|
||||
|
||||
echo "Background corpus split summary: ${BACKGROUND_SPLIT_OUTPUT_DIR}/background_corpus_split_summary.json"
|
||||
@@ -46,6 +46,7 @@ ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
|
||||
${PYTHON_BIN} -m py_compile scripts/build_background_corpus_split_report.py
|
||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m compileall backend/app
|
||||
@@ -67,6 +68,7 @@ bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh
|
||||
bash -n scripts/run_detection_quality_matrix.sh
|
||||
bash -n scripts/run_multi_sample_detection_quality_matrix.sh
|
||||
bash -n scripts/run_operator_hard_negative_detection_matrix.sh
|
||||
bash -n scripts/run_background_corpus_split_matrix.sh
|
||||
bash -n scripts/train_operator_yolo_detector.sh
|
||||
bash -n scripts/verify_workbench_default_state.sh
|
||||
bash -n scripts/verify_workbench_interactions.sh
|
||||
|
||||
Reference in New Issue
Block a user