Add background split matrix runner
This commit is contained in:
@@ -7,6 +7,13 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 157 Background split matrix runner (2026-07-10)
|
||||||
|
|
||||||
|
- Added `scripts/run_background_corpus_split_matrix.sh` to run pure-empty and sparse-context hard-negative matrices separately from one operator command.
|
||||||
|
- Added `scripts/build_background_corpus_split_report.py` to combine both hard-negative summaries into `background_corpus_split_summary.json` and `.md`.
|
||||||
|
- Added readiness coverage and tests for the split runner/report contract.
|
||||||
|
- No model default, backend API, database migration, provider fetching, fake detection output or model download behavior changed.
|
||||||
|
|
||||||
## Sprint 156 Background corpus classification (2026-07-10)
|
## Sprint 156 Background corpus classification (2026-07-10)
|
||||||
|
|
||||||
- Added explicit operator background categories to prepared sample manifests: `pure_empty_negative` when GRB returns zero reference buildings and `sparse_building_context` when contextual GRB buildings are present.
|
- Added explicit operator background categories to prepared sample manifests: `pure_empty_negative` when GRB returns zero reference buildings and `sparse_building_context` when contextual GRB buildings are present.
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def load_split_report_builder():
|
||||||
|
script_path = ROOT / "scripts" / "build_background_corpus_split_report.py"
|
||||||
|
assert script_path.exists()
|
||||||
|
spec = importlib.util.spec_from_file_location("background_split_report_builder", script_path)
|
||||||
|
assert spec is not None
|
||||||
|
assert spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def write_summary(path: Path, *, category: str, detections: list[int]) -> None:
|
||||||
|
items = [
|
||||||
|
{
|
||||||
|
"sample_slug": f"{category}_{index}",
|
||||||
|
"background_category": category,
|
||||||
|
"model_asset_id": "candidate-model",
|
||||||
|
"tile_size": 512,
|
||||||
|
"tile_overlap": 64,
|
||||||
|
"threshold": 0.35,
|
||||||
|
"tile_count": 4,
|
||||||
|
"detection_count": detection_count,
|
||||||
|
"false_positive_pressure": detection_count / 4,
|
||||||
|
}
|
||||||
|
for index, detection_count in enumerate(detections, start=1)
|
||||||
|
]
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"generated_at": "2026-07-10T00:00:00+00:00",
|
||||||
|
"sample_count": len(items),
|
||||||
|
"run_count": len(items),
|
||||||
|
"background_category_counts": {category: len(items)},
|
||||||
|
"best_by_lowest_pressure": min(items, key=lambda item: item["false_positive_pressure"]),
|
||||||
|
"items": items,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_report_builder_creates_strict_gate_and_context_review(tmp_path: Path) -> None:
|
||||||
|
module = load_split_report_builder()
|
||||||
|
pure_summary = tmp_path / "pure_empty.json"
|
||||||
|
sparse_summary = tmp_path / "sparse_context.json"
|
||||||
|
output_dir = tmp_path / "split-report"
|
||||||
|
write_summary(pure_summary, category="pure_empty_negative", detections=[0, 2])
|
||||||
|
write_summary(sparse_summary, category="sparse_building_context", detections=[1, 5])
|
||||||
|
|
||||||
|
report = module.build_split_report(
|
||||||
|
pure_empty_summary_path=pure_summary,
|
||||||
|
sparse_context_summary_path=sparse_summary,
|
||||||
|
output_dir=output_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report["strict_default_gate"]["category"] == "pure_empty_negative"
|
||||||
|
assert report["strict_default_gate"]["passes_zero_detection_gate"] is False
|
||||||
|
assert report["strict_default_gate"]["max_detection_count"] == 2
|
||||||
|
assert report["context_review"]["category"] == "sparse_building_context"
|
||||||
|
assert report["context_review"]["review_only"] is True
|
||||||
|
assert report["context_review"]["max_detection_count"] == 5
|
||||||
|
assert report["recommended_next_step"] == "retrain_or_recalibrate_after_review"
|
||||||
|
assert (output_dir / "background_corpus_split_summary.json").exists()
|
||||||
|
markdown = (output_dir / "background_corpus_split_summary.md").read_text(encoding="utf-8")
|
||||||
|
assert "Strict default gate" in markdown
|
||||||
|
assert "Sparse-context review" in markdown
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_report_builder_rejects_wrong_summary_category(tmp_path: Path) -> None:
|
||||||
|
module = load_split_report_builder()
|
||||||
|
wrong_summary = tmp_path / "wrong.json"
|
||||||
|
sparse_summary = tmp_path / "sparse.json"
|
||||||
|
write_summary(wrong_summary, category="sparse_building_context", detections=[0])
|
||||||
|
write_summary(sparse_summary, category="sparse_building_context", detections=[0])
|
||||||
|
|
||||||
|
try:
|
||||||
|
module.build_split_report(
|
||||||
|
pure_empty_summary_path=wrong_summary,
|
||||||
|
sparse_context_summary_path=sparse_summary,
|
||||||
|
output_dir=tmp_path / "out",
|
||||||
|
)
|
||||||
|
except SystemExit as exc:
|
||||||
|
assert "pure_empty_negative" in str(exc)
|
||||||
|
else: # pragma: no cover - defensive assertion for the contract.
|
||||||
|
raise AssertionError("wrong category summary should fail")
|
||||||
|
|
||||||
|
|
||||||
|
def test_split_matrix_runner_invokes_both_background_categories() -> None:
|
||||||
|
runner = ROOT / "scripts" / "run_background_corpus_split_matrix.sh"
|
||||||
|
assert runner.exists()
|
||||||
|
source = runner.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "run_operator_hard_negative_detection_matrix.sh" in source
|
||||||
|
assert "OPERATOR_BACKGROUND_CATEGORIES=\"pure_empty_negative\"" in source
|
||||||
|
assert "OPERATOR_BACKGROUND_CATEGORIES=\"sparse_building_context\"" in source
|
||||||
|
assert "build_background_corpus_split_report.py" in source
|
||||||
|
assert "background_corpus_split_summary.json" in source
|
||||||
|
assert "/qa/reference" not in source
|
||||||
|
assert "fixture_mode" not in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_covers_split_matrix_runner() -> None:
|
||||||
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "py_compile scripts/build_background_corpus_split_report.py" in readiness
|
||||||
|
assert "bash -n scripts/run_background_corpus_split_matrix.sh" in readiness
|
||||||
@@ -245,6 +245,23 @@ detections or treat AI detections as ground truth without QA/QC. The same
|
|||||||
candidate should also pass the background false-positive matrix before it is
|
candidate should also pass the background false-positive matrix before it is
|
||||||
considered as a default:
|
considered as a default:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||||
|
OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \
|
||||||
|
QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
|
||||||
|
QUALITY_TILE_SIZES="512" \
|
||||||
|
QUALITY_TILE_OVERLAPS="64" \
|
||||||
|
QUALITY_THRESHOLDS="0.35 0.15" \
|
||||||
|
BACKGROUND_SPLIT_OUTPUT_DIR=artifacts/detection-hard-negatives/aoi1024bg512r3e50-split \
|
||||||
|
bash scripts/run_background_corpus_split_matrix.sh http://192.168.10.150:1202
|
||||||
|
```
|
||||||
|
|
||||||
|
The split runner writes `background_corpus_split_summary.json` and Markdown
|
||||||
|
handoff output with a strict `pure_empty_negative` gate and a separate
|
||||||
|
review-only `sparse_building_context` block.
|
||||||
|
|
||||||
|
The underlying single-category matrix remains available:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||||
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
||||||
|
|||||||
@@ -6235,3 +6235,43 @@ Open:
|
|||||||
- `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` for the strict default-promotion false-positive gate.
|
- `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` for the strict default-promotion false-positive gate.
|
||||||
- `OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` for contextual review evidence.
|
- `OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` for contextual review evidence.
|
||||||
- Retrain or recalibrate the inactive AOI1024 local model candidate only after those two matrices are available.
|
- Retrain or recalibrate the inactive AOI1024 local model candidate only after those two matrices are available.
|
||||||
|
|
||||||
|
# Sprint 157 - Background split matrix runner
|
||||||
|
|
||||||
|
## What changed
|
||||||
|
|
||||||
|
- Added `scripts/run_background_corpus_split_matrix.sh` as the operator wrapper for the next Tower run.
|
||||||
|
- The wrapper runs `scripts/run_operator_hard_negative_detection_matrix.sh` twice:
|
||||||
|
- `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"` for the strict default-promotion false-positive gate.
|
||||||
|
- `OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` for review-only contextual evidence.
|
||||||
|
- Added `scripts/build_background_corpus_split_report.py` to combine both summaries into:
|
||||||
|
- `background_corpus_split_summary.json`
|
||||||
|
- `background_corpus_split_summary.md`
|
||||||
|
- The combined report records `strict_default_gate`, `context_review`, `passes_zero_detection_gate`, max detection counts and the recommended next step.
|
||||||
|
- Added readiness coverage for the new Python and Bash scripts.
|
||||||
|
- Updated operator pipeline docs, TODO and changelog.
|
||||||
|
|
||||||
|
## What was tested
|
||||||
|
|
||||||
|
- Added regression coverage in `backend/tests/test_sprint157_background_split_matrix_runner.py`.
|
||||||
|
- Ran `python -m pytest tests/test_sprint157_background_split_matrix_runner.py -q`.
|
||||||
|
- Ran `python -m pytest tests/test_sprint157_background_split_matrix_runner.py tests/test_sprint156_background_corpus_classification.py tests/test_sprint132_operator_hard_negative_matrix.py -q`: 9 passed.
|
||||||
|
- Ran `python -m py_compile scripts/build_background_corpus_split_report.py`.
|
||||||
|
- Ran `bash -n scripts/run_background_corpus_split_matrix.sh`.
|
||||||
|
- Ran `python -m compileall backend/app`.
|
||||||
|
- Ran `python -m pytest` in `backend`: 443 passed.
|
||||||
|
- Ran `cd frontend && npm run typecheck`.
|
||||||
|
- Ran `cd frontend && npm run build`.
|
||||||
|
- Ran `bash scripts/run_readiness_check.sh`.
|
||||||
|
- Ran `cd backend && python -m alembic heads` and `cd backend && python -m alembic upgrade head --sql`.
|
||||||
|
- Ran `bash -n scripts/live_migration_smoke.sh` and `bash -n scripts/run_background_corpus_split_matrix.sh`.
|
||||||
|
|
||||||
|
## Known limitations
|
||||||
|
|
||||||
|
- This pass adds orchestration/report tooling only. It does not run live inference on Tower, retrain YOLO, rerun the split matrices or change any model default.
|
||||||
|
- No backend API contract, database migration, provider fetching, fake detection output or model download behavior changed.
|
||||||
|
|
||||||
|
## Next recommended pass
|
||||||
|
|
||||||
|
- Rebuild/redeploy the runtime, regenerate the operator manifest if needed, then run `scripts/run_background_corpus_split_matrix.sh` against `http://192.168.10.150:1202`.
|
||||||
|
- Use the emitted split report to decide whether to retrain, recalibrate thresholds or keep the AOI1024 candidate operator-only.
|
||||||
|
|||||||
+2
-1
@@ -122,7 +122,8 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Train and gate background-aware `geointel-building-yolov8s-aoi1024bg512r3e50-pt`; it is the strongest positive-AOI candidate so far but remains inactive because full background-candidate false-positive pressure still blocks default promotion.
|
- [x] Train and gate background-aware `geointel-building-yolov8s-aoi1024bg512r3e50-pt`; it is the strongest positive-AOI candidate so far but remains inactive because full background-candidate false-positive pressure still blocks default promotion.
|
||||||
- [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass.
|
- [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass.
|
||||||
- [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance.
|
- [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance.
|
||||||
- [ ] Retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix.
|
- [x] Add a split background-corpus matrix runner and report builder that runs pure-empty and sparse-context matrices separately.
|
||||||
|
- [ ] Rerun split background matrices on Tower after rebuild, then retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix.
|
||||||
- [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads.
|
- [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads.
|
||||||
|
|
||||||
## Sprint 8 status
|
## Sprint 8 status
|
||||||
|
|||||||
@@ -473,6 +473,26 @@ as the best current experimental dense-AOI candidate, not as a V1 default.
|
|||||||
Run a dedicated hard-negative matrix against documented background candidates
|
Run a dedicated hard-negative matrix against documented background candidates
|
||||||
before changing model defaults:
|
before changing model defaults:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||||
|
OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos dessel_heide ravels_bos meerhout_bos geel_bel arendonk_heide herenthout_bos" \
|
||||||
|
QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
|
||||||
|
QUALITY_TILE_SIZES="512" \
|
||||||
|
QUALITY_TILE_OVERLAPS="64" \
|
||||||
|
QUALITY_THRESHOLDS="0.35 0.15" \
|
||||||
|
BACKGROUND_SPLIT_OUTPUT_DIR=artifacts/detection-hard-negatives/aoi1024bg512r3e50-split \
|
||||||
|
bash scripts/run_background_corpus_split_matrix.sh http://192.168.10.150:1202
|
||||||
|
```
|
||||||
|
|
||||||
|
The split runner executes the strict `pure_empty_negative` matrix and the
|
||||||
|
review-only `sparse_building_context` matrix as separate runs, then writes
|
||||||
|
`background_corpus_split_summary.json` and
|
||||||
|
`background_corpus_split_summary.md`. Use the pure-empty block for the
|
||||||
|
default-promotion false-positive gate; use sparse-context results as review
|
||||||
|
evidence only.
|
||||||
|
|
||||||
|
The lower-level hard-negative matrix can still be run directly:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||||
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
||||||
|
|||||||
@@ -0,0 +1,174 @@
|
|||||||
|
"""Build a split background-corpus detection summary.
|
||||||
|
|
||||||
|
This operator helper combines two hard-negative matrix summaries:
|
||||||
|
|
||||||
|
- pure-empty negatives: strict false-positive gate for default promotion.
|
||||||
|
- sparse-building context: review-only evidence, not a precision/recall proxy.
|
||||||
|
|
||||||
|
It does not run inference, fetch providers, mutate models or promote defaults.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
PURE_EMPTY_CATEGORY = "pure_empty_negative"
|
||||||
|
SPARSE_CONTEXT_CATEGORY = "sparse_building_context"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description="Combine split background-corpus hard-negative summaries.")
|
||||||
|
parser.add_argument("--pure-empty-summary", type=Path, required=True)
|
||||||
|
parser.add_argument("--sparse-context-summary", type=Path, required=True)
|
||||||
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def load_summary(path: Path) -> dict[str, Any]:
|
||||||
|
if not path.exists():
|
||||||
|
raise SystemExit(f"Summary file is not readable: {path}")
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||||
|
items = payload.get("items") or []
|
||||||
|
if not isinstance(items, list) or not items:
|
||||||
|
raise SystemExit(f"Summary has no items: {path}")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def validate_category(summary: dict[str, Any], expected_category: str, label: str) -> None:
|
||||||
|
categories = {
|
||||||
|
str(item.get("background_category") or "")
|
||||||
|
for item in summary.get("items") or []
|
||||||
|
}
|
||||||
|
if categories != {expected_category}:
|
||||||
|
raise SystemExit(
|
||||||
|
f"{label} summary must contain only {expected_category} items; found {sorted(categories)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def max_number(items: list[dict[str, Any]], key: str) -> float:
|
||||||
|
values = [float(item.get(key) or 0) for item in items]
|
||||||
|
return max(values, default=0.0)
|
||||||
|
|
||||||
|
|
||||||
|
def total_int(items: list[dict[str, Any]], key: str) -> int:
|
||||||
|
return sum(int(item.get(key) or 0) for item in items)
|
||||||
|
|
||||||
|
|
||||||
|
def category_block(summary: dict[str, Any], category: str, *, review_only: bool) -> dict[str, Any]:
|
||||||
|
items = list(summary.get("items") or [])
|
||||||
|
sample_slugs = sorted({str(item.get("sample_slug") or "") for item in items if item.get("sample_slug")})
|
||||||
|
max_detection_count = int(max_number(items, "detection_count"))
|
||||||
|
block: dict[str, Any] = {
|
||||||
|
"category": category,
|
||||||
|
"review_only": review_only,
|
||||||
|
"sample_count": len(sample_slugs),
|
||||||
|
"run_count": len(items),
|
||||||
|
"sample_slugs": sample_slugs,
|
||||||
|
"total_detection_count": total_int(items, "detection_count"),
|
||||||
|
"max_detection_count": max_detection_count,
|
||||||
|
"max_false_positive_pressure": max_number(items, "false_positive_pressure"),
|
||||||
|
"best_by_lowest_pressure": summary.get("best_by_lowest_pressure"),
|
||||||
|
"background_category_counts": summary.get("background_category_counts") or {},
|
||||||
|
}
|
||||||
|
if not review_only:
|
||||||
|
block["passes_zero_detection_gate"] = max_detection_count == 0
|
||||||
|
return block
|
||||||
|
|
||||||
|
|
||||||
|
def build_markdown(report: dict[str, Any]) -> str:
|
||||||
|
strict = report["strict_default_gate"]
|
||||||
|
context = report["context_review"]
|
||||||
|
lines = [
|
||||||
|
"# Background corpus split summary",
|
||||||
|
"",
|
||||||
|
f"- Generated: `{report['generated_at']}`",
|
||||||
|
f"- Recommended next step: `{report['recommended_next_step']}`",
|
||||||
|
"",
|
||||||
|
"## Strict default gate",
|
||||||
|
"",
|
||||||
|
f"- Category: `{strict['category']}`",
|
||||||
|
f"- Samples: `{strict['sample_count']}`",
|
||||||
|
f"- Runs: `{strict['run_count']}`",
|
||||||
|
f"- Max detections: `{strict['max_detection_count']}`",
|
||||||
|
f"- Total detections: `{strict['total_detection_count']}`",
|
||||||
|
f"- Max false-positive pressure: `{strict['max_false_positive_pressure']}`",
|
||||||
|
f"- Passes zero-detection gate: `{strict['passes_zero_detection_gate']}`",
|
||||||
|
"",
|
||||||
|
"## Sparse-context review",
|
||||||
|
"",
|
||||||
|
f"- Category: `{context['category']}`",
|
||||||
|
f"- Samples: `{context['sample_count']}`",
|
||||||
|
f"- Runs: `{context['run_count']}`",
|
||||||
|
f"- Max detections: `{context['max_detection_count']}`",
|
||||||
|
f"- Total detections: `{context['total_detection_count']}`",
|
||||||
|
f"- Max false-positive pressure: `{context['max_false_positive_pressure']}`",
|
||||||
|
"- Interpretation: review-only evidence, not a default-promotion precision/recall gate.",
|
||||||
|
"",
|
||||||
|
"## Source summaries",
|
||||||
|
"",
|
||||||
|
f"- Pure-empty summary: `{report['source_summaries']['pure_empty_negative']}`",
|
||||||
|
f"- Sparse-context summary: `{report['source_summaries']['sparse_building_context']}`",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def build_split_report(
|
||||||
|
*,
|
||||||
|
pure_empty_summary_path: Path,
|
||||||
|
sparse_context_summary_path: Path,
|
||||||
|
output_dir: Path,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
pure_summary = load_summary(pure_empty_summary_path)
|
||||||
|
sparse_summary = load_summary(sparse_context_summary_path)
|
||||||
|
validate_category(pure_summary, PURE_EMPTY_CATEGORY, "pure-empty")
|
||||||
|
validate_category(sparse_summary, SPARSE_CONTEXT_CATEGORY, "sparse-context")
|
||||||
|
|
||||||
|
strict_block = category_block(pure_summary, PURE_EMPTY_CATEGORY, review_only=False)
|
||||||
|
context_block = category_block(sparse_summary, SPARSE_CONTEXT_CATEGORY, review_only=True)
|
||||||
|
recommended_next_step = (
|
||||||
|
"retrain_or_recalibrate_after_review"
|
||||||
|
if not strict_block["passes_zero_detection_gate"] or context_block["max_detection_count"] > 0
|
||||||
|
else "eligible_for_positive_aoi_gate_review"
|
||||||
|
)
|
||||||
|
report = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"source_summaries": {
|
||||||
|
PURE_EMPTY_CATEGORY: str(pure_empty_summary_path),
|
||||||
|
SPARSE_CONTEXT_CATEGORY: str(sparse_context_summary_path),
|
||||||
|
},
|
||||||
|
"strict_default_gate": strict_block,
|
||||||
|
"context_review": context_block,
|
||||||
|
"recommended_next_step": recommended_next_step,
|
||||||
|
}
|
||||||
|
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(output_dir / "background_corpus_split_summary.json").write_text(
|
||||||
|
json.dumps(report, indent=2, sort_keys=True),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(output_dir / "background_corpus_split_summary.md").write_text(build_markdown(report), encoding="utf-8")
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
report = build_split_report(
|
||||||
|
pure_empty_summary_path=args.pure_empty_summary,
|
||||||
|
sparse_context_summary_path=args.sparse_context_summary,
|
||||||
|
output_dir=args.output_dir,
|
||||||
|
)
|
||||||
|
print(args.output_dir / "background_corpus_split_summary.json")
|
||||||
|
print(f"strict_default_gate_passed={report['strict_default_gate']['passes_zero_detection_gate']}")
|
||||||
|
print(f"recommended_next_step={report['recommended_next_step']}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat >&2 <<'EOF'
|
||||||
|
Usage:
|
||||||
|
OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
|
||||||
|
QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8s-aoi1024bg512r3e50-pt" \
|
||||||
|
QUALITY_THRESHOLDS="0.35 0.15" \
|
||||||
|
bash scripts/run_background_corpus_split_matrix.sh [base_url]
|
||||||
|
|
||||||
|
Optional environment:
|
||||||
|
BACKGROUND_SPLIT_OUTPUT_DIR Output directory, default: artifacts/detection-hard-negatives/background-split/<timestamp>.
|
||||||
|
OPERATOR_SAMPLE_MANIFEST_PATH Manifest from prepare_operator_real_data_samples.py.
|
||||||
|
OPERATOR_BACKGROUND_SAMPLE_SLUGS Optional comma/space separated slug filter applied to both categories.
|
||||||
|
QUALITY_MODEL_ASSET_IDS Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||||
|
QUALITY_TILE_SIZES Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||||
|
QUALITY_TILE_OVERLAPS Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||||
|
QUALITY_THRESHOLDS Forwarded to run_operator_hard_negative_detection_matrix.sh.
|
||||||
|
|
||||||
|
Runs two live hard-negative matrices from the same manifest:
|
||||||
|
1. pure_empty_negative: strict default-promotion false-positive gate.
|
||||||
|
2. sparse_building_context: review-only contextual evidence.
|
||||||
|
|
||||||
|
The script does not upload reference vectors, run QA/QC, use fixture detections,
|
||||||
|
fetch providers, download model weights or promote model defaults.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
|
||||||
|
BACKGROUND_SPLIT_OUTPUT_DIR="${BACKGROUND_SPLIT_OUTPUT_DIR:-artifacts/detection-hard-negatives/background-split/$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||||
|
|
||||||
|
if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -n "${PYTHON_BIN:-}" ]; then
|
||||||
|
PYTHON_BIN="${PYTHON_BIN}"
|
||||||
|
else
|
||||||
|
PYTHON_BIN=""
|
||||||
|
for candidate in python3 python.exe python; do
|
||||||
|
if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
|
||||||
|
PYTHON_BIN="${candidate}"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${PYTHON_BIN}" ]; then
|
||||||
|
echo "A Python interpreter is required for background split reporting" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "${BACKGROUND_SPLIT_OUTPUT_DIR}"
|
||||||
|
|
||||||
|
pure_output="${BACKGROUND_SPLIT_OUTPUT_DIR}/pure_empty_negative"
|
||||||
|
sparse_output="${BACKGROUND_SPLIT_OUTPUT_DIR}/sparse_building_context"
|
||||||
|
|
||||||
|
echo "== GeoIntel background corpus split matrix =="
|
||||||
|
echo "Base URL: ${BASE_URL}"
|
||||||
|
echo "Output: ${BACKGROUND_SPLIT_OUTPUT_DIR}"
|
||||||
|
echo "-- Running strict pure-empty default gate --"
|
||||||
|
OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
|
||||||
|
HARD_NEGATIVE_OUTPUT_DIR="${pure_output}" \
|
||||||
|
bash scripts/run_operator_hard_negative_detection_matrix.sh "${BASE_URL}"
|
||||||
|
|
||||||
|
echo "-- Running sparse-context review matrix --"
|
||||||
|
OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context" \
|
||||||
|
HARD_NEGATIVE_OUTPUT_DIR="${sparse_output}" \
|
||||||
|
bash scripts/run_operator_hard_negative_detection_matrix.sh "${BASE_URL}"
|
||||||
|
|
||||||
|
"${PYTHON_BIN}" scripts/build_background_corpus_split_report.py \
|
||||||
|
--pure-empty-summary "${pure_output}/hard_negative_matrix_summary.json" \
|
||||||
|
--sparse-context-summary "${sparse_output}/hard_negative_matrix_summary.json" \
|
||||||
|
--output-dir "${BACKGROUND_SPLIT_OUTPUT_DIR}"
|
||||||
|
|
||||||
|
echo "Background corpus split summary: ${BACKGROUND_SPLIT_OUTPUT_DIR}/background_corpus_split_summary.json"
|
||||||
@@ -46,6 +46,7 @@ ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
|||||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
|
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/build_background_corpus_split_report.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m compileall backend/app
|
${PYTHON_BIN} -m compileall backend/app
|
||||||
@@ -67,6 +68,7 @@ bash -n scripts/assemble_detection_calibration_evidence_portfolio.sh
|
|||||||
bash -n scripts/run_detection_quality_matrix.sh
|
bash -n scripts/run_detection_quality_matrix.sh
|
||||||
bash -n scripts/run_multi_sample_detection_quality_matrix.sh
|
bash -n scripts/run_multi_sample_detection_quality_matrix.sh
|
||||||
bash -n scripts/run_operator_hard_negative_detection_matrix.sh
|
bash -n scripts/run_operator_hard_negative_detection_matrix.sh
|
||||||
|
bash -n scripts/run_background_corpus_split_matrix.sh
|
||||||
bash -n scripts/train_operator_yolo_detector.sh
|
bash -n scripts/train_operator_yolo_detector.sh
|
||||||
bash -n scripts/verify_workbench_default_state.sh
|
bash -n scripts/verify_workbench_default_state.sh
|
||||||
bash -n scripts/verify_workbench_interactions.sh
|
bash -n scripts/verify_workbench_interactions.sh
|
||||||
|
|||||||
Reference in New Issue
Block a user