diff --git a/CHANGELOG.md b/CHANGELOG.md index 41ffd4a5..fc8cdbe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 158 Split-aware promotion report (2026-07-10) + +- Added `--background-split-summary` support to `scripts/build_detection_model_promotion_report.py`. +- The promotion report now resolves the split summary's `pure_empty_negative` source as the strict default-promotion background gate and records `sparse_building_context` as review-only evidence. +- Added regression coverage proving sparse-context detections do not block default promotion when the pure-empty gate passes. +- No model default, backend API, database migration, provider fetching, fake detection output or model download behavior changed. + ## Sprint 157 Background split matrix runner (2026-07-10) - Added `scripts/run_background_corpus_split_matrix.sh` to run pure-empty and sparse-context hard-negative matrices separately from one operator command. diff --git a/backend/tests/test_sprint158_promotion_report_split_background.py b/backend/tests/test_sprint158_promotion_report_split_background.py new file mode 100644 index 00000000..1d0cdfc9 --- /dev/null +++ b/backend/tests/test_sprint158_promotion_report_split_background.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def write_hard_negative_summary( + path: Path, + *, + category: str, + detection_counts: list[int], +) -> None: + items = [ + { + "sample_slug": f"{category}_{index}", + "background_category": category, + "model_asset_id": "candidate-context-sensitive", + "tile_size": 512, + "tile_overlap": 64, + "threshold": 0.35, + "tile_count": 4, + "detection_count": detection_count, + "false_positive_pressure": detection_count / 4, + } + for index, detection_count in enumerate(detection_counts, start=1) + ] + path.write_text( + json.dumps( + { + "generated_at": "2026-07-10T00:00:00+00:00", + "background_category_counts": {category: len(items)}, + "items": items, + } + ), + encoding="utf-8", + ) + + +def test_promotion_report_uses_split_pure_empty_as_gate_and_sparse_context_as_review( + tmp_path: Path, +) -> None: + script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py" + positive_path = tmp_path / "positive_portfolio.json" + pure_summary_path = tmp_path / "pure_empty_summary.json" + sparse_summary_path = tmp_path / "sparse_context_summary.json" + split_summary_path = tmp_path / "background_corpus_split_summary.json" + output_dir = tmp_path / "promotion-report" + + positive_path.write_text( + json.dumps( + { + "items": [ + { + "sample_slug": "geel", + "model_asset_id": "candidate-context-sensitive", + "tile_size": 512, + "tile_overlap": 64, + "threshold": 0.35, + "precision": 0.72, + "recall": 0.5, + "f1_score": 0.59, + }, + { + "sample_slug": "mol", + "model_asset_id": "candidate-context-sensitive", + "tile_size": 512, + "tile_overlap": 64, + "threshold": 0.35, + "precision": 0.68, + "recall": 0.48, + "f1_score": 0.56, + }, + ] + } + ), + encoding="utf-8", + ) + write_hard_negative_summary( + pure_summary_path, + category="pure_empty_negative", + detection_counts=[0, 0], + ) + write_hard_negative_summary( + sparse_summary_path, + category="sparse_building_context", + detection_counts=[4, 7], + ) + split_summary_path.write_text( + json.dumps( + { + "schema_version": 1, + "source_summaries": { + "pure_empty_negative": str(pure_summary_path), + "sparse_building_context": str(sparse_summary_path), + }, + "strict_default_gate": { + "category": "pure_empty_negative", + "review_only": False, + "sample_count": 2, + "run_count": 2, + "total_detection_count": 0, + "max_detection_count": 0, + "passes_zero_detection_gate": True, + }, + "context_review": { + "category": "sparse_building_context", + "review_only": True, + "sample_count": 2, + "run_count": 2, + "total_detection_count": 11, + "max_detection_count": 7, + }, + } + ), + encoding="utf-8", + ) + + result = subprocess.run( + [ + "python", + str(script_path), + "--positive-portfolio", + str(positive_path), + "--background-split-summary", + str(split_summary_path), + "--output-dir", + str(output_dir), + "--min-positive-samples", + "2", + "--min-background-samples", + "2", + "--min-mean-f1", + "0.5", + "--max-background-detections-per-sample", + "0", + ], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + + assert "Detection model promotion report passed" in result.stdout + report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8")) + decision = report["candidate_decisions"][0] + + assert report["hard_negative_summary_paths"] == [str(pure_summary_path)] + assert report["background_split_summary_paths"] == [str(split_summary_path)] + assert report["background_context_reviews"] == [ + { + "source_split_summary_path": str(split_summary_path), + "source_summary_path": str(sparse_summary_path), + "category": "sparse_building_context", + "review_only": True, + "sample_count": 2, + "run_count": 2, + "total_detection_count": 11, + "max_detection_count": 7, + } + ] + assert decision["candidate_key"] == "candidate-context-sensitive|512|64|0.35" + assert decision["background_sample_count"] == 2 + assert decision["max_background_detections"] == 0 + assert decision["promotion_status"] == "promote_candidate" + assert report["recommended_candidate"]["candidate_key"] == decision["candidate_key"] + + markdown = (output_dir / "detection_model_promotion_report.md").read_text(encoding="utf-8") + assert "Background split summaries: 1" in markdown + assert "Sparse-context review evidence" in markdown + assert "not used as a default-promotion gate" in markdown diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 3dfa2169..08c3deea 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -260,6 +260,24 @@ The split runner writes `background_corpus_split_summary.json` and Markdown handoff output with a strict `pure_empty_negative` gate and a separate review-only `sparse_building_context` block. +Use that split summary directly in the model promotion report: + +```bash +python scripts/build_detection_model_promotion_report.py \ + --positive-portfolio artifacts/detection-quality-matrix/multi-sample/aoi1024bg512r3e50-positive/multi_sample_quality_summary.json \ + --background-split-summary artifacts/detection-hard-negatives/aoi1024bg512r3e50-split/background_corpus_split_summary.json \ + --output-dir artifacts/detection-model-promotion/aoi1024bg512r3e50-split-aware \ + --min-positive-samples 7 \ + --min-background-samples 2 \ + --min-mean-f1 0.25 \ + --max-background-detections-per-sample 0 +``` + +The promotion report follows the split contract: `pure_empty_negative` is the +only strict background gate for default promotion, while +`sparse_building_context` remains review-only evidence in the report. This keeps +contextual buildings from being treated as empty-background false positives. + The underlying single-category matrix remains available: ```bash diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 81930710..c052c47f 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -6275,3 +6275,40 @@ Open: - Rebuild/redeploy the runtime, regenerate the operator manifest if needed, then run `scripts/run_background_corpus_split_matrix.sh` against `http://192.168.10.150:1202`. - Use the emitted split report to decide whether to retrain, recalibrate thresholds or keep the AOI1024 candidate operator-only. + +# Sprint 158 - Split-aware promotion report + +## What changed + +- Added `--background-split-summary` support to `scripts/build_detection_model_promotion_report.py`. +- The promotion report now resolves a split report's `pure_empty_negative` source summary as the strict default-promotion background gate. +- The same report records `sparse_building_context` as review-only evidence, including source path and detection-pressure context, without counting it as a default-promotion blocker. +- Kept direct `--hard-negative-summary` support unchanged for older operator workflows. +- Updated operator pipeline docs, TODO and changelog. + +## What was tested + +- Red step: `python -m pytest tests/test_sprint158_promotion_report_split_background.py -q` failed because the promotion report required `--hard-negative-summary` and did not yet accept `--background-split-summary`. +- Ran `python -m pytest tests/test_sprint158_promotion_report_split_background.py -q`: 1 passed. +- Ran `python -m pytest tests/test_sprint143_detection_model_promotion_report.py tests/test_sprint157_background_split_matrix_runner.py -q`: 7 passed. +- Ran `python -m py_compile scripts/build_detection_model_promotion_report.py scripts/build_background_corpus_split_report.py`. +- Ran `python -m pytest tests/test_sprint158_promotion_report_split_background.py tests/test_sprint143_detection_model_promotion_report.py tests/test_sprint157_background_split_matrix_runner.py -q`: 8 passed. +- Ran `python -m compileall backend/app`. +- Ran `python -m pytest` in `backend`: 444 passed, 17 existing Pydantic namespace warnings. +- Ran `cd frontend && npm run typecheck`. +- Ran `cd frontend && npm run build`. +- Ran `bash scripts/run_readiness_check.sh`: passed. +- Ran `cd backend && python -m alembic heads`: `202606120900 (head)`. +- Ran `cd backend && python -m alembic upgrade head --sql`. +- Ran `bash -n scripts/live_migration_smoke.sh`. + +## Known limitations + +- This pass is report/tooling only. It does not run live split matrices on Tower, retrain YOLO, change a model default, change API contracts, change migrations, fetch providers, download model weights or create fake detections. +- The AOI1024 background-aware local model remains explicit operator-review only until the split matrices plus positive-AOI QA/QC evidence pass the documented gates. + +## Next recommended pass + +- After redeploy, run `scripts/run_background_corpus_split_matrix.sh` on Tower for `geointel-building-yolov8s-aoi1024bg512r3e50-pt`. +- Feed the generated `background_corpus_split_summary.json` into `scripts/build_detection_model_promotion_report.py --background-split-summary` together with the seven-AOI positive summary. +- If pure-empty false-positive pressure still fails, retrain or recalibrate before any default activation. If pure-empty passes, inspect sparse-context review evidence before deciding whether to keep the model operator-only or prepare a guarded default-candidate decision. diff --git a/docs/TODO.md b/docs/TODO.md index 8c07e764..7bca0d45 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -123,6 +123,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add explicit operator detection profiles for local model assets: balanced review around threshold `0.15` and conservative high-precision review around threshold `0.35`, both clearly marked as non-default-approved until promotion gates pass. - [x] Add pure-empty versus sparse-building contextual background corpus classification to operator manifests, hard-negative matrix filters and YOLO tile provenance. - [x] Add a split background-corpus matrix runner and report builder that runs pure-empty and sparse-context matrices separately. +- [x] Teach the model promotion report to consume split background summaries so only `pure_empty_negative` blocks default promotion and `sparse_building_context` stays review-only. - [ ] Rerun split background matrices on Tower after rebuild, then retrain or recalibrate against the cleaner pure-empty gate plus separate sparse-context inspection matrix. - [ ] Promote a V1 default building detector only after it passes seven positive AOIs, clean hard-negative/background gates and persisted QA/QC evidence without fake detections or model downloads. diff --git a/scripts/README.md b/scripts/README.md index 4d2ec26c..170aeba7 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -679,6 +679,26 @@ and maximum background detections per sample. It is evidence/report tooling only: it does not run inference, mutate application data, download models or change the active YOLO configuration. +When the background corpus has been split with +`run_background_corpus_split_matrix.sh`, pass the combined split summary instead +of manually wiring both category summaries: + +```bash +python scripts/build_detection_model_promotion_report.py \ + --positive-portfolio /mnt/user/appdata/geointel/artifacts/detection-quality-matrix/multi-sample/aoi1024bg512r3e50-positive/multi_sample_quality_summary.json \ + --background-split-summary /mnt/user/appdata/geointel/artifacts/detection-hard-negatives/aoi1024bg512r3e50-split/background_corpus_split_summary.json \ + --output-dir /mnt/user/appdata/geointel/artifacts/detection-model-promotion/aoi1024bg512r3e50-split-aware \ + --min-positive-samples 7 \ + --min-background-samples 2 \ + --min-mean-f1 0.25 \ + --max-background-detections-per-sample 0 +``` + +The promotion report resolves the split summary's `pure_empty_negative` source +summary as the strict default-promotion false-positive gate. The +`sparse_building_context` source remains visible in the JSON/Markdown report as +review evidence only and is not counted as a default-promotion gate. + If a legacy positive evidence portfolio records `model_asset_id` at portfolio level but does not include per-run tile size/overlap, pass explicit tile defaults instead of letting the report guess: diff --git a/scripts/build_detection_model_promotion_report.py b/scripts/build_detection_model_promotion_report.py index 2dd36534..d4101404 100644 --- a/scripts/build_detection_model_promotion_report.py +++ b/scripts/build_detection_model_promotion_report.py @@ -18,6 +18,10 @@ from statistics import mean from typing import Any +PURE_EMPTY_CATEGORY = "pure_empty_negative" +SPARSE_CONTEXT_CATEGORY = "sparse_building_context" + + @dataclass(frozen=True) class CandidateKey: model_asset_id: str @@ -39,9 +43,18 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--hard-negative-summary", action="append", - required=True, + default=[], help="Path to hard_negative_matrix_summary.json. May be supplied multiple times.", ) + parser.add_argument( + "--background-split-summary", + action="append", + default=[], + help=( + "Path to background_corpus_split_summary.json. The pure-empty source summary is used as the " + "strict default gate; sparse-context evidence is kept as review-only context." + ), + ) parser.add_argument("--output-dir", required=True, help="Directory for JSON and Markdown report output.") parser.add_argument("--min-positive-samples", type=int, default=3) parser.add_argument("--min-background-samples", type=int, default=3) @@ -59,7 +72,10 @@ def parse_args() -> argparse.Namespace: default=None, help="Tile overlap to use for positive portfolio runs that do not record tile_overlap.", ) - return parser.parse_args() + args = parser.parse_args() + if not args.hard_negative_summary and not args.background_split_summary: + parser.error("at least one --hard-negative-summary or --background-split-summary is required") + return args def read_json(path: Path) -> dict[str, Any]: @@ -168,6 +184,65 @@ def collect_background_runs(summary_paths: list[Path]) -> dict[CandidateKey, lis return grouped +def resolve_split_source_path(source_path: Any, split_summary_path: Path) -> Path: + path = Path(str(source_path)) + if path.is_file(): + return path + if not path.is_absolute(): + candidate = split_summary_path.parent / path + if candidate.is_file(): + return candidate + return path + + +def collect_split_background_inputs( + split_summary_paths: list[Path], +) -> tuple[list[Path], list[dict[str, Any]]]: + strict_background_paths: list[Path] = [] + context_reviews: list[dict[str, Any]] = [] + + for split_summary_path in split_summary_paths: + payload = read_json(split_summary_path) + source_summaries = payload.get("source_summaries") or {} + pure_empty_source = source_summaries.get(PURE_EMPTY_CATEGORY) + if not pure_empty_source: + raise SystemExit(f"Split summary has no {PURE_EMPTY_CATEGORY} source: {split_summary_path}") + + strict_block = payload.get("strict_default_gate") or {} + strict_category = strict_block.get("category") + if strict_category and strict_category != PURE_EMPTY_CATEGORY: + raise SystemExit( + f"Split summary strict gate must be {PURE_EMPTY_CATEGORY}; found {strict_category}: {split_summary_path}" + ) + + strict_background_paths.append(resolve_split_source_path(pure_empty_source, split_summary_path)) + + context_block = payload.get("context_review") or {} + sparse_context_source = source_summaries.get(SPARSE_CONTEXT_CATEGORY) + if context_block or sparse_context_source: + context_entry = dict(context_block) + context_entry["source_split_summary_path"] = str(split_summary_path) + if sparse_context_source: + context_entry["source_summary_path"] = str( + resolve_split_source_path(sparse_context_source, split_summary_path) + ) + context_reviews.append(context_entry) + + return strict_background_paths, context_reviews + + +def unique_paths(paths: list[Path]) -> list[Path]: + seen: set[str] = set() + unique: list[Path] = [] + for path in paths: + marker = str(path) + if marker in seen: + continue + seen.add(marker) + unique.append(path) + return unique + + def average_metric(runs: list[dict[str, Any]], metric: str) -> float | None: values = [numeric(run, metric) for run in runs] values = [value for value in values if value is not None] @@ -177,7 +252,10 @@ def average_metric(runs: list[dict[str, Any]], metric: str) -> float | None: def build_decisions(args: argparse.Namespace) -> dict[str, Any]: portfolio_path = Path(args.positive_portfolio) positive_portfolio = read_json(portfolio_path) - background_paths = [Path(path) for path in args.hard_negative_summary] + direct_background_paths = [Path(path) for path in args.hard_negative_summary] + split_summary_paths = [Path(path) for path in args.background_split_summary] + split_background_paths, context_reviews = collect_split_background_inputs(split_summary_paths) + background_paths = unique_paths([*direct_background_paths, *split_background_paths]) positive = collect_positive_runs( positive_portfolio, default_tile_size=args.default_positive_tile_size, @@ -249,6 +327,8 @@ def build_decisions(args: argparse.Namespace) -> dict[str, Any]: "generated_at": datetime.now(timezone.utc).isoformat(), "positive_portfolio_path": str(portfolio_path), "hard_negative_summary_paths": [str(path) for path in background_paths], + "background_split_summary_paths": [str(path) for path in split_summary_paths], + "background_context_reviews": context_reviews, "gates": { "min_positive_samples": args.min_positive_samples, "min_background_samples": args.min_background_samples, @@ -268,6 +348,7 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: f"- Generated: {report['generated_at']}", f"- Positive portfolio: `{report['positive_portfolio_path']}`", f"- Hard-negative summaries: {len(report['hard_negative_summary_paths'])}", + f"- Background split summaries: {len(report['background_split_summary_paths'])}", f"- Candidates: {report['candidate_count']}", "", "## Gates", @@ -281,6 +362,25 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: lines.append(f"- Promote candidate for operator review: `{recommended['candidate_key']}`") else: lines.append("- No candidate passed all positive and hard-negative gates.") + if report.get("background_context_reviews"): + lines.extend( + [ + "", + "## Sparse-context review evidence", + "", + ( + f"- Sparse-context review evidence: {len(report['background_context_reviews'])} " + "(not used as a default-promotion gate)" + ), + ] + ) + for item in report["background_context_reviews"]: + lines.append( + "- " + f"`{item.get('category', SPARSE_CONTEXT_CATEGORY)}` from " + f"`{item.get('source_summary_path', 'unknown')}`; " + f"max detections `{item.get('max_detection_count')}`" + ) lines.extend(["", "## Candidate Decisions", ""]) for item in report["candidate_decisions"]: reasons = ", ".join(item["rejection_reasons"]) or "none"