From 4e9fe6ccb1e27f33a69b88b167b1fc28eb24bfdc Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 03:54:48 +0200 Subject: [PATCH] Record unique hard-negative YOLO candidate gate --- CHANGELOG.md | 9 ++ ...int143_detection_model_promotion_report.py | 85 +++++++++++++++++++ docs/CODEX_EXECUTION_LOG.md | 48 +++++++++++ docs/TODO.md | 1 + scripts/README.md | 17 ++++ .../build_detection_model_promotion_report.py | 49 +++++++++-- 6 files changed, 202 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec37003..f815f017 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ # Changelog +## Sprint 146 Unique hard-negative YOLOv8s candidate gate (2026-07-09) + +- Fixed the all-in-one Docker image so the operator YOLO training wrapper is available at `/app/scripts/train_operator_yolo_detector.sh`. +- Trained the Tower-local `geointel-building-yolov8s-uniquehardneg160e50-pt` candidate from the `yolo-building-tile-uniquehardneg160` dataset and preserved it as an explicit local model asset. +- Ran 7 positive AOI calibration sweeps, a 17,008-feature evidence portfolio, a 9-sample hard-negative/background matrix and a promotion report. +- Result: the candidate is rejected. Mean positive F1 remains around `0.16` and background false-positive pressure reaches `58` detections at threshold `0.25`, `85` at `0.15` and `172` at `0.05`. +- Hardened the operator promotion report so older positive evidence portfolios can be compared with explicit positive tile-size/overlap defaults. +- No API contract, migration, frontend behavior, model download, provider-fetching behavior or active model configuration changed. + ## Sprint 145 YOLOv8s hardneg r8 e60 full candidate evaluation (2026-07-08) - Completed the Tower-local YOLOv8s hard-negative r8 training run through 60 CPU epochs and published local model asset `geointel-building-yolov8s-hardneg160r8e60-pt`. diff --git a/backend/tests/test_sprint143_detection_model_promotion_report.py b/backend/tests/test_sprint143_detection_model_promotion_report.py index ef3d9289..ade6d76e 100644 --- a/backend/tests/test_sprint143_detection_model_promotion_report.py +++ b/backend/tests/test_sprint143_detection_model_promotion_report.py @@ -167,3 +167,88 @@ def test_detection_model_promotion_report_combines_positive_and_background_gates assert "candidate-clean" in markdown assert "candidate-leaky" in markdown assert "background_false_positive_pressure" in markdown + + +def test_detection_model_promotion_report_uses_portfolio_and_tile_defaults( + tmp_path: Path, +) -> None: + script_path = ROOT / "scripts" / "build_detection_model_promotion_report.py" + + positive_path = tmp_path / "positive_portfolio.json" + positive_path.write_text( + json.dumps( + { + "portfolio_name": "Positive AOI portfolio", + "model_asset_id": "candidate-from-portfolio", + "samples": [ + { + "sample_slug": "geel", + "runs": [ + { + "model_asset_id": None, + "tile_size": None, + "tile_overlap": None, + "threshold": 0.25, + "precision": 0.7, + "recall": 0.42, + "f1_score": 0.525, + } + ], + } + ], + } + ), + encoding="utf-8", + ) + + background_path = tmp_path / "hard_negative_matrix_summary.json" + background_path.write_text( + json.dumps( + { + "items": [ + { + "sample_slug": "postel_bos", + "model_asset_id": "candidate-from-portfolio", + "tile_size": 640, + "tile_overlap": 64, + "threshold": 0.25, + "detection_count": 0, + } + ] + } + ), + encoding="utf-8", + ) + + output_dir = tmp_path / "promotion-report" + subprocess.run( + [ + "python", + str(script_path), + "--positive-portfolio", + str(positive_path), + "--hard-negative-summary", + str(background_path), + "--output-dir", + str(output_dir), + "--min-positive-samples", + "1", + "--min-background-samples", + "1", + "--min-mean-f1", + "0.35", + "--max-background-detections-per-sample", + "0", + "--default-positive-tile-size", + "640", + "--default-positive-tile-overlap", + "64", + ], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + + report = json.loads((output_dir / "detection_model_promotion_report.json").read_text(encoding="utf-8")) + assert report["recommended_candidate"]["candidate_key"] == "candidate-from-portfolio|640|64|0.25" diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 330387fb..df2be3e7 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,51 @@ +## Sprint 146 Unique hard-negative YOLOv8s candidate gate (2026-07-09) + +Changed: +- Fixed the all-in-one Docker runtime so `/app/scripts/train_operator_yolo_detector.sh` is copied into the image and made executable. +- Added regression coverage in `backend/tests/test_docker_runtime_config.py` so the all-in-one image must include the operator YOLO training wrapper. +- Hardened `scripts/build_detection_model_promotion_report.py` with explicit `--default-positive-tile-size` and `--default-positive-tile-overlap` options for older positive evidence portfolios that record the model at portfolio level but omit per-run tile provenance. +- Added regression coverage in `backend/tests/test_sprint143_detection_model_promotion_report.py` for portfolio-level model fallback plus explicit positive tile defaults. + +Tested: +- Red step: `python -m pytest backend\tests\test_docker_runtime_config.py::test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use -q` failed because the all-in-one Dockerfile did not copy the training wrapper. +- `python -m pytest backend\tests\test_docker_runtime_config.py -q` (`22 passed`) +- `bash scripts/run_readiness_check.sh` (`423 passed`; frontend typecheck/build passed; Alembic head `202606120900`) +- Pushed commit `afd2cba` and redeployed Tower with `.\scripts\deploy_tower.ps1 -InstallAi true`; browser runtime verification and live migration smoke passed. +- Verified the wrapper in the live container: `/app/scripts/train_operator_yolo_detector.sh` exists and is executable. +- Trained `geointel-building-yolov8s-uniquehardneg160e50-pt` on Tower from `/app/storage/operator-data/yolo-building-tile-uniquehardneg160/dataset.yaml` using local `/app/models/yolov8s.pt`, 50 CPU epochs, image size `160`, batch `8`. +- Training completed with final validation precision `0.38`, recall `0.365`, mAP50 `0.278`, mAP50-95 `0.0803`. +- Live model asset: + - `/app/models/geointel-building-yolov8s-uniquehardneg160e50.pt` + - model asset id `geointel-building-yolov8s-uniquehardneg160e50-pt` + - SHA256 `c4e480273d3da5fc27532cd8bdc3fa7786582e06848ea11b56714ab8cb1750b3` +- Ran 7-AOI positive calibration sweeps for Geel, Mol, Turnhout, Herentals, Balen, Retie and Westerlo: + - output root `/mnt/user/appdata/geointel/artifacts/detection-calibration/uniquehardneg160e50-positive` + - best observed AOI result: Westerlo threshold `0.25`, F1 `0.384180790960452`, precision `0.5177664974619289`, recall `0.30538922155688625`, detections `197` + - other positive AOIs remained weak, with best F1 roughly `0.10` to `0.17`. +- Assembled positive evidence portfolio: + - output `/mnt/user/appdata/geointel/artifacts/detection-calibration-portfolio/uniquehardneg160e50-positive/calibration_evidence_portfolio.json` + - sample count `7` + - evidence features `17008` + - role counts `false_negative=9710`, `false_positive=4734`, `match_candidate=1282`, `match_reference=1282` +- Ran 9-sample hard-negative/background matrix: + - output `/mnt/user/appdata/geointel/artifacts/detection-hard-negatives/uniquehardneg160e50/hard_negative_matrix_summary.json` + - threshold `0.25`: total background detections `98`, max sample detections `58` + - threshold `0.15`: total background detections `137`, max sample detections `85` + - threshold `0.05`: total background detections `276`, max sample detections `172` +- Red step: `python -m pytest backend\tests\test_sprint143_detection_model_promotion_report.py -q` failed because the promotion report could not yet accept explicit positive tile defaults. +- `python -m pytest backend\tests\test_sprint143_detection_model_promotion_report.py -q` (`2 passed`) +- Rebuilt the promotion report using explicit positive tile defaults: + - output `/mnt/user/appdata/geointel/artifacts/detection-model-promotion/uniquehardneg160e50-positive-vs-hard-negative-v2/detection_model_promotion_report.json` + - threshold `0.05`: rejected for `positive_mean_f1_below_gate` and `background_false_positive_pressure`, mean F1 `0.15797188547918842`, max background detections `172` + - threshold `0.15`: rejected for the same reasons, mean F1 `0.1598974047548654`, max background detections `85` + - threshold `0.25`: rejected for the same reasons, mean F1 `0.15967766715169612`, max background detections `58` + - recommended candidate `none` + +Open: +- Do not activate `geointel-building-yolov8s-uniquehardneg160e50-pt` as the V1 default. +- The candidate improves some individual AOIs but still combines low mean positive F1 with unacceptable false-positive pressure on background samples. +- Next model work should focus on materially better training data/model strategy, not another default activation attempt with this artifact. + ## Sprint 145 YOLOv8s hardneg r8 e60 full candidate evaluation (2026-07-08) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index e845efff..f940187f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -108,6 +108,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add a hard-negative model-quality pass with sparse/background AOIs and explicit false-positive scoring. - [x] Train a hard-negative-balanced YOLO candidate and rerun dense QA plus background false-positive matrices. - [x] Benchmark an external remote-sensing YOLOv8l building candidate as an explicit local model asset. +- [x] Train and gate the `uniquehardneg160e50` YOLOv8s candidate through 7 positive AOIs and 9 hard-negative/background samples. - [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default. - [ ] Train a higher-capacity local aerial-building detector with stronger positive recall while preserving the hard-negative false-positive gate. diff --git a/scripts/README.md b/scripts/README.md index b6ddf6fc..12d704ed 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -603,6 +603,23 @@ and maximum background detections per sample. It is evidence/report tooling only: it does not run inference, mutate application data, download models or change the active YOLO configuration. +If a legacy positive evidence portfolio records `model_asset_id` at portfolio +level but does not include per-run tile size/overlap, pass explicit tile +defaults instead of letting the report guess: + +```bash +python scripts/build_detection_model_promotion_report.py \ + --positive-portfolio /mnt/user/appdata/geointel/artifacts/detection-calibration-portfolio/uniquehardneg160e50-positive/calibration_evidence_portfolio.json \ + --hard-negative-summary /mnt/user/appdata/geointel/artifacts/detection-hard-negatives/uniquehardneg160e50/hard_negative_matrix_summary.json \ + --output-dir /mnt/user/appdata/geointel/artifacts/detection-model-promotion/uniquehardneg160e50-positive-vs-hard-negative \ + --min-positive-samples 7 \ + --min-background-samples 9 \ + --min-mean-f1 0.25 \ + --max-background-detections-per-sample 0 \ + --default-positive-tile-size 640 \ + --default-positive-tile-overlap 64 +``` + Clean old offline demo export artifacts without touching uploaded source data: ```bash diff --git a/scripts/build_detection_model_promotion_report.py b/scripts/build_detection_model_promotion_report.py index d0ec9299..6781b7ba 100644 --- a/scripts/build_detection_model_promotion_report.py +++ b/scripts/build_detection_model_promotion_report.py @@ -47,6 +47,18 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--min-background-samples", type=int, default=3) parser.add_argument("--min-mean-f1", type=float, default=0.25) parser.add_argument("--max-background-detections-per-sample", type=int, default=0) + parser.add_argument( + "--default-positive-tile-size", + type=int, + default=None, + help="Tile size to use for positive portfolio runs that do not record tile_size.", + ) + parser.add_argument( + "--default-positive-tile-overlap", + type=int, + default=None, + help="Tile overlap to use for positive portfolio runs that do not record tile_overlap.", + ) return parser.parse_args() @@ -56,15 +68,23 @@ def read_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8-sig")) -def as_candidate_key(item: dict[str, Any]) -> CandidateKey | None: - model_asset_id = item.get("model_asset_id") or item.get("model_request") +def as_candidate_key( + item: dict[str, Any], + *, + fallback_model_asset_id: str | None = None, + fallback_tile_size: int | None = None, + fallback_tile_overlap: int | None = None, +) -> CandidateKey | None: + model_asset_id = item.get("model_asset_id") or item.get("model_request") or fallback_model_asset_id if not model_asset_id: return None try: return CandidateKey( model_asset_id=str(model_asset_id), - tile_size=int(item.get("tile_size")), - tile_overlap=int(item.get("tile_overlap")), + tile_size=int(item.get("tile_size") if item.get("tile_size") is not None else fallback_tile_size), + tile_overlap=int( + item.get("tile_overlap") if item.get("tile_overlap") is not None else fallback_tile_overlap + ), threshold=float(item.get("threshold")), ) except (TypeError, ValueError): @@ -81,12 +101,23 @@ def numeric(item: dict[str, Any], key: str) -> float | None: return None -def collect_positive_runs(portfolio: dict[str, Any]) -> dict[CandidateKey, list[dict[str, Any]]]: +def collect_positive_runs( + portfolio: dict[str, Any], + *, + default_tile_size: int | None = None, + default_tile_overlap: int | None = None, +) -> dict[CandidateKey, list[dict[str, Any]]]: best_by_candidate_sample: dict[tuple[CandidateKey, str], dict[str, Any]] = {} + portfolio_model_asset_id = portfolio.get("model_asset_id") for sample in portfolio.get("samples") or []: sample_slug = str(sample.get("sample_slug") or "unknown") for run in sample.get("runs") or []: - key = as_candidate_key(run) + key = as_candidate_key( + run, + fallback_model_asset_id=str(portfolio_model_asset_id) if portfolio_model_asset_id else None, + fallback_tile_size=default_tile_size, + fallback_tile_overlap=default_tile_overlap, + ) f1 = numeric(run, "f1_score") if key is None or f1 is None: continue @@ -136,7 +167,11 @@ def build_decisions(args: argparse.Namespace) -> dict[str, Any]: portfolio_path = Path(args.positive_portfolio) positive_portfolio = read_json(portfolio_path) background_paths = [Path(path) for path in args.hard_negative_summary] - positive = collect_positive_runs(positive_portfolio) + positive = collect_positive_runs( + positive_portfolio, + default_tile_size=args.default_positive_tile_size, + default_tile_overlap=args.default_positive_tile_overlap, + ) background = collect_background_runs(background_paths) decisions = []