From e8530ae4764c01fc6fa9463ba3b5bf42b1ef9f48 Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 15:31:55 +0200 Subject: [PATCH] Unblock objective Belgian training loop --- backend/tests/test_belgium_training_loop.py | 29 +++++++++++++- docs/BELGIUM_BUILDING_TRAINING_LOOP.md | 7 +++- docs/CODEX_EXECUTION_LOG.md | 38 +++++++++++++++++++ docs/TODO.md | 3 ++ scripts/run_belgium_building_training_loop.py | 22 +++++++++-- 5 files changed, 92 insertions(+), 7 deletions(-) diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 2e3a42f9..0d4e5a6e 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -91,7 +91,34 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: check=False, ) assert result.returncode != 0 - assert "Dataset audit is not ok" in result.stderr + assert "Dataset audit is not eligible for training" in result.stderr + + +def test_pending_human_review_does_not_block_objective_training() -> None: + audit = { + "status": "needs_human_review", + "failures": [], + "manifest_immutable": True, + "spatial_leakage_status": "ok", + "low_variance_positive_tile_count": 0, + "review_complete": False, + } + assert MODULE.dataset_audit_failures(audit) == [] + + +def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None: + audit = { + "status": "needs_human_review", + "failures": ["wallonia/test below minimum"], + "manifest_immutable": False, + "spatial_leakage_status": "failed", + "low_variance_positive_tile_count": 2, + } + failures = MODULE.dataset_audit_failures(audit) + assert "wallonia/test below minimum" in failures + assert "corpus manifest is not immutable" in failures + assert "spatial leakage audit is not ok" in failures + assert "dataset contains blank/low-variance positive tiles" in failures def test_calibration_failure_blocks_protected_evaluation() -> None: diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index cdec6cbd..988fa74a 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -92,8 +92,11 @@ recall are repeated, while true negative train tiles are repeated when a regional precision gate or the pure-background gate fails. Calibration, test, background-test and validation AOIs are excluded by their frozen corpus split; the generated evidence records that no protected sample entered training. -The orchestrator refuses to start unless the frozen dataset audit is `ok` and -contains zero blank/low-variance positive tiles. +The orchestrator refuses to start unless every automated frozen-dataset gate +passes and the corpus contains zero blank/low-variance positive tiles. The +audit status may remain `needs_human_review` while training and objective +evaluation continue: final human sign-off is deliberately the last gate and +can never be interpreted as model promotion approval in advance. For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the end of the imagery period. A feature created afterward is retained in the audit but excluded from training as `created_after_imagery_period`. UrbIS does diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 67309a6f..ff00c416 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11747,3 +11747,41 @@ Deployment evidence: - Repeated the live landing, guest bootstrap, map navigation and quality navigation against `http://192.168.10.150:1202`; the inspected live flow produced no browser-console errors. + +## 2026-07-29 - V36 assessment and objective-loop repair + +- Audited the current Tower evidence rather than relying on the earlier v6 + checkpoint. The governed v30 rotated-holdout corpus contains 124 samples, + 21,830 accepted labels, zero temporal-unknown samples, an immutable manifest + and no spatial leakage. Its split composition exceeds the frozen regional + minima and protected samples remain outside training. +- Assessed the completed YOLO11x v36 checkpoint calibration-first on the RTX + 4080. At the selected threshold `0.10`, aggregate F1 is `0.552`, Brussels F1 + is `0.662`, Wallonia F1 is `0.532`, and Flanders F1 is `0.265`. The candidate + failed Flanders F1/precision/recall and Wallonia precision, so test and + background evidence were not opened and the production model was unchanged. +- Built checksummed v37 failure-driven sampling from that rejection: 3,900 + train entries across 94 train AOIs, with extra Flanders recall and + Flanders/Wallonia precision evidence. The sampling audit records zero + protected samples in training. +- Started the inactive v37 YOLO11x fine-tune on Tower CUDA device 0 with + deterministic seed `20260806`, `max_det=1000`, AdamW and aerial rotation + augmentation. NVIDIA process evidence confirmed GPU execution. +- Corrected a loop deadlock: an automatically clean corpus awaiting the final + human review has status `needs_human_review`, while the orchestrator formerly + required `ok`. Training now accepts that status only when the manifest is + immutable, automated failures are empty, spatial leakage is `ok`, and blank + positive-tile count is zero. Human sign-off remains a separate mandatory + final promotion gate. + +Verified in this pass: + +- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_belgium_training_iteration_assessment.py backend/tests/test_belgium_training_portfolio.py` + (`12 passed`). + +Open: + +- Let v37 finish, run calibration-only assessment, and expose protected test + and pure-background results only if every regional calibration gate passes. +- Continue failure-driven, train-only corpus iterations until all objective + gates pass; only then request the queued representative human review. diff --git a/docs/TODO.md b/docs/TODO.md index e7adba73..203a8deb 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -971,6 +971,9 @@ This file now starts with the current implementation status. Older preparation/b - [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and complete 2023 SPW imagery; retain exact flight-day limitations. - [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign. - [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence. +- [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate. +- [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. +- [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. # Sprint 229 - Visual release correction and Tower redeploy diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index 11e8c8f6..897493c9 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -29,6 +29,21 @@ def write_json(path: Path, value: dict[str, Any]) -> None: temporary.replace(path) +def dataset_audit_failures(audit: dict[str, Any]) -> list[str]: + """Return automated corpus blockers while leaving final human review deferred.""" + failures = [str(item) for item in audit.get("failures") or []] + status = audit.get("status") + if status not in {"ok", "needs_human_review"}: + failures.append(f"unsupported audit status: {status}") + if audit.get("manifest_immutable") is not True: + failures.append("corpus manifest is not immutable") + if audit.get("spatial_leakage_status") != "ok": + failures.append("spatial leakage audit is not ok") + if int(audit.get("low_variance_positive_tile_count") or 0) != 0: + failures.append("dataset contains blank/low-variance positive tiles") + return failures + + def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]: """Choose a threshold without consulting test or background evidence.""" eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0] @@ -161,10 +176,9 @@ def main() -> int: if args.iterations < 1: raise SystemExit("--iterations must be positive") dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8")) - if dataset_audit.get("status") != "ok": - raise SystemExit(f"Dataset audit is not ok: {args.dataset_audit}") - if int(dataset_audit.get("low_variance_positive_tile_count") or 0) != 0: - raise SystemExit("Dataset audit contains blank/low-variance positive tiles") + audit_failures = dataset_audit_failures(dataset_audit) + if audit_failures: + raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}") state_path = args.output_dir / "training-loop-state.json" state: dict[str, Any] = {