Unblock objective Belgian training loop
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-29 15:31:55 +02:00
parent a5980127b9
commit e8530ae476
5 changed files with 92 additions and 7 deletions
+28 -1
View File
@@ -91,7 +91,34 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
check=False, check=False,
) )
assert result.returncode != 0 assert result.returncode != 0
assert "Dataset audit is not ok" in result.stderr assert "Dataset audit is not eligible for training" in result.stderr
def test_pending_human_review_does_not_block_objective_training() -> None:
audit = {
"status": "needs_human_review",
"failures": [],
"manifest_immutable": True,
"spatial_leakage_status": "ok",
"low_variance_positive_tile_count": 0,
"review_complete": False,
}
assert MODULE.dataset_audit_failures(audit) == []
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
audit = {
"status": "needs_human_review",
"failures": ["wallonia/test below minimum"],
"manifest_immutable": False,
"spatial_leakage_status": "failed",
"low_variance_positive_tile_count": 2,
}
failures = MODULE.dataset_audit_failures(audit)
assert "wallonia/test below minimum" in failures
assert "corpus manifest is not immutable" in failures
assert "spatial leakage audit is not ok" in failures
assert "dataset contains blank/low-variance positive tiles" in failures
def test_calibration_failure_blocks_protected_evaluation() -> None: def test_calibration_failure_blocks_protected_evaluation() -> None:
+5 -2
View File
@@ -92,8 +92,11 @@ recall are repeated, while true negative train tiles are repeated when a
regional precision gate or the pure-background gate fails. Calibration, test, regional precision gate or the pure-background gate fails. Calibration, test,
background-test and validation AOIs are excluded by their frozen corpus split; background-test and validation AOIs are excluded by their frozen corpus split;
the generated evidence records that no protected sample entered training. the generated evidence records that no protected sample entered training.
The orchestrator refuses to start unless the frozen dataset audit is `ok` and The orchestrator refuses to start unless every automated frozen-dataset gate
contains zero blank/low-variance positive tiles. passes and the corpus contains zero blank/low-variance positive tiles. The
audit status may remain `needs_human_review` while training and objective
evaluation continue: final human sign-off is deliberately the last gate and
can never be interpreted as model promotion approval in advance.
For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the
end of the imagery period. A feature created afterward is retained in the end of the imagery period. A feature created afterward is retained in the
audit but excluded from training as `created_after_imagery_period`. UrbIS does audit but excluded from training as `created_after_imagery_period`. UrbIS does
+38
View File
@@ -11747,3 +11747,41 @@ Deployment evidence:
- Repeated the live landing, guest bootstrap, map navigation and quality - Repeated the live landing, guest bootstrap, map navigation and quality
navigation against `http://192.168.10.150:1202`; the inspected live flow navigation against `http://192.168.10.150:1202`; the inspected live flow
produced no browser-console errors. produced no browser-console errors.
## 2026-07-29 - V36 assessment and objective-loop repair
- Audited the current Tower evidence rather than relying on the earlier v6
checkpoint. The governed v30 rotated-holdout corpus contains 124 samples,
21,830 accepted labels, zero temporal-unknown samples, an immutable manifest
and no spatial leakage. Its split composition exceeds the frozen regional
minima and protected samples remain outside training.
- Assessed the completed YOLO11x v36 checkpoint calibration-first on the RTX
4080. At the selected threshold `0.10`, aggregate F1 is `0.552`, Brussels F1
is `0.662`, Wallonia F1 is `0.532`, and Flanders F1 is `0.265`. The candidate
failed Flanders F1/precision/recall and Wallonia precision, so test and
background evidence were not opened and the production model was unchanged.
- Built checksummed v37 failure-driven sampling from that rejection: 3,900
train entries across 94 train AOIs, with extra Flanders recall and
Flanders/Wallonia precision evidence. The sampling audit records zero
protected samples in training.
- Started the inactive v37 YOLO11x fine-tune on Tower CUDA device 0 with
deterministic seed `20260806`, `max_det=1000`, AdamW and aerial rotation
augmentation. NVIDIA process evidence confirmed GPU execution.
- Corrected a loop deadlock: an automatically clean corpus awaiting the final
human review has status `needs_human_review`, while the orchestrator formerly
required `ok`. Training now accepts that status only when the manifest is
immutable, automated failures are empty, spatial leakage is `ok`, and blank
positive-tile count is zero. Human sign-off remains a separate mandatory
final promotion gate.
Verified in this pass:
- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_belgium_training_iteration_assessment.py backend/tests/test_belgium_training_portfolio.py`
(`12 passed`).
Open:
- Let v37 finish, run calibration-only assessment, and expose protected test
and pure-background results only if every regional calibration gate passes.
- Continue failure-driven, train-only corpus iterations until all objective
gates pass; only then request the queued representative human review.
+3
View File
@@ -971,6 +971,9 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and complete 2023 SPW imagery; retain exact flight-day limitations. - [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and complete 2023 SPW imagery; retain exact flight-day limitations.
- [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign. - [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign.
- [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence. - [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence.
- [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate.
- [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed.
- [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes.
# Sprint 229 - Visual release correction and Tower redeploy # Sprint 229 - Visual release correction and Tower redeploy
+18 -4
View File
@@ -29,6 +29,21 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
temporary.replace(path) temporary.replace(path)
def dataset_audit_failures(audit: dict[str, Any]) -> list[str]:
"""Return automated corpus blockers while leaving final human review deferred."""
failures = [str(item) for item in audit.get("failures") or []]
status = audit.get("status")
if status not in {"ok", "needs_human_review"}:
failures.append(f"unsupported audit status: {status}")
if audit.get("manifest_immutable") is not True:
failures.append("corpus manifest is not immutable")
if audit.get("spatial_leakage_status") != "ok":
failures.append("spatial leakage audit is not ok")
if int(audit.get("low_variance_positive_tile_count") or 0) != 0:
failures.append("dataset contains blank/low-variance positive tiles")
return failures
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]: def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
"""Choose a threshold without consulting test or background evidence.""" """Choose a threshold without consulting test or background evidence."""
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0] eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
@@ -161,10 +176,9 @@ def main() -> int:
if args.iterations < 1: if args.iterations < 1:
raise SystemExit("--iterations must be positive") raise SystemExit("--iterations must be positive")
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8")) dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
if dataset_audit.get("status") != "ok": audit_failures = dataset_audit_failures(dataset_audit)
raise SystemExit(f"Dataset audit is not ok: {args.dataset_audit}") if audit_failures:
if int(dataset_audit.get("low_variance_positive_tile_count") or 0) != 0: raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
raise SystemExit("Dataset audit contains blank/low-variance positive tiles")
state_path = args.output_dir / "training-loop-state.json" state_path = args.output_dir / "training-loop-state.json"
state: dict[str, Any] = { state: dict[str, Any] = {