Unblock objective Belgian training loop
This commit is contained in:
@@ -91,7 +91,34 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
|||||||
check=False,
|
check=False,
|
||||||
)
|
)
|
||||||
assert result.returncode != 0
|
assert result.returncode != 0
|
||||||
assert "Dataset audit is not ok" in result.stderr
|
assert "Dataset audit is not eligible for training" in result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_human_review_does_not_block_objective_training() -> None:
|
||||||
|
audit = {
|
||||||
|
"status": "needs_human_review",
|
||||||
|
"failures": [],
|
||||||
|
"manifest_immutable": True,
|
||||||
|
"spatial_leakage_status": "ok",
|
||||||
|
"low_variance_positive_tile_count": 0,
|
||||||
|
"review_complete": False,
|
||||||
|
}
|
||||||
|
assert MODULE.dataset_audit_failures(audit) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
|
||||||
|
audit = {
|
||||||
|
"status": "needs_human_review",
|
||||||
|
"failures": ["wallonia/test below minimum"],
|
||||||
|
"manifest_immutable": False,
|
||||||
|
"spatial_leakage_status": "failed",
|
||||||
|
"low_variance_positive_tile_count": 2,
|
||||||
|
}
|
||||||
|
failures = MODULE.dataset_audit_failures(audit)
|
||||||
|
assert "wallonia/test below minimum" in failures
|
||||||
|
assert "corpus manifest is not immutable" in failures
|
||||||
|
assert "spatial leakage audit is not ok" in failures
|
||||||
|
assert "dataset contains blank/low-variance positive tiles" in failures
|
||||||
|
|
||||||
|
|
||||||
def test_calibration_failure_blocks_protected_evaluation() -> None:
|
def test_calibration_failure_blocks_protected_evaluation() -> None:
|
||||||
|
|||||||
@@ -92,8 +92,11 @@ recall are repeated, while true negative train tiles are repeated when a
|
|||||||
regional precision gate or the pure-background gate fails. Calibration, test,
|
regional precision gate or the pure-background gate fails. Calibration, test,
|
||||||
background-test and validation AOIs are excluded by their frozen corpus split;
|
background-test and validation AOIs are excluded by their frozen corpus split;
|
||||||
the generated evidence records that no protected sample entered training.
|
the generated evidence records that no protected sample entered training.
|
||||||
The orchestrator refuses to start unless the frozen dataset audit is `ok` and
|
The orchestrator refuses to start unless every automated frozen-dataset gate
|
||||||
contains zero blank/low-variance positive tiles.
|
passes and the corpus contains zero blank/low-variance positive tiles. The
|
||||||
|
audit status may remain `needs_human_review` while training and objective
|
||||||
|
evaluation continue: final human sign-off is deliberately the last gate and
|
||||||
|
can never be interpreted as model promotion approval in advance.
|
||||||
For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the
|
For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the
|
||||||
end of the imagery period. A feature created afterward is retained in the
|
end of the imagery period. A feature created afterward is retained in the
|
||||||
audit but excluded from training as `created_after_imagery_period`. UrbIS does
|
audit but excluded from training as `created_after_imagery_period`. UrbIS does
|
||||||
|
|||||||
@@ -11747,3 +11747,41 @@ Deployment evidence:
|
|||||||
- Repeated the live landing, guest bootstrap, map navigation and quality
|
- Repeated the live landing, guest bootstrap, map navigation and quality
|
||||||
navigation against `http://192.168.10.150:1202`; the inspected live flow
|
navigation against `http://192.168.10.150:1202`; the inspected live flow
|
||||||
produced no browser-console errors.
|
produced no browser-console errors.
|
||||||
|
|
||||||
|
## 2026-07-29 - V36 assessment and objective-loop repair
|
||||||
|
|
||||||
|
- Audited the current Tower evidence rather than relying on the earlier v6
|
||||||
|
checkpoint. The governed v30 rotated-holdout corpus contains 124 samples,
|
||||||
|
21,830 accepted labels, zero temporal-unknown samples, an immutable manifest
|
||||||
|
and no spatial leakage. Its split composition exceeds the frozen regional
|
||||||
|
minima and protected samples remain outside training.
|
||||||
|
- Assessed the completed YOLO11x v36 checkpoint calibration-first on the RTX
|
||||||
|
4080. At the selected threshold `0.10`, aggregate F1 is `0.552`, Brussels F1
|
||||||
|
is `0.662`, Wallonia F1 is `0.532`, and Flanders F1 is `0.265`. The candidate
|
||||||
|
failed Flanders F1/precision/recall and Wallonia precision, so test and
|
||||||
|
background evidence were not opened and the production model was unchanged.
|
||||||
|
- Built checksummed v37 failure-driven sampling from that rejection: 3,900
|
||||||
|
train entries across 94 train AOIs, with extra Flanders recall and
|
||||||
|
Flanders/Wallonia precision evidence. The sampling audit records zero
|
||||||
|
protected samples in training.
|
||||||
|
- Started the inactive v37 YOLO11x fine-tune on Tower CUDA device 0 with
|
||||||
|
deterministic seed `20260806`, `max_det=1000`, AdamW and aerial rotation
|
||||||
|
augmentation. NVIDIA process evidence confirmed GPU execution.
|
||||||
|
- Corrected a loop deadlock: an automatically clean corpus awaiting the final
|
||||||
|
human review has status `needs_human_review`, while the orchestrator formerly
|
||||||
|
required `ok`. Training now accepts that status only when the manifest is
|
||||||
|
immutable, automated failures are empty, spatial leakage is `ok`, and blank
|
||||||
|
positive-tile count is zero. Human sign-off remains a separate mandatory
|
||||||
|
final promotion gate.
|
||||||
|
|
||||||
|
Verified in this pass:
|
||||||
|
|
||||||
|
- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_belgium_training_iteration_assessment.py backend/tests/test_belgium_training_portfolio.py`
|
||||||
|
(`12 passed`).
|
||||||
|
|
||||||
|
Open:
|
||||||
|
|
||||||
|
- Let v37 finish, run calibration-only assessment, and expose protected test
|
||||||
|
and pure-background results only if every regional calibration gate passes.
|
||||||
|
- Continue failure-driven, train-only corpus iterations until all objective
|
||||||
|
gates pass; only then request the queued representative human review.
|
||||||
|
|||||||
@@ -971,6 +971,9 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and complete 2023 SPW imagery; retain exact flight-day limitations.
|
- [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and complete 2023 SPW imagery; retain exact flight-day limitations.
|
||||||
- [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign.
|
- [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign.
|
||||||
- [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence.
|
- [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence.
|
||||||
|
- [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate.
|
||||||
|
- [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed.
|
||||||
|
- [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes.
|
||||||
|
|
||||||
# Sprint 229 - Visual release correction and Tower redeploy
|
# Sprint 229 - Visual release correction and Tower redeploy
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,21 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
|
|||||||
temporary.replace(path)
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def dataset_audit_failures(audit: dict[str, Any]) -> list[str]:
|
||||||
|
"""Return automated corpus blockers while leaving final human review deferred."""
|
||||||
|
failures = [str(item) for item in audit.get("failures") or []]
|
||||||
|
status = audit.get("status")
|
||||||
|
if status not in {"ok", "needs_human_review"}:
|
||||||
|
failures.append(f"unsupported audit status: {status}")
|
||||||
|
if audit.get("manifest_immutable") is not True:
|
||||||
|
failures.append("corpus manifest is not immutable")
|
||||||
|
if audit.get("spatial_leakage_status") != "ok":
|
||||||
|
failures.append("spatial leakage audit is not ok")
|
||||||
|
if int(audit.get("low_variance_positive_tile_count") or 0) != 0:
|
||||||
|
failures.append("dataset contains blank/low-variance positive tiles")
|
||||||
|
return failures
|
||||||
|
|
||||||
|
|
||||||
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
|
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
|
||||||
"""Choose a threshold without consulting test or background evidence."""
|
"""Choose a threshold without consulting test or background evidence."""
|
||||||
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
|
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
|
||||||
@@ -161,10 +176,9 @@ def main() -> int:
|
|||||||
if args.iterations < 1:
|
if args.iterations < 1:
|
||||||
raise SystemExit("--iterations must be positive")
|
raise SystemExit("--iterations must be positive")
|
||||||
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
|
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
|
||||||
if dataset_audit.get("status") != "ok":
|
audit_failures = dataset_audit_failures(dataset_audit)
|
||||||
raise SystemExit(f"Dataset audit is not ok: {args.dataset_audit}")
|
if audit_failures:
|
||||||
if int(dataset_audit.get("low_variance_positive_tile_count") or 0) != 0:
|
raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
|
||||||
raise SystemExit("Dataset audit contains blank/low-variance positive tiles")
|
|
||||||
|
|
||||||
state_path = args.output_dir / "training-loop-state.json"
|
state_path = args.output_dir / "training-loop-state.json"
|
||||||
state: dict[str, Any] = {
|
state: dict[str, Any] = {
|
||||||
|
|||||||
Reference in New Issue
Block a user