diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 78c2b52d..18888342 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -98,6 +98,11 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) - "status": "needs_human_review", "failures": [], "manifest_immutable": True, "spatial_leakage_status": "ok", })) + quality = tmp_path / "quality.json" + quality.write_text(json.dumps({ + "status": "ok", "low_variance_positive_tile_count": 0, + "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, + })) result = subprocess.run( [ sys.executable, str(SCRIPT), @@ -105,6 +110,7 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) - "--train-yaml", str(tmp_path / "dataset.yaml"), "--train-summary", str(tmp_path / "train-summary.json"), "--dataset-audit", str(audit), + "--train-quality-audit", str(quality), "--calibration-summary", str(tmp_path / "cal.json"), "--test-summary", str(tmp_path / "test.json"), "--background-summary", str(tmp_path / "background.json"), @@ -128,6 +134,11 @@ def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: audit = tmp_path / "audit.json" audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4})) + quality = tmp_path / "quality.json" + quality.write_text(json.dumps({ + "status": "ok", "low_variance_positive_tile_count": 0, + "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, + })) result = subprocess.run( [ sys.executable, @@ -140,6 +151,8 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: str(tmp_path / "train-summary.json"), "--dataset-audit", str(audit), + "--train-quality-audit", + str(quality), "--calibration-summary", str(tmp_path / "cal.json"), "--test-summary", @@ -168,7 +181,11 @@ def test_pending_human_review_does_not_block_objective_training() -> None: "low_variance_positive_tile_count": 0, "review_complete": False, } - assert MODULE.dataset_audit_failures(audit) == [] + quality = { + "status": "ok", "low_variance_positive_tile_count": 0, + "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, + } + assert MODULE.dataset_audit_failures(audit, quality) == [] def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None: @@ -179,11 +196,30 @@ def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> Non "spatial_leakage_status": "failed", "low_variance_positive_tile_count": 2, } - failures = MODULE.dataset_audit_failures(audit) + quality = { + "status": "failed", "low_variance_positive_tile_count": 2, + "label_stats": {"invalid_label_count": 1, "missing_label_file_count": 1}, + } + failures = MODULE.dataset_audit_failures(audit, quality) assert "wallonia/test below minimum" in failures assert "corpus manifest is not immutable" in failures assert "spatial leakage audit is not ok" in failures assert "dataset contains blank/low-variance positive tiles" in failures + assert "train tile quality audit is not ok" in failures + assert "train tile quality audit contains invalid labels" in failures + assert "train tile quality audit contains missing label files" in failures + + +def test_missing_tile_quality_evidence_fails_closed() -> None: + audit = { + "status": "needs_human_review", "failures": [], + "manifest_immutable": True, "spatial_leakage_status": "ok", + } + failures = MODULE.dataset_audit_failures(audit, {}) + assert "train tile quality audit is not ok" in failures + assert "train tile quality audit contains invalid labels" in failures + assert "train tile quality audit contains missing label files" in failures + assert "dataset contains blank/low-variance positive tiles" in failures def test_calibration_failure_blocks_protected_evaluation() -> None: diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index b83891b9..87123e3b 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -131,6 +131,9 @@ the versioned JSON argv command. Invalid state and bounded launch exhaustion fail closed. The orchestrator refuses to start unless every automated frozen-dataset gate passes and the corpus contains zero blank/low-variance positive tiles. The +separate train tile-quality audit is a required checksummed loop input; missing +invalid-label, missing-label-file or low-variance evidence fails closed rather +than being interpreted as zero. The audit status may remain `needs_human_review` while training and objective evaluation continue: final human sign-off is deliberately the last gate and can never be interpreted as model promotion approval in advance. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index d2bb824f..d3879446 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11,6 +11,10 @@ and nine Walloon dense/rural/architecture or difficult-negative samples. Added mode-0600 session-token-file support to the official provisioner so authenticated Tower acquisition does not expose session material in argv. +- Closed a fail-open evidence gap between corpus and tile audits. The training + loop now requires and hashes the train tile-quality report, and refuses + missing counters instead of treating absent invalid/blank-label evidence as + zero. ## 2026-07-27 - Guest demo and product professionalization diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index 3c23b919..f1eef4ef 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -29,7 +29,10 @@ def write_json(path: Path, value: dict[str, Any]) -> None: temporary.replace(path) -def dataset_audit_failures(audit: dict[str, Any]) -> list[str]: +def dataset_audit_failures( + audit: dict[str, Any], + train_quality_audit: dict[str, Any], +) -> list[str]: """Return automated corpus blockers while leaving final human review deferred.""" failures = [str(item) for item in audit.get("failures") or []] status = audit.get("status") @@ -39,7 +42,13 @@ def dataset_audit_failures(audit: dict[str, Any]) -> list[str]: failures.append("corpus manifest is not immutable") if audit.get("spatial_leakage_status") != "ok": failures.append("spatial leakage audit is not ok") - if int(audit.get("low_variance_positive_tile_count") or 0) != 0: + if train_quality_audit.get("status") != "ok": + failures.append("train tile quality audit is not ok") + if int(train_quality_audit.get("label_stats", {}).get("invalid_label_count", -1)) != 0: + failures.append("train tile quality audit contains invalid labels") + if int(train_quality_audit.get("label_stats", {}).get("missing_label_file_count", -1)) != 0: + failures.append("train tile quality audit contains missing label files") + if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0: failures.append("dataset contains blank/low-variance positive tiles") return failures @@ -188,6 +197,7 @@ def main() -> int: parser.add_argument("--train-yaml", type=Path, required=True) parser.add_argument("--train-summary", type=Path, required=True) parser.add_argument("--dataset-audit", type=Path, required=True) + parser.add_argument("--train-quality-audit", type=Path, required=True) parser.add_argument("--calibration-summary", type=Path, required=True) parser.add_argument("--test-summary", type=Path, required=True) parser.add_argument("--background-summary", type=Path, required=True) @@ -230,7 +240,8 @@ def main() -> int: if args.iterations < 1: raise SystemExit("--iterations must be positive") dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8")) - audit_failures = dataset_audit_failures(dataset_audit) + train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8")) + audit_failures = dataset_audit_failures(dataset_audit, train_quality_audit) if audit_failures: raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}") @@ -242,6 +253,9 @@ def main() -> int: "initial_model": str(args.initial_model), "train_yaml": str(args.train_yaml), "dataset_audit": str(args.dataset_audit), + "dataset_audit_sha256": sha256(args.dataset_audit), + "train_quality_audit": str(args.train_quality_audit), + "train_quality_audit_sha256": sha256(args.train_quality_audit), "corpus_manifest": str(args.corpus_manifest), "iterations": [], } diff --git a/scripts/tower-v37-completion-command.json b/scripts/tower-v37-completion-command.json index 1801084c..fcf90b26 100644 --- a/scripts/tower-v37-completion-command.json +++ b/scripts/tower-v37-completion-command.json @@ -13,6 +13,8 @@ "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/train/yolo_tile_dataset_summary.json", "--dataset-audit", "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/audit/belgium-building-corpus-audit.json", + "--train-quality-audit", + "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/train/audit/operator_yolo_dataset_quality_audit.json", "--calibration-summary", "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/calibration/yolo_tile_dataset_summary.json", "--test-summary",