Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
511 lines
19 KiB
Python
511 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
SCRIPT = Path(__file__).parents[2] / "scripts" / "run_belgium_building_training_loop.py"
|
|
SPEC = importlib.util.spec_from_file_location("training_loop", SCRIPT)
|
|
assert SPEC and SPEC.loader
|
|
MODULE = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(MODULE)
|
|
|
|
from training_release_manifest import create_training_release_manifest # noqa: E402
|
|
|
|
|
|
def write_fixture_manifest(path: Path) -> None:
|
|
policy = "geointel-training-source-eligibility/v1"
|
|
def eligible(sample_slug: str) -> dict[str, object]:
|
|
return {
|
|
"policy_version": policy,
|
|
"eligible": True,
|
|
"fixture_mode": True,
|
|
"raster": {
|
|
"eligible": True,
|
|
"reasons": [],
|
|
"evidence": {
|
|
"dataset_id": f"raster:{sample_slug}",
|
|
"checksum_sha256": "a" * 64,
|
|
"source_registry_id": "fixture-raster",
|
|
"source_snapshot_id": "fixture-raster-snapshot",
|
|
},
|
|
},
|
|
"reference": {
|
|
"eligible": True,
|
|
"reasons": [],
|
|
"evidence": {
|
|
"dataset_id": f"reference:{sample_slug}",
|
|
"checksum_sha256": "b" * 64,
|
|
"source_registry_id": "fixture-reference",
|
|
"source_snapshot_id": "fixture-reference-snapshot",
|
|
},
|
|
},
|
|
}
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"training_eligibility": {
|
|
"policy_version": policy,
|
|
"status": "eligible",
|
|
"fixture_mode": True,
|
|
},
|
|
"samples": [
|
|
{
|
|
"sample_slug": sample_slug,
|
|
"split": split,
|
|
"raster_dataset_id": f"raster:{sample_slug}",
|
|
"reference_dataset_id": f"reference:{sample_slug}",
|
|
"training_eligibility": eligible(sample_slug),
|
|
}
|
|
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val"))
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(path.parent / "corpus-freeze.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 2,
|
|
"manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
"immutable": True,
|
|
"training_eligibility_policy": policy,
|
|
"fixture_mode": True,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path:
|
|
dataset_dir = tmp_path / "fixture-dataset"
|
|
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
|
|
image = dataset_dir / "images" / split / f"{sample_slug}.png"
|
|
label = dataset_dir / "labels" / split / f"{sample_slug}.txt"
|
|
image.parent.mkdir(parents=True, exist_ok=True)
|
|
label.parent.mkdir(parents=True, exist_ok=True)
|
|
image.write_bytes(split.encode("utf-8"))
|
|
label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
|
yaml_path = dataset_dir / "dataset.yaml"
|
|
yaml_path.write_text(
|
|
f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
|
encoding="utf-8",
|
|
)
|
|
create_training_release_manifest(
|
|
train_yaml=yaml_path,
|
|
corpus_manifest=manifest,
|
|
fixture_mode=True,
|
|
)
|
|
return yaml_path
|
|
|
|
|
|
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
|
|
command = MODULE.training_command(
|
|
"yolo",
|
|
model=tmp_path / "base.pt",
|
|
data=tmp_path / "dataset.yaml",
|
|
project=tmp_path / "runs",
|
|
name="iteration-001",
|
|
epochs=160,
|
|
seed=42,
|
|
batch=2,
|
|
workers=4,
|
|
)
|
|
assert command[:2] == ["yolo", "train"]
|
|
assert "device=0" in command
|
|
assert "deterministic=True" in command
|
|
assert "seed=42" in command
|
|
assert "epochs=160" in command
|
|
assert "patience=18" in command
|
|
assert "max_det=1000" in command
|
|
assert "imgsz=640" in command
|
|
assert "optimizer=auto" in command
|
|
assert "mosaic=1.0" in command
|
|
|
|
|
|
def test_training_command_supports_conservative_aerial_finetuning(tmp_path: Path) -> None:
|
|
command = MODULE.training_command(
|
|
"yolo",
|
|
model=tmp_path / "base.pt",
|
|
data=tmp_path / "dataset.yaml",
|
|
project=tmp_path / "runs",
|
|
name="aerial",
|
|
epochs=50,
|
|
seed=42,
|
|
batch=2,
|
|
workers=0,
|
|
optimizer="AdamW",
|
|
lr0=0.0001,
|
|
mosaic=0.0,
|
|
scale=0.2,
|
|
translate=0.05,
|
|
)
|
|
assert "optimizer=AdamW" in command
|
|
assert "lr0=0.0001" in command
|
|
assert "mosaic=0.0" in command
|
|
assert "scale=0.2" in command
|
|
assert "translate=0.05" in command
|
|
assert "degrees=0.0" in command
|
|
assert "flipud=0.0" in command
|
|
assert "fliplr=0.5" in command
|
|
assert "warmup_epochs=1.0" in command
|
|
assert "warmup_bias_lr=0.01" in command
|
|
assert "hsv_h=0.01" in command
|
|
assert "hsv_s=0.2" in command
|
|
assert "hsv_v=0.15" in command
|
|
assert f"data={tmp_path / 'dataset.yaml'}" in command
|
|
|
|
|
|
def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_path: Path) -> None:
|
|
command = MODULE.failure_sampling_command(
|
|
scripts_dir=tmp_path / "scripts",
|
|
train_summary=tmp_path / "train-summary.json",
|
|
corpus_manifest=tmp_path / "manifest.json",
|
|
assessment=tmp_path / "assessment.json",
|
|
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
|
|
review_audit=tmp_path / "review-audit.json",
|
|
)
|
|
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
|
|
assert command[command.index("--summary") + 1].endswith("train-summary.json")
|
|
assert command[command.index("--assessment") + 1].endswith("assessment.json")
|
|
assert command[command.index("--output-dir") + 1].endswith("failure-driven-training")
|
|
|
|
|
|
def test_protected_assessment_uses_frozen_threshold_and_configured_gates(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
command = MODULE.protected_assessment_command(
|
|
scripts_dir=tmp_path / "scripts",
|
|
calibration=tmp_path / "calibration.json",
|
|
test=tmp_path / "test.json",
|
|
background=tmp_path / "background.json",
|
|
output=tmp_path / "assessment.json",
|
|
selected_threshold=0.275,
|
|
min_aggregate_f1=0.71,
|
|
min_region_f1=0.62,
|
|
min_region_precision=0.73,
|
|
min_region_recall=0.58,
|
|
max_pure_empty_fp=1,
|
|
)
|
|
|
|
assert command[command.index("--selected-threshold") + 1] == "0.275"
|
|
assert command[command.index("--min-aggregate-f1") + 1] == "0.71"
|
|
assert command[command.index("--min-region-f1") + 1] == "0.62"
|
|
assert command[command.index("--min-region-precision") + 1] == "0.73"
|
|
assert command[command.index("--min-region-recall") + 1] == "0.58"
|
|
assert command[command.index("--max-pure-empty-fp") + 1] == "1"
|
|
|
|
|
|
def test_partial_iteration_resume_uses_exact_checkpoint_and_cuda(tmp_path: Path) -> None:
|
|
checkpoint = tmp_path / "runs" / "iteration-002" / "weights" / "last.pt"
|
|
assert MODULE.resumable_training_command("yolo", checkpoint) == [
|
|
"yolo", "train", f"resume={checkpoint}", "device=0"
|
|
]
|
|
|
|
|
|
def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -> None:
|
|
audit = tmp_path / "audit.json"
|
|
audit.write_text(json.dumps({
|
|
"status": "needs_human_review", "failures": [],
|
|
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
|
}))
|
|
quality = tmp_path / "quality.json"
|
|
quality.write_text(json.dumps({
|
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
|
}))
|
|
manifest = tmp_path / "manifest.json"
|
|
write_fixture_manifest(manifest)
|
|
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable, str(SCRIPT),
|
|
"--initial-model", str(tmp_path / "candidate.pt"),
|
|
"--train-yaml", str(train_yaml),
|
|
"--train-summary", str(tmp_path / "train-summary.json"),
|
|
"--dataset-audit", str(audit),
|
|
"--train-quality-audit", str(quality),
|
|
"--calibration-summary", str(tmp_path / "cal.json"),
|
|
"--test-summary", str(tmp_path / "test.json"),
|
|
"--background-summary", str(tmp_path / "background.json"),
|
|
"--corpus-manifest", str(manifest),
|
|
"--output-dir", str(tmp_path / "output"),
|
|
"--evaluate-initial-model", "--fixture-mode", "--dry-run",
|
|
], capture_output=True, text=True, check=False,
|
|
)
|
|
assert result.returncode == 0
|
|
assert json.loads(result.stdout) == {
|
|
"training_command": None, "evaluate_existing": True, "resume_partial": False
|
|
}
|
|
|
|
|
|
def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp_path: Path) -> None:
|
|
iteration_dir = tmp_path / "closed-loop" / "iteration-001"
|
|
iteration_dir.mkdir(parents=True, exist_ok=True)
|
|
assert iteration_dir.is_dir()
|
|
|
|
|
|
def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
|
audit = tmp_path / "audit.json"
|
|
audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4}))
|
|
quality = tmp_path / "quality.json"
|
|
quality.write_text(json.dumps({
|
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
|
}))
|
|
manifest = tmp_path / "manifest.json"
|
|
write_fixture_manifest(manifest)
|
|
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--initial-model",
|
|
str(tmp_path / "base.pt"),
|
|
"--train-yaml",
|
|
str(train_yaml),
|
|
"--train-summary",
|
|
str(tmp_path / "train-summary.json"),
|
|
"--dataset-audit",
|
|
str(audit),
|
|
"--train-quality-audit",
|
|
str(quality),
|
|
"--calibration-summary",
|
|
str(tmp_path / "cal.json"),
|
|
"--test-summary",
|
|
str(tmp_path / "test.json"),
|
|
"--background-summary",
|
|
str(tmp_path / "background.json"),
|
|
"--corpus-manifest",
|
|
str(manifest),
|
|
"--output-dir",
|
|
str(tmp_path / "output"),
|
|
"--fixture-mode",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
assert result.returncode != 0
|
|
assert "Dataset audit is not eligible for training" in result.stderr
|
|
|
|
|
|
def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None:
|
|
audit = tmp_path / "audit.json"
|
|
audit.write_text(json.dumps({
|
|
"status": "needs_human_review", "failures": [],
|
|
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
|
}))
|
|
quality = tmp_path / "quality.json"
|
|
quality.write_text(json.dumps({
|
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
|
}))
|
|
manifest = tmp_path / "manifest.json"
|
|
manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8")
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable, str(SCRIPT),
|
|
"--initial-model", str(tmp_path / "candidate.pt"),
|
|
"--train-yaml", str(tmp_path / "dataset.yaml"),
|
|
"--train-summary", str(tmp_path / "train-summary.json"),
|
|
"--dataset-audit", str(audit),
|
|
"--train-quality-audit", str(quality),
|
|
"--calibration-summary", str(tmp_path / "cal.json"),
|
|
"--test-summary", str(tmp_path / "test.json"),
|
|
"--background-summary", str(tmp_path / "background.json"),
|
|
"--corpus-manifest", str(manifest),
|
|
"--output-dir", str(tmp_path / "output"),
|
|
"--evaluate-initial-model", "--dry-run",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert "manifest_training_eligibility_missing" in result.stderr
|
|
|
|
|
|
def test_pending_human_review_blocks_operational_training() -> None:
|
|
audit = {
|
|
"status": "needs_human_review",
|
|
"failures": [],
|
|
"manifest_immutable": True,
|
|
"spatial_leakage_status": "ok",
|
|
"low_variance_positive_tile_count": 0,
|
|
"review_complete": False,
|
|
}
|
|
quality = {
|
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
|
}
|
|
failures = MODULE.dataset_audit_failures(audit, quality)
|
|
assert "unsupported audit status: needs_human_review" in failures
|
|
assert "review_complete_not_true" in failures
|
|
assert "accepted_human_review_evidence_missing" in failures
|
|
|
|
|
|
def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None:
|
|
audit = {
|
|
"status": "needs_human_review",
|
|
"failures": [],
|
|
"manifest_immutable": True,
|
|
"spatial_leakage_status": "ok",
|
|
"review_complete": False,
|
|
}
|
|
quality = {
|
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
|
}
|
|
assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == []
|
|
|
|
|
|
def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None:
|
|
bound = tmp_path / "bound-audit.json"
|
|
other = tmp_path / "other-audit.json"
|
|
bound.write_text("{}", encoding="utf-8")
|
|
other.write_text("{}", encoding="utf-8")
|
|
release = {"human_review": {"audit_path": str(bound.resolve())}}
|
|
|
|
MODULE.assert_dataset_audit_bound_to_release(
|
|
release=release,
|
|
dataset_audit=bound,
|
|
fixture_mode=False,
|
|
)
|
|
try:
|
|
MODULE.assert_dataset_audit_bound_to_release(
|
|
release=release,
|
|
dataset_audit=other,
|
|
fixture_mode=False,
|
|
)
|
|
except MODULE.TrainingReleaseError as exc:
|
|
assert "does not match" in str(exc)
|
|
else:
|
|
raise AssertionError("unbound dataset audit was accepted")
|
|
|
|
|
|
def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None:
|
|
assert MODULE.protected_feedback_roles(
|
|
{"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None}
|
|
) == ["test"]
|
|
assert MODULE.protected_feedback_roles(
|
|
{"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}}
|
|
) == ["background"]
|
|
|
|
|
|
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
|
|
audit = {
|
|
"status": "needs_human_review",
|
|
"failures": ["wallonia/test below minimum"],
|
|
"manifest_immutable": False,
|
|
"spatial_leakage_status": "failed",
|
|
"low_variance_positive_tile_count": 2,
|
|
}
|
|
quality = {
|
|
"status": "failed", "low_variance_positive_tile_count": 2,
|
|
"label_stats": {"invalid_label_count": 1, "missing_label_file_count": 1},
|
|
}
|
|
failures = MODULE.dataset_audit_failures(audit, quality)
|
|
assert "wallonia/test below minimum" in failures
|
|
assert "corpus manifest is not immutable" in failures
|
|
assert "spatial leakage audit is not ok" in failures
|
|
assert "dataset contains blank/low-variance positive tiles" in failures
|
|
assert "train tile quality audit is not ok" in failures
|
|
assert "train tile quality audit contains invalid labels" in failures
|
|
assert "train tile quality audit contains missing label files" in failures
|
|
|
|
|
|
def test_missing_tile_quality_evidence_fails_closed() -> None:
|
|
audit = {
|
|
"status": "needs_human_review", "failures": [],
|
|
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
|
}
|
|
failures = MODULE.dataset_audit_failures(audit, {})
|
|
assert "train tile quality audit is not ok" in failures
|
|
assert "train tile quality audit contains invalid labels" in failures
|
|
assert "train tile quality audit contains missing label files" in failures
|
|
assert "dataset contains blank/low-variance positive tiles" in failures
|
|
|
|
|
|
def test_calibration_failure_blocks_protected_evaluation() -> None:
|
|
chosen = {
|
|
"threshold": 0.1,
|
|
"aggregate": {"f1": 0.54},
|
|
"regions": {
|
|
"flanders": {"f1": 0.44, "precision": 0.49, "recall": 0.39},
|
|
"wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
|
|
},
|
|
"pure_empty_false_positives": 0,
|
|
}
|
|
failures = MODULE.calibration_failures(
|
|
chosen,
|
|
min_aggregate_f1=0.55,
|
|
min_region_f1=0.45,
|
|
min_region_precision=0.5,
|
|
min_region_recall=0.4,
|
|
max_pure_empty_fp=0,
|
|
)
|
|
assert failures == [
|
|
"calibration_aggregate_f1_below_gate",
|
|
"calibration_flanders_f1_below_gate",
|
|
"calibration_flanders_precision_below_gate",
|
|
"calibration_flanders_recall_below_gate",
|
|
]
|
|
|
|
|
|
def test_threshold_selection_uses_worst_region_then_aggregate() -> None:
|
|
report = {
|
|
"sweeps": [
|
|
{
|
|
"threshold": 0.1,
|
|
"aggregate": {"f1": 0.8},
|
|
"regions": {"a": {"f1": 0.4}, "b": {"f1": 0.7}},
|
|
"pure_empty_false_positives": 0,
|
|
},
|
|
{
|
|
"threshold": 0.2,
|
|
"aggregate": {"f1": 0.6},
|
|
"regions": {"a": {"f1": 0.5}, "b": {"f1": 0.5}},
|
|
"pure_empty_false_positives": 0,
|
|
},
|
|
]
|
|
}
|
|
assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2
|
|
|
|
|
|
def test_rejected_candidate_score_prioritizes_weakest_release_gate() -> None:
|
|
gates = {
|
|
"min_aggregate_f1": 0.55,
|
|
"min_region_f1": 0.45,
|
|
"min_region_precision": 0.5,
|
|
"min_region_recall": 0.4,
|
|
}
|
|
incumbent = {
|
|
"gates": gates,
|
|
"calibration": {
|
|
"aggregate": {"f1": 0.58},
|
|
"regions": {
|
|
"flanders": {"f1": 0.34, "precision": 0.38, "recall": 0.31},
|
|
"wallonia": {"f1": 0.60, "precision": 0.50, "recall": 0.75},
|
|
},
|
|
},
|
|
}
|
|
regressed = {
|
|
"gates": gates,
|
|
"calibration": {
|
|
"aggregate": {"f1": 0.60},
|
|
"regions": {
|
|
"flanders": {"f1": 0.31, "precision": 0.45, "recall": 0.24},
|
|
"wallonia": {"f1": 0.62, "precision": 0.52, "recall": 0.77},
|
|
},
|
|
},
|
|
}
|
|
|
|
assert MODULE.rejected_candidate_score(incumbent) > MODULE.rejected_candidate_score(regressed)
|