Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_belgium_training_loop.py
+193 -7
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import importlib.util
import hashlib
import json
import subprocess
import sys
@@ -13,6 +14,94 @@ assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
from training_release_manifest import create_training_release_manifest # noqa: E402
def write_fixture_manifest(path: Path) -> None:
policy = "geointel-training-source-eligibility/v1"
def eligible(sample_slug: str) -> dict[str, object]:
return {
"policy_version": policy,
"eligible": True,
"fixture_mode": True,
"raster": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": f"raster:{sample_slug}",
"checksum_sha256": "a" * 64,
"source_registry_id": "fixture-raster",
"source_snapshot_id": "fixture-raster-snapshot",
},
},
"reference": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": f"reference:{sample_slug}",
"checksum_sha256": "b" * 64,
"source_registry_id": "fixture-reference",
"source_snapshot_id": "fixture-reference-snapshot",
},
},
}
path.write_text(
json.dumps(
{
"training_eligibility": {
"policy_version": policy,
"status": "eligible",
"fixture_mode": True,
},
"samples": [
{
"sample_slug": sample_slug,
"split": split,
"raster_dataset_id": f"raster:{sample_slug}",
"reference_dataset_id": f"reference:{sample_slug}",
"training_eligibility": eligible(sample_slug),
}
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val"))
],
}
),
encoding="utf-8",
)
(path.parent / "corpus-freeze.json").write_text(
json.dumps(
{
"schema_version": 2,
"manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"immutable": True,
"training_eligibility_policy": policy,
"fixture_mode": True,
}
),
encoding="utf-8",
)
def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path:
dataset_dir = tmp_path / "fixture-dataset"
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
image = dataset_dir / "images" / split / f"{sample_slug}.png"
label = dataset_dir / "labels" / split / f"{sample_slug}.txt"
image.parent.mkdir(parents=True, exist_ok=True)
label.parent.mkdir(parents=True, exist_ok=True)
image.write_bytes(split.encode("utf-8"))
label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
yaml_path = dataset_dir / "dataset.yaml"
yaml_path.write_text(
f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=manifest,
fixture_mode=True,
)
return yaml_path
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
command = MODULE.training_command(
@@ -78,6 +167,7 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat
corpus_manifest=tmp_path / "manifest.json",
assessment=tmp_path / "assessment.json",
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
review_audit=tmp_path / "review-audit.json",
)
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
assert command[command.index("--summary") + 1].endswith("train-summary.json")
@@ -103,20 +193,23 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
write_fixture_manifest(manifest)
train_yaml = write_fixture_training_release(tmp_path, manifest)
result = subprocess.run(
[
sys.executable, str(SCRIPT),
"--initial-model", str(tmp_path / "candidate.pt"),
"--train-yaml", str(tmp_path / "dataset.yaml"),
"--train-yaml", str(train_yaml),
"--train-summary", str(tmp_path / "train-summary.json"),
"--dataset-audit", str(audit),
"--train-quality-audit", str(quality),
"--calibration-summary", str(tmp_path / "cal.json"),
"--test-summary", str(tmp_path / "test.json"),
"--background-summary", str(tmp_path / "background.json"),
"--corpus-manifest", str(tmp_path / "manifest.json"),
"--corpus-manifest", str(manifest),
"--output-dir", str(tmp_path / "output"),
"--evaluate-initial-model", "--dry-run",
"--evaluate-initial-model", "--fixture-mode", "--dry-run",
], capture_output=True, text=True, check=False,
)
assert result.returncode == 0
@@ -139,6 +232,9 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
write_fixture_manifest(manifest)
train_yaml = write_fixture_training_release(tmp_path, manifest)
result = subprocess.run(
[
sys.executable,
@@ -146,7 +242,7 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
"--initial-model",
str(tmp_path / "base.pt"),
"--train-yaml",
str(tmp_path / "dataset.yaml"),
str(train_yaml),
"--train-summary",
str(tmp_path / "train-summary.json"),
"--dataset-audit",
@@ -160,9 +256,10 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
"--background-summary",
str(tmp_path / "background.json"),
"--corpus-manifest",
str(tmp_path / "manifest.json"),
str(manifest),
"--output-dir",
str(tmp_path / "output"),
"--fixture-mode",
],
capture_output=True,
text=True,
@@ -172,7 +269,45 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
assert "Dataset audit is not eligible for training" in result.stderr
def test_pending_human_review_does_not_block_objective_training() -> None:
def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None:
audit = tmp_path / "audit.json"
audit.write_text(json.dumps({
"status": "needs_human_review", "failures": [],
"manifest_immutable": True, "spatial_leakage_status": "ok",
}))
quality = tmp_path / "quality.json"
quality.write_text(json.dumps({
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8")
result = subprocess.run(
[
sys.executable, str(SCRIPT),
"--initial-model", str(tmp_path / "candidate.pt"),
"--train-yaml", str(tmp_path / "dataset.yaml"),
"--train-summary", str(tmp_path / "train-summary.json"),
"--dataset-audit", str(audit),
"--train-quality-audit", str(quality),
"--calibration-summary", str(tmp_path / "cal.json"),
"--test-summary", str(tmp_path / "test.json"),
"--background-summary", str(tmp_path / "background.json"),
"--corpus-manifest", str(manifest),
"--output-dir", str(tmp_path / "output"),
"--evaluate-initial-model", "--dry-run",
],
capture_output=True,
text=True,
check=False,
)
assert result.returncode != 0
assert "manifest_training_eligibility_missing" in result.stderr
def test_pending_human_review_blocks_operational_training() -> None:
audit = {
"status": "needs_human_review",
"failures": [],
@@ -185,7 +320,58 @@ def test_pending_human_review_does_not_block_objective_training() -> None:
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}
assert MODULE.dataset_audit_failures(audit, quality) == []
failures = MODULE.dataset_audit_failures(audit, quality)
assert "unsupported audit status: needs_human_review" in failures
assert "review_complete_not_true" in failures
assert "accepted_human_review_evidence_missing" in failures
def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None:
audit = {
"status": "needs_human_review",
"failures": [],
"manifest_immutable": True,
"spatial_leakage_status": "ok",
"review_complete": False,
}
quality = {
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}
assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == []
def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None:
bound = tmp_path / "bound-audit.json"
other = tmp_path / "other-audit.json"
bound.write_text("{}", encoding="utf-8")
other.write_text("{}", encoding="utf-8")
release = {"human_review": {"audit_path": str(bound.resolve())}}
MODULE.assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=bound,
fixture_mode=False,
)
try:
MODULE.assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=other,
fixture_mode=False,
)
except MODULE.TrainingReleaseError as exc:
assert "does not match" in str(exc)
else:
raise AssertionError("unbound dataset audit was accepted")
def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None:
assert MODULE.protected_feedback_roles(
{"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None}
) == ["test"]
assert MODULE.protected_feedback_roles(
{"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}}
) == ["background"]
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None: