Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_belgium_building_training_loop.py
+172 -5
View File
@@ -13,6 +13,21 @@ from datetime import UTC, datetime
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_training_release_eligible,
human_review_audit_failures,
training_release_paths,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -32,11 +47,14 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
def dataset_audit_failures(
audit: dict[str, Any],
train_quality_audit: dict[str, Any],
*,
fixture_mode: bool = False,
) -> list[str]:
"""Return automated corpus blockers while leaving final human review deferred."""
"""Return fail-closed corpus blockers, including human review in normal mode."""
failures = [str(item) for item in audit.get("failures") or []]
status = audit.get("status")
if status not in {"ok", "needs_human_review"}:
permitted_statuses = {"ok", "needs_human_review"} if fixture_mode else {"ok"}
if status not in permitted_statuses:
failures.append(f"unsupported audit status: {status}")
if audit.get("manifest_immutable") is not True:
failures.append("corpus manifest is not immutable")
@@ -50,9 +68,56 @@ def dataset_audit_failures(
failures.append("train tile quality audit contains missing label files")
if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0:
failures.append("dataset contains blank/low-variance positive tiles")
if not fixture_mode:
failures.extend(human_review_audit_failures(audit))
return failures
def verify_training_inputs(
*,
train_yaml: Path,
corpus_manifest: Path,
fixture_mode: bool,
) -> dict[str, Any]:
"""Re-check every immutable input before initial, retry or resume training."""
try:
assert_frozen_manifest_training_eligible(
corpus_manifest,
fixture_mode=fixture_mode,
verify_live=True,
)
return assert_training_release_eligible(
train_yaml=train_yaml,
corpus_manifest=corpus_manifest,
fixture_mode=fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise TrainingReleaseError(str(exc)) from exc
def assert_dataset_audit_bound_to_release(
*,
release: dict[str, Any],
dataset_audit: Path,
fixture_mode: bool,
) -> None:
"""Do not let a caller swap the reviewed corpus audit after release sealing."""
if fixture_mode:
return
review = release.get("human_review")
if not isinstance(review, dict):
raise TrainingReleaseError("Training release has no human-review audit binding")
review_audit_path = review.get("audit_path")
if not isinstance(review_audit_path, str) or not review_audit_path:
raise TrainingReleaseError("Training release human-review audit path is missing")
if Path(review_audit_path).resolve(strict=False) != dataset_audit.resolve(strict=False):
raise TrainingReleaseError(
"--dataset-audit does not match the immutable training-release human-review audit"
)
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
"""Choose a threshold without consulting test or background evidence."""
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
@@ -111,6 +176,16 @@ def rejected_candidate_score(assessment: dict[str, Any]) -> tuple[float, ...]:
return tuple(normalized)
def protected_feedback_roles(assessment: dict[str, Any]) -> list[str]:
"""Return protected evidence roles that make iterative retraining illegal."""
return [
role
for role in ("test", "background")
if assessment.get(role) is not None
]
def training_command(
yolo: str,
*,
@@ -183,17 +258,23 @@ def failure_sampling_command(
corpus_manifest: Path,
assessment: Path,
output_dir: Path,
review_audit: Path,
sampling_round: int = 0,
fixture_mode: bool = False,
) -> list[str]:
return [
command = [
sys.executable,
str(scripts_dir / "build_failure_driven_yolo_sampling.py"),
"--summary", str(train_summary),
"--corpus-manifest", str(corpus_manifest),
"--assessment", str(assessment),
"--output-dir", str(output_dir),
"--review-audit", str(review_audit),
"--sampling-round", str(sampling_round),
]
if fixture_mode:
command.append("--fixture-mode")
return command
def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]:
@@ -259,6 +340,14 @@ def main() -> int:
parser.add_argument("--min-region-recall", type=float, default=0.4)
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
parser.add_argument("--dry-run", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help=(
"Accept an explicitly fixture-only corpus manifest. "
"This mode is prohibited for operational training."
),
)
parser.add_argument(
"--evaluate-initial-model",
action="store_true",
@@ -267,9 +356,29 @@ def main() -> int:
args = parser.parse_args()
if args.iterations < 1:
raise SystemExit("--iterations must be positive")
try:
initial_release = verify_training_inputs(
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
try:
assert_dataset_audit_bound_to_release(
release=initial_release,
dataset_audit=args.dataset_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8"))
audit_failures = dataset_audit_failures(dataset_audit, train_quality_audit)
audit_failures = dataset_audit_failures(
dataset_audit,
train_quality_audit,
fixture_mode=args.fixture_mode,
)
if audit_failures:
raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
@@ -285,6 +394,13 @@ def main() -> int:
"train_quality_audit": str(args.train_quality_audit),
"train_quality_audit_sha256": sha256(args.train_quality_audit),
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"initial_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"initial_training_release_sha256": sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"initial_training_release_contract": initial_release["contract_version"],
"fixture_mode": bool(args.fixture_mode),
"iterations": [],
}
if state_path.is_file():
@@ -304,6 +420,26 @@ def main() -> int:
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
partial_checkpoint = train_run / "weights" / "last.pt"
resume_partial = not evaluate_existing and partial_checkpoint.is_file()
try:
release = verify_training_inputs(
train_yaml=train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Training inputs changed before {name}; refusing initial/retry/resume execution: {exc}"
) from exc
try:
assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=args.dataset_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Training audit changed before {name}; refusing initial/retry/resume execution: {exc}"
) from exc
command = None if evaluate_existing else (
resumable_training_command(args.yolo, partial_checkpoint)
if resume_partial else training_command(
@@ -454,11 +590,27 @@ def main() -> int:
"candidate_sha256": sha256(candidate),
"training_skipped_for_existing_checkpoint": evaluate_existing,
"training_resumed_from_partial_checkpoint": resume_partial,
"training_release": str(training_release_paths(train_yaml)["release_manifest"]),
"training_release_sha256": sha256(training_release_paths(train_yaml)["release_manifest"]),
"training_release_asset_manifest_sha256": release["asset_manifest"]["sha256"],
"assessment": str(assessment),
"status": decision["status"],
"failures": decision["failures"],
}
state["iterations"].append(record)
protected_feedback = protected_feedback_roles(decision)
if decision["status"] != "training_complete" and protected_feedback:
record["protected_feedback_blocked"] = protected_feedback
record["retraining_prohibited"] = True
state["status"] = "protected_evaluation_rejected"
state["stopped_at"] = datetime.now(UTC).isoformat()
state["stop_reason"] = (
"Protected test/background evidence was opened for a rejected candidate; "
"its results cannot generate another training YAML."
)
write_json(state_path, state)
print(json.dumps(state, indent=2))
return 3
score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else ()
incumbent_score = tuple(state.get("incumbent_rejected_score", ()))
if not incumbent_score or score > incumbent_score:
@@ -482,14 +634,29 @@ def main() -> int:
corpus_manifest=args.corpus_manifest,
assessment=assessment,
output_dir=sampling_dir,
review_audit=args.dataset_audit,
sampling_round=index,
fixture_mode=args.fixture_mode,
),
iteration_dir / "failure-driven-sampling.log",
)
sampling_evidence = sampling_dir / "failure-driven-sampling.json"
next_train_yaml = sampling_dir / "dataset.yaml"
if not sampling_evidence.is_file() or not next_train_yaml.is_file():
next_release_paths = training_release_paths(next_train_yaml)
if not sampling_evidence.is_file() or not next_train_yaml.is_file() or not all(
path.is_file() for path in next_release_paths.values()
):
raise RuntimeError("Failure-driven sampling produced incomplete evidence")
try:
verify_training_inputs(
train_yaml=next_train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Failure-driven sampling produced an unbound training release: {exc}"
) from exc
record["failure_driven_sampling"] = str(sampling_evidence)
record["failure_driven_sampling_sha256"] = sha256(sampling_evidence)
record["next_train_yaml"] = str(next_train_yaml)