from __future__ import annotations import importlib.util import hashlib import json from pathlib import Path from types import SimpleNamespace from uuid import uuid4 ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "training_dataset_eligibility.py" SPEC = importlib.util.spec_from_file_location("training_dataset_eligibility", SCRIPT) assert SPEC and SPEC.loader MODULE = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(MODULE) CHECKSUM = "a" * 64 UNSET = object() def source_registry( *, classification: str = "authoritative", training_allowed: bool = True, ground_truth_allowed: bool = True, allowed_tasks: list[str] | None = None, building_validation_authority: str = "primary", ) -> SimpleNamespace: return SimpleNamespace( id="source-registry-1", source_key="governed-source", classification=classification, freshness_status="current", ingest_status="ingested", usage_policy_json={ "training_allowed": training_allowed, "ground_truth_allowed": ground_truth_allowed, "allowed_tasks": allowed_tasks or ["building_validation", "building_labels"], "validation_authority": { "building_validation": building_validation_authority, }, }, ) def source_snapshot(*, checksum: str = CHECKSUM) -> SimpleNamespace: return SimpleNamespace( id="source-snapshot-1", snapshot_key="2026-08-01", checksum_sha256=checksum, freshness_status="current", ingest_status="ingested", ) def governed_dataset( *, role: str, registry: SimpleNamespace | None | object = UNSET, snapshot: SimpleNamespace | None | object = UNSET, **overrides: object, ) -> SimpleNamespace: dataset_type = "raster" if role == "raster" else "vector" values: dict[str, object] = { "id": f"dataset-{role}", "dataset_type": dataset_type, "dataset_role": "source" if role == "raster" else "reference", "source": "governed_import", "source_name": "governed-source", "checksum_sha256": CHECKSUM, "data_contract_key": f"{role}-contract", "data_contract_version": "1.0.0", "validation_status": "passed", "provenance_status": "complete", "lineage_status": "not_applicable", "quarantine_status": "not_quarantined", "status": "ready", "metadata_json": {}, "provenance_metadata": {}, "source_registry": source_registry() if registry is UNSET else registry, "source_snapshot": source_snapshot() if snapshot is UNSET else snapshot, } values.update(overrides) return SimpleNamespace(**values) def test_governed_authoritative_pair_is_eligible_for_operational_training() -> None: raster = governed_dataset( role="raster", registry=source_registry(ground_truth_allowed=False), ) reference = governed_dataset(role="reference") decision = MODULE.training_pair_evidence(raster=raster, reference=reference) assert decision["eligible"] is True assert decision["raster"]["reasons"] == [] assert decision["reference"]["evidence"]["source_ground_truth_allowed"] is True def test_operational_training_rejects_invalid_quarantined_incomplete_and_untrusted_inputs() -> None: dataset = governed_dataset( role="reference", validation_status="failed", provenance_status="incomplete", lineage_status="incomplete", quarantine_status="quarantined", source_registry=source_registry( classification="contextual", training_allowed=False, ground_truth_allowed=False, ), ) decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") assert decision.eligible is False assert set(decision.reasons) >= { "validation_failed", "dataset_quarantined", "provenance_not_complete", "lineage_not_complete", "source_not_allowed_for_training", "reference_source_not_authoritative", "reference_source_not_ground_truth_allowed", } def test_operational_training_rejects_a_due_source_snapshot() -> None: snapshot = source_snapshot() snapshot.freshness_status = "due" dataset = governed_dataset(role="reference", snapshot=snapshot) decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") assert decision.eligible is False assert "source_snapshot_freshness_not_approved" in decision.reasons def test_osm_like_context_is_never_accepted_as_building_ground_truth() -> None: dataset = governed_dataset( role="reference", source_name="osm", source_registry=source_registry( classification="contextual", training_allowed=False, ground_truth_allowed=False, ), ) decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") assert decision.eligible is False assert "source_not_allowed_for_training" in decision.reasons assert "reference_source_not_authoritative" in decision.reasons def test_regional_building_sources_pending_primary_authority_cannot_enter_training_labels() -> None: for source_key in ("spw_picc", "urbis"): dataset = governed_dataset( role="reference", source_name=source_key, source_registry=source_registry( allowed_tasks=["building_validation", "building_labels"], building_validation_authority="regional_primary_pending_contract", ), ) decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") assert decision.eligible is False assert "reference_building_validation_not_primary" in decision.reasons def test_authoritative_source_without_building_validation_task_cannot_be_used_as_a_label_reference() -> None: dataset = governed_dataset( role="reference", source_registry=source_registry( allowed_tasks=["elevation_validation"], building_validation_authority="corroborative", ), ) decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") assert decision.eligible is False assert set(decision.reasons) >= { "reference_source_not_approved_for_building_validation", "reference_building_validation_not_primary", } def test_dataset_and_snapshot_registry_bindings_cannot_be_forged() -> None: snapshot = source_snapshot() snapshot.source_registry_id = "different-registry" dataset = governed_dataset( role="reference", registry=source_registry(), snapshot=snapshot, source_registry_id="different-registry", source_snapshot_id="different-snapshot", ) decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") assert decision.eligible is False assert set(decision.reasons) >= { "dataset_source_registry_binding_mismatch", "dataset_source_snapshot_binding_mismatch", "source_snapshot_registry_mismatch", } def test_fixture_mode_only_relaxes_legacy_provenance_for_explicit_fixtures() -> None: fixture = governed_dataset( role="reference", source="fixture", source_name="fixture", validation_status=None, provenance_status="incomplete", lineage_status="incomplete", data_contract_key=None, data_contract_version=None, checksum_sha256=None, source_registry=None, source_snapshot=None, metadata_json={"fixture": True}, ) unmarked = governed_dataset( role="reference", validation_status=None, provenance_status="incomplete", lineage_status="incomplete", source_registry=None, source_snapshot=None, ) assert MODULE.evaluate_dataset_training_eligibility( fixture, role="reference", fixture_mode=True, ).eligible is True rejected = MODULE.evaluate_dataset_training_eligibility( unmarked, role="reference", fixture_mode=True, ) assert rejected.eligible is False assert "fixture_mode_requires_explicit_fixture" in rejected.reasons def test_fixture_mode_never_allows_failed_validation_or_quarantine() -> None: fixture = governed_dataset( role="raster", source="fixture", source_name="fixture", validation_status="failed", quarantine_status="quarantined", source_registry=None, source_snapshot=None, ) decision = MODULE.evaluate_dataset_training_eligibility(fixture, role="raster", fixture_mode=True) assert decision.eligible is False assert set(decision.reasons) >= {"validation_failed", "dataset_quarantined"} def test_manifest_gate_rejects_missing_or_tampered_pair_decisions() -> None: raster = governed_dataset( role="raster", registry=source_registry(ground_truth_allowed=False), ) reference = governed_dataset(role="reference") pair = MODULE.training_pair_evidence(raster=raster, reference=reference) manifest = { "samples": [{"sample_slug": "governed", "training_eligibility": pair}], "training_eligibility": { "policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, "status": "eligible", "fixture_mode": False, }, } assert MODULE.manifest_training_eligibility_failures(manifest) == [] tampered = { **manifest, "samples": [{"sample_slug": "governed", "training_eligibility": {**pair, "eligible": False}}], } failures = MODULE.manifest_training_eligibility_failures(tampered) assert "governed:training_pair_not_eligible" in failures assert MODULE.manifest_training_eligibility_failures({"samples": []}) == [ "manifest_training_eligibility_missing" ] def test_frozen_manifest_gate_detects_checksum_tampering(tmp_path: Path) -> None: raster = governed_dataset( role="raster", registry=source_registry(ground_truth_allowed=False), ) reference = governed_dataset(role="reference") pair = MODULE.training_pair_evidence(raster=raster, reference=reference) manifest_path = tmp_path / "operator_samples_manifest.json" manifest_path.write_text( json.dumps( { "immutable": True, "training_eligibility": { "policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, "status": "eligible", "fixture_mode": False, }, "samples": [{"sample_slug": "governed", "training_eligibility": pair}], } ), encoding="utf-8", ) (tmp_path / "corpus-freeze.json").write_text( json.dumps( { "schema_version": 2, "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), "immutable": True, "training_eligibility_policy": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, "fixture_mode": False, } ), encoding="utf-8", ) assert MODULE.frozen_manifest_training_eligibility_failures(manifest_path) == [] manifest_path.write_text(manifest_path.read_text(encoding="utf-8") + "\n", encoding="utf-8") assert "corpus_manifest_checksum_mismatch" in MODULE.frozen_manifest_training_eligibility_failures( manifest_path ) def test_live_manifest_gate_revokes_a_frozen_pair_when_an_upstream_dataset_is_quarantined() -> None: raster_id = uuid4() reference_id = uuid4() raster_registry = source_registry(ground_truth_allowed=False) reference_registry = source_registry() raster_snapshot = source_snapshot() reference_snapshot = source_snapshot() raster_snapshot.source_registry_id = raster_registry.id reference_snapshot.source_registry_id = reference_registry.id raster = governed_dataset( role="raster", id=raster_id, source_registry=raster_registry, source_snapshot=raster_snapshot, source_registry_id=raster_registry.id, source_snapshot_id=raster_snapshot.id, ) reference = governed_dataset( role="reference", id=reference_id, source_registry=reference_registry, source_snapshot=reference_snapshot, source_registry_id=reference_registry.id, source_snapshot_id=reference_snapshot.id, ) pair = MODULE.training_pair_evidence(raster=raster, reference=reference) manifest = { "training_eligibility": { "policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, "status": "eligible", "fixture_mode": False, }, "samples": [ { "sample_slug": "governed-aoi", "raster_dataset_id": str(raster_id), "reference_dataset_id": str(reference_id), "training_eligibility": pair, } ], } class DatasetModel: pass class Session: def __init__(self) -> None: self.closed = False @staticmethod def get(model, item_id): assert model is DatasetModel return {raster_id: raster, reference_id: reference}.get(item_id) def close(self) -> None: self.closed = True assert MODULE.live_manifest_training_eligibility_failures( manifest, session_factory=Session, dataset_model=DatasetModel, ) == [] raster.quarantine_status = "quarantined" failures = MODULE.live_manifest_training_eligibility_failures( manifest, session_factory=Session, dataset_model=DatasetModel, ) assert "governed-aoi:raster_live_revoked:dataset_quarantined" in failures