feat(provenance): govern source snapshots and data inputs
This commit is contained in:
Jens
@@ -0,0 +1,410 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "training_dataset_eligibility.py"
|
||||
SPEC = importlib.util.spec_from_file_location("training_dataset_eligibility", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
CHECKSUM = "a" * 64
|
||||
UNSET = object()
|
||||
|
||||
|
||||
def source_registry(
|
||||
*,
|
||||
classification: str = "authoritative",
|
||||
training_allowed: bool = True,
|
||||
ground_truth_allowed: bool = True,
|
||||
allowed_tasks: list[str] | None = None,
|
||||
building_validation_authority: str = "primary",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id="source-registry-1",
|
||||
source_key="governed-source",
|
||||
classification=classification,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
usage_policy_json={
|
||||
"training_allowed": training_allowed,
|
||||
"ground_truth_allowed": ground_truth_allowed,
|
||||
"allowed_tasks": allowed_tasks or ["building_validation", "building_labels"],
|
||||
"validation_authority": {
|
||||
"building_validation": building_validation_authority,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def source_snapshot(*, checksum: str = CHECKSUM) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id="source-snapshot-1",
|
||||
snapshot_key="2026-08-01",
|
||||
checksum_sha256=checksum,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
|
||||
|
||||
def governed_dataset(
|
||||
*,
|
||||
role: str,
|
||||
registry: SimpleNamespace | None | object = UNSET,
|
||||
snapshot: SimpleNamespace | None | object = UNSET,
|
||||
**overrides: object,
|
||||
) -> SimpleNamespace:
|
||||
dataset_type = "raster" if role == "raster" else "vector"
|
||||
values: dict[str, object] = {
|
||||
"id": f"dataset-{role}",
|
||||
"dataset_type": dataset_type,
|
||||
"dataset_role": "source" if role == "raster" else "reference",
|
||||
"source": "governed_import",
|
||||
"source_name": "governed-source",
|
||||
"checksum_sha256": CHECKSUM,
|
||||
"data_contract_key": f"{role}-contract",
|
||||
"data_contract_version": "1.0.0",
|
||||
"validation_status": "passed",
|
||||
"provenance_status": "complete",
|
||||
"lineage_status": "not_applicable",
|
||||
"quarantine_status": "not_quarantined",
|
||||
"status": "ready",
|
||||
"metadata_json": {},
|
||||
"provenance_metadata": {},
|
||||
"source_registry": source_registry() if registry is UNSET else registry,
|
||||
"source_snapshot": source_snapshot() if snapshot is UNSET else snapshot,
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def test_governed_authoritative_pair_is_eligible_for_operational_training() -> None:
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
registry=source_registry(ground_truth_allowed=False),
|
||||
)
|
||||
reference = governed_dataset(role="reference")
|
||||
|
||||
decision = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
|
||||
assert decision["eligible"] is True
|
||||
assert decision["raster"]["reasons"] == []
|
||||
assert decision["reference"]["evidence"]["source_ground_truth_allowed"] is True
|
||||
|
||||
|
||||
def test_operational_training_rejects_invalid_quarantined_incomplete_and_untrusted_inputs() -> None:
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
validation_status="failed",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
quarantine_status="quarantined",
|
||||
source_registry=source_registry(
|
||||
classification="contextual",
|
||||
training_allowed=False,
|
||||
ground_truth_allowed=False,
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {
|
||||
"validation_failed",
|
||||
"dataset_quarantined",
|
||||
"provenance_not_complete",
|
||||
"lineage_not_complete",
|
||||
"source_not_allowed_for_training",
|
||||
"reference_source_not_authoritative",
|
||||
"reference_source_not_ground_truth_allowed",
|
||||
}
|
||||
|
||||
|
||||
def test_operational_training_rejects_a_due_source_snapshot() -> None:
|
||||
snapshot = source_snapshot()
|
||||
snapshot.freshness_status = "due"
|
||||
dataset = governed_dataset(role="reference", snapshot=snapshot)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert "source_snapshot_freshness_not_approved" in decision.reasons
|
||||
|
||||
|
||||
def test_osm_like_context_is_never_accepted_as_building_ground_truth() -> None:
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
source_name="osm",
|
||||
source_registry=source_registry(
|
||||
classification="contextual",
|
||||
training_allowed=False,
|
||||
ground_truth_allowed=False,
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert "source_not_allowed_for_training" in decision.reasons
|
||||
assert "reference_source_not_authoritative" in decision.reasons
|
||||
|
||||
|
||||
def test_regional_building_sources_pending_primary_authority_cannot_enter_training_labels() -> None:
|
||||
for source_key in ("spw_picc", "urbis"):
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
source_name=source_key,
|
||||
source_registry=source_registry(
|
||||
allowed_tasks=["building_validation", "building_labels"],
|
||||
building_validation_authority="regional_primary_pending_contract",
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert "reference_building_validation_not_primary" in decision.reasons
|
||||
|
||||
|
||||
def test_authoritative_source_without_building_validation_task_cannot_be_used_as_a_label_reference() -> None:
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
source_registry=source_registry(
|
||||
allowed_tasks=["elevation_validation"],
|
||||
building_validation_authority="corroborative",
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {
|
||||
"reference_source_not_approved_for_building_validation",
|
||||
"reference_building_validation_not_primary",
|
||||
}
|
||||
|
||||
|
||||
def test_dataset_and_snapshot_registry_bindings_cannot_be_forged() -> None:
|
||||
snapshot = source_snapshot()
|
||||
snapshot.source_registry_id = "different-registry"
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
registry=source_registry(),
|
||||
snapshot=snapshot,
|
||||
source_registry_id="different-registry",
|
||||
source_snapshot_id="different-snapshot",
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {
|
||||
"dataset_source_registry_binding_mismatch",
|
||||
"dataset_source_snapshot_binding_mismatch",
|
||||
"source_snapshot_registry_mismatch",
|
||||
}
|
||||
|
||||
|
||||
def test_fixture_mode_only_relaxes_legacy_provenance_for_explicit_fixtures() -> None:
|
||||
fixture = governed_dataset(
|
||||
role="reference",
|
||||
source="fixture",
|
||||
source_name="fixture",
|
||||
validation_status=None,
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
data_contract_key=None,
|
||||
data_contract_version=None,
|
||||
checksum_sha256=None,
|
||||
source_registry=None,
|
||||
source_snapshot=None,
|
||||
metadata_json={"fixture": True},
|
||||
)
|
||||
unmarked = governed_dataset(
|
||||
role="reference",
|
||||
validation_status=None,
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
source_registry=None,
|
||||
source_snapshot=None,
|
||||
)
|
||||
|
||||
assert MODULE.evaluate_dataset_training_eligibility(
|
||||
fixture,
|
||||
role="reference",
|
||||
fixture_mode=True,
|
||||
).eligible is True
|
||||
rejected = MODULE.evaluate_dataset_training_eligibility(
|
||||
unmarked,
|
||||
role="reference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
assert rejected.eligible is False
|
||||
assert "fixture_mode_requires_explicit_fixture" in rejected.reasons
|
||||
|
||||
|
||||
def test_fixture_mode_never_allows_failed_validation_or_quarantine() -> None:
|
||||
fixture = governed_dataset(
|
||||
role="raster",
|
||||
source="fixture",
|
||||
source_name="fixture",
|
||||
validation_status="failed",
|
||||
quarantine_status="quarantined",
|
||||
source_registry=None,
|
||||
source_snapshot=None,
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(fixture, role="raster", fixture_mode=True)
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {"validation_failed", "dataset_quarantined"}
|
||||
|
||||
|
||||
def test_manifest_gate_rejects_missing_or_tampered_pair_decisions() -> None:
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
registry=source_registry(ground_truth_allowed=False),
|
||||
)
|
||||
reference = governed_dataset(role="reference")
|
||||
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
manifest = {
|
||||
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
||||
"training_eligibility": {
|
||||
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "eligible",
|
||||
"fixture_mode": False,
|
||||
},
|
||||
}
|
||||
|
||||
assert MODULE.manifest_training_eligibility_failures(manifest) == []
|
||||
tampered = {
|
||||
**manifest,
|
||||
"samples": [{"sample_slug": "governed", "training_eligibility": {**pair, "eligible": False}}],
|
||||
}
|
||||
failures = MODULE.manifest_training_eligibility_failures(tampered)
|
||||
assert "governed:training_pair_not_eligible" in failures
|
||||
assert MODULE.manifest_training_eligibility_failures({"samples": []}) == [
|
||||
"manifest_training_eligibility_missing"
|
||||
]
|
||||
|
||||
|
||||
def test_frozen_manifest_gate_detects_checksum_tampering(tmp_path: Path) -> None:
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
registry=source_registry(ground_truth_allowed=False),
|
||||
)
|
||||
reference = governed_dataset(role="reference")
|
||||
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
manifest_path = tmp_path / "operator_samples_manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"immutable": True,
|
||||
"training_eligibility": {
|
||||
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "eligible",
|
||||
"fixture_mode": False,
|
||||
},
|
||||
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "corpus-freeze.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(),
|
||||
"immutable": True,
|
||||
"training_eligibility_policy": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"fixture_mode": False,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert MODULE.frozen_manifest_training_eligibility_failures(manifest_path) == []
|
||||
manifest_path.write_text(manifest_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
||||
assert "corpus_manifest_checksum_mismatch" in MODULE.frozen_manifest_training_eligibility_failures(
|
||||
manifest_path
|
||||
)
|
||||
|
||||
|
||||
def test_live_manifest_gate_revokes_a_frozen_pair_when_an_upstream_dataset_is_quarantined() -> None:
|
||||
raster_id = uuid4()
|
||||
reference_id = uuid4()
|
||||
raster_registry = source_registry(ground_truth_allowed=False)
|
||||
reference_registry = source_registry()
|
||||
raster_snapshot = source_snapshot()
|
||||
reference_snapshot = source_snapshot()
|
||||
raster_snapshot.source_registry_id = raster_registry.id
|
||||
reference_snapshot.source_registry_id = reference_registry.id
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
id=raster_id,
|
||||
source_registry=raster_registry,
|
||||
source_snapshot=raster_snapshot,
|
||||
source_registry_id=raster_registry.id,
|
||||
source_snapshot_id=raster_snapshot.id,
|
||||
)
|
||||
reference = governed_dataset(
|
||||
role="reference",
|
||||
id=reference_id,
|
||||
source_registry=reference_registry,
|
||||
source_snapshot=reference_snapshot,
|
||||
source_registry_id=reference_registry.id,
|
||||
source_snapshot_id=reference_snapshot.id,
|
||||
)
|
||||
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
manifest = {
|
||||
"training_eligibility": {
|
||||
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "eligible",
|
||||
"fixture_mode": False,
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"sample_slug": "governed-aoi",
|
||||
"raster_dataset_id": str(raster_id),
|
||||
"reference_dataset_id": str(reference_id),
|
||||
"training_eligibility": pair,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
class DatasetModel:
|
||||
pass
|
||||
|
||||
class Session:
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
|
||||
@staticmethod
|
||||
def get(model, item_id):
|
||||
assert model is DatasetModel
|
||||
return {raster_id: raster, reference_id: reference}.get(item_id)
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
assert MODULE.live_manifest_training_eligibility_failures(
|
||||
manifest,
|
||||
session_factory=Session,
|
||||
dataset_model=DatasetModel,
|
||||
) == []
|
||||
|
||||
raster.quarantine_status = "quarantined"
|
||||
failures = MODULE.live_manifest_training_eligibility_failures(
|
||||
manifest,
|
||||
session_factory=Session,
|
||||
dataset_model=DatasetModel,
|
||||
)
|
||||
|
||||
assert "governed-aoi:raster_live_revoked:dataset_quarantined" in failures
|
||||
Reference in New Issue
Block a user