Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
411 lines
14 KiB
Python
411 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from uuid import uuid4
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "training_dataset_eligibility.py"
|
|
SPEC = importlib.util.spec_from_file_location("training_dataset_eligibility", SCRIPT)
|
|
assert SPEC and SPEC.loader
|
|
MODULE = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(MODULE)
|
|
|
|
CHECKSUM = "a" * 64
|
|
UNSET = object()
|
|
|
|
|
|
def source_registry(
|
|
*,
|
|
classification: str = "authoritative",
|
|
training_allowed: bool = True,
|
|
ground_truth_allowed: bool = True,
|
|
allowed_tasks: list[str] | None = None,
|
|
building_validation_authority: str = "primary",
|
|
) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
id="source-registry-1",
|
|
source_key="governed-source",
|
|
classification=classification,
|
|
freshness_status="current",
|
|
ingest_status="ingested",
|
|
usage_policy_json={
|
|
"training_allowed": training_allowed,
|
|
"ground_truth_allowed": ground_truth_allowed,
|
|
"allowed_tasks": allowed_tasks or ["building_validation", "building_labels"],
|
|
"validation_authority": {
|
|
"building_validation": building_validation_authority,
|
|
},
|
|
},
|
|
)
|
|
|
|
|
|
def source_snapshot(*, checksum: str = CHECKSUM) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
id="source-snapshot-1",
|
|
snapshot_key="2026-08-01",
|
|
checksum_sha256=checksum,
|
|
freshness_status="current",
|
|
ingest_status="ingested",
|
|
)
|
|
|
|
|
|
def governed_dataset(
|
|
*,
|
|
role: str,
|
|
registry: SimpleNamespace | None | object = UNSET,
|
|
snapshot: SimpleNamespace | None | object = UNSET,
|
|
**overrides: object,
|
|
) -> SimpleNamespace:
|
|
dataset_type = "raster" if role == "raster" else "vector"
|
|
values: dict[str, object] = {
|
|
"id": f"dataset-{role}",
|
|
"dataset_type": dataset_type,
|
|
"dataset_role": "source" if role == "raster" else "reference",
|
|
"source": "governed_import",
|
|
"source_name": "governed-source",
|
|
"checksum_sha256": CHECKSUM,
|
|
"data_contract_key": f"{role}-contract",
|
|
"data_contract_version": "1.0.0",
|
|
"validation_status": "passed",
|
|
"provenance_status": "complete",
|
|
"lineage_status": "not_applicable",
|
|
"quarantine_status": "not_quarantined",
|
|
"status": "ready",
|
|
"metadata_json": {},
|
|
"provenance_metadata": {},
|
|
"source_registry": source_registry() if registry is UNSET else registry,
|
|
"source_snapshot": source_snapshot() if snapshot is UNSET else snapshot,
|
|
}
|
|
values.update(overrides)
|
|
return SimpleNamespace(**values)
|
|
|
|
|
|
def test_governed_authoritative_pair_is_eligible_for_operational_training() -> None:
|
|
raster = governed_dataset(
|
|
role="raster",
|
|
registry=source_registry(ground_truth_allowed=False),
|
|
)
|
|
reference = governed_dataset(role="reference")
|
|
|
|
decision = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
|
|
|
assert decision["eligible"] is True
|
|
assert decision["raster"]["reasons"] == []
|
|
assert decision["reference"]["evidence"]["source_ground_truth_allowed"] is True
|
|
|
|
|
|
def test_operational_training_rejects_invalid_quarantined_incomplete_and_untrusted_inputs() -> None:
|
|
dataset = governed_dataset(
|
|
role="reference",
|
|
validation_status="failed",
|
|
provenance_status="incomplete",
|
|
lineage_status="incomplete",
|
|
quarantine_status="quarantined",
|
|
source_registry=source_registry(
|
|
classification="contextual",
|
|
training_allowed=False,
|
|
ground_truth_allowed=False,
|
|
),
|
|
)
|
|
|
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
|
|
|
assert decision.eligible is False
|
|
assert set(decision.reasons) >= {
|
|
"validation_failed",
|
|
"dataset_quarantined",
|
|
"provenance_not_complete",
|
|
"lineage_not_complete",
|
|
"source_not_allowed_for_training",
|
|
"reference_source_not_authoritative",
|
|
"reference_source_not_ground_truth_allowed",
|
|
}
|
|
|
|
|
|
def test_operational_training_rejects_a_due_source_snapshot() -> None:
|
|
snapshot = source_snapshot()
|
|
snapshot.freshness_status = "due"
|
|
dataset = governed_dataset(role="reference", snapshot=snapshot)
|
|
|
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
|
|
|
assert decision.eligible is False
|
|
assert "source_snapshot_freshness_not_approved" in decision.reasons
|
|
|
|
|
|
def test_osm_like_context_is_never_accepted_as_building_ground_truth() -> None:
|
|
dataset = governed_dataset(
|
|
role="reference",
|
|
source_name="osm",
|
|
source_registry=source_registry(
|
|
classification="contextual",
|
|
training_allowed=False,
|
|
ground_truth_allowed=False,
|
|
),
|
|
)
|
|
|
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
|
|
|
assert decision.eligible is False
|
|
assert "source_not_allowed_for_training" in decision.reasons
|
|
assert "reference_source_not_authoritative" in decision.reasons
|
|
|
|
|
|
def test_regional_building_sources_pending_primary_authority_cannot_enter_training_labels() -> None:
|
|
for source_key in ("spw_picc", "urbis"):
|
|
dataset = governed_dataset(
|
|
role="reference",
|
|
source_name=source_key,
|
|
source_registry=source_registry(
|
|
allowed_tasks=["building_validation", "building_labels"],
|
|
building_validation_authority="regional_primary_pending_contract",
|
|
),
|
|
)
|
|
|
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
|
|
|
assert decision.eligible is False
|
|
assert "reference_building_validation_not_primary" in decision.reasons
|
|
|
|
|
|
def test_authoritative_source_without_building_validation_task_cannot_be_used_as_a_label_reference() -> None:
|
|
dataset = governed_dataset(
|
|
role="reference",
|
|
source_registry=source_registry(
|
|
allowed_tasks=["elevation_validation"],
|
|
building_validation_authority="corroborative",
|
|
),
|
|
)
|
|
|
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
|
|
|
assert decision.eligible is False
|
|
assert set(decision.reasons) >= {
|
|
"reference_source_not_approved_for_building_validation",
|
|
"reference_building_validation_not_primary",
|
|
}
|
|
|
|
|
|
def test_dataset_and_snapshot_registry_bindings_cannot_be_forged() -> None:
|
|
snapshot = source_snapshot()
|
|
snapshot.source_registry_id = "different-registry"
|
|
dataset = governed_dataset(
|
|
role="reference",
|
|
registry=source_registry(),
|
|
snapshot=snapshot,
|
|
source_registry_id="different-registry",
|
|
source_snapshot_id="different-snapshot",
|
|
)
|
|
|
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
|
|
|
assert decision.eligible is False
|
|
assert set(decision.reasons) >= {
|
|
"dataset_source_registry_binding_mismatch",
|
|
"dataset_source_snapshot_binding_mismatch",
|
|
"source_snapshot_registry_mismatch",
|
|
}
|
|
|
|
|
|
def test_fixture_mode_only_relaxes_legacy_provenance_for_explicit_fixtures() -> None:
|
|
fixture = governed_dataset(
|
|
role="reference",
|
|
source="fixture",
|
|
source_name="fixture",
|
|
validation_status=None,
|
|
provenance_status="incomplete",
|
|
lineage_status="incomplete",
|
|
data_contract_key=None,
|
|
data_contract_version=None,
|
|
checksum_sha256=None,
|
|
source_registry=None,
|
|
source_snapshot=None,
|
|
metadata_json={"fixture": True},
|
|
)
|
|
unmarked = governed_dataset(
|
|
role="reference",
|
|
validation_status=None,
|
|
provenance_status="incomplete",
|
|
lineage_status="incomplete",
|
|
source_registry=None,
|
|
source_snapshot=None,
|
|
)
|
|
|
|
assert MODULE.evaluate_dataset_training_eligibility(
|
|
fixture,
|
|
role="reference",
|
|
fixture_mode=True,
|
|
).eligible is True
|
|
rejected = MODULE.evaluate_dataset_training_eligibility(
|
|
unmarked,
|
|
role="reference",
|
|
fixture_mode=True,
|
|
)
|
|
assert rejected.eligible is False
|
|
assert "fixture_mode_requires_explicit_fixture" in rejected.reasons
|
|
|
|
|
|
def test_fixture_mode_never_allows_failed_validation_or_quarantine() -> None:
|
|
fixture = governed_dataset(
|
|
role="raster",
|
|
source="fixture",
|
|
source_name="fixture",
|
|
validation_status="failed",
|
|
quarantine_status="quarantined",
|
|
source_registry=None,
|
|
source_snapshot=None,
|
|
)
|
|
|
|
decision = MODULE.evaluate_dataset_training_eligibility(fixture, role="raster", fixture_mode=True)
|
|
|
|
assert decision.eligible is False
|
|
assert set(decision.reasons) >= {"validation_failed", "dataset_quarantined"}
|
|
|
|
|
|
def test_manifest_gate_rejects_missing_or_tampered_pair_decisions() -> None:
|
|
raster = governed_dataset(
|
|
role="raster",
|
|
registry=source_registry(ground_truth_allowed=False),
|
|
)
|
|
reference = governed_dataset(role="reference")
|
|
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
|
manifest = {
|
|
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
|
"training_eligibility": {
|
|
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
|
"status": "eligible",
|
|
"fixture_mode": False,
|
|
},
|
|
}
|
|
|
|
assert MODULE.manifest_training_eligibility_failures(manifest) == []
|
|
tampered = {
|
|
**manifest,
|
|
"samples": [{"sample_slug": "governed", "training_eligibility": {**pair, "eligible": False}}],
|
|
}
|
|
failures = MODULE.manifest_training_eligibility_failures(tampered)
|
|
assert "governed:training_pair_not_eligible" in failures
|
|
assert MODULE.manifest_training_eligibility_failures({"samples": []}) == [
|
|
"manifest_training_eligibility_missing"
|
|
]
|
|
|
|
|
|
def test_frozen_manifest_gate_detects_checksum_tampering(tmp_path: Path) -> None:
|
|
raster = governed_dataset(
|
|
role="raster",
|
|
registry=source_registry(ground_truth_allowed=False),
|
|
)
|
|
reference = governed_dataset(role="reference")
|
|
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
|
manifest_path = tmp_path / "operator_samples_manifest.json"
|
|
manifest_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"immutable": True,
|
|
"training_eligibility": {
|
|
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
|
"status": "eligible",
|
|
"fixture_mode": False,
|
|
},
|
|
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "corpus-freeze.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 2,
|
|
"manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(),
|
|
"immutable": True,
|
|
"training_eligibility_policy": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
|
"fixture_mode": False,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
assert MODULE.frozen_manifest_training_eligibility_failures(manifest_path) == []
|
|
manifest_path.write_text(manifest_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
|
assert "corpus_manifest_checksum_mismatch" in MODULE.frozen_manifest_training_eligibility_failures(
|
|
manifest_path
|
|
)
|
|
|
|
|
|
def test_live_manifest_gate_revokes_a_frozen_pair_when_an_upstream_dataset_is_quarantined() -> None:
|
|
raster_id = uuid4()
|
|
reference_id = uuid4()
|
|
raster_registry = source_registry(ground_truth_allowed=False)
|
|
reference_registry = source_registry()
|
|
raster_snapshot = source_snapshot()
|
|
reference_snapshot = source_snapshot()
|
|
raster_snapshot.source_registry_id = raster_registry.id
|
|
reference_snapshot.source_registry_id = reference_registry.id
|
|
raster = governed_dataset(
|
|
role="raster",
|
|
id=raster_id,
|
|
source_registry=raster_registry,
|
|
source_snapshot=raster_snapshot,
|
|
source_registry_id=raster_registry.id,
|
|
source_snapshot_id=raster_snapshot.id,
|
|
)
|
|
reference = governed_dataset(
|
|
role="reference",
|
|
id=reference_id,
|
|
source_registry=reference_registry,
|
|
source_snapshot=reference_snapshot,
|
|
source_registry_id=reference_registry.id,
|
|
source_snapshot_id=reference_snapshot.id,
|
|
)
|
|
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
|
manifest = {
|
|
"training_eligibility": {
|
|
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
|
"status": "eligible",
|
|
"fixture_mode": False,
|
|
},
|
|
"samples": [
|
|
{
|
|
"sample_slug": "governed-aoi",
|
|
"raster_dataset_id": str(raster_id),
|
|
"reference_dataset_id": str(reference_id),
|
|
"training_eligibility": pair,
|
|
}
|
|
],
|
|
}
|
|
|
|
class DatasetModel:
|
|
pass
|
|
|
|
class Session:
|
|
def __init__(self) -> None:
|
|
self.closed = False
|
|
|
|
@staticmethod
|
|
def get(model, item_id):
|
|
assert model is DatasetModel
|
|
return {raster_id: raster, reference_id: reference}.get(item_id)
|
|
|
|
def close(self) -> None:
|
|
self.closed = True
|
|
|
|
assert MODULE.live_manifest_training_eligibility_failures(
|
|
manifest,
|
|
session_factory=Session,
|
|
dataset_model=DatasetModel,
|
|
) == []
|
|
|
|
raster.quarantine_status = "quarantined"
|
|
failures = MODULE.live_manifest_training_eligibility_failures(
|
|
manifest,
|
|
session_factory=Session,
|
|
dataset_model=DatasetModel,
|
|
)
|
|
|
|
assert "governed-aoi:raster_live_revoked:dataset_quarantined" in failures
|