Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/assemble_belgium_building_corpus.py
+52 -4
View File
@@ -26,6 +26,10 @@ from app.db.session import SessionLocal # noqa: E402
from app.models import Dataset # noqa: E402
from normalize_belgium_building_labels import normalize # noqa: E402
from training_dataset_eligibility import ( # noqa: E402
TRAINING_ELIGIBILITY_POLICY_VERSION,
training_pair_evidence,
)
REGION_SOURCES = {
"flanders": ({"digitaal_vlaanderen_orthophoto"}, "grb"),
@@ -52,7 +56,13 @@ def _dataset_path(dataset: Dataset) -> Path:
return path
def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]:
def _validate_pair(
sample: dict[str, Any],
raster: Dataset,
reference: Dataset,
*,
fixture_mode: bool,
) -> tuple[str, str, dict[str, Any]]:
region = str(sample.get("region") or "").lower()
if region not in REGION_SOURCES:
raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}")
@@ -66,7 +76,23 @@ def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset)
raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}")
if raster.status != "ready" or reference.status != "ready":
raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}")
return region, expected_reference
eligibility = training_pair_evidence(
raster=raster,
reference=reference,
fixture_mode=fixture_mode,
)
if not eligibility["eligible"]:
reasons = sorted(
{
reason
for role in ("raster", "reference")
for reason in eligibility[role]["reasons"]
}
)
raise SystemExit(
f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}"
)
return region, expected_reference, eligibility
def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]:
@@ -104,6 +130,14 @@ def main() -> int:
parser.add_argument("--min-label-px", type=float, default=3.0)
parser.add_argument("--merge-touching-roofs", action="store_true")
parser.add_argument("--freeze", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help=(
"Allow only explicitly marked fixture datasets with legacy provenance. "
"Never use this mode for an operational corpus."
),
)
args = parser.parse_args()
spec = json.loads(args.spec.read_text(encoding="utf-8-sig"))
@@ -129,7 +163,12 @@ def main() -> int:
reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"])))
if raster is None or reference is None:
raise SystemExit(f"Persisted Dataset pair not found for {slug}")
region, reference_source = _validate_pair(sample, raster, reference)
region, reference_source, eligibility = _validate_pair(
sample,
raster,
reference,
fixture_mode=args.fixture_mode,
)
raster_source = _dataset_path(raster)
reference_source_path = _dataset_path(reference)
sample_dir = pairs_dir / slug
@@ -173,14 +212,20 @@ def main() -> int:
"raster_sha256": sha256(raster_target),
"reference_sha256": sha256(normalized_target),
"label_audit_sha256": sha256(audit_target),
"training_eligibility": eligibility,
"bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326")
or sample.get("bbox_epsg4326"),
}
)
manifest = {
"schema_version": 1,
"schema_version": 2,
"dataset_version": args.version,
"immutable": bool(args.freeze),
"training_eligibility": {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "eligible",
"fixture_mode": bool(args.fixture_mode),
},
"samples": manifest_samples,
}
manifest_path = output_dir / "operator_samples_manifest.json"
@@ -192,10 +237,13 @@ def main() -> int:
if leakage_audit["status"] != "ok":
raise SystemExit("Spatial split leakage audit failed")
freeze = {
"schema_version": 2,
"dataset_version": args.version,
"manifest_sha256": sha256(manifest_path),
"sample_count": len(manifest_samples),
"immutable": bool(args.freeze),
"training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION,
"fixture_mode": bool(args.fixture_mode),
}
(output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
print(json.dumps(freeze))