Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build_building_proposal_classifier_dataset.py
+281 -18
View File
@@ -9,7 +9,7 @@ import json
import sys
from collections import Counter
from pathlib import Path
from typing import Any
from typing import Any, Mapping
from PIL import Image
@@ -17,10 +17,21 @@ SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from evaluate_belgium_building_candidate import iou, read_references
from evaluate_belgium_building_candidate import iou, read_references # noqa: E402
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
)
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json"
PROPOSAL_DATASET_SCHEMA_VERSION = 1
def sha256(path: Path) -> str:
@@ -31,6 +42,125 @@ def sha256(path: Path) -> str:
return digest.hexdigest()
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str:
normalized = dict(payload)
normalized.pop(field, None)
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None:
"""Persist one immutable sidecar without silently replacing prior evidence."""
encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n"
if path.exists():
if path.read_text(encoding="utf-8") != encoded:
raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}")
return
path.write_text(encoded, encoding="utf-8")
def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str:
"""Require the proposal summary to name the exact frozen corpus bytes."""
expected = sha256(corpus_manifest_path)
observed = summary.get("source_manifest_sha256")
if observed != expected:
raise ValueError(
"proposal source summary is not bound to the supplied governed corpus manifest "
f"(expected {expected}, observed {observed!r})"
)
return expected
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
"""Validate frozen corpus evidence and re-check the live Dataset state.
This must run before importing Ultralytics/PyTorch so a revocation cannot
consume GPU work or create proposal crops.
"""
try:
manifest = assert_frozen_manifest_training_eligible(
corpus_manifest_path,
fixture_mode=fixture_mode,
verify_live=True,
)
except TrainingEligibilityError as exc:
raise ValueError(str(exc)) from exc
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
raise ValueError("governed corpus manifest must be a JSON object")
return manifest
def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]:
raw_path = tile.get(field)
if not isinstance(raw_path, str) or not raw_path.strip():
raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}")
path = Path(raw_path).expanduser().resolve(strict=False)
if not path.is_file():
raise ValueError(f"proposal source tile {field} is unavailable: {path}")
return path, sha256(path)
def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]:
"""Return the canonical source identity used for the pre-inference cache."""
return (
str(tile.get("sample_slug") or ""),
str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)),
str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)),
)
def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None:
"""Reject a mutable source changing between evidence capture and use."""
if sha256(path) != expected_sha256:
raise RuntimeError(f"proposal {role} changed after evidence capture: {path}")
def _crop_entry(
*,
output_dir: Path,
crop_path: Path,
tile: Mapping[str, Any],
label: str,
proposal_index: int,
proposal_score: float,
source_box_xyxy: tuple[float, float, float, float],
source_image_path: Path,
source_image_sha256: str,
source_label_path: Path,
source_label_sha256: str,
) -> dict[str, Any]:
relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix()
sample_slug = tile.get("sample_slug")
split = tile.get("split")
if not isinstance(sample_slug, str) or not sample_slug.strip():
raise ValueError("proposal source tile has no sample_slug")
if split not in {"train", "val"}:
raise ValueError(f"proposal crop has unsupported split: {split!r}")
return {
"relative_path": relative_path,
"sha256": sha256(crop_path),
"size_bytes": crop_path.stat().st_size,
"split": split,
"label": label,
"sample_slug": sample_slug,
"proposal_index": proposal_index,
"proposal_score": proposal_score,
"source_box_xyxy": [float(value) for value in source_box_xyxy],
"source_image_path": str(source_image_path),
"source_image_sha256": source_image_sha256,
"source_label_path": str(source_label_path),
"source_label_sha256": source_label_sha256,
}
def classify_proposals(
predictions: list[tuple[tuple[float, float, float, float], float]],
references: list[tuple[float, float, float, float]],
@@ -99,29 +229,74 @@ def main() -> int:
parser.add_argument("--max-negative-per-tile", type=int, default=24)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--imgsz", type=int, default=640)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
)
args = parser.parse_args()
if args.output_dir.exists():
parser.error(f"output already exists: {args.output_dir}")
from ultralytics import YOLO
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
summary = json.loads(args.summary.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc
if not isinstance(summary, dict):
raise SystemExit("proposal source summary must be a JSON object")
try:
source_release = assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode)
corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest)
except (ValueError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
tiles = eligible_tiles(summary, manifest, args.region)
model_path = args.model.expanduser().resolve(strict=False)
if not model_path.is_file():
raise SystemExit(f"proposal model is unavailable: {model_path}")
model_sha256 = sha256(model_path)
# Hash every input image/label before GPU inference. A crop sidecar then
# binds each generated JPEG to the exact source tile rather than its
# filename or an unverified aggregate summary.
source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {}
for tile in tiles:
image_path, image_sha256 = _source_file_evidence(tile, field="image_path")
label_path, label_sha256 = _source_file_evidence(tile, field="label_path")
key = _source_tile_key(tile)
source_evidence[key] = (image_path, image_sha256, label_path, label_sha256)
args.output_dir.mkdir(parents=True)
counts: Counter[str] = Counter()
sample_counts: Counter[str] = Counter()
model = YOLO(str(args.model))
crop_entries: list[dict[str, Any]] = []
# Importing the model is deliberately after the source-manifest and live
# Dataset checks above. A revoked corpus must not start GPU work.
from ultralytics import YOLO
model = YOLO(str(model_path))
for start in range(0, len(tiles), 16):
batch_tiles = tiles[start : start + 16]
batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles]
for image_path, image_sha256, label_path, label_sha256 in batch_sources:
_assert_file_unchanged(image_path, image_sha256, role="source image")
_assert_file_unchanged(label_path, label_sha256, role="source label")
results = model.predict(
[tile["image_path"] for tile in batch_tiles], conf=args.confidence,
[str(source[0]) for source in batch_sources], conf=args.confidence,
device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False,
)
for tile, result in zip(batch_tiles, results, strict=True):
with Image.open(tile["image_path"]) as opened:
image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)]
_assert_file_unchanged(image_path, image_sha256, role="source image")
_assert_file_unchanged(label_path, label_sha256, role="source label")
with Image.open(image_path) as opened:
source = opened.convert("RGB")
references = read_references(Path(tile["label_path"]), source.width, source.height)
references = read_references(label_path, source.width, source.height)
proposals = [
(tuple(map(float, box)), float(score))
for box, score in zip(
@@ -139,25 +314,113 @@ def main() -> int:
target_dir = args.output_dir / split / label
target_dir.mkdir(parents=True, exist_ok=True)
name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg"
crop_square(source, box, args.crop_scale).save(target_dir / name, quality=92)
crop_path = target_dir / name
if crop_path.exists():
raise RuntimeError(f"proposal crop identity collision: {crop_path}")
crop_square(source, box, args.crop_scale).save(crop_path, quality=92)
crop_entries.append(
_crop_entry(
output_dir=args.output_dir,
crop_path=crop_path,
tile=tile,
label=label,
proposal_index=proposal_index,
proposal_score=score,
source_box_xyxy=box,
source_image_path=image_path,
source_image_sha256=image_sha256,
source_label_path=label_path,
source_label_sha256=label_sha256,
)
)
counts[f"{split}/{label}"] += 1
sample_counts[tile["sample_slug"]] += 1
for split in ("train", "val"):
for label in ("positive", "negative"):
if counts[f"{split}/{label}"] == 0:
raise RuntimeError(f"empty proposal class: {split}/{label}")
crop_entries.sort(key=lambda item: str(item["relative_path"]))
if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries):
raise RuntimeError("proposal crop manifest contains duplicate relative paths")
source_summary_sha256 = sha256(args.summary)
corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json"
if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output
raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}")
provenance: dict[str, Any] = {
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
"status": "ok",
"immutable": True,
"dataset_kind": "building_proposal_classifier_crops",
"fixture_mode": bool(args.fixture_mode),
"governed_corpus_live_recheck": True,
"source": {
"corpus_manifest": {
"path": str(args.corpus_manifest.expanduser().resolve(strict=False)),
"sha256": corpus_manifest_sha256,
},
"corpus_freeze": {
"path": str(corpus_freeze_path.resolve(strict=False)),
"sha256": sha256(corpus_freeze_path),
},
"summary": {
"path": str(args.summary.expanduser().resolve(strict=False)),
"sha256": source_summary_sha256,
"source_manifest_sha256": summary["source_manifest_sha256"],
"training_release_manifest": summary["training_release_manifest"],
"training_release_manifest_sha256": summary["training_release_manifest_sha256"],
"training_asset_manifest": summary["training_asset_manifest"],
},
"training_release": {
"dataset_yaml_path": source_release["dataset_yaml"]["path"],
"dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"],
"corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
},
"proposal_model": {
"path": str(model_path),
"sha256": model_sha256,
},
},
"parameters": {
"region": args.region,
"confidence": args.confidence,
"match_iou": args.match_iou,
"crop_scale": args.crop_scale,
"max_positive_per_tile": args.max_positive_per_tile,
"max_negative_per_tile": args.max_negative_per_tile,
"device": args.device,
"imgsz": args.imgsz,
},
"counts": dict(sorted(counts.items())),
"sample_counts": dict(sorted(sample_counts.items())),
"tile_count": len(tiles),
"crop_count": len(crop_entries),
"crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(),
"crops": crop_entries,
}
provenance["manifest_sha256"] = _immutable_payload_sha256(provenance)
provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME
_write_immutable_json(provenance_path, provenance)
evidence = {
"schema_version": 1, "status": "ok", "model": str(args.model),
"model_sha256": sha256(args.model), "summary": str(args.summary),
"summary_sha256": sha256(args.summary), "corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest), "region": args.region,
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
"status": "ok",
"model": str(model_path),
"model_sha256": model_sha256,
"summary": str(args.summary),
"summary_sha256": source_summary_sha256,
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": corpus_manifest_sha256,
"region": args.region,
"confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale,
"counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())),
"protected_samples_in_training": [], "tile_count": len(tiles),
"fixture_mode": bool(args.fixture_mode),
"proposal_dataset_provenance": str(provenance_path),
"proposal_dataset_provenance_sha256": sha256(provenance_path),
"proposal_dataset_manifest_sha256": provenance["manifest_sha256"],
}
(args.output_dir / "proposal-dataset-evidence.json").write_text(
json.dumps(evidence, indent=2), encoding="utf-8"
)
_write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence)
print(json.dumps(evidence, indent=2))
return 0