#!/usr/bin/env python3 """Mine detector proposals into a leak-free binary crop dataset.""" from __future__ import annotations import argparse import hashlib import json import sys from collections import Counter from pathlib import Path from typing import Any, Mapping from PIL import Image SCRIPT_DIR = Path(__file__).resolve().parent if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) from evaluate_belgium_building_candidate import iou, read_references # noqa: E402 from training_dataset_eligibility import ( # noqa: E402 TrainingEligibilityError, assert_frozen_manifest_training_eligible, ) from training_release_manifest import ( # noqa: E402 TrainingReleaseError, assert_yolo_summary_bound_to_embedded_training_release, ) PROTECTED_SPLITS = {"calibration", "test", "background-test", "challenge"} PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json" PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json" PROPOSAL_DATASET_SCHEMA_VERSION = 1 def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes: return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str: normalized = dict(payload) normalized.pop(field, None) return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest() def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None: """Persist one immutable sidecar without silently replacing prior evidence.""" encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n" if path.exists(): if path.read_text(encoding="utf-8") != encoded: raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}") return path.write_text(encoded, encoding="utf-8") def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str: """Require the proposal summary to name the exact frozen corpus bytes.""" expected = sha256(corpus_manifest_path) observed = summary.get("source_manifest_sha256") if observed != expected: raise ValueError( "proposal source summary is not bound to the supplied governed corpus manifest " f"(expected {expected}, observed {observed!r})" ) return expected def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]: """Validate frozen corpus evidence and re-check the live Dataset state. This must run before importing Ultralytics/PyTorch so a revocation cannot consume GPU work or create proposal crops. """ try: manifest = assert_frozen_manifest_training_eligible( corpus_manifest_path, fixture_mode=fixture_mode, verify_live=True, ) except TrainingEligibilityError as exc: raise ValueError(str(exc)) from exc if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary raise ValueError("governed corpus manifest must be a JSON object") return manifest def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]: raw_path = tile.get(field) if not isinstance(raw_path, str) or not raw_path.strip(): raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}") path = Path(raw_path).expanduser().resolve(strict=False) if not path.is_file(): raise ValueError(f"proposal source tile {field} is unavailable: {path}") return path, sha256(path) def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]: """Return the canonical source identity used for the pre-inference cache.""" return ( str(tile.get("sample_slug") or ""), str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)), str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)), ) def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None: """Reject a mutable source changing between evidence capture and use.""" if sha256(path) != expected_sha256: raise RuntimeError(f"proposal {role} changed after evidence capture: {path}") def _crop_entry( *, output_dir: Path, crop_path: Path, tile: Mapping[str, Any], label: str, proposal_index: int, proposal_score: float, source_box_xyxy: tuple[float, float, float, float], source_image_path: Path, source_image_sha256: str, source_label_path: Path, source_label_sha256: str, ) -> dict[str, Any]: relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix() sample_slug = tile.get("sample_slug") split = tile.get("split") if not isinstance(sample_slug, str) or not sample_slug.strip(): raise ValueError("proposal source tile has no sample_slug") if split not in {"train", "val"}: raise ValueError(f"proposal crop has unsupported split: {split!r}") return { "relative_path": relative_path, "sha256": sha256(crop_path), "size_bytes": crop_path.stat().st_size, "split": split, "label": label, "sample_slug": sample_slug, "proposal_index": proposal_index, "proposal_score": proposal_score, "source_box_xyxy": [float(value) for value in source_box_xyxy], "source_image_path": str(source_image_path), "source_image_sha256": source_image_sha256, "source_label_path": str(source_label_path), "source_label_sha256": source_label_sha256, } def classify_proposals( predictions: list[tuple[tuple[float, float, float, float], float]], references: list[tuple[float, float, float, float]], match_iou: float, ) -> list[tuple[str, tuple[float, float, float, float], float]]: unmatched = set(range(len(references))) classified = [] for box, score in sorted(predictions, key=lambda item: -item[1]): candidates = [(iou(box, references[index]), index) for index in unmatched] overlap, index = max(candidates, default=(0.0, -1)) label = "positive" if overlap >= match_iou else "negative" if label == "positive": unmatched.remove(index) classified.append((label, box, score)) return classified def eligible_tiles( summary: dict[str, Any], manifest: dict[str, Any], region: str | None ) -> list[dict[str, Any]]: samples = {item["sample_slug"]: item for item in manifest["samples"]} selected = [] for tile in summary["tiles"]: sample = samples.get(tile["sample_slug"]) if sample is None: raise ValueError(f"unknown sample: {tile['sample_slug']}") tile_split = str(tile.get("split") or "") manifest_split = str(sample.get("split") or "") if tile_split in PROTECTED_SPLITS or manifest_split in PROTECTED_SPLITS: raise ValueError(f"protected split in proposal source: {tile['sample_slug']}") if tile_split != manifest_split: raise ValueError(f"tile/manifest split mismatch: {tile['sample_slug']}") if tile.get("kept", True) and tile_split in {"train", "val"} and ( region is None or sample.get("region") == region ): selected.append(tile) if not selected or not any(tile["split"] == "val" for tile in selected): raise ValueError("proposal dataset requires eligible train and validation tiles") return selected def crop_square( source: Image.Image, box: tuple[float, float, float, float], scale: float ) -> Image.Image: x1, y1, x2, y2 = box cx, cy = (x1 + x2) / 2, (y1 + y2) / 2 side = max(x2 - x1, y2 - y1, 8.0) * scale left = max(0, min(source.width - 1, int(cx - side / 2))) top = max(0, min(source.height - 1, int(cy - side / 2))) right = max(left + 1, min(source.width, int(cx + side / 2))) bottom = max(top + 1, min(source.height, int(cy + side / 2))) return source.crop((left, top, right, bottom)).convert("RGB") def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--model", type=Path, required=True) parser.add_argument("--summary", type=Path, required=True) parser.add_argument("--corpus-manifest", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--region") parser.add_argument("--confidence", type=float, default=0.05) parser.add_argument("--match-iou", type=float, default=0.25) parser.add_argument("--crop-scale", type=float, default=1.4) parser.add_argument("--max-positive-per-tile", type=int, default=24) parser.add_argument("--max-negative-per-tile", type=int, default=24) parser.add_argument("--device", default="cuda:0") parser.add_argument("--imgsz", type=int, default=640) parser.add_argument( "--fixture-mode", action="store_true", help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.", ) args = parser.parse_args() if args.output_dir.exists(): parser.error(f"output already exists: {args.output_dir}") try: summary = json.loads(args.summary.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc if not isinstance(summary, dict): raise SystemExit("proposal source summary must be a JSON object") try: source_release = assert_yolo_summary_bound_to_embedded_training_release( summary_path=args.summary, corpus_manifest=args.corpus_manifest, fixture_mode=args.fixture_mode, ) manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode) corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest) except (ValueError, TrainingReleaseError) as exc: raise SystemExit(str(exc)) from exc tiles = eligible_tiles(summary, manifest, args.region) model_path = args.model.expanduser().resolve(strict=False) if not model_path.is_file(): raise SystemExit(f"proposal model is unavailable: {model_path}") model_sha256 = sha256(model_path) # Hash every input image/label before GPU inference. A crop sidecar then # binds each generated JPEG to the exact source tile rather than its # filename or an unverified aggregate summary. source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {} for tile in tiles: image_path, image_sha256 = _source_file_evidence(tile, field="image_path") label_path, label_sha256 = _source_file_evidence(tile, field="label_path") key = _source_tile_key(tile) source_evidence[key] = (image_path, image_sha256, label_path, label_sha256) args.output_dir.mkdir(parents=True) counts: Counter[str] = Counter() sample_counts: Counter[str] = Counter() crop_entries: list[dict[str, Any]] = [] # Importing the model is deliberately after the source-manifest and live # Dataset checks above. A revoked corpus must not start GPU work. from ultralytics import YOLO model = YOLO(str(model_path)) for start in range(0, len(tiles), 16): batch_tiles = tiles[start : start + 16] batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles] for image_path, image_sha256, label_path, label_sha256 in batch_sources: _assert_file_unchanged(image_path, image_sha256, role="source image") _assert_file_unchanged(label_path, label_sha256, role="source label") results = model.predict( [str(source[0]) for source in batch_sources], conf=args.confidence, device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False, ) for tile, result in zip(batch_tiles, results, strict=True): image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)] _assert_file_unchanged(image_path, image_sha256, role="source image") _assert_file_unchanged(label_path, label_sha256, role="source label") with Image.open(image_path) as opened: source = opened.convert("RGB") references = read_references(label_path, source.width, source.height) proposals = [ (tuple(map(float, box)), float(score)) for box, score in zip( result.boxes.xyxy.cpu().tolist(), result.boxes.conf.cpu().tolist(), strict=True ) ] classified = classify_proposals(proposals, references, args.match_iou) limits = {"positive": args.max_positive_per_tile, "negative": args.max_negative_per_tile} per_label: Counter[str] = Counter() for proposal_index, (label, box, score) in enumerate(classified): if per_label[label] >= limits[label]: continue per_label[label] += 1 split = tile["split"] target_dir = args.output_dir / split / label target_dir.mkdir(parents=True, exist_ok=True) name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg" crop_path = target_dir / name if crop_path.exists(): raise RuntimeError(f"proposal crop identity collision: {crop_path}") crop_square(source, box, args.crop_scale).save(crop_path, quality=92) crop_entries.append( _crop_entry( output_dir=args.output_dir, crop_path=crop_path, tile=tile, label=label, proposal_index=proposal_index, proposal_score=score, source_box_xyxy=box, source_image_path=image_path, source_image_sha256=image_sha256, source_label_path=label_path, source_label_sha256=label_sha256, ) ) counts[f"{split}/{label}"] += 1 sample_counts[tile["sample_slug"]] += 1 for split in ("train", "val"): for label in ("positive", "negative"): if counts[f"{split}/{label}"] == 0: raise RuntimeError(f"empty proposal class: {split}/{label}") crop_entries.sort(key=lambda item: str(item["relative_path"])) if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries): raise RuntimeError("proposal crop manifest contains duplicate relative paths") source_summary_sha256 = sha256(args.summary) corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json" if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}") provenance: dict[str, Any] = { "schema_version": PROPOSAL_DATASET_SCHEMA_VERSION, "status": "ok", "immutable": True, "dataset_kind": "building_proposal_classifier_crops", "fixture_mode": bool(args.fixture_mode), "governed_corpus_live_recheck": True, "source": { "corpus_manifest": { "path": str(args.corpus_manifest.expanduser().resolve(strict=False)), "sha256": corpus_manifest_sha256, }, "corpus_freeze": { "path": str(corpus_freeze_path.resolve(strict=False)), "sha256": sha256(corpus_freeze_path), }, "summary": { "path": str(args.summary.expanduser().resolve(strict=False)), "sha256": source_summary_sha256, "source_manifest_sha256": summary["source_manifest_sha256"], "training_release_manifest": summary["training_release_manifest"], "training_release_manifest_sha256": summary["training_release_manifest_sha256"], "training_asset_manifest": summary["training_asset_manifest"], }, "training_release": { "dataset_yaml_path": source_release["dataset_yaml"]["path"], "dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"], "corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"], }, "proposal_model": { "path": str(model_path), "sha256": model_sha256, }, }, "parameters": { "region": args.region, "confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale, "max_positive_per_tile": args.max_positive_per_tile, "max_negative_per_tile": args.max_negative_per_tile, "device": args.device, "imgsz": args.imgsz, }, "counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())), "tile_count": len(tiles), "crop_count": len(crop_entries), "crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(), "crops": crop_entries, } provenance["manifest_sha256"] = _immutable_payload_sha256(provenance) provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME _write_immutable_json(provenance_path, provenance) evidence = { "schema_version": PROPOSAL_DATASET_SCHEMA_VERSION, "status": "ok", "model": str(model_path), "model_sha256": model_sha256, "summary": str(args.summary), "summary_sha256": source_summary_sha256, "corpus_manifest": str(args.corpus_manifest), "corpus_manifest_sha256": corpus_manifest_sha256, "region": args.region, "confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale, "counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())), "protected_samples_in_training": [], "tile_count": len(tiles), "fixture_mode": bool(args.fixture_mode), "proposal_dataset_provenance": str(provenance_path), "proposal_dataset_provenance_sha256": sha256(provenance_path), "proposal_dataset_manifest_sha256": provenance["manifest_sha256"], } _write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence) print(json.dumps(evidence, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())