430 lines
19 KiB
Python
430 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Mine detector proposals into a leak-free binary crop dataset."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
|
|
from PIL import Image
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
if str(SCRIPT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPT_DIR))
|
|
|
|
from evaluate_belgium_building_candidate import iou, read_references # noqa: E402
|
|
from training_dataset_eligibility import ( # noqa: E402
|
|
TrainingEligibilityError,
|
|
assert_frozen_manifest_training_eligible,
|
|
)
|
|
from training_release_manifest import ( # noqa: E402
|
|
TrainingReleaseError,
|
|
assert_yolo_summary_bound_to_embedded_training_release,
|
|
)
|
|
|
|
|
|
PROTECTED_SPLITS = {"calibration", "test", "background-test", "challenge"}
|
|
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
|
|
PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json"
|
|
PROPOSAL_DATASET_SCHEMA_VERSION = 1
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
|
|
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
|
|
|
|
|
def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str:
|
|
normalized = dict(payload)
|
|
normalized.pop(field, None)
|
|
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
|
|
|
|
|
|
def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None:
|
|
"""Persist one immutable sidecar without silently replacing prior evidence."""
|
|
|
|
encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
|
if path.exists():
|
|
if path.read_text(encoding="utf-8") != encoded:
|
|
raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}")
|
|
return
|
|
path.write_text(encoded, encoding="utf-8")
|
|
|
|
|
|
def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str:
|
|
"""Require the proposal summary to name the exact frozen corpus bytes."""
|
|
|
|
expected = sha256(corpus_manifest_path)
|
|
observed = summary.get("source_manifest_sha256")
|
|
if observed != expected:
|
|
raise ValueError(
|
|
"proposal source summary is not bound to the supplied governed corpus manifest "
|
|
f"(expected {expected}, observed {observed!r})"
|
|
)
|
|
return expected
|
|
|
|
|
|
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
|
|
"""Validate frozen corpus evidence and re-check the live Dataset state.
|
|
|
|
This must run before importing Ultralytics/PyTorch so a revocation cannot
|
|
consume GPU work or create proposal crops.
|
|
"""
|
|
|
|
try:
|
|
manifest = assert_frozen_manifest_training_eligible(
|
|
corpus_manifest_path,
|
|
fixture_mode=fixture_mode,
|
|
verify_live=True,
|
|
)
|
|
except TrainingEligibilityError as exc:
|
|
raise ValueError(str(exc)) from exc
|
|
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
|
|
raise ValueError("governed corpus manifest must be a JSON object")
|
|
return manifest
|
|
|
|
|
|
def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]:
|
|
raw_path = tile.get(field)
|
|
if not isinstance(raw_path, str) or not raw_path.strip():
|
|
raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}")
|
|
path = Path(raw_path).expanduser().resolve(strict=False)
|
|
if not path.is_file():
|
|
raise ValueError(f"proposal source tile {field} is unavailable: {path}")
|
|
return path, sha256(path)
|
|
|
|
|
|
def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]:
|
|
"""Return the canonical source identity used for the pre-inference cache."""
|
|
|
|
return (
|
|
str(tile.get("sample_slug") or ""),
|
|
str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)),
|
|
str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)),
|
|
)
|
|
|
|
|
|
def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None:
|
|
"""Reject a mutable source changing between evidence capture and use."""
|
|
|
|
if sha256(path) != expected_sha256:
|
|
raise RuntimeError(f"proposal {role} changed after evidence capture: {path}")
|
|
|
|
|
|
def _crop_entry(
|
|
*,
|
|
output_dir: Path,
|
|
crop_path: Path,
|
|
tile: Mapping[str, Any],
|
|
label: str,
|
|
proposal_index: int,
|
|
proposal_score: float,
|
|
source_box_xyxy: tuple[float, float, float, float],
|
|
source_image_path: Path,
|
|
source_image_sha256: str,
|
|
source_label_path: Path,
|
|
source_label_sha256: str,
|
|
) -> dict[str, Any]:
|
|
relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix()
|
|
sample_slug = tile.get("sample_slug")
|
|
split = tile.get("split")
|
|
if not isinstance(sample_slug, str) or not sample_slug.strip():
|
|
raise ValueError("proposal source tile has no sample_slug")
|
|
if split not in {"train", "val"}:
|
|
raise ValueError(f"proposal crop has unsupported split: {split!r}")
|
|
return {
|
|
"relative_path": relative_path,
|
|
"sha256": sha256(crop_path),
|
|
"size_bytes": crop_path.stat().st_size,
|
|
"split": split,
|
|
"label": label,
|
|
"sample_slug": sample_slug,
|
|
"proposal_index": proposal_index,
|
|
"proposal_score": proposal_score,
|
|
"source_box_xyxy": [float(value) for value in source_box_xyxy],
|
|
"source_image_path": str(source_image_path),
|
|
"source_image_sha256": source_image_sha256,
|
|
"source_label_path": str(source_label_path),
|
|
"source_label_sha256": source_label_sha256,
|
|
}
|
|
|
|
|
|
def classify_proposals(
|
|
predictions: list[tuple[tuple[float, float, float, float], float]],
|
|
references: list[tuple[float, float, float, float]],
|
|
match_iou: float,
|
|
) -> list[tuple[str, tuple[float, float, float, float], float]]:
|
|
unmatched = set(range(len(references)))
|
|
classified = []
|
|
for box, score in sorted(predictions, key=lambda item: -item[1]):
|
|
candidates = [(iou(box, references[index]), index) for index in unmatched]
|
|
overlap, index = max(candidates, default=(0.0, -1))
|
|
label = "positive" if overlap >= match_iou else "negative"
|
|
if label == "positive":
|
|
unmatched.remove(index)
|
|
classified.append((label, box, score))
|
|
return classified
|
|
|
|
|
|
def eligible_tiles(
|
|
summary: dict[str, Any], manifest: dict[str, Any], region: str | None
|
|
) -> list[dict[str, Any]]:
|
|
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
|
selected = []
|
|
for tile in summary["tiles"]:
|
|
sample = samples.get(tile["sample_slug"])
|
|
if sample is None:
|
|
raise ValueError(f"unknown sample: {tile['sample_slug']}")
|
|
tile_split = str(tile.get("split") or "")
|
|
manifest_split = str(sample.get("split") or "")
|
|
if tile_split in PROTECTED_SPLITS or manifest_split in PROTECTED_SPLITS:
|
|
raise ValueError(f"protected split in proposal source: {tile['sample_slug']}")
|
|
if tile_split != manifest_split:
|
|
raise ValueError(f"tile/manifest split mismatch: {tile['sample_slug']}")
|
|
if tile.get("kept", True) and tile_split in {"train", "val"} and (
|
|
region is None or sample.get("region") == region
|
|
):
|
|
selected.append(tile)
|
|
if not selected or not any(tile["split"] == "val" for tile in selected):
|
|
raise ValueError("proposal dataset requires eligible train and validation tiles")
|
|
return selected
|
|
|
|
|
|
def crop_square(
|
|
source: Image.Image, box: tuple[float, float, float, float], scale: float
|
|
) -> Image.Image:
|
|
x1, y1, x2, y2 = box
|
|
cx, cy = (x1 + x2) / 2, (y1 + y2) / 2
|
|
side = max(x2 - x1, y2 - y1, 8.0) * scale
|
|
left = max(0, min(source.width - 1, int(cx - side / 2)))
|
|
top = max(0, min(source.height - 1, int(cy - side / 2)))
|
|
right = max(left + 1, min(source.width, int(cx + side / 2)))
|
|
bottom = max(top + 1, min(source.height, int(cy + side / 2)))
|
|
return source.crop((left, top, right, bottom)).convert("RGB")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--model", type=Path, required=True)
|
|
parser.add_argument("--summary", type=Path, required=True)
|
|
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--region")
|
|
parser.add_argument("--confidence", type=float, default=0.05)
|
|
parser.add_argument("--match-iou", type=float, default=0.25)
|
|
parser.add_argument("--crop-scale", type=float, default=1.4)
|
|
parser.add_argument("--max-positive-per-tile", type=int, default=24)
|
|
parser.add_argument("--max-negative-per-tile", type=int, default=24)
|
|
parser.add_argument("--device", default="cuda:0")
|
|
parser.add_argument("--imgsz", type=int, default=640)
|
|
parser.add_argument(
|
|
"--fixture-mode",
|
|
action="store_true",
|
|
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
|
|
)
|
|
args = parser.parse_args()
|
|
if args.output_dir.exists():
|
|
parser.error(f"output already exists: {args.output_dir}")
|
|
|
|
try:
|
|
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc
|
|
if not isinstance(summary, dict):
|
|
raise SystemExit("proposal source summary must be a JSON object")
|
|
try:
|
|
source_release = assert_yolo_summary_bound_to_embedded_training_release(
|
|
summary_path=args.summary,
|
|
corpus_manifest=args.corpus_manifest,
|
|
fixture_mode=args.fixture_mode,
|
|
)
|
|
manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode)
|
|
corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest)
|
|
except (ValueError, TrainingReleaseError) as exc:
|
|
raise SystemExit(str(exc)) from exc
|
|
tiles = eligible_tiles(summary, manifest, args.region)
|
|
model_path = args.model.expanduser().resolve(strict=False)
|
|
if not model_path.is_file():
|
|
raise SystemExit(f"proposal model is unavailable: {model_path}")
|
|
model_sha256 = sha256(model_path)
|
|
|
|
# Hash every input image/label before GPU inference. A crop sidecar then
|
|
# binds each generated JPEG to the exact source tile rather than its
|
|
# filename or an unverified aggregate summary.
|
|
source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {}
|
|
for tile in tiles:
|
|
image_path, image_sha256 = _source_file_evidence(tile, field="image_path")
|
|
label_path, label_sha256 = _source_file_evidence(tile, field="label_path")
|
|
key = _source_tile_key(tile)
|
|
source_evidence[key] = (image_path, image_sha256, label_path, label_sha256)
|
|
|
|
args.output_dir.mkdir(parents=True)
|
|
counts: Counter[str] = Counter()
|
|
sample_counts: Counter[str] = Counter()
|
|
crop_entries: list[dict[str, Any]] = []
|
|
|
|
# Importing the model is deliberately after the source-manifest and live
|
|
# Dataset checks above. A revoked corpus must not start GPU work.
|
|
from ultralytics import YOLO
|
|
|
|
model = YOLO(str(model_path))
|
|
for start in range(0, len(tiles), 16):
|
|
batch_tiles = tiles[start : start + 16]
|
|
batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles]
|
|
for image_path, image_sha256, label_path, label_sha256 in batch_sources:
|
|
_assert_file_unchanged(image_path, image_sha256, role="source image")
|
|
_assert_file_unchanged(label_path, label_sha256, role="source label")
|
|
results = model.predict(
|
|
[str(source[0]) for source in batch_sources], conf=args.confidence,
|
|
device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False,
|
|
)
|
|
for tile, result in zip(batch_tiles, results, strict=True):
|
|
image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)]
|
|
_assert_file_unchanged(image_path, image_sha256, role="source image")
|
|
_assert_file_unchanged(label_path, label_sha256, role="source label")
|
|
with Image.open(image_path) as opened:
|
|
source = opened.convert("RGB")
|
|
references = read_references(label_path, source.width, source.height)
|
|
proposals = [
|
|
(tuple(map(float, box)), float(score))
|
|
for box, score in zip(
|
|
result.boxes.xyxy.cpu().tolist(), result.boxes.conf.cpu().tolist(), strict=True
|
|
)
|
|
]
|
|
classified = classify_proposals(proposals, references, args.match_iou)
|
|
limits = {"positive": args.max_positive_per_tile, "negative": args.max_negative_per_tile}
|
|
per_label: Counter[str] = Counter()
|
|
for proposal_index, (label, box, score) in enumerate(classified):
|
|
if per_label[label] >= limits[label]:
|
|
continue
|
|
per_label[label] += 1
|
|
split = tile["split"]
|
|
target_dir = args.output_dir / split / label
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg"
|
|
crop_path = target_dir / name
|
|
if crop_path.exists():
|
|
raise RuntimeError(f"proposal crop identity collision: {crop_path}")
|
|
crop_square(source, box, args.crop_scale).save(crop_path, quality=92)
|
|
crop_entries.append(
|
|
_crop_entry(
|
|
output_dir=args.output_dir,
|
|
crop_path=crop_path,
|
|
tile=tile,
|
|
label=label,
|
|
proposal_index=proposal_index,
|
|
proposal_score=score,
|
|
source_box_xyxy=box,
|
|
source_image_path=image_path,
|
|
source_image_sha256=image_sha256,
|
|
source_label_path=label_path,
|
|
source_label_sha256=label_sha256,
|
|
)
|
|
)
|
|
counts[f"{split}/{label}"] += 1
|
|
sample_counts[tile["sample_slug"]] += 1
|
|
for split in ("train", "val"):
|
|
for label in ("positive", "negative"):
|
|
if counts[f"{split}/{label}"] == 0:
|
|
raise RuntimeError(f"empty proposal class: {split}/{label}")
|
|
|
|
crop_entries.sort(key=lambda item: str(item["relative_path"]))
|
|
if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries):
|
|
raise RuntimeError("proposal crop manifest contains duplicate relative paths")
|
|
source_summary_sha256 = sha256(args.summary)
|
|
corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json"
|
|
if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output
|
|
raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}")
|
|
provenance: dict[str, Any] = {
|
|
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
|
|
"status": "ok",
|
|
"immutable": True,
|
|
"dataset_kind": "building_proposal_classifier_crops",
|
|
"fixture_mode": bool(args.fixture_mode),
|
|
"governed_corpus_live_recheck": True,
|
|
"source": {
|
|
"corpus_manifest": {
|
|
"path": str(args.corpus_manifest.expanduser().resolve(strict=False)),
|
|
"sha256": corpus_manifest_sha256,
|
|
},
|
|
"corpus_freeze": {
|
|
"path": str(corpus_freeze_path.resolve(strict=False)),
|
|
"sha256": sha256(corpus_freeze_path),
|
|
},
|
|
"summary": {
|
|
"path": str(args.summary.expanduser().resolve(strict=False)),
|
|
"sha256": source_summary_sha256,
|
|
"source_manifest_sha256": summary["source_manifest_sha256"],
|
|
"training_release_manifest": summary["training_release_manifest"],
|
|
"training_release_manifest_sha256": summary["training_release_manifest_sha256"],
|
|
"training_asset_manifest": summary["training_asset_manifest"],
|
|
},
|
|
"training_release": {
|
|
"dataset_yaml_path": source_release["dataset_yaml"]["path"],
|
|
"dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"],
|
|
"corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
|
|
},
|
|
"proposal_model": {
|
|
"path": str(model_path),
|
|
"sha256": model_sha256,
|
|
},
|
|
},
|
|
"parameters": {
|
|
"region": args.region,
|
|
"confidence": args.confidence,
|
|
"match_iou": args.match_iou,
|
|
"crop_scale": args.crop_scale,
|
|
"max_positive_per_tile": args.max_positive_per_tile,
|
|
"max_negative_per_tile": args.max_negative_per_tile,
|
|
"device": args.device,
|
|
"imgsz": args.imgsz,
|
|
},
|
|
"counts": dict(sorted(counts.items())),
|
|
"sample_counts": dict(sorted(sample_counts.items())),
|
|
"tile_count": len(tiles),
|
|
"crop_count": len(crop_entries),
|
|
"crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(),
|
|
"crops": crop_entries,
|
|
}
|
|
provenance["manifest_sha256"] = _immutable_payload_sha256(provenance)
|
|
provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME
|
|
_write_immutable_json(provenance_path, provenance)
|
|
|
|
evidence = {
|
|
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
|
|
"status": "ok",
|
|
"model": str(model_path),
|
|
"model_sha256": model_sha256,
|
|
"summary": str(args.summary),
|
|
"summary_sha256": source_summary_sha256,
|
|
"corpus_manifest": str(args.corpus_manifest),
|
|
"corpus_manifest_sha256": corpus_manifest_sha256,
|
|
"region": args.region,
|
|
"confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale,
|
|
"counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())),
|
|
"protected_samples_in_training": [], "tile_count": len(tiles),
|
|
"fixture_mode": bool(args.fixture_mode),
|
|
"proposal_dataset_provenance": str(provenance_path),
|
|
"proposal_dataset_provenance_sha256": sha256(provenance_path),
|
|
"proposal_dataset_manifest_sha256": provenance["manifest_sha256"],
|
|
}
|
|
_write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence)
|
|
print(json.dumps(evidence, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|