feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+52 -4
View File
@@ -26,6 +26,10 @@ from app.db.session import SessionLocal # noqa: E402
from app.models import Dataset # noqa: E402
from normalize_belgium_building_labels import normalize # noqa: E402
from training_dataset_eligibility import ( # noqa: E402
TRAINING_ELIGIBILITY_POLICY_VERSION,
training_pair_evidence,
)
REGION_SOURCES = {
"flanders": ({"digitaal_vlaanderen_orthophoto"}, "grb"),
@@ -52,7 +56,13 @@ def _dataset_path(dataset: Dataset) -> Path:
return path
def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]:
def _validate_pair(
sample: dict[str, Any],
raster: Dataset,
reference: Dataset,
*,
fixture_mode: bool,
) -> tuple[str, str, dict[str, Any]]:
region = str(sample.get("region") or "").lower()
if region not in REGION_SOURCES:
raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}")
@@ -66,7 +76,23 @@ def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset)
raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}")
if raster.status != "ready" or reference.status != "ready":
raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}")
return region, expected_reference
eligibility = training_pair_evidence(
raster=raster,
reference=reference,
fixture_mode=fixture_mode,
)
if not eligibility["eligible"]:
reasons = sorted(
{
reason
for role in ("raster", "reference")
for reason in eligibility[role]["reasons"]
}
)
raise SystemExit(
f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}"
)
return region, expected_reference, eligibility
def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]:
@@ -104,6 +130,14 @@ def main() -> int:
parser.add_argument("--min-label-px", type=float, default=3.0)
parser.add_argument("--merge-touching-roofs", action="store_true")
parser.add_argument("--freeze", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help=(
"Allow only explicitly marked fixture datasets with legacy provenance. "
"Never use this mode for an operational corpus."
),
)
args = parser.parse_args()
spec = json.loads(args.spec.read_text(encoding="utf-8-sig"))
@@ -129,7 +163,12 @@ def main() -> int:
reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"])))
if raster is None or reference is None:
raise SystemExit(f"Persisted Dataset pair not found for {slug}")
region, reference_source = _validate_pair(sample, raster, reference)
region, reference_source, eligibility = _validate_pair(
sample,
raster,
reference,
fixture_mode=args.fixture_mode,
)
raster_source = _dataset_path(raster)
reference_source_path = _dataset_path(reference)
sample_dir = pairs_dir / slug
@@ -173,14 +212,20 @@ def main() -> int:
"raster_sha256": sha256(raster_target),
"reference_sha256": sha256(normalized_target),
"label_audit_sha256": sha256(audit_target),
"training_eligibility": eligibility,
"bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326")
or sample.get("bbox_epsg4326"),
}
)
manifest = {
"schema_version": 1,
"schema_version": 2,
"dataset_version": args.version,
"immutable": bool(args.freeze),
"training_eligibility": {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "eligible",
"fixture_mode": bool(args.fixture_mode),
},
"samples": manifest_samples,
}
manifest_path = output_dir / "operator_samples_manifest.json"
@@ -192,10 +237,13 @@ def main() -> int:
if leakage_audit["status"] != "ok":
raise SystemExit("Spatial split leakage audit failed")
freeze = {
"schema_version": 2,
"dataset_version": args.version,
"manifest_sha256": sha256(manifest_path),
"sample_count": len(manifest_samples),
"immutable": bool(args.freeze),
"training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION,
"fixture_mode": bool(args.fixture_mode),
}
(output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
print(json.dumps(freeze))
+113 -4
View File
@@ -4,13 +4,49 @@
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from collections import Counter
from datetime import datetime
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import frozen_manifest_training_eligibility_failures # noqa: E402
REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test")
REGIONS = ("flanders", "wallonia", "brussels")
SHA256_LENGTH = 64
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def valid_review_timestamp(value: Any) -> bool:
if not isinstance(value, str) or not value.strip():
return False
try:
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
except ValueError:
return False
return parsed.tzinfo is not None
def valid_sha256(value: Any) -> bool:
return (
isinstance(value, str)
and len(value) == SHA256_LENGTH
and all(character in "0123456789abcdefABCDEF" for character in value)
)
def main() -> int:
@@ -19,8 +55,15 @@ def main() -> int:
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--review-decisions", type=Path)
args = parser.parse_args()
manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8"))
manifest_path = args.corpus_dir / "operator_samples_manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8"))
manifest_policy = manifest.get("training_eligibility")
fixture_mode = bool(manifest_policy.get("fixture_mode")) if isinstance(manifest_policy, dict) else False
eligibility_failures = frozen_manifest_training_eligibility_failures(
manifest_path,
fixture_mode=fixture_mode,
)
samples = manifest["samples"]
split_counts = Counter((sample["region"], sample["split"]) for sample in samples)
decision_counts: Counter[str] = Counter()
@@ -28,7 +71,7 @@ def main() -> int:
total_input = 0
total_accepted = 0
temporal_unknown = 0
failures: list[str] = []
failures: list[str] = list(eligibility_failures)
for region in REGIONS:
for split in REQUIRED_SPLITS:
minimum = 4 if split == "train" else 2
@@ -62,8 +105,12 @@ def main() -> int:
failures.append("spatial leakage audit failed")
reviewed = 0
review_complete = False
review_evidence_failures: list[str] = []
accepted_review_evidence: dict[str, dict[str, Any]] = {}
review_decisions_path: Path | None = None
if args.review_decisions and args.review_decisions.is_file():
decisions = json.loads(args.review_decisions.read_text(encoding="utf-8"))
review_decisions_path = args.review_decisions.resolve(strict=False)
decisions = json.loads(review_decisions_path.read_text(encoding="utf-8"))
by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])}
for item in review_queue:
decision = by_slug.get(item["sample_slug"])
@@ -71,13 +118,70 @@ def main() -> int:
item["decision"] = decision.get("decision")
item["reviewer"] = decision.get("reviewer")
item["notes"] = decision.get("notes")
if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"):
item["reviewed_at"] = decision.get("reviewed_at")
item["reviewed_artifact_path"] = decision.get("reviewed_artifact_path")
item["reviewed_artifact_sha256"] = decision.get("reviewed_artifact_sha256")
reviewed_artifact = (
Path(item["reviewed_artifact_path"]).expanduser().resolve(strict=False)
if isinstance(item.get("reviewed_artifact_path"), str)
and item["reviewed_artifact_path"].strip()
else None
)
has_evidence = (
item["decision"] == "accepted"
and isinstance(item.get("reviewer"), str)
and bool(item["reviewer"].strip())
and valid_review_timestamp(item.get("reviewed_at"))
and isinstance(item.get("reviewed_artifact_path"), str)
and bool(item["reviewed_artifact_path"].strip())
and valid_sha256(item.get("reviewed_artifact_sha256"))
and reviewed_artifact is not None
and reviewed_artifact.is_file()
and item["reviewed_artifact_sha256"] == sha256(reviewed_artifact)
)
if has_evidence:
reviewed += 1
accepted_review_evidence[str(item["sample_slug"])] = {
"reviewer": item["reviewer"].strip(),
"reviewed_at": item["reviewed_at"],
"reviewed_artifact_path": item["reviewed_artifact_path"],
"reviewed_artifact_sha256": item["reviewed_artifact_sha256"],
}
elif item["decision"] in {"accepted", "rejected"}:
review_evidence_failures.append(
f"{item['sample_slug']}:accepted review lacks reviewer/timestamp/artifact evidence"
)
review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue)
human_review_evidence: dict[str, Any] | None = None
if review_decisions_path is not None:
human_review_evidence = {
"review_decisions_path": str(review_decisions_path),
"review_decisions_sha256": sha256(review_decisions_path),
"required_sample_count": len(review_queue),
"accepted_sample_count": reviewed,
"accepted_sample_slugs": sorted(accepted_review_evidence),
"reviewer_ids": sorted(
{item["reviewer"] for item in accepted_review_evidence.values()}
),
"reviewed_at_by_sample": {
slug: accepted_review_evidence[slug]["reviewed_at"]
for slug in sorted(accepted_review_evidence)
},
"reviewed_artifact_path_by_sample": {
slug: accepted_review_evidence[slug]["reviewed_artifact_path"]
for slug in sorted(accepted_review_evidence)
},
"reviewed_artifact_sha256_by_sample": {
slug: accepted_review_evidence[slug]["reviewed_artifact_sha256"]
for slug in sorted(accepted_review_evidence)
},
}
status = "failed" if failures else ("ok" if review_complete else "needs_human_review")
report = {
"status": status,
"dataset_version": manifest["dataset_version"],
"corpus_manifest_path": str(manifest_path.resolve(strict=False)),
"corpus_manifest_sha256": sha256(manifest_path),
"manifest_immutable": manifest["immutable"],
"sample_count": len(samples),
"split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS},
@@ -86,8 +190,13 @@ def main() -> int:
"decision_counts": dict(sorted(decision_counts.items())),
"temporal_unknown_sample_count": temporal_unknown,
"spatial_leakage_status": leakage.get("status"),
"training_eligibility_status": manifest_policy.get("status") if isinstance(manifest_policy, dict) else None,
"training_eligibility_fixture_mode": fixture_mode,
"training_eligibility_failures": eligibility_failures,
"reviewed_sample_count": reviewed,
"review_complete": review_complete,
"human_review_evidence": human_review_evidence,
"review_evidence_failures": sorted(set(review_evidence_failures)),
"failures": failures,
"review_queue": review_queue,
}
@@ -9,7 +9,7 @@ import json
import sys
from collections import Counter
from pathlib import Path
from typing import Any
from typing import Any, Mapping
from PIL import Image
@@ -17,10 +17,21 @@ SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from evaluate_belgium_building_candidate import iou, read_references
from evaluate_belgium_building_candidate import iou, read_references # noqa: E402
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
)
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json"
PROPOSAL_DATASET_SCHEMA_VERSION = 1
def sha256(path: Path) -> str:
@@ -31,6 +42,125 @@ def sha256(path: Path) -> str:
return digest.hexdigest()
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str:
normalized = dict(payload)
normalized.pop(field, None)
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None:
"""Persist one immutable sidecar without silently replacing prior evidence."""
encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n"
if path.exists():
if path.read_text(encoding="utf-8") != encoded:
raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}")
return
path.write_text(encoded, encoding="utf-8")
def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str:
"""Require the proposal summary to name the exact frozen corpus bytes."""
expected = sha256(corpus_manifest_path)
observed = summary.get("source_manifest_sha256")
if observed != expected:
raise ValueError(
"proposal source summary is not bound to the supplied governed corpus manifest "
f"(expected {expected}, observed {observed!r})"
)
return expected
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
"""Validate frozen corpus evidence and re-check the live Dataset state.
This must run before importing Ultralytics/PyTorch so a revocation cannot
consume GPU work or create proposal crops.
"""
try:
manifest = assert_frozen_manifest_training_eligible(
corpus_manifest_path,
fixture_mode=fixture_mode,
verify_live=True,
)
except TrainingEligibilityError as exc:
raise ValueError(str(exc)) from exc
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
raise ValueError("governed corpus manifest must be a JSON object")
return manifest
def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]:
raw_path = tile.get(field)
if not isinstance(raw_path, str) or not raw_path.strip():
raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}")
path = Path(raw_path).expanduser().resolve(strict=False)
if not path.is_file():
raise ValueError(f"proposal source tile {field} is unavailable: {path}")
return path, sha256(path)
def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]:
"""Return the canonical source identity used for the pre-inference cache."""
return (
str(tile.get("sample_slug") or ""),
str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)),
str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)),
)
def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None:
"""Reject a mutable source changing between evidence capture and use."""
if sha256(path) != expected_sha256:
raise RuntimeError(f"proposal {role} changed after evidence capture: {path}")
def _crop_entry(
*,
output_dir: Path,
crop_path: Path,
tile: Mapping[str, Any],
label: str,
proposal_index: int,
proposal_score: float,
source_box_xyxy: tuple[float, float, float, float],
source_image_path: Path,
source_image_sha256: str,
source_label_path: Path,
source_label_sha256: str,
) -> dict[str, Any]:
relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix()
sample_slug = tile.get("sample_slug")
split = tile.get("split")
if not isinstance(sample_slug, str) or not sample_slug.strip():
raise ValueError("proposal source tile has no sample_slug")
if split not in {"train", "val"}:
raise ValueError(f"proposal crop has unsupported split: {split!r}")
return {
"relative_path": relative_path,
"sha256": sha256(crop_path),
"size_bytes": crop_path.stat().st_size,
"split": split,
"label": label,
"sample_slug": sample_slug,
"proposal_index": proposal_index,
"proposal_score": proposal_score,
"source_box_xyxy": [float(value) for value in source_box_xyxy],
"source_image_path": str(source_image_path),
"source_image_sha256": source_image_sha256,
"source_label_path": str(source_label_path),
"source_label_sha256": source_label_sha256,
}
def classify_proposals(
predictions: list[tuple[tuple[float, float, float, float], float]],
references: list[tuple[float, float, float, float]],
@@ -99,29 +229,74 @@ def main() -> int:
parser.add_argument("--max-negative-per-tile", type=int, default=24)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--imgsz", type=int, default=640)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
)
args = parser.parse_args()
if args.output_dir.exists():
parser.error(f"output already exists: {args.output_dir}")
from ultralytics import YOLO
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
summary = json.loads(args.summary.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc
if not isinstance(summary, dict):
raise SystemExit("proposal source summary must be a JSON object")
try:
source_release = assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode)
corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest)
except (ValueError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
tiles = eligible_tiles(summary, manifest, args.region)
model_path = args.model.expanduser().resolve(strict=False)
if not model_path.is_file():
raise SystemExit(f"proposal model is unavailable: {model_path}")
model_sha256 = sha256(model_path)
# Hash every input image/label before GPU inference. A crop sidecar then
# binds each generated JPEG to the exact source tile rather than its
# filename or an unverified aggregate summary.
source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {}
for tile in tiles:
image_path, image_sha256 = _source_file_evidence(tile, field="image_path")
label_path, label_sha256 = _source_file_evidence(tile, field="label_path")
key = _source_tile_key(tile)
source_evidence[key] = (image_path, image_sha256, label_path, label_sha256)
args.output_dir.mkdir(parents=True)
counts: Counter[str] = Counter()
sample_counts: Counter[str] = Counter()
model = YOLO(str(args.model))
crop_entries: list[dict[str, Any]] = []
# Importing the model is deliberately after the source-manifest and live
# Dataset checks above. A revoked corpus must not start GPU work.
from ultralytics import YOLO
model = YOLO(str(model_path))
for start in range(0, len(tiles), 16):
batch_tiles = tiles[start : start + 16]
batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles]
for image_path, image_sha256, label_path, label_sha256 in batch_sources:
_assert_file_unchanged(image_path, image_sha256, role="source image")
_assert_file_unchanged(label_path, label_sha256, role="source label")
results = model.predict(
[tile["image_path"] for tile in batch_tiles], conf=args.confidence,
[str(source[0]) for source in batch_sources], conf=args.confidence,
device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False,
)
for tile, result in zip(batch_tiles, results, strict=True):
with Image.open(tile["image_path"]) as opened:
image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)]
_assert_file_unchanged(image_path, image_sha256, role="source image")
_assert_file_unchanged(label_path, label_sha256, role="source label")
with Image.open(image_path) as opened:
source = opened.convert("RGB")
references = read_references(Path(tile["label_path"]), source.width, source.height)
references = read_references(label_path, source.width, source.height)
proposals = [
(tuple(map(float, box)), float(score))
for box, score in zip(
@@ -139,25 +314,113 @@ def main() -> int:
target_dir = args.output_dir / split / label
target_dir.mkdir(parents=True, exist_ok=True)
name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg"
crop_square(source, box, args.crop_scale).save(target_dir / name, quality=92)
crop_path = target_dir / name
if crop_path.exists():
raise RuntimeError(f"proposal crop identity collision: {crop_path}")
crop_square(source, box, args.crop_scale).save(crop_path, quality=92)
crop_entries.append(
_crop_entry(
output_dir=args.output_dir,
crop_path=crop_path,
tile=tile,
label=label,
proposal_index=proposal_index,
proposal_score=score,
source_box_xyxy=box,
source_image_path=image_path,
source_image_sha256=image_sha256,
source_label_path=label_path,
source_label_sha256=label_sha256,
)
)
counts[f"{split}/{label}"] += 1
sample_counts[tile["sample_slug"]] += 1
for split in ("train", "val"):
for label in ("positive", "negative"):
if counts[f"{split}/{label}"] == 0:
raise RuntimeError(f"empty proposal class: {split}/{label}")
crop_entries.sort(key=lambda item: str(item["relative_path"]))
if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries):
raise RuntimeError("proposal crop manifest contains duplicate relative paths")
source_summary_sha256 = sha256(args.summary)
corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json"
if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output
raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}")
provenance: dict[str, Any] = {
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
"status": "ok",
"immutable": True,
"dataset_kind": "building_proposal_classifier_crops",
"fixture_mode": bool(args.fixture_mode),
"governed_corpus_live_recheck": True,
"source": {
"corpus_manifest": {
"path": str(args.corpus_manifest.expanduser().resolve(strict=False)),
"sha256": corpus_manifest_sha256,
},
"corpus_freeze": {
"path": str(corpus_freeze_path.resolve(strict=False)),
"sha256": sha256(corpus_freeze_path),
},
"summary": {
"path": str(args.summary.expanduser().resolve(strict=False)),
"sha256": source_summary_sha256,
"source_manifest_sha256": summary["source_manifest_sha256"],
"training_release_manifest": summary["training_release_manifest"],
"training_release_manifest_sha256": summary["training_release_manifest_sha256"],
"training_asset_manifest": summary["training_asset_manifest"],
},
"training_release": {
"dataset_yaml_path": source_release["dataset_yaml"]["path"],
"dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"],
"corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
},
"proposal_model": {
"path": str(model_path),
"sha256": model_sha256,
},
},
"parameters": {
"region": args.region,
"confidence": args.confidence,
"match_iou": args.match_iou,
"crop_scale": args.crop_scale,
"max_positive_per_tile": args.max_positive_per_tile,
"max_negative_per_tile": args.max_negative_per_tile,
"device": args.device,
"imgsz": args.imgsz,
},
"counts": dict(sorted(counts.items())),
"sample_counts": dict(sorted(sample_counts.items())),
"tile_count": len(tiles),
"crop_count": len(crop_entries),
"crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(),
"crops": crop_entries,
}
provenance["manifest_sha256"] = _immutable_payload_sha256(provenance)
provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME
_write_immutable_json(provenance_path, provenance)
evidence = {
"schema_version": 1, "status": "ok", "model": str(args.model),
"model_sha256": sha256(args.model), "summary": str(args.summary),
"summary_sha256": sha256(args.summary), "corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest), "region": args.region,
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
"status": "ok",
"model": str(model_path),
"model_sha256": model_sha256,
"summary": str(args.summary),
"summary_sha256": source_summary_sha256,
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": corpus_manifest_sha256,
"region": args.region,
"confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale,
"counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())),
"protected_samples_in_training": [], "tile_count": len(tiles),
"fixture_mode": bool(args.fixture_mode),
"proposal_dataset_provenance": str(provenance_path),
"proposal_dataset_provenance_sha256": sha256(provenance_path),
"proposal_dataset_manifest_sha256": provenance["manifest_sha256"],
}
(args.output_dir / "proposal-dataset-evidence.json").write_text(
json.dumps(evidence, indent=2), encoding="utf-8"
)
_write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence)
print(json.dumps(evidence, indent=2))
return 0
+67 -4
View File
@@ -8,10 +8,25 @@ import hashlib
import json
import math
import re
import sys
from collections import Counter
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
create_training_release_manifest,
)
PRECISION_NEGATIVE_CONTEXTS = {
"coastal-urban": {"port-hard-negative", "dunes-negative"},
@@ -80,6 +95,17 @@ def build_sampling(
) -> tuple[list[str], dict[str, Any]]:
if assessment.get("status") != "continue_training_loop":
raise ValueError("Failure-driven sampling requires a failed assessment")
protected_feedback = [
role
for role in ("test", "background")
if assessment.get(role) is not None
]
if protected_feedback:
raise ValueError(
"Failure-driven sampling is prohibited after protected "
+ "/".join(protected_feedback)
+ " evidence was opened"
)
if min(
positive_repeat,
negative_repeat,
@@ -97,9 +123,9 @@ def build_sampling(
samples = {item["sample_slug"]: item for item in manifest["samples"]}
gates = assessment["gates"]
evaluation = assessment.get("test") or assessment.get("calibration")
evaluation = assessment.get("calibration")
if not evaluation or "regions" not in evaluation:
raise ValueError("Assessment has no regional calibration or test evidence")
raise ValueError("Assessment has no regional calibration evidence")
regions = evaluation["regions"]
weak_recall_regions = {
region
@@ -246,7 +272,7 @@ def build_sampling(
"schema_version": 1,
"status": "ok",
"strategy": "failed-region-positive-and-hard-negative-repeat",
"failure_evidence_source": "test" if assessment.get("test") else "calibration",
"failure_evidence_source": "calibration",
"weak_recall_regions": sorted(weak_recall_regions),
"weak_precision_regions": sorted(weak_precision_regions),
"recall_dominant_regions": sorted(recall_dominant_regions),
@@ -290,6 +316,11 @@ def main() -> int:
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--assessment", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument(
"--review-audit",
type=Path,
help="Passed corpus audit containing accepted human-review evidence for the frozen corpus.",
)
parser.add_argument("--positive-repeat", type=int, default=3)
parser.add_argument("--negative-repeat", type=int, default=4)
parser.add_argument("--precision-positive-repeat", type=int, default=1)
@@ -299,8 +330,26 @@ def main() -> int:
parser.add_argument("--sampling-round", type=int)
parser.add_argument("--precision-guard-band", type=float, default=0.03)
parser.add_argument("--recall-guard-band", type=float, default=0.03)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational sampling.",
)
args = parser.parse_args()
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
source_release = assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
assessment = json.loads(args.assessment.read_text(encoding="utf-8"))
@@ -325,7 +374,7 @@ def main() -> int:
args.output_dir.mkdir(parents=True, exist_ok=True)
train_list = args.output_dir / "train-failure-driven.txt"
train_list.write_text("\n".join(paths) + "\n", encoding="utf-8")
source_yaml = args.summary.parent / "dataset.yaml"
source_yaml = Path(str(source_release["dataset_yaml"]["path"]))
val_source = dataset_validation_source(source_yaml)
dataset_yaml = args.output_dir / "dataset.yaml"
dataset_yaml.write_text(
@@ -335,6 +384,15 @@ def main() -> int:
"names:\n 0: building\n",
encoding="utf-8",
)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.corpus_manifest,
review_audit_path=args.review_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
metadata.update(
{
"summary": str(args.summary),
@@ -346,6 +404,11 @@ def main() -> int:
"source_dataset_yaml": str(source_yaml),
"train_list": str(train_list),
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"fixture_mode": bool(args.fixture_mode),
}
)
output = args.output_dir / "failure-driven-sampling.json"
+48
View File
@@ -7,11 +7,24 @@ import argparse
import hashlib
import json
import shutil
import sys
from pathlib import Path
from PIL import Image
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
file_sha256,
training_release_paths,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
@@ -23,9 +36,25 @@ def sha256(path: Path) -> str:
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only release; never creates a production-ready derived dataset.",
)
parser.add_argument("--force", action="store_true")
args = parser.parse_args()
try:
release = assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -54,6 +83,16 @@ def main() -> int:
f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
# The source release binds the original image bytes. These transformed
# bytes cannot inherit it, so remove its operational release pointers and
# retain them only as explicitly non-trainable parent evidence below.
for field_name in (
"training_release_manifest",
"training_release_manifest_sha256",
"training_release_freeze",
"training_asset_manifest",
):
summary.pop(field_name, None)
summary.update(
{
"output_dir": str(args.output_dir),
@@ -71,10 +110,19 @@ def main() -> int:
"preprocessing": "luminance_rgb_replicated",
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"source_training_release_sha256": file_sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"source_corpus_manifest_sha256": release["corpus"]["manifest_sha256"],
"converted_tile_count": converted,
"output_summary": str(output_summary),
"output_summary_sha256": sha256(output_summary),
"dataset_yaml": str(dataset_yaml),
"training_eligible": False,
"training_eligibility_reason": (
"Derived image bytes require a new governed corpus, validation report and immutable training release."
),
}
(args.output_dir / "grayscale-dataset-evidence.json").write_text(
json.dumps(evidence, indent=2), encoding="utf-8"
+52
View File
@@ -6,10 +6,25 @@ from __future__ import annotations
import argparse
import hashlib
import json
import sys
from collections import Counter
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
create_training_release_manifest,
)
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
@@ -138,11 +153,34 @@ def main() -> int:
parser.add_argument("--priority-context", action="append", default=[])
parser.add_argument("--priority-repeat", type=int, default=2)
parser.add_argument("--negative-repeat", type=int, default=2)
parser.add_argument(
"--review-audit",
type=Path,
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
)
args = parser.parse_args()
if args.priority_repeat < 1 or args.negative_repeat < 1:
parser.error("repeat counts must be positive")
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
train, val, evidence = build(
summary=summary,
manifest=manifest,
@@ -161,6 +199,15 @@ def main() -> int:
f"path: /\ntrain: {train_path}\nval: {val_path}\nnames:\n 0: building\n",
encoding="utf-8",
)
try:
release_paths = create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=args.corpus_manifest,
review_audit_path=args.review_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
evidence.update({
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
@@ -169,6 +216,11 @@ def main() -> int:
"train_sha256": sha256(train_path),
"validation_sha256": sha256(val_path),
"dataset_yaml": str(yaml_path),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"fixture_mode": bool(args.fixture_mode),
})
encoded_evidence = json.dumps(evidence, indent=2)
(args.output_dir / "regional-dataset-evidence.json").write_text(encoded_evidence, encoding="utf-8")
@@ -6,8 +6,23 @@ from __future__ import annotations
import argparse
import hashlib
import json
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
create_training_release_manifest,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -31,12 +46,35 @@ def main() -> int:
default=0,
help="Include each train tile outside the expert region this many times.",
)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
)
parser.add_argument(
"--review-audit",
type=Path,
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
)
args = parser.parse_args()
if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
samples = {item["sample_slug"]: item for item in manifest["samples"]}
paths: list[str] = []
selected_samples: set[str] = set()
@@ -69,6 +107,15 @@ def main() -> int:
f"val: {args.summary.parent / 'images' / 'val'}\nnames:\n 0: building\n",
encoding="utf-8",
)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.corpus_manifest,
review_audit_path=args.review_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
evidence = {
"schema_version": 1,
"status": "ok",
@@ -88,6 +135,11 @@ def main() -> int:
"protected_samples_in_training": [],
"train_list": str(train_list),
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"fixture_mode": bool(args.fixture_mode),
}
evidence_path = args.output_dir / "regional-expert-dataset.json"
evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8")
+47 -1
View File
@@ -9,6 +9,7 @@ new provider data.
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
@@ -16,6 +17,19 @@ import sys
from pathlib import Path
from typing import Any, Iterable
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
create_training_release_manifest,
)
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
@@ -24,6 +38,14 @@ Transformer: Any = None
Image: Any = None
def file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Export operator real-data samples to a YOLO detection dataset.",
@@ -45,6 +67,12 @@ def parse_args() -> argparse.Namespace:
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
)
parser.add_argument(
"--review-audit",
type=Path,
required=True,
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
)
parser.add_argument(
"--force",
action="store_true",
@@ -218,12 +246,16 @@ def ensure_yolo_directories(output_dir: Path) -> None:
def main() -> int:
args = parse_args()
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
try:
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
except TrainingEligibilityError as exc:
raise SystemExit(str(exc)) from exc
ensure_dependencies()
if args.force and args.output_dir.exists():
shutil.rmtree(args.output_dir)
args.output_dir.mkdir(parents=True, exist_ok=True)
ensure_yolo_directories(args.output_dir)
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
samples = manifest.get("samples") or []
if not samples:
raise SystemExit("Operator sample manifest contains no samples")
@@ -234,12 +266,26 @@ def main() -> int:
if not any(item["split"] == "val" for item in exported):
raise SystemExit("YOLO dataset export produced no validation samples")
dataset_yaml = write_dataset_yaml(args.output_dir)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.manifest_path,
review_audit_path=args.review_audit,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
summary = {
"status": "ok",
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"output_dir": str(args.output_dir),
"class_names": ["building"],
"sample_count": len(exported),
"source_manifest": str(args.manifest_path),
"source_manifest_sha256": file_sha256(args.manifest_path),
"train_sample_count": sum(1 for item in exported if item["split"] == "train"),
"val_sample_count": sum(1 for item in exported if item["split"] == "val"),
"label_count": sum(item["label_count"] for item in exported),
+46 -1
View File
@@ -18,6 +18,19 @@ import sys
from pathlib import Path
from typing import Any, Iterable
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
create_training_release_manifest,
)
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
@@ -45,6 +58,14 @@ Transformer: Any = None
Image: Any = None
def file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
@dataclass(frozen=True)
class TileWindow:
row_off: int
@@ -87,6 +108,12 @@ def parse_args() -> argparse.Namespace:
default=os.environ.get("OPERATOR_YOLO_REFERENCE_SOURCE", DEFAULT_REFERENCE_SOURCE),
help="Required source_name in reference GeoJSON features.",
)
parser.add_argument(
"--review-audit",
type=Path,
required=True,
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
)
parser.add_argument(
"--reference-layer",
default=os.environ.get("OPERATOR_YOLO_REFERENCE_LAYER", DEFAULT_REFERENCE_LAYER),
@@ -626,13 +653,17 @@ def main() -> int:
):
if not value or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in value):
raise SystemExit(f"YOLO {label} must be a non-empty canonical slug")
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
try:
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
except TrainingEligibilityError as exc:
raise SystemExit(str(exc)) from exc
ensure_dependencies()
if args.force and args.output_dir.exists():
shutil.rmtree(args.output_dir)
args.output_dir.mkdir(parents=True, exist_ok=True)
ensure_yolo_directories(args.output_dir)
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
manifest_samples = manifest.get("samples") or []
if not manifest_samples:
raise SystemExit("Operator sample manifest contains no samples")
@@ -672,6 +703,14 @@ def main() -> int:
if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles):
raise SystemExit("YOLO tile dataset export produced no validation tiles")
dataset_yaml = write_dataset_yaml(args.output_dir, class_name)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.manifest_path,
review_audit_path=args.review_audit,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
@@ -684,6 +723,10 @@ def main() -> int:
summary = {
"status": "ok",
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"output_dir": str(args.output_dir),
"class_names": [class_name],
"reference_source": reference_source,
@@ -697,6 +740,8 @@ def main() -> int:
"drop_low_variance_negatives": args.drop_low_variance_negatives,
"blank_range_threshold": args.blank_range_threshold,
"source_manifest_sample_count": len(manifest_samples),
"source_manifest": str(args.manifest_path),
"source_manifest_sha256": file_sha256(args.manifest_path),
"source_sample_count": len(samples),
"selected_sample_slugs": sorted(
str(sample.get("sample_slug") or "").strip().lower() for sample in samples
@@ -16,6 +16,12 @@ from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import TRAINING_ELIGIBILITY_POLICY_VERSION # noqa: E402
WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
@@ -803,6 +809,14 @@ def main() -> int:
manifest = {
"schema_version": 2,
"description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
"training_eligibility": {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "not_eligible",
"reason": (
"Direct provider downloads are QA-only until re-ingested through the governed "
"dataset source registry, contract validator and provenance snapshot flow."
),
},
"output_dir": str(output_dir),
"sample_width": args.width,
"sample_height": args.height,
+1 -1
View File
@@ -626,7 +626,7 @@ def provision_dataset(
from app.services.dataset_service import DatasetService
partition_checksums = {
summary["nis_code"]: summary["sha256"]
summary["filename"]: summary["sha256"]
for summary in manifest["partitions"]
}
metadata_json = {
+1 -2
View File
@@ -43,7 +43,6 @@ from provision_regional_grb_buildings import (
list_paginated_items,
next_page_url,
observed_at,
response_data,
reusable_manifest,
safe_slug,
sha256_file,
@@ -758,7 +757,7 @@ def provision_dataset(
"source_urls": manifest["grb_source_urls"],
"artifact_sha256": manifest["artifact_sha256"],
"artifact_size_bytes": manifest["artifact_size_bytes"],
"partition_checksums": {summary["nis_code"]: summary["sha256"] for summary in manifest["partitions"]},
"partition_checksums": {summary["filename"]: summary["sha256"] for summary in manifest["partitions"]},
"partition_assignment_rule": manifest["partition_assignment_rule"],
"reference_truncated": False,
}
+50 -1
View File
@@ -10,8 +10,20 @@ import json
import math
import os
import shutil
import sys
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
file_sha256,
training_release_paths,
)
def sha256(path: Path) -> str:
@@ -97,6 +109,8 @@ def link_or_copy(source: Path, target: Path) -> None:
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--device", default="cuda:0")
@@ -110,7 +124,21 @@ def main() -> int:
parser.add_argument("--max-prompts-per-pass", type=int, default=96)
parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain")
parser.add_argument("--force", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only source release; refined labels remain non-trainable.",
)
args = parser.parse_args()
try:
source_release = assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -194,6 +222,16 @@ def main() -> int:
print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True)
output_summary = dict(summary)
# SAM changes label bytes and semantics. It therefore cannot inherit the
# source release; keep the parent evidence separately and require a new
# governed corpus/review/release before any training consumer can use it.
for field_name in (
"training_release_manifest",
"training_release_manifest_sha256",
"training_release_freeze",
"training_asset_manifest",
):
output_summary.pop(field_name, None)
output_summary.update(
{
"output_dir": str(args.output_dir),
@@ -207,6 +245,10 @@ def main() -> int:
"label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"],
"positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles),
"negative_tile_count": sum(tile["is_negative"] for tile in output_tiles),
"training_eligible": False,
"training_eligibility_reason": (
"SAM-refined labels require a new governed corpus, validation report, human review and immutable training release."
),
}
)
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
@@ -220,6 +262,11 @@ def main() -> int:
"status": "ok",
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"source_training_release_sha256": file_sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"source_corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
"sam_model": str(args.model),
"sam_model_sha256": sha256(args.model),
"device": args.device,
@@ -236,6 +283,8 @@ def main() -> int:
"fallback_label_count": fallback_count,
"dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0,
"fallback_reason_counts": reason_counts,
"training_eligible": False,
"fixture_mode": bool(args.fixture_mode),
}
(args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8")
print(json.dumps(evidence, indent=2))
@@ -7,6 +7,7 @@ import argparse
import hashlib
import json
import shutil
import sys
from collections import Counter
from pathlib import Path
from typing import Any
@@ -15,6 +16,19 @@ from pyproj import Transformer
from shapely.geometry import box
from shapely.ops import transform as shapely_transform
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -73,6 +87,11 @@ def main() -> int:
parser.add_argument("--internal-val-samples", required=True)
parser.add_argument("--version", required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only source corpus; never use for operational holdout rotation.",
)
args = parser.parse_args()
role_slugs = {
@@ -86,6 +105,20 @@ def main() -> int:
raise SystemExit("rotated holdout lists overlap")
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
for path in args.source_summary:
assert_yolo_summary_bound_to_embedded_training_release(
summary_path=path,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
source_samples = {item["sample_slug"]: item for item in manifest["samples"]}
missing = sorted(all_holdouts - source_samples.keys())
if missing:
@@ -126,6 +159,18 @@ def main() -> int:
args.output_dir.mkdir(parents=True, exist_ok=True)
manifest_path = args.output_dir / "operator_samples_manifest.json"
write_json(manifest_path, rotated_manifest)
write_json(
args.output_dir / "corpus-freeze.json",
{
"schema_version": 2,
"dataset_version": rotated_manifest["dataset_version"],
"manifest_sha256": sha256(manifest_path),
"sample_count": len(rotated_manifest["samples"]),
"immutable": True,
"training_eligibility_policy": rotated_manifest["training_eligibility"]["policy_version"],
"fixture_mode": bool(args.fixture_mode),
},
)
leakage = audit_spatial_leakage(rotated_manifest["samples"])
write_json(args.output_dir / "spatial-leakage-audit.json", leakage)
if leakage["status"] != "ok":
@@ -193,6 +238,11 @@ def main() -> int:
"fit_samples_in_internal_validation": sorted(
{tile["sample_slug"] for tile in train_tiles} & {tile["sample_slug"] for tile in internal_val_tiles}
),
"fixture_mode": bool(args.fixture_mode),
"training_eligible": False,
"training_eligibility_reason": (
"A rotated split needs a new human review, label contract validation and immutable training release."
),
}
if evidence["protected_samples_in_training"]:
raise SystemExit("protected samples leaked into rotated training lists")
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Emit reproducible local evidence for the Phase-2 data foundation.
The collector does not connect to PostgreSQL, source providers, a GPU or any
training corpus. It inventories the versioned source-policy and contract
definitions that are present in this checkout. Runtime migration/application
evidence is recorded separately because it needs an explicitly chosen target
database and must never be inferred from this static report.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from collections import Counter
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
BACKEND = ROOT / "backend"
if str(BACKEND) not in sys.path:
sys.path.insert(0, str(BACKEND))
from app.services.data_contract_validation import build_default_data_contract_registry # noqa: E402
from app.services.source_registry_service import SERVER_OWNED_SOURCE_DEFINITIONS # noqa: E402
def _git_value(*args: str) -> str | None:
try:
return subprocess.check_output(
["git", *args],
cwd=ROOT,
text=True,
stderr=subprocess.DEVNULL,
).strip() or None
except (OSError, subprocess.CalledProcessError):
return None
def collect() -> dict[str, Any]:
definitions = list(SERVER_OWNED_SOURCE_DEFINITIONS.values())
contracts = build_default_data_contract_registry().registered_contracts()
classification_counts = Counter(item.classification for item in definitions)
source_items = [
{
"source_key": item.source_key,
"classification": item.classification,
"authority_name": item.authority_name,
"authority_scope": item.authority_scope,
"provider_adapter_key": item.provider_adapter_key,
"default_crs": item.default_crs,
"default_units": item.default_units,
"freshness_status": item.freshness_status,
"ingest_status": item.ingest_status,
"ground_truth_allowed": bool((item.usage_policy or {}).get("ground_truth_allowed")),
"training_allowed": bool((item.usage_policy or {}).get("training_allowed")),
}
for item in sorted(definitions, key=lambda definition: definition.source_key)
]
contract_items = [
{
"key": item.key,
"version": item.version,
"kind": item.kind.value,
"fingerprint_sha256": item.fingerprint(),
"canonical_storage_crs": item.canonical_storage_crs,
"accepted_source_crs": sorted(item.accepted_source_crs),
"required_metadata_fields": list(item.required_metadata_fields),
"requires_source_registry": item.lineage_rules.require_source_registry,
"requires_source_snapshot": item.lineage_rules.require_source_snapshot,
"requires_upstream_assets": item.lineage_rules.require_upstream_assets,
}
for item in sorted(contracts, key=lambda contract: (contract.kind.value, contract.key, contract.version))
]
return {
"schema_version": 1,
"program": "GeoIntel Accuracy Improvement Program",
"phase": "P2",
"collected_at": datetime.now(UTC).isoformat(),
"repository": {
"branch": _git_value("branch", "--show-current"),
"head": _git_value("rev-parse", "HEAD"),
"dirty": bool(_git_value("status", "--porcelain")),
},
"scope": "Belgium and the Belgian North Sea",
"migration_revision": "202608010001",
"source_registry": {
"definition_count": len(source_items),
"classification_counts": dict(sorted(classification_counts.items())),
"required_building_policy": {
"grb_primary_building_validation": SERVER_OWNED_SOURCE_DEFINITIONS["grb"].usage_policy[
"validation_authority"
].get("building_validation"),
"buildings_register_classification": SERVER_OWNED_SOURCE_DEFINITIONS[
"digitaal_vlaanderen_buildings_addresses_register"
].classification,
"sentinel_2_classification": SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"].classification,
"dhmv_classification": SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"].classification,
"osm_ground_truth_allowed": SERVER_OWNED_SOURCE_DEFINITIONS["osm"].usage_policy[
"ground_truth_allowed"
],
},
"definitions": source_items,
},
"data_contracts": contract_items,
"claim_boundary": (
"Static policy/contract inventory only. It does not attest that a database migration ran, "
"that legacy rows are complete, or that a model/corpus is release-ready."
),
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"--output",
type=Path,
default=ROOT / "artifacts" / "evidence" / "accuracy" / "P2" / "source-contract-inventory.json",
)
args = parser.parse_args()
result = collect()
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "source_count": result["source_registry"]["definition_count"]}))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+172 -5
View File
@@ -13,6 +13,21 @@ from datetime import UTC, datetime
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_training_release_eligible,
human_review_audit_failures,
training_release_paths,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -32,11 +47,14 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
def dataset_audit_failures(
audit: dict[str, Any],
train_quality_audit: dict[str, Any],
*,
fixture_mode: bool = False,
) -> list[str]:
"""Return automated corpus blockers while leaving final human review deferred."""
"""Return fail-closed corpus blockers, including human review in normal mode."""
failures = [str(item) for item in audit.get("failures") or []]
status = audit.get("status")
if status not in {"ok", "needs_human_review"}:
permitted_statuses = {"ok", "needs_human_review"} if fixture_mode else {"ok"}
if status not in permitted_statuses:
failures.append(f"unsupported audit status: {status}")
if audit.get("manifest_immutable") is not True:
failures.append("corpus manifest is not immutable")
@@ -50,9 +68,56 @@ def dataset_audit_failures(
failures.append("train tile quality audit contains missing label files")
if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0:
failures.append("dataset contains blank/low-variance positive tiles")
if not fixture_mode:
failures.extend(human_review_audit_failures(audit))
return failures
def verify_training_inputs(
*,
train_yaml: Path,
corpus_manifest: Path,
fixture_mode: bool,
) -> dict[str, Any]:
"""Re-check every immutable input before initial, retry or resume training."""
try:
assert_frozen_manifest_training_eligible(
corpus_manifest,
fixture_mode=fixture_mode,
verify_live=True,
)
return assert_training_release_eligible(
train_yaml=train_yaml,
corpus_manifest=corpus_manifest,
fixture_mode=fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise TrainingReleaseError(str(exc)) from exc
def assert_dataset_audit_bound_to_release(
*,
release: dict[str, Any],
dataset_audit: Path,
fixture_mode: bool,
) -> None:
"""Do not let a caller swap the reviewed corpus audit after release sealing."""
if fixture_mode:
return
review = release.get("human_review")
if not isinstance(review, dict):
raise TrainingReleaseError("Training release has no human-review audit binding")
review_audit_path = review.get("audit_path")
if not isinstance(review_audit_path, str) or not review_audit_path:
raise TrainingReleaseError("Training release human-review audit path is missing")
if Path(review_audit_path).resolve(strict=False) != dataset_audit.resolve(strict=False):
raise TrainingReleaseError(
"--dataset-audit does not match the immutable training-release human-review audit"
)
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
"""Choose a threshold without consulting test or background evidence."""
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
@@ -111,6 +176,16 @@ def rejected_candidate_score(assessment: dict[str, Any]) -> tuple[float, ...]:
return tuple(normalized)
def protected_feedback_roles(assessment: dict[str, Any]) -> list[str]:
"""Return protected evidence roles that make iterative retraining illegal."""
return [
role
for role in ("test", "background")
if assessment.get(role) is not None
]
def training_command(
yolo: str,
*,
@@ -183,17 +258,23 @@ def failure_sampling_command(
corpus_manifest: Path,
assessment: Path,
output_dir: Path,
review_audit: Path,
sampling_round: int = 0,
fixture_mode: bool = False,
) -> list[str]:
return [
command = [
sys.executable,
str(scripts_dir / "build_failure_driven_yolo_sampling.py"),
"--summary", str(train_summary),
"--corpus-manifest", str(corpus_manifest),
"--assessment", str(assessment),
"--output-dir", str(output_dir),
"--review-audit", str(review_audit),
"--sampling-round", str(sampling_round),
]
if fixture_mode:
command.append("--fixture-mode")
return command
def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]:
@@ -259,6 +340,14 @@ def main() -> int:
parser.add_argument("--min-region-recall", type=float, default=0.4)
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
parser.add_argument("--dry-run", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help=(
"Accept an explicitly fixture-only corpus manifest. "
"This mode is prohibited for operational training."
),
)
parser.add_argument(
"--evaluate-initial-model",
action="store_true",
@@ -267,9 +356,29 @@ def main() -> int:
args = parser.parse_args()
if args.iterations < 1:
raise SystemExit("--iterations must be positive")
try:
initial_release = verify_training_inputs(
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
try:
assert_dataset_audit_bound_to_release(
release=initial_release,
dataset_audit=args.dataset_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8"))
audit_failures = dataset_audit_failures(dataset_audit, train_quality_audit)
audit_failures = dataset_audit_failures(
dataset_audit,
train_quality_audit,
fixture_mode=args.fixture_mode,
)
if audit_failures:
raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
@@ -285,6 +394,13 @@ def main() -> int:
"train_quality_audit": str(args.train_quality_audit),
"train_quality_audit_sha256": sha256(args.train_quality_audit),
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"initial_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"initial_training_release_sha256": sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"initial_training_release_contract": initial_release["contract_version"],
"fixture_mode": bool(args.fixture_mode),
"iterations": [],
}
if state_path.is_file():
@@ -304,6 +420,26 @@ def main() -> int:
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
partial_checkpoint = train_run / "weights" / "last.pt"
resume_partial = not evaluate_existing and partial_checkpoint.is_file()
try:
release = verify_training_inputs(
train_yaml=train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Training inputs changed before {name}; refusing initial/retry/resume execution: {exc}"
) from exc
try:
assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=args.dataset_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Training audit changed before {name}; refusing initial/retry/resume execution: {exc}"
) from exc
command = None if evaluate_existing else (
resumable_training_command(args.yolo, partial_checkpoint)
if resume_partial else training_command(
@@ -454,11 +590,27 @@ def main() -> int:
"candidate_sha256": sha256(candidate),
"training_skipped_for_existing_checkpoint": evaluate_existing,
"training_resumed_from_partial_checkpoint": resume_partial,
"training_release": str(training_release_paths(train_yaml)["release_manifest"]),
"training_release_sha256": sha256(training_release_paths(train_yaml)["release_manifest"]),
"training_release_asset_manifest_sha256": release["asset_manifest"]["sha256"],
"assessment": str(assessment),
"status": decision["status"],
"failures": decision["failures"],
}
state["iterations"].append(record)
protected_feedback = protected_feedback_roles(decision)
if decision["status"] != "training_complete" and protected_feedback:
record["protected_feedback_blocked"] = protected_feedback
record["retraining_prohibited"] = True
state["status"] = "protected_evaluation_rejected"
state["stopped_at"] = datetime.now(UTC).isoformat()
state["stop_reason"] = (
"Protected test/background evidence was opened for a rejected candidate; "
"its results cannot generate another training YAML."
)
write_json(state_path, state)
print(json.dumps(state, indent=2))
return 3
score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else ()
incumbent_score = tuple(state.get("incumbent_rejected_score", ()))
if not incumbent_score or score > incumbent_score:
@@ -482,14 +634,29 @@ def main() -> int:
corpus_manifest=args.corpus_manifest,
assessment=assessment,
output_dir=sampling_dir,
review_audit=args.dataset_audit,
sampling_round=index,
fixture_mode=args.fixture_mode,
),
iteration_dir / "failure-driven-sampling.log",
)
sampling_evidence = sampling_dir / "failure-driven-sampling.json"
next_train_yaml = sampling_dir / "dataset.yaml"
if not sampling_evidence.is_file() or not next_train_yaml.is_file():
next_release_paths = training_release_paths(next_train_yaml)
if not sampling_evidence.is_file() or not next_train_yaml.is_file() or not all(
path.is_file() for path in next_release_paths.values()
):
raise RuntimeError("Failure-driven sampling produced incomplete evidence")
try:
verify_training_inputs(
train_yaml=next_train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Failure-driven sampling produced an unbound training release: {exc}"
) from exc
record["failure_driven_sampling"] = str(sampling_evidence)
record["failure_driven_sampling_sha256"] = sha256(sampling_evidence)
record["next_train_yaml"] = str(next_train_yaml)
+48 -5
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import argparse
import json
import sys
from datetime import UTC, datetime
from hashlib import sha256
from pathlib import Path
from uuid import uuid4
@@ -12,9 +14,10 @@ BACKEND_ROOT = ROOT / "backend"
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from app.models import Area, Dataset, Metric, QualityCheck # noqa: E402
from app.models import Dataset, Metric, QualityCheck, SourceRegistry, SourceSnapshot # noqa: E402
from app.services.qa_service import QaService # noqa: E402
from app.services.quality_service import QualityService # noqa: E402
from app.services.source_registry_service import SourceRegistryService # noqa: E402
class BenchmarkSession:
@@ -55,18 +58,58 @@ def _load_manifest() -> dict:
def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
# The benchmark is still synthetic evidence, but it intentionally models
# the same complete Phase 2 provenance binding required at the QA service
# boundary. The candidate is a derived fixture; the reference simulates
# a governed GRB building snapshot. No legacy fixture exception is used.
source_key = "grb" if role == "reference" else "derived"
source = SourceRegistry(
id=uuid4(),
**SourceRegistryService.definition_for(source_key).as_model_values(),
)
checksum = sha256(path.read_bytes()).hexdigest()
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=source.id,
snapshot_key=f"golden-qa:{source_key}:{checksum}",
checksum_sha256=checksum,
fetched_at=datetime.now(UTC),
crs="EPSG:4326",
units=source.default_units,
spatial_resolution_json={"status": "fixture"},
temporal_coverage_json={"status": "fixture"},
geographic_coverage_json={"scope": "golden-qa-fixture"},
observed_schema_json={"dataset_type": "vector", "fixture_mode": True},
freshness_status="current",
ingest_status="ingested",
known_limitations_json=["Synthetic golden benchmark fixture; not production evidence."],
snapshot_metadata_json={"fixture_mode": True, "benchmark": "golden-qa"},
)
return Dataset(
id=dataset_id,
project_id=project_id,
name=name,
dataset_type="vector",
source="golden_fixture",
dataset_role=role,
source_name="fixture",
source="governed golden benchmark fixture",
dataset_role="reference" if role == "reference" else "derived",
source_name=source.source_key,
reference_layer_name="buildings" if role == "reference" else None,
storage_path=str(path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
checksum_sha256=checksum,
source_registry_id=source.id,
source_snapshot_id=snapshot.id,
source_registry=source,
source_snapshot=snapshot,
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
metadata_json={"crs_assumed": False, "fixture_mode": True},
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
status="ready",
)
+4
View File
@@ -109,6 +109,10 @@ ${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py
${PYTHON_BIN} -m py_compile scripts/manage_grb_refresh.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
${PYTHON_BIN} -m py_compile scripts/training_dataset_eligibility.py
${PYTHON_BIN} -m py_compile scripts/training_release_manifest.py
${PYTHON_BIN} -m py_compile scripts/run_belgium_building_training_loop.py
${PYTHON_BIN} -m py_compile scripts/build_failure_driven_yolo_sampling.py
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
${PYTHON_BIN} -m py_compile scripts/build_mol_operational_benchmark_report.py
@@ -6,10 +6,20 @@ from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_training_release_eligible,
)
def container_running(container: str) -> bool:
result = subprocess.run(
@@ -44,11 +54,33 @@ def load_completion_command(path: Path) -> list[str]:
return command
def verify_training_release(
*,
train_yaml: Path,
corpus_manifest: Path,
fixture_mode: bool,
) -> None:
"""Re-check the exact release before each detached resume command."""
assert_training_release_eligible(
train_yaml=train_yaml,
corpus_manifest=corpus_manifest,
fixture_mode=fixture_mode,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--container", required=True)
parser.add_argument("--host-run-dir", type=Path, required=True)
parser.add_argument("--container-checkpoint", required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Only valid for an explicitly fixture-only frozen corpus release.",
)
parser.add_argument("--run-marker", required=True)
parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
parser.add_argument("--poll-seconds", type=int, default=30)
@@ -101,6 +133,17 @@ def main() -> int:
state["status"] = "resume_budget_exhausted"
write_state(state_path, state)
return 3
try:
verify_training_release(
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
state["status"] = "training_release_verification_failed"
state["training_release_error"] = str(exc)
write_state(state_path, state)
return 6
result = subprocess.run(
["docker", "exec", "-d", args.container, args.yolo, "train",
f"resume={args.container_checkpoint}", "device=0"],
+302 -7
View File
@@ -4,8 +4,248 @@
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import sys
from collections import Counter
from pathlib import Path, PurePosixPath
from typing import Any, Mapping
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
)
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
PROPOSAL_DATASET_SCHEMA_VERSION = 1
_CROP_IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"}
class ProposalDatasetProvenanceError(ValueError):
"""Raised when proposal crops no longer match their governed provenance."""
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
def _payload_sha256(payload: Mapping[str, Any]) -> str:
normalized = dict(payload)
normalized.pop("manifest_sha256", None)
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
def _is_sha256(value: Any) -> bool:
return isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value)
def _require_mapping(value: Any, *, field: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be an object")
return value
def _require_sha256(value: Any, *, field: str) -> str:
if not _is_sha256(value):
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a lowercase SHA-256")
return str(value)
def _require_nonempty_path(value: Any, *, field: str) -> str:
if not isinstance(value, str) or not value.strip():
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a non-empty path")
return value
def _safe_relative_crop_path(value: Any, *, dataset_root: Path) -> tuple[str, Path]:
if not isinstance(value, str) or not value or "\\" in value:
raise ProposalDatasetProvenanceError("proposal crop relative_path must be a non-empty POSIX relative path")
relative = PurePosixPath(value)
if relative.is_absolute() or ".." in relative.parts or "." in relative.parts:
raise ProposalDatasetProvenanceError(f"proposal crop has an unsafe relative path: {value!r}")
path = (dataset_root / Path(*relative.parts)).resolve(strict=False)
try:
path.relative_to(dataset_root)
except ValueError as exc: # pragma: no cover - resolved-path defence
raise ProposalDatasetProvenanceError(f"proposal crop escapes dataset directory: {value!r}") from exc
return relative.as_posix(), path
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
"""Assert frozen evidence and current live Dataset eligibility before PyTorch."""
try:
manifest = assert_frozen_manifest_training_eligible(
corpus_manifest_path,
fixture_mode=fixture_mode,
verify_live=True,
)
except TrainingEligibilityError as exc:
raise ProposalDatasetProvenanceError(str(exc)) from exc
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
raise ProposalDatasetProvenanceError("governed corpus manifest must be a JSON object")
return manifest
def validate_proposal_dataset_provenance(
dataset_dir: Path,
corpus_manifest_path: Path,
*,
fixture_mode: bool,
) -> dict[str, Any]:
"""Fail closed unless every trainable crop still matches its source evidence.
ImageFolder discovers files from the directory tree. Merely verifying a
manifest is insufficient: any unbound image placed under train/ or val/
would otherwise enter PyTorch training. This validator compares the
complete tree with the immutable crop manifest and re-binds it to the
supplied frozen corpus and summary.
"""
root = dataset_dir.expanduser().resolve(strict=False)
if not root.is_dir():
raise ProposalDatasetProvenanceError(f"proposal dataset directory is unavailable: {root}")
manifest_path = root / PROPOSAL_DATASET_PROVENANCE_NAME
try:
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ProposalDatasetProvenanceError(f"proposal dataset provenance is unreadable: {manifest_path}") from exc
if not isinstance(payload, dict):
raise ProposalDatasetProvenanceError("proposal dataset provenance must be a JSON object")
if payload.get("schema_version") != PROPOSAL_DATASET_SCHEMA_VERSION:
raise ProposalDatasetProvenanceError("proposal dataset provenance schema_version is unsupported")
if payload.get("status") != "ok" or payload.get("immutable") is not True:
raise ProposalDatasetProvenanceError("proposal dataset provenance is not an immutable successful release")
if payload.get("governed_corpus_live_recheck") is not True:
raise ProposalDatasetProvenanceError("proposal dataset provenance lacks the governed live-source recheck")
if bool(payload.get("fixture_mode")) != fixture_mode:
raise ProposalDatasetProvenanceError("proposal dataset provenance fixture-mode binding mismatches this invocation")
if payload.get("manifest_sha256") != _payload_sha256(payload):
raise ProposalDatasetProvenanceError("proposal dataset provenance manifest checksum mismatches its content")
expected_corpus_sha256 = sha256(corpus_manifest_path)
source = _require_mapping(payload.get("source"), field="source")
recorded_corpus = _require_mapping(source.get("corpus_manifest"), field="source.corpus_manifest")
if _require_sha256(recorded_corpus.get("sha256"), field="source.corpus_manifest.sha256") != expected_corpus_sha256:
raise ProposalDatasetProvenanceError("proposal crops are not bound to the supplied governed corpus manifest")
corpus_freeze_path = corpus_manifest_path.parent / "corpus-freeze.json"
recorded_freeze = _require_mapping(source.get("corpus_freeze"), field="source.corpus_freeze")
if not corpus_freeze_path.is_file() or _require_sha256(
recorded_freeze.get("sha256"), field="source.corpus_freeze.sha256"
) != sha256(corpus_freeze_path):
raise ProposalDatasetProvenanceError("proposal crops are not bound to the current frozen corpus sidecar")
summary = _require_mapping(source.get("summary"), field="source.summary")
summary_path_value = _require_nonempty_path(summary.get("path"), field="source.summary.path")
summary_path = Path(summary_path_value).expanduser().resolve(strict=False)
if not summary_path.is_file():
raise ProposalDatasetProvenanceError(f"proposal source summary is unavailable: {summary_path}")
if _require_sha256(summary.get("sha256"), field="source.summary.sha256") != sha256(summary_path):
raise ProposalDatasetProvenanceError("proposal source summary checksum no longer matches the crop provenance")
try:
summary_payload = json.loads(summary_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ProposalDatasetProvenanceError(f"proposal source summary is unreadable: {summary_path}") from exc
if not isinstance(summary_payload, Mapping) or summary_payload.get("source_manifest_sha256") != expected_corpus_sha256:
raise ProposalDatasetProvenanceError("proposal source summary is not bound to the supplied governed corpus manifest")
if summary.get("source_manifest_sha256") != expected_corpus_sha256:
raise ProposalDatasetProvenanceError("proposal crop provenance summary binding is invalid")
try:
source_release = assert_yolo_summary_bound_to_embedded_training_release(
summary_path=summary_path,
corpus_manifest=corpus_manifest_path,
fixture_mode=fixture_mode,
)
except TrainingReleaseError as exc:
raise ProposalDatasetProvenanceError(
"proposal source summary has no eligible immutable training release"
) from exc
recorded_release = _require_mapping(source.get("training_release"), field="source.training_release")
if (
recorded_release.get("dataset_yaml_path") != source_release["dataset_yaml"]["path"]
or recorded_release.get("dataset_yaml_sha256") != source_release["dataset_yaml"]["sha256"]
or recorded_release.get("corpus_manifest_sha256") != source_release["corpus"]["manifest_sha256"]
):
raise ProposalDatasetProvenanceError("proposal crop provenance release binding is invalid")
proposal_model = _require_mapping(source.get("proposal_model"), field="source.proposal_model")
_require_nonempty_path(proposal_model.get("path"), field="source.proposal_model.path")
_require_sha256(proposal_model.get("sha256"), field="source.proposal_model.sha256")
entries = payload.get("crops")
if not isinstance(entries, list) or not entries:
raise ProposalDatasetProvenanceError("proposal dataset provenance has no crop entries")
expected_paths: set[str] = set()
calculated_counts: Counter[str] = Counter()
normalized_entries: list[dict[str, Any]] = []
for entry in entries:
item = _require_mapping(entry, field="crops[]")
relative_path, crop_path = _safe_relative_crop_path(item.get("relative_path"), dataset_root=root)
if relative_path in expected_paths:
raise ProposalDatasetProvenanceError(f"proposal crop provenance has duplicate path: {relative_path}")
expected_paths.add(relative_path)
split = item.get("split")
label = item.get("label")
if split not in {"train", "val"} or label not in {"negative", "positive"}:
raise ProposalDatasetProvenanceError(f"proposal crop has invalid split/class: {relative_path}")
if not isinstance(item.get("sample_slug"), str) or not item["sample_slug"].strip():
raise ProposalDatasetProvenanceError(f"proposal crop has no sample identity: {relative_path}")
if not crop_path.is_file():
raise ProposalDatasetProvenanceError(f"proposal crop is missing: {relative_path}")
if _require_sha256(item.get("sha256"), field=f"crops[{relative_path}].sha256") != sha256(crop_path):
raise ProposalDatasetProvenanceError(f"proposal crop checksum mismatches provenance: {relative_path}")
if item.get("size_bytes") != crop_path.stat().st_size:
raise ProposalDatasetProvenanceError(f"proposal crop byte size mismatches provenance: {relative_path}")
_require_nonempty_path(item.get("source_image_path"), field=f"crops[{relative_path}].source_image_path")
_require_sha256(item.get("source_image_sha256"), field=f"crops[{relative_path}].source_image_sha256")
_require_nonempty_path(item.get("source_label_path"), field=f"crops[{relative_path}].source_label_path")
_require_sha256(item.get("source_label_sha256"), field=f"crops[{relative_path}].source_label_sha256")
calculated_counts[f"{split}/{label}"] += 1
normalized_entries.append(dict(item))
canonical_entries = sorted(normalized_entries, key=lambda item: str(item["relative_path"]))
expected_crops_sha256 = hashlib.sha256(_canonical_json_bytes({"crops": canonical_entries})).hexdigest()
if payload.get("crops_sha256") != expected_crops_sha256:
raise ProposalDatasetProvenanceError("proposal crop collection checksum mismatches provenance")
if payload.get("crop_count") != len(entries):
raise ProposalDatasetProvenanceError("proposal crop count mismatches provenance")
if payload.get("counts") != dict(sorted(calculated_counts.items())):
raise ProposalDatasetProvenanceError("proposal crop class counts mismatch provenance")
if any(calculated_counts[f"{split}/{label}"] < 1 for split in ("train", "val") for label in ("negative", "positive")):
raise ProposalDatasetProvenanceError("proposal dataset has an empty train/validation class")
actual_paths = {
file_path.relative_to(root).as_posix()
for file_path in root.rglob("*")
if file_path.is_file() and file_path.suffix.lower() in _CROP_IMAGE_SUFFIXES
}
if actual_paths != expected_paths:
unexpected = sorted(actual_paths - expected_paths)
missing = sorted(expected_paths - actual_paths)
raise ProposalDatasetProvenanceError(
"proposal dataset files are not exactly the immutable crop manifest "
f"(unexpected={unexpected}, missing={missing})"
)
return payload
def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.5) -> dict[str, float | int]:
@@ -21,16 +261,44 @@ def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--dataset-dir", type=Path, required=True)
parser.add_argument(
"--corpus-manifest",
type=Path,
required=True,
help="The exact frozen governed corpus manifest that produced the proposal crops.",
)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--epochs", type=int, default=12)
parser.add_argument("--batch", type=int, default=64)
parser.add_argument("--lr", type=float, default=1e-4)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--export-existing-best", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Permit only an explicitly fixture-only corpus; operational runs always resolve live dataset evidence.",
)
args = parser.parse_args()
if args.output_dir.exists() and not args.export_existing_best:
parser.error(f"output already exists: {args.output_dir}")
# All governed-source checks run before importing torch/torchvision or
# touching CUDA. A source revocation therefore cannot start PyTorch work.
try:
governed_corpus = load_governed_corpus_manifest(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
proposal_provenance = validate_proposal_dataset_provenance(
args.dataset_dir,
args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except ProposalDatasetProvenanceError as exc:
raise SystemExit(str(exc)) from exc
proposal_provenance_path = args.dataset_dir / PROPOSAL_DATASET_PROVENANCE_NAME
proposal_provenance_sha256 = sha256(proposal_provenance_path)
import torch
from torch import nn
from torch.utils.data import DataLoader
@@ -54,7 +322,18 @@ def main() -> int:
model.load_state_dict(torch.load(state_path, map_location=device))
model.eval()
torch.jit.script(model).save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
print(json.dumps({"status": "exported_existing_best", "model": str(state_path)}))
print(
json.dumps(
{
"status": "exported_existing_best",
"model": str(state_path),
"proposal_dataset_provenance": str(proposal_provenance_path),
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
"governed_corpus_manifest_sha256": sha256(args.corpus_manifest),
"governed_corpus_live_recheck": True,
}
)
)
return 0
train_loader = DataLoader(train_ds, batch_size=args.batch, shuffle=True, num_workers=0)
val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=0)
@@ -93,11 +372,27 @@ def main() -> int:
model.load_state_dict(torch.load(args.output_dir / "best-state.pt", map_location=device))
model.eval()
scripted = torch.jit.script(model)
scripted.save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
report = {"schema_version": 1, "status": "ok", "classes": train_ds.class_to_idx,
"train_count": len(train_ds), "validation_count": len(val_ds),
"best_validation_f1": best_f1, "history": history,
"model": str(args.output_dir / "proposal-classifier.torchscript.pt")}
model_path = args.output_dir / "proposal-classifier.torchscript.pt"
scripted.save(str(model_path))
report = {
"schema_version": 1,
"status": "ok",
"classes": train_ds.class_to_idx,
"train_count": len(train_ds),
"validation_count": len(val_ds),
"best_validation_f1": best_f1,
"history": history,
"model": str(model_path),
"model_sha256": sha256(model_path),
"proposal_dataset_provenance": str(proposal_provenance_path),
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
"proposal_dataset_manifest_sha256": proposal_provenance["manifest_sha256"],
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"fixture_mode": bool(args.fixture_mode),
"governed_corpus_live_recheck": True,
"governed_corpus_sample_count": len(governed_corpus.get("samples", [])),
}
(args.output_dir / "training-report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
return 0
+54 -2
View File
@@ -8,9 +8,23 @@ import hashlib
import json
import random
import shutil
import sys
from pathlib import Path
from typing import Any, TypeVar
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
)
T = TypeVar("T")
@@ -111,6 +125,18 @@ def choose_threshold(probabilities: list[float], labels: list[int]) -> tuple[flo
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument(
"--corpus-manifest",
type=Path,
required=True,
help="Immutable governed corpus manifest that produced the tile summary.",
)
parser.add_argument(
"--train-yaml",
type=Path,
required=True,
help="YOLO dataset YAML with an eligible immutable GeoIntel training release.",
)
parser.add_argument("--proposal-model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--device", default="cuda:0")
@@ -124,7 +150,31 @@ def main() -> int:
parser.add_argument("--proposal-chunk-size", type=int, default=16)
parser.add_argument("--seed", type=int, default=20260727)
parser.add_argument("--force", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training.",
)
args = parser.parse_args()
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
summary = json.loads(args.summary.read_text(encoding="utf-8"))
if summary.get("source_manifest_sha256") != sha256(args.corpus_manifest):
raise SystemExit(
"Proposal-filter tile summary is not bound to the supplied governed corpus manifest"
)
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -133,12 +183,11 @@ def main() -> int:
import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from torchvision import datasets
from torchvision.models import ResNet18_Weights, resnet18
from ultralytics import YOLO
torch.manual_seed(args.seed)
summary = json.loads(args.summary.read_text(encoding="utf-8"))
split_tiles = {
split: [tile for tile in summary["tiles"] if tile.get("kept", True) and tile["split"] == split]
for split in ("train", "val")
@@ -214,6 +263,9 @@ def main() -> int:
evidence = {
"schema_version": 1, "status": "ok", "architecture": "resnet18_binary_proposal_filter",
"summary": str(args.summary), "summary_sha256": sha256(args.summary),
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"fixture_mode": bool(args.fixture_mode),
"proposal_model": str(args.proposal_model), "proposal_model_sha256": sha256(args.proposal_model),
"device": args.device, "proposal_confidence": args.proposal_confidence, "crop_scale": args.crop_scale,
"proposal_chunk_size": args.proposal_chunk_size,
+7
View File
@@ -53,6 +53,8 @@ if [[ -z "${PYTHON_BIN:-}" ]]; then
fi
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
export DATASET_YAML
@@ -79,6 +81,11 @@ if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
exit 1
fi
# A filename is not a training identity. Verify the immutable YAML/corpus/
# asset/review release before importing Ultralytics or allocating CUDA work.
"${PYTHON_BIN}" "${SCRIPT_DIR}/training_release_manifest.py" verify \
--train-yaml "${DATASET_YAML}"
mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
"${PYTHON_BIN}" - <<'PY'
+545
View File
@@ -0,0 +1,545 @@
"""Fail-closed provenance gate for Belgian building-training inputs.
The database data-contract validator decides whether a dataset can be stored.
This module is intentionally a second, independent gate at the point where
persisted datasets become irreversible training pairs. It has no database
side effects, making the decision reproducible in a corpus manifest and easy
to re-check before CUDA training starts.
Legacy data may only pass through this module in explicit fixture mode. That
mode is deliberately limited to records marked as fixtures and must never be
used for an operational corpus.
"""
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Callable, Literal
from uuid import UUID
TRAINING_ELIGIBILITY_POLICY_VERSION = "geointel-training-source-eligibility/v1"
DatasetRole = Literal["raster", "reference"]
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
_ALLOWED_SOURCE_CLASSIFICATIONS = {
"authoritative",
"corroborative",
"contextual",
"derived",
}
_FIXTURE_SOURCES = {"fixture", "test", "test_fixture", "test-fixture"}
class TrainingEligibilityError(ValueError):
"""Raised when persisted inputs cannot be used in an operational corpus."""
class TrainingEligibilityResult:
"""Stable, JSON-serializable decision for one persisted dataset."""
__slots__ = ("role", "eligible", "fixture_mode", "reasons", "evidence")
def __init__(
self,
*,
role: DatasetRole,
eligible: bool,
fixture_mode: bool,
reasons: tuple[str, ...],
evidence: dict[str, Any],
) -> None:
self.role = role
self.eligible = eligible
self.fixture_mode = fixture_mode
self.reasons = reasons
self.evidence = evidence
def as_dict(self) -> dict[str, Any]:
return {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"role": self.role,
"eligible": self.eligible,
"fixture_mode": self.fixture_mode,
"reasons": list(self.reasons),
"evidence": self.evidence,
}
def _value(record: Any, field: str, default: Any = None) -> Any:
if isinstance(record, Mapping):
return record.get(field, default)
return getattr(record, field, default)
def _normalise_status(value: Any) -> str:
return str(value or "").strip().lower()
def _normalise_mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _source_identity(dataset: Any) -> tuple[str, str]:
return (
_normalise_status(_value(dataset, "source")),
_normalise_status(_value(dataset, "source_name")),
)
def _is_explicit_fixture(dataset: Any, registry: Any) -> bool:
source, source_name = _source_identity(dataset)
metadata = _normalise_mapping(_value(dataset, "metadata_json"))
provenance = _normalise_mapping(_value(dataset, "provenance_metadata"))
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
return bool(
source in _FIXTURE_SOURCES
or source_name in _FIXTURE_SOURCES
or metadata.get("fixture") is True
or provenance.get("fixture") is True
or usage_policy.get("fixture_only") is True
)
def _dataset_evidence(dataset: Any, registry: Any, snapshot: Any) -> dict[str, Any]:
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
allowed_tasks = usage_policy.get("allowed_tasks")
validation_authority = _normalise_mapping(usage_policy.get("validation_authority"))
return {
"dataset_id": str(_value(dataset, "id") or ""),
"dataset_type": _normalise_status(_value(dataset, "dataset_type")),
"dataset_role": _normalise_status(_value(dataset, "dataset_role")),
"source": _normalise_status(_value(dataset, "source")),
"source_name": _normalise_status(_value(dataset, "source_name")),
"checksum_sha256": _value(dataset, "checksum_sha256"),
"data_contract_key": _value(dataset, "data_contract_key"),
"data_contract_version": _value(dataset, "data_contract_version"),
"validation_status": _normalise_status(_value(dataset, "validation_status")),
"provenance_status": _normalise_status(_value(dataset, "provenance_status")),
"lineage_status": _normalise_status(_value(dataset, "lineage_status")),
"quarantine_status": _normalise_status(_value(dataset, "quarantine_status")),
"dataset_status": _normalise_status(_value(dataset, "status")),
"dataset_source_registry_id": str(_value(dataset, "source_registry_id") or ""),
"dataset_source_snapshot_id": str(_value(dataset, "source_snapshot_id") or ""),
"source_registry_id": str(_value(registry, "id") or ""),
"source_key": _normalise_status(_value(registry, "source_key")),
"source_classification": _normalise_status(_value(registry, "classification")),
"source_freshness_status": _normalise_status(_value(registry, "freshness_status")),
"source_ingest_status": _normalise_status(_value(registry, "ingest_status")),
"source_training_allowed": usage_policy.get("training_allowed"),
"source_ground_truth_allowed": usage_policy.get("ground_truth_allowed"),
"source_allowed_tasks": list(allowed_tasks) if isinstance(allowed_tasks, list) else [],
"source_validation_authority": validation_authority,
"source_building_validation_authority": validation_authority.get("building_validation"),
"source_snapshot_id": str(_value(snapshot, "id") or _value(dataset, "source_snapshot_id") or ""),
"snapshot_source_registry_id": str(_value(snapshot, "source_registry_id") or ""),
"snapshot_key": _value(snapshot, "snapshot_key"),
"snapshot_checksum_sha256": _value(snapshot, "checksum_sha256"),
"snapshot_freshness_status": _normalise_status(_value(snapshot, "freshness_status")),
"snapshot_ingest_status": _normalise_status(_value(snapshot, "ingest_status")),
}
def evaluate_dataset_training_eligibility(
dataset: Any,
*,
role: DatasetRole,
fixture_mode: bool = False,
) -> TrainingEligibilityResult:
"""Evaluate a source Dataset without mutating it.
Operational calls require server-attested source registry and snapshot
relationships. ``fixture_mode`` can relax only legacy provenance fields,
and only for an explicitly marked fixture. A failed validation or a
quarantine is never relaxed.
"""
registry = _value(dataset, "source_registry")
snapshot = _value(dataset, "source_snapshot")
evidence = _dataset_evidence(dataset, registry, snapshot)
reasons: list[str] = []
dataset_type = evidence["dataset_type"]
dataset_role = evidence["dataset_role"]
if role == "raster" and dataset_type != "raster":
reasons.append("dataset_type_not_raster")
if role == "reference":
if dataset_type != "vector":
reasons.append("dataset_type_not_vector")
if dataset_role != "reference":
reasons.append("dataset_role_not_reference")
if evidence["dataset_status"] != "ready":
reasons.append("dataset_not_ready")
if evidence["quarantine_status"] == "quarantined":
reasons.append("dataset_quarantined")
explicit_fixture = _is_explicit_fixture(dataset, registry)
if fixture_mode and not explicit_fixture:
reasons.append("fixture_mode_requires_explicit_fixture")
validation_status = evidence["validation_status"]
if validation_status == "failed":
reasons.append("validation_failed")
elif validation_status != "passed" and not (fixture_mode and explicit_fixture):
reasons.append("validation_not_passed")
if not fixture_mode:
if evidence["provenance_status"] != "complete":
reasons.append("provenance_not_complete")
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
reasons.append("lineage_not_complete")
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
reasons.append("data_contract_not_versioned")
checksum = str(evidence["checksum_sha256"] or "")
if not _SHA256_RE.fullmatch(checksum):
reasons.append("dataset_checksum_invalid")
if registry is None:
reasons.append("source_registry_missing")
if snapshot is None:
reasons.append("source_snapshot_missing")
if registry is not None:
if (
evidence["dataset_source_registry_id"]
and evidence["dataset_source_registry_id"] != evidence["source_registry_id"]
):
reasons.append("dataset_source_registry_binding_mismatch")
classification = evidence["source_classification"]
if classification not in _ALLOWED_SOURCE_CLASSIFICATIONS:
reasons.append("source_classification_not_allowed")
if evidence["source_training_allowed"] is not True:
reasons.append("source_not_allowed_for_training")
if evidence["source_ingest_status"] == "quarantined":
reasons.append("source_registry_quarantined")
if snapshot is not None:
if (
evidence["dataset_source_snapshot_id"]
and evidence["dataset_source_snapshot_id"] != evidence["source_snapshot_id"]
):
reasons.append("dataset_source_snapshot_binding_mismatch")
if (
registry is not None
and evidence["snapshot_source_registry_id"]
and evidence["snapshot_source_registry_id"] != evidence["source_registry_id"]
):
reasons.append("source_snapshot_registry_mismatch")
if evidence["snapshot_ingest_status"] != "ingested":
reasons.append("source_snapshot_not_ingested")
if evidence["snapshot_freshness_status"] not in {"current", "not_applicable"}:
reasons.append("source_snapshot_freshness_not_approved")
snapshot_checksum = str(evidence["snapshot_checksum_sha256"] or "")
if not _SHA256_RE.fullmatch(snapshot_checksum):
reasons.append("source_snapshot_checksum_invalid")
elif snapshot_checksum.lower() != checksum.lower():
reasons.append("source_snapshot_checksum_mismatch")
if role == "reference":
if evidence["source_classification"] != "authoritative":
reasons.append("reference_source_not_authoritative")
if evidence["source_ground_truth_allowed"] is not True:
reasons.append("reference_source_not_ground_truth_allowed")
if "building_validation" not in evidence["source_allowed_tasks"]:
reasons.append("reference_source_not_approved_for_building_validation")
if evidence["source_building_validation_authority"] != "primary":
reasons.append("reference_building_validation_not_primary")
return TrainingEligibilityResult(
role=role,
eligible=not reasons,
fixture_mode=fixture_mode,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
def assert_dataset_training_eligible(
dataset: Any,
*,
role: DatasetRole,
fixture_mode: bool = False,
sample_slug: str | None = None,
) -> TrainingEligibilityResult:
result = evaluate_dataset_training_eligibility(dataset, role=role, fixture_mode=fixture_mode)
if result.eligible:
return result
label = f" for sample {sample_slug!r}" if sample_slug else ""
raise TrainingEligibilityError(
f"{role} dataset is not eligible for training{label}: {', '.join(result.reasons)}"
)
def training_pair_evidence(
*,
raster: Any,
reference: Any,
fixture_mode: bool = False,
) -> dict[str, Any]:
"""Return manifest-ready evidence for one raster/reference pair."""
raster_result = evaluate_dataset_training_eligibility(
raster,
role="raster",
fixture_mode=fixture_mode,
)
reference_result = evaluate_dataset_training_eligibility(
reference,
role="reference",
fixture_mode=fixture_mode,
)
return {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"eligible": raster_result.eligible and reference_result.eligible,
"fixture_mode": fixture_mode,
"raster": raster_result.as_dict(),
"reference": reference_result.as_dict(),
}
def manifest_training_eligibility_failures(
manifest: Mapping[str, Any],
*,
fixture_mode: bool = False,
) -> list[str]:
"""Re-check immutable eligibility evidence before an actual training run."""
failures: list[str] = []
eligibility = manifest.get("training_eligibility")
if not isinstance(eligibility, Mapping):
return ["manifest_training_eligibility_missing"]
if eligibility.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
failures.append("manifest_training_eligibility_policy_invalid")
if eligibility.get("status") != "eligible":
failures.append("manifest_training_eligibility_not_eligible")
manifest_fixture_mode = eligibility.get("fixture_mode") is True
if manifest_fixture_mode != fixture_mode:
failures.append("manifest_fixture_mode_mismatch")
samples = manifest.get("samples")
if not isinstance(samples, list) or not samples:
failures.append("manifest_samples_missing")
return failures
for sample in samples:
if not isinstance(sample, Mapping):
failures.append("manifest_sample_invalid")
continue
slug = str(sample.get("sample_slug") or "<unknown>")
pair = sample.get("training_eligibility")
if not isinstance(pair, Mapping):
failures.append(f"{slug}:training_eligibility_missing")
continue
if pair.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
failures.append(f"{slug}:training_eligibility_policy_invalid")
if pair.get("fixture_mode") is not fixture_mode:
failures.append(f"{slug}:fixture_mode_mismatch")
if pair.get("eligible") is not True:
failures.append(f"{slug}:training_pair_not_eligible")
for role in ("raster", "reference"):
decision = pair.get(role)
if not isinstance(decision, Mapping):
failures.append(f"{slug}:{role}_eligibility_missing")
continue
if decision.get("eligible") is not True:
failures.append(f"{slug}:{role}_not_eligible")
reasons = decision.get("reasons")
if isinstance(reasons, list) and reasons:
failures.append(f"{slug}:{role}_has_rejection_reasons")
return sorted(set(failures))
def _default_live_dataset_access() -> tuple[Callable[[], Any], type[Any]]:
"""Load the database dependencies only for an operational live re-check.
This module is also imported by pure filesystem tooling and fixture tests.
Keeping the import lazy means those paths do not accidentally open a
database connection, while a normal release/verify invocation still fails
closed if the live registry cannot be checked.
"""
import sys
repo_root = Path(__file__).resolve().parents[1]
app_root = repo_root / "backend"
if str(app_root) not in sys.path:
sys.path.insert(0, str(app_root))
from app.db.session import SessionLocal
from app.models import Dataset
return SessionLocal, Dataset
def live_manifest_training_eligibility_failures(
manifest: Mapping[str, Any],
*,
fixture_mode: bool = False,
session_factory: Callable[[], Any] | None = None,
dataset_model: type[Any] | None = None,
) -> list[str]:
"""Re-evaluate every frozen corpus parent against the live database.
A frozen manifest proves what was eligible when it was created, not what
remains eligible now. Operational releases therefore resolve the exact
raster/reference Dataset ids again immediately before seal, retry, resume
and CUDA training. A later quarantine, failed contract, snapshot change
or missing record is a revocation and cannot be masked by the old manifest.
Fixture-only corpora deliberately have no operational authority and are
never used for a production release; their isolated tests may skip this
live database boundary.
"""
if fixture_mode:
return []
failures: list[str] = []
samples = manifest.get("samples")
if not isinstance(samples, list) or not samples:
return ["live_manifest_samples_missing"]
if session_factory is None or dataset_model is None:
try:
default_factory, default_model = _default_live_dataset_access()
except Exception:
return ["live_training_dataset_access_unavailable"]
session_factory = session_factory or default_factory
dataset_model = dataset_model or default_model
try:
db = session_factory()
except Exception:
return ["live_training_dataset_access_unavailable"]
try:
for sample in samples:
if not isinstance(sample, Mapping):
failures.append("live_manifest_sample_invalid")
continue
slug = str(sample.get("sample_slug") or "<unknown>")
recorded_pair = sample.get("training_eligibility")
for role, field_name in (("raster", "raster_dataset_id"), ("reference", "reference_dataset_id")):
raw_id = sample.get(field_name)
if not isinstance(raw_id, str) or not raw_id.strip():
failures.append(f"{slug}:{role}_dataset_id_missing_for_live_check")
continue
try:
dataset_id = UUID(raw_id)
except (TypeError, ValueError, AttributeError):
failures.append(f"{slug}:{role}_dataset_id_invalid_for_live_check")
continue
try:
dataset = db.get(dataset_model, dataset_id)
except Exception:
failures.append(f"{slug}:{role}_live_lookup_failed")
continue
if dataset is None:
failures.append(f"{slug}:{role}_live_dataset_missing")
continue
result = evaluate_dataset_training_eligibility(
dataset,
role=role, # type: ignore[arg-type]
fixture_mode=False,
)
if not result.eligible:
for reason in result.reasons:
failures.append(f"{slug}:{role}_live_revoked:{reason}")
if isinstance(recorded_pair, Mapping):
recorded_role = recorded_pair.get(role)
if isinstance(recorded_role, Mapping):
recorded_evidence = recorded_role.get("evidence")
if isinstance(recorded_evidence, Mapping) and str(recorded_evidence.get("dataset_id") or "") != raw_id:
failures.append(f"{slug}:{role}_manifest_dataset_binding_mismatch")
finally:
close = getattr(db, "close", None)
if callable(close):
close()
return sorted(set(failures))
def assert_manifest_training_eligible(
manifest: Mapping[str, Any],
*,
fixture_mode: bool = False,
) -> None:
failures = manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode)
if failures:
raise TrainingEligibilityError(
"Corpus manifest is not eligible for training: " + ", ".join(failures)
)
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def frozen_manifest_training_eligibility_failures(
manifest_path: Path,
*,
fixture_mode: bool = False,
verify_live: bool = False,
session_factory: Callable[[], Any] | None = None,
dataset_model: type[Any] | None = None,
) -> list[str]:
"""Verify an immutable manifest and its source-eligibility decision together."""
failures: list[str] = []
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return ["corpus_manifest_unreadable"]
if not isinstance(manifest, Mapping):
return ["corpus_manifest_invalid"]
failures.extend(manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode))
freeze_path = manifest_path.parent / "corpus-freeze.json"
try:
freeze = json.loads(freeze_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
if not isinstance(freeze, Mapping):
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
if freeze.get("immutable") is not True:
failures.append("corpus_freeze_not_immutable")
if freeze.get("manifest_sha256") != _file_sha256(manifest_path):
failures.append("corpus_manifest_checksum_mismatch")
if freeze.get("training_eligibility_policy") != TRAINING_ELIGIBILITY_POLICY_VERSION:
failures.append("corpus_freeze_policy_invalid")
if bool(freeze.get("fixture_mode")) != fixture_mode:
failures.append("corpus_freeze_fixture_mode_mismatch")
if verify_live:
failures.extend(
live_manifest_training_eligibility_failures(
manifest,
fixture_mode=fixture_mode,
session_factory=session_factory,
dataset_model=dataset_model,
)
)
return sorted(set(failures))
def assert_frozen_manifest_training_eligible(
manifest_path: Path,
*,
fixture_mode: bool = False,
verify_live: bool = False,
session_factory: Callable[[], Any] | None = None,
dataset_model: type[Any] | None = None,
) -> dict[str, Any]:
"""Load only a frozen, policy-valid corpus manifest for a training entrypoint."""
failures = frozen_manifest_training_eligibility_failures(
manifest_path,
fixture_mode=fixture_mode,
verify_live=verify_live,
session_factory=session_factory,
dataset_model=dataset_model,
)
if failures:
raise TrainingEligibilityError(
"Corpus manifest is not eligible for training: " + ", ".join(failures)
)
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
assert isinstance(payload, dict)
return payload
File diff suppressed because it is too large Load Diff
+738
View File
@@ -0,0 +1,738 @@
#!/usr/bin/env python3
"""Exercise Phase-2 PostgreSQL migration guards on a disposable database.
This is intentionally *not* a general migration runner. It refuses every
database whose name does not start with ``geointel_phase2_`` so it cannot be
pointed accidentally at a developer, staging or production database. The
script upgrades the complete Alembic chain, proves the new trigger guards with
real DML, and optionally downgrades again.
"""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import UTC, datetime
import json
import os
from pathlib import Path
import subprocess
import sys
from typing import Any, Iterator
from uuid import UUID, uuid4
from sqlalchemy import create_engine, text
from sqlalchemy.engine import Connection, Engine, make_url
from sqlalchemy.exc import IntegrityError
ROOT = Path(__file__).resolve().parents[1]
BACKEND = ROOT / "backend"
SAFE_DATABASE_PREFIX = "geointel_phase2_"
def _json_dump(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
def _require_disposable_database(database_url: str) -> None:
parsed = make_url(database_url)
database_name = str(parsed.database or "")
if not database_name.startswith(SAFE_DATABASE_PREFIX):
raise SystemExit(
"Refusing migration guard test: database name must start with "
f"{SAFE_DATABASE_PREFIX!r}, received {database_name!r}."
)
def _run_alembic(database_url: str, *arguments: str) -> str:
environment = dict(os.environ)
environment["DATABASE_URL"] = database_url
completed = subprocess.run(
[sys.executable, "-m", "alembic", *arguments],
cwd=BACKEND,
env=environment,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=False,
)
if completed.returncode != 0:
raise RuntimeError(
f"Alembic {' '.join(arguments)} failed with exit code {completed.returncode}:\n{completed.stdout}"
)
return completed.stdout
@contextmanager
def _transaction(engine: Engine) -> Iterator[Connection]:
with engine.begin() as connection:
yield connection
def _insert_project(connection: Connection) -> UUID:
project_id = uuid4()
connection.execute(
text("INSERT INTO projects (id, name) VALUES (:id, :name)"),
{"id": project_id, "name": "Phase 2 migration guard fixture"},
)
return project_id
def _source_id(connection: Connection, source_key: str) -> UUID:
value = connection.execute(
text("SELECT id FROM source_registry WHERE source_key = :source_key"),
{"source_key": source_key},
).scalar_one()
return UUID(str(value))
def _insert_snapshot(
connection: Connection, source_registry_id: UUID, *, key: str, checksum: str
) -> UUID:
snapshot_id = uuid4()
connection.execute(
text(
"""
INSERT INTO source_snapshots (
id, source_registry_id, snapshot_key, checksum_sha256,
crs, units, freshness_status, ingest_status
) VALUES (
:id, :source_registry_id, :snapshot_key, :checksum_sha256,
'EPSG:4326', 'metres', 'current', 'ingested'
)
"""
),
{
"id": snapshot_id,
"source_registry_id": source_registry_id,
"snapshot_key": key,
"checksum_sha256": checksum,
},
)
return snapshot_id
def _accepted_report(
*, contract_key: str = "geointel.vector.geojson", contract_version: str = "1.0.0"
) -> str:
"""Produce a structurally complete persisted validation report fixture."""
return json.dumps(
{
"asset_id": "phase2-guard-fixture",
"data_contract_key": contract_key,
"data_contract_version": contract_version,
"contract_fingerprint_sha256": "d" * 64,
"validation_status": "passed",
"provenance_status": "complete",
"lineage_status": "complete",
"quarantine_status": "not_quarantined",
"validation_scope": ["contract"],
"report_sha256": "e" * 64,
}
)
def _insert_dataset(
connection: Connection,
*,
project_id: UUID,
source_registry_id: UUID,
source_snapshot_id: UUID,
suffix: str,
checksum_sha256: str,
) -> UUID:
dataset_id = uuid4()
connection.execute(
text(
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name,
status, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status, quarantine_status,
checksum_sha256
) VALUES (
:id, :project_id, :name, 'vector', 'governed', 'grb',
'ready', :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
)
"""
),
{
"id": dataset_id,
"project_id": project_id,
"name": f"phase2-{suffix}.geojson",
"source_registry_id": source_registry_id,
"source_snapshot_id": source_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": checksum_sha256,
},
)
return dataset_id
def _assert_constraint_rejection(
engine: Engine, statement: str, parameters: dict[str, Any], *, label: str
) -> None:
try:
with _transaction(engine) as connection:
connection.execute(text(statement), parameters)
except IntegrityError as exc:
sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", None)
if sqlstate == "23514":
return
raise AssertionError(
f"{label} failed with unexpected SQLSTATE {sqlstate!r}"
) from exc
raise AssertionError(f"{label} was accepted unexpectedly")
def verify(database_url: str, *, verify_downgrade: bool) -> dict[str, Any]:
_require_disposable_database(database_url)
started_at = datetime.now(UTC).isoformat()
upgrade_output = _run_alembic(database_url, "upgrade", "head")
engine = create_engine(database_url)
try:
edge_a = uuid4()
edge_b = uuid4()
with _transaction(engine) as connection:
project_id = _insert_project(connection)
grb_id = _source_id(connection, "grb")
osm_id = _source_id(connection, "osm")
grb_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-grb", checksum="a" * 64
)
osm_snapshot_id = _insert_snapshot(
connection, osm_id, key="phase2-osm", checksum="b" * 64
)
derived_b_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-derived-b", checksum="c" * 64
)
derived_c_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-derived-c", checksum="d" * 64
)
mutation_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-mutation", checksum="e" * 64
)
dataset_a = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=grb_snapshot_id,
suffix="a",
checksum_sha256="a" * 64,
)
dataset_b = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=derived_b_snapshot_id,
suffix="b",
checksum_sha256="c" * 64,
)
dataset_c = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=derived_c_snapshot_id,
suffix="c",
checksum_sha256="d" * 64,
)
shared_dataset = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=grb_snapshot_id,
suffix="shared",
checksum_sha256="a" * 64,
)
mutation_dataset = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=mutation_snapshot_id,
suffix="mutation",
checksum_sha256="e" * 64,
)
version_id = uuid4()
connection.execute(
text(
"""
INSERT INTO dataset_versions (
id, dataset_id, version, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status, checksum_sha256
) VALUES (
:id, :dataset_id, 1, :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete', :checksum_sha256
)
"""
),
{
"id": version_id,
"dataset_id": dataset_a,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "a" * 64,
},
)
connection.execute(
text(
"""
INSERT INTO dataset_lineage_edges (
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
) VALUES
(:edge_a, :dataset_a, :dataset_b, 'derived_from', 'clip'),
(:edge_b, :dataset_b, :dataset_c, 'derived_from', 'buffer')
"""
),
{
"edge_a": edge_a,
"edge_b": edge_b,
"dataset_a": dataset_a,
"dataset_b": dataset_b,
"dataset_c": dataset_c,
},
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
data_contract_key, data_contract_version, validation_report_json,
provenance_status, lineage_status, quarantine_status, checksum_sha256
) VALUES (
:id, :project_id, 'mismatched.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed',
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'complete', 'complete', 'not_quarantined', :checksum_sha256
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": osm_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "b" * 64,
},
label="snapshot-registry mismatch guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
provenance_status, lineage_status, quarantine_status, checksum_sha256
) VALUES (
:id, :project_id, 'unreported.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"checksum_sha256": "a" * 64,
},
label="passed contract report guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
data_contract_key, data_contract_version, validation_report_json,
provenance_status, lineage_status, quarantine_status
) VALUES (
:id, :project_id, 'missing-checksum.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed',
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'complete', 'complete', 'not_quarantined'
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
},
label="passed dataset checksum required guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
data_contract_key, data_contract_version, validation_report_json,
provenance_status, lineage_status, quarantine_status, checksum_sha256
) VALUES (
:id, :project_id, 'checksum-mismatch.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed',
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'complete', 'complete', 'not_quarantined', :checksum_sha256
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "f" * 64,
},
label="passed dataset snapshot checksum binding guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO dataset_versions (
id, dataset_id, version, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status
) VALUES (
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete'
)
""",
{
"id": uuid4(),
"dataset_id": dataset_a,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
},
label="passed dataset version checksum required guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO dataset_versions (
id, dataset_id, version, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status, checksum_sha256
) VALUES (
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete', :checksum_sha256
)
""",
{
"id": uuid4(),
"dataset_id": dataset_a,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "f" * 64,
},
label="passed dataset version snapshot checksum binding guard",
)
_assert_constraint_rejection(
engine,
"UPDATE datasets SET data_contract_version = '9.9.9' WHERE id = :dataset_id",
{"dataset_id": dataset_b},
label="accepted contract evidence immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE dataset_versions SET data_contract_version = '9.9.9' WHERE id = :dataset_version_id",
{"dataset_version_id": version_id},
label="accepted dataset-version contract evidence immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE datasets SET storage_path = '/tampered/asset.geojson' WHERE id = :dataset_id",
{"dataset_id": mutation_dataset},
label="accepted dataset artifact immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE datasets SET observed_at = CURRENT_TIMESTAMP WHERE id = :dataset_id",
{"dataset_id": mutation_dataset},
label="accepted dataset temporal evidence immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE dataset_versions SET storage_path = '/tampered/version.geojson' WHERE id = :dataset_version_id",
{"dataset_version_id": version_id},
label="accepted dataset-version artifact immutability guard",
)
_assert_constraint_rejection(
engine,
"""
UPDATE datasets
SET validation_status = 'failed', checksum_sha256 = 'f' || repeat('0', 63)
WHERE id = :dataset_id
""",
{"dataset_id": mutation_dataset},
label="accepted dataset requires invalidation before artifact replacement",
)
with _transaction(engine) as connection:
connection.execute(
text(
"UPDATE datasets SET validation_status = 'failed' WHERE id = :dataset_id"
),
{"dataset_id": mutation_dataset},
)
connection.execute(
text(
"UPDATE datasets SET storage_path = '/replacement/asset.geojson' WHERE id = :dataset_id"
),
{"dataset_id": mutation_dataset},
)
_assert_constraint_rejection(
engine,
"UPDATE source_snapshots SET source_registry_id = :registry_id WHERE id = :snapshot_id",
{"registry_id": osm_id, "snapshot_id": grb_snapshot_id},
label="immutable snapshot registry guard",
)
_assert_constraint_rejection(
engine,
"UPDATE source_registry SET display_name = 'tampered' WHERE id = :registry_id",
{"registry_id": grb_id},
label="server-owned source registry guard",
)
_assert_constraint_rejection(
engine,
"UPDATE source_snapshots SET checksum_sha256 = :checksum_sha256 WHERE id = :snapshot_id",
{"checksum_sha256": "c" * 64, "snapshot_id": grb_snapshot_id},
label="immutable snapshot evidence guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO dataset_lineage_edges (
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
) VALUES (:id, :parent_dataset_id, :child_dataset_id, 'derived_from', 'cycle')
""",
{
"id": uuid4(),
"parent_dataset_id": dataset_c,
"child_dataset_id": dataset_a,
},
label="lineage cycle guard",
)
_assert_constraint_rejection(
engine,
"UPDATE dataset_lineage_edges SET transformation_name = 'tampered' WHERE id = :id",
{"id": edge_a},
label="lineage edge immutability update guard",
)
_assert_constraint_rejection(
engine,
"DELETE FROM dataset_lineage_edges WHERE id = :id",
{"id": edge_b},
label="lineage edge immutability delete guard",
)
with _transaction(engine) as connection:
connection.execute(
text(
"""
INSERT INTO dataset_quarantines (
id, dataset_version_id, stage, reason_code, status
) VALUES (:id, :dataset_version_id, 'integration_test', 'CHECKSUM_MISMATCH', 'quarantined')
"""
),
{"id": uuid4(), "dataset_version_id": version_id},
)
dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": dataset_a},
)
.mappings()
.one()
)
version_state = (
connection.execute(
text(
"""
SELECT validation_status, provenance_status, lineage_status
FROM dataset_versions WHERE id = :dataset_version_id
"""
),
{"dataset_version_id": version_id},
)
.mappings()
.one()
)
snapshot_state = connection.execute(
text(
"SELECT ingest_status FROM source_snapshots WHERE id = :snapshot_id"
),
{"snapshot_id": grb_snapshot_id},
).scalar_one()
child_dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": dataset_b},
)
.mappings()
.one()
)
grandchild_dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": dataset_c},
)
.mappings()
.one()
)
shared_dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": shared_dataset},
)
.mappings()
.one()
)
expected_dataset_state = {
"status": "quarantined",
"quarantine_status": "quarantined",
"validation_status": "failed",
"provenance_status": "incomplete",
"lineage_status": "incomplete",
}
if dict(dataset_state) != expected_dataset_state:
raise AssertionError(
f"quarantine parent propagation mismatch: {dict(dataset_state)}"
)
expected_version_state = {
"validation_status": "failed",
"provenance_status": "incomplete",
"lineage_status": "incomplete",
}
if dict(version_state) != expected_version_state:
raise AssertionError(
f"quarantine version propagation mismatch: {dict(version_state)}"
)
if snapshot_state != "quarantined":
raise AssertionError(
f"quarantine snapshot propagation mismatch: {snapshot_state}"
)
if dict(child_dataset_state) != expected_dataset_state:
raise AssertionError(
"quarantine transitive-child propagation mismatch: "
f"{dict(child_dataset_state)}"
)
if dict(grandchild_dataset_state) != expected_dataset_state:
raise AssertionError(
"quarantine transitive-grandchild propagation mismatch: "
f"{dict(grandchild_dataset_state)}"
)
if dict(shared_dataset_state) != expected_dataset_state:
raise AssertionError(
"quarantine shared-snapshot propagation mismatch: "
f"{dict(shared_dataset_state)}"
)
finally:
engine.dispose()
downgrade_output = (
_run_alembic(database_url, "downgrade", "base")
if verify_downgrade
else "not requested"
)
return {
"schema_version": 1,
"phase": "P2",
"started_at": started_at,
"completed_at": datetime.now(UTC).isoformat(),
"migration_revision": "202608010001",
"database_name": str(make_url(database_url).database),
"result": "passed",
"guards": {
"snapshot_registry_pairing": "passed",
"snapshot_registry_immutable": "passed",
"source_registry_immutable": "passed",
"snapshot_evidence_immutable": "passed",
"lineage_cycle": "passed",
"lineage_edge_immutable": "passed",
"version_quarantine_propagation": "passed",
"snapshot_quarantine_fanout": "passed",
"transitive_lineage_quarantine": "passed",
"passed_contract_report_required": "passed",
"passed_dataset_checksum_required_and_snapshot_bound": "passed",
"passed_dataset_version_checksum_required_and_snapshot_bound": "passed",
"accepted_contract_evidence_immutable": "passed",
"accepted_dataset_version_contract_evidence_immutable": "passed",
"accepted_artifact_and_temporal_evidence_immutable": "passed",
},
"upgrade_output_tail": upgrade_output.splitlines()[-8:],
"downgrade_output_tail": downgrade_output.splitlines()[-8:],
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--database-url", required=True)
parser.add_argument(
"--output",
type=Path,
default=ROOT
/ "artifacts"
/ "evidence"
/ "accuracy"
/ "P2"
/ "postgres-migration-guards.json",
)
parser.add_argument("--skip-downgrade", action="store_true")
args = parser.parse_args()
try:
result = verify(args.database_url, verify_downgrade=not args.skip_downgrade)
except Exception as exc:
result = {
"schema_version": 1,
"phase": "P2",
"completed_at": datetime.now(UTC).isoformat(),
"migration_revision": "202608010001",
"result": "failed",
"error_type": type(exc).__name__,
"error": str(exc),
}
_json_dump(args.output, result)
print(json.dumps(result, indent=2))
return 1
_json_dump(args.output, result)
print(json.dumps(result, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())